commit 6d978fe483f3f8a528ee92136bb8370ab7c47451 Author: wehub-resource-sync Date: Mon Jul 13 12:35:24 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..85f75e4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.gitignore +.DS_Store +node_modules +frontend/node_modules +frontend/dist +backend/bin +backend/.cache +dev_docs +docs +scripts +*.log \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfdb8b7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/.github/scripts/compute_image_fingerprint.py b/.github/scripts/compute_image_fingerprint.py new file mode 100644 index 0000000..bcc86d5 --- /dev/null +++ b/.github/scripts/compute_image_fingerprint.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Compute a stable fingerprint for the runtime contents of a local Docker image.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path + + +def run(*args: str) -> str: + completed = subprocess.run( + list(args), + check=True, + capture_output=True, + text=True, + ) + return completed.stdout + + +def hash_stream(fileobj) -> str: + digest = hashlib.sha256() + while True: + chunk = fileobj.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def normalized_config(image_ref: str) -> bytes: + inspect_payload = json.loads(run("docker", "image", "inspect", image_ref))[0] + config_payload = { + "architecture": inspect_payload.get("Architecture"), + "os": inspect_payload.get("Os"), + "variant": inspect_payload.get("Variant"), + "config": inspect_payload.get("Config", {}), + } + return json.dumps( + config_payload, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def normalized_tar_entries(tar_path: Path) -> list[str]: + records: list[str] = [] + with tarfile.open(tar_path, mode="r") as archive: + for member in archive.getmembers(): + path = member.name.rstrip("/") or "." + record: dict[str, object] = { + "path": path, + "mode": member.mode, + } + + if member.isdir(): + record["type"] = "dir" + elif member.isfile(): + extracted = archive.extractfile(member) + if extracted is None: + raise RuntimeError(f"Unable to read file contents for {path}") + with extracted: + record["type"] = "file" + record["sha256"] = hash_stream(extracted) + elif member.issym(): + record["type"] = "symlink" + record["target"] = member.linkname + elif member.islnk(): + record["type"] = "hardlink" + record["target"] = member.linkname + elif member.ischr(): + record["type"] = "char" + record["device"] = [member.devmajor, member.devminor] + elif member.isblk(): + record["type"] = "block" + record["device"] = [member.devmajor, member.devminor] + elif member.isfifo(): + record["type"] = "fifo" + else: + record["type"] = f"other:{member.type!r}" + if member.linkname: + record["target"] = member.linkname + + records.append( + json.dumps(record, sort_keys=True, separators=(",", ":")) + ) + + records.sort() + return records + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: compute_image_fingerprint.py ", file=sys.stderr) + return 2 + + image_ref = sys.argv[1] + container_id = run("docker", "create", image_ref).strip() + + try: + with tempfile.TemporaryDirectory(prefix="clawmanager-image-export-") as temp_dir: + tar_path = Path(temp_dir) / "filesystem.tar" + subprocess.run( + ["docker", "export", "-o", str(tar_path), container_id], + check=True, + ) + + digest = hashlib.sha256() + digest.update(normalized_config(image_ref)) + digest.update(b"\n") + + for record in normalized_tar_entries(tar_path): + digest.update(record.encode("utf-8")) + digest.update(b"\n") + + print(digest.hexdigest()) + finally: + subprocess.run( + ["docker", "rm", "-f", container_id], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..e64f70a --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,213 @@ +name: Build + +on: + push: + branches: + - main + - 'codex/**' + - 'feature/**' + - 'fix/**' + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + frontend: + name: Frontend Build + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Build frontend + run: npm run build + + backend: + name: Backend Build + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum + + - name: Download dependencies + run: go mod download + + - name: Build backend + run: go build -o bin/server ./cmd/server + + - name: Run backend tests + run: go test ./... + + docker: + name: Docker Build + runs-on: ubuntu-latest + needs: + - frontend + - backend + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build multi-platform application image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + push: false + platforms: linux/amd64,linux/arm64 + tags: clawmanager:ci + + k8s-smoke: + name: Kubernetes Deploy Smoke Test + runs-on: ubuntu-latest + needs: + - frontend + - backend + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build local application image + run: docker build -t clawmanager:ci . + + - name: Create kind cluster + uses: helm/kind-action@v1.12.0 + with: + cluster_name: clawmanager-ci + ignore_failed_clean: true + + - name: Load image into kind + run: kind load docker-image clawmanager:ci --name clawmanager-ci + + - name: Deploy manifest to cluster + shell: bash + run: | + kubectl label node clawmanager-ci-control-plane clawmanager.io/storage-node=true --overwrite + cp deployments/k8s/single-node/clawmanager.yaml /tmp/clawmanager-ci.yaml + + python - <<'PY' + from pathlib import Path + import re + + path = Path("/tmp/clawmanager-ci.yaml") + text = path.read_text(encoding="utf-8") + + text = text.replace( + "image: ghcr.io/yuan-lab-llm/clawmanager:latest", + "image: clawmanager:ci", + ) + + text = re.sub( + r"(kind: Deployment\s+metadata:\s+name: mysql\s+namespace: clawmanager-system\s+spec:\s+replicas: 1\s+selector:\s+matchLabels:\s+app: mysql\s+template:\s+metadata:\s+labels:\s+app: mysql\s+spec:\s+containers:\s+- name: mysql.*?volumeMounts:\s+- name: mysql-data\s+mountPath: /var/lib/mysql\s+- name: mysql-init\s+mountPath: /docker-entrypoint-initdb.d\s+readinessProbe:.*?periodSeconds: 10\s+)volumes:\s+- name: mysql-data\s+persistentVolumeClaim:\s+claimName: mysql-data\s+- name: mysql-init\s+configMap:\s+name: clawmanager-mysql-init", + r"\1volumes:\n - name: mysql-data\n emptyDir: {}\n - name: mysql-init\n configMap:\n name: clawmanager-mysql-init", + text, + flags=re.S, + ) + + text = re.sub( + r"(kind: Deployment\s+metadata:\s+name: clawmanager-app\s+namespace: clawmanager-system\s+spec:\s+)replicas: 1", + r"\1replicas: 0", + text, + count=1, + flags=re.S, + ) + + text = re.sub( + r"(kind: Service\s+metadata:\s+name: clawmanager-frontend\s+namespace: clawmanager-system\s+spec:\s+)type: NodePort", + r"\1type: ClusterIP", + text, + count=1, + flags=re.S, + ) + + text = re.sub( + r"\n\s+nodePort:\s+\d+", + "", + text, + count=1, + ) + + text = re.sub( + r"(\n\s+- name: workspaces\s+)nfs:\s+server: workspace-store\.clawmanager-system\.svc\.cluster\.local\s+path: /exports/workspaces", + r"\1emptyDir: {}", + text, + ) + + path.write_text(text, encoding="utf-8") + PY + + kubectl apply -f /tmp/clawmanager-ci.yaml + + - name: Wait for MySQL deployment + run: kubectl -n clawmanager-system rollout status deployment/mysql --timeout=300s + + - name: Start ClawManager after MySQL is ready + run: kubectl -n clawmanager-system scale deployment/clawmanager-app --replicas=1 + + - name: Wait for ClawManager deployment + run: kubectl -n clawmanager-system rollout status deployment/clawmanager-app --timeout=300s + + - name: Show cluster status on failure + if: failure() + run: | + kubectl get pods -A || true + kubectl get pvc -A || true + kubectl get events -A --sort-by=.lastTimestamp || true + kubectl -n clawmanager-system describe deployment mysql || true + kubectl -n clawmanager-system describe deployment clawmanager-app || true + kubectl -n clawmanager-system describe pod -l app=mysql || true + kubectl -n clawmanager-system describe pod -l app=clawmanager-app || true + kubectl -n clawmanager-system logs deployment/mysql --tail=200 || true + kubectl -n clawmanager-system logs deployment/clawmanager-app --tail=200 || true + + - name: Start port-forward + run: | + kubectl -n clawmanager-system port-forward svc/clawmanager-frontend 8443:443 > /tmp/clawmanager-port-forward.log 2>&1 & + echo $! > /tmp/clawmanager-port-forward.pid + sleep 10 + + - name: Verify health endpoint + run: curl --fail --insecure https://127.0.0.1:8443/healthz + + - name: Verify web page is reachable + run: curl --fail --insecure https://127.0.0.1:8443/ | grep -i "" + + - name: Stop port-forward + if: always() + run: | + if [ -f /tmp/clawmanager-port-forward.pid ]; then + kill "$(cat /tmp/clawmanager-port-forward.pid)" || true + fi diff --git a/.github/workflows/manual-lgtm.yml b/.github/workflows/manual-lgtm.yml new file mode 100644 index 0000000..5428c82 --- /dev/null +++ b/.github/workflows/manual-lgtm.yml @@ -0,0 +1,55 @@ +name: Manual LGTM + +on: + issue_comment: + types: + - created + +permissions: + contents: read + issues: read + pull-requests: read + statuses: write + +jobs: + lgtm: + name: Apply Manual LGTM + if: github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/lgtm') + runs-on: ubuntu-latest + steps: + - name: Validate commenter permission and set status + uses: actions/github-script@v7 + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const issueNumber = context.issue.number; + const commenter = context.payload.comment.user.login; + + const permission = await github.rest.repos.getCollaboratorPermissionLevel({ + owner, + repo, + username: commenter, + }); + + const allowed = new Set(['admin', 'maintain', 'write']); + if (!allowed.has(permission.data.permission)) { + core.notice(`Skipping /lgtm from ${commenter}: permission=${permission.data.permission}`); + return; + } + + const pull = await github.rest.pulls.get({ + owner, + repo, + pull_number: issueNumber, + }); + + await github.rest.repos.createCommitStatus({ + owner, + repo, + sha: pull.data.head.sha, + state: 'success', + context: 'Manual LGTM', + description: `Approved by /lgtm from ${commenter}`, + target_url: context.payload.comment.html_url, + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a41f66e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,243 @@ +name: Release + +on: + schedule: + # 09:00 Asia/Shanghai = 01:00 UTC + - cron: '0 1 * * *' + workflow_dispatch: + inputs: + force_release: + description: Force create a release even if the image fingerprint is unchanged + required: false + default: false + type: boolean + push: + tags: + - 'v*' + +permissions: + contents: write + packages: write + +jobs: + prepare-scheduled-release: + name: Prepare Scheduled Release + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + outputs: + should_release: ${{ steps.decision.outputs.should_release }} + tag_name: ${{ steps.decision.outputs.tag_name }} + image_fingerprint: ${{ steps.compute-fingerprint.outputs.image_fingerprint }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build amd64 candidate image + shell: bash + run: | + docker buildx build --pull --platform linux/amd64 --load -t clawmanager:release-candidate-amd64 . + + - name: Build arm64 candidate image + shell: bash + run: | + docker buildx build --pull --platform linux/arm64 --load -t clawmanager:release-candidate-arm64 . + + - name: Compute release fingerprint + id: compute-fingerprint + shell: bash + run: | + AMD64_FINGERPRINT="$(python3 .github/scripts/compute_image_fingerprint.py clawmanager:release-candidate-amd64)" + ARM64_FINGERPRINT="$(python3 .github/scripts/compute_image_fingerprint.py clawmanager:release-candidate-arm64)" + IMAGE_FINGERPRINT="$(printf '%s\n%s\n' "${AMD64_FINGERPRINT}" "${ARM64_FINGERPRINT}" | sha256sum | cut -d' ' -f1)" + echo "IMAGE_FINGERPRINT=${IMAGE_FINGERPRINT}" >> "$GITHUB_ENV" + echo "image_fingerprint=${IMAGE_FINGERPRINT}" >> "$GITHUB_OUTPUT" + + - name: Read latest release fingerprint + id: latest-release + uses: actions/github-script@v7 + with: + script: | + try { + const release = await github.rest.repos.getLatestRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + const body = release.data.body || ""; + const match = body.match(/Build-Fingerprint:\s*`?([^\s`]+)`?/); + core.setOutput("tag", release.data.tag_name || ""); + core.setOutput("fingerprint", match ? match[1] : ""); + } catch (error) { + if (error.status === 404) { + core.setOutput("tag", ""); + core.setOutput("fingerprint", ""); + return; + } + throw error; + } + + - name: Decide whether to create a new release tag + id: decision + shell: bash + env: + FORCE_RELEASE: ${{ github.event.inputs.force_release || 'false' }} + LAST_FINGERPRINT: ${{ steps.latest-release.outputs.fingerprint }} + run: | + CURRENT_FINGERPRINT="${IMAGE_FINGERPRINT}" + TAG_NAME="v$(date -u +'%Y.%-m.%-d')" + + LAST_FINGERPRINT="${LAST_FINGERPRINT//\`/}" + CURRENT_FINGERPRINT="${CURRENT_FINGERPRINT//\`/}" + + if git rev-parse "$TAG_NAME" >/dev/null 2>&1; then + echo "should_release=false" >> "$GITHUB_OUTPUT" + echo "tag_name=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ "$FORCE_RELEASE" = "true" ]; then + echo "should_release=true" >> "$GITHUB_OUTPUT" + echo "tag_name=$TAG_NAME" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ -n "$LAST_FINGERPRINT" ] && [ "$LAST_FINGERPRINT" = "$CURRENT_FINGERPRINT" ]; then + echo "should_release=false" >> "$GITHUB_OUTPUT" + echo "tag_name=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "should_release=true" >> "$GITHUB_OUTPUT" + echo "tag_name=$TAG_NAME" >> "$GITHUB_OUTPUT" + + - name: Create and push release tag + if: steps.decision.outputs.should_release == 'true' + shell: bash + run: | + TAG_NAME="${{ steps.decision.outputs.tag_name }}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG_NAME" -m "Release $TAG_NAME" + git push origin "$TAG_NAME" + + release: + name: Publish Release + needs: + - prepare-scheduled-release + if: | + always() && ( + github.event_name == 'push' || + needs.prepare-scheduled-release.outputs.should_release == 'true' + ) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set release metadata + shell: bash + run: | + REPOSITORY_OWNER_LOWER="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" + if [ "${{ github.event_name }}" = "push" ]; then + echo "TAG_NAME=${GITHUB_REF_NAME}" >> "$GITHUB_ENV" + else + echo "TAG_NAME=${{ needs.prepare-scheduled-release.outputs.tag_name }}" >> "$GITHUB_ENV" + fi + echo "IMAGE_URI=ghcr.io/${REPOSITORY_OWNER_LOWER}/clawmanager" >> "$GITHUB_ENV" + + - name: Checkout release tag + if: github.event_name != 'push' + run: git checkout "${TAG_NAME}" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build amd64 release candidate + shell: bash + run: | + docker buildx build --pull --platform linux/amd64 --load -t clawmanager:release-candidate-amd64 . + + - name: Build arm64 release candidate + shell: bash + run: | + docker buildx build --pull --platform linux/arm64 --load -t clawmanager:release-candidate-arm64 . + + - name: Compute release fingerprint + shell: bash + run: | + if [ -n "${{ needs.prepare-scheduled-release.outputs.image_fingerprint }}" ]; then + echo "IMAGE_FINGERPRINT=${{ needs.prepare-scheduled-release.outputs.image_fingerprint }}" >> "$GITHUB_ENV" + exit 0 + fi + + AMD64_FINGERPRINT="$(python3 .github/scripts/compute_image_fingerprint.py clawmanager:release-candidate-amd64)" + ARM64_FINGERPRINT="$(python3 .github/scripts/compute_image_fingerprint.py clawmanager:release-candidate-arm64)" + IMAGE_FINGERPRINT="$(printf '%s\n%s\n' "${AMD64_FINGERPRINT}" "${ARM64_FINGERPRINT}" | sha256sum | cut -d' ' -f1)" + echo "IMAGE_FINGERPRINT=${IMAGE_FINGERPRINT}" >> "$GITHUB_ENV" + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push multi-platform release image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + push: true + platforms: linux/amd64,linux/arm64 + tags: | + ${{ env.IMAGE_URI }}:${{ env.TAG_NAME }} + ${{ env.IMAGE_URI }}:latest + + - name: Read pushed image digest + shell: bash + run: | + IMAGE_DIGEST="$(docker buildx imagetools inspect "${IMAGE_URI}:${TAG_NAME}" | sed -n 's/^Digest:[[:space:]]*//p' | head -n1)" + if [ -z "${IMAGE_DIGEST}" ]; then + echo "Failed to resolve image digest for ${IMAGE_URI}:${TAG_NAME}" >&2 + exit 1 + fi + echo "IMAGE_DIGEST=${IMAGE_DIGEST}" >> "$GITHUB_ENV" + + - name: Build release notes + shell: bash + run: | + cat < /tmp/release-highlights.md + ## Highlights + + - Added automated release flow for scheduled checks and \`v*\` tag publishing. + - Added in-app delete confirmation dialogs instead of browser-native prompts. + - Fixed WebSocket authentication so real-time status updates can connect in browser environments. + - Fixed Linux container startup script handling and enforced LF line endings for shell scripts. + + ## Release Assets + + Image: \`${IMAGE_URI}:${TAG_NAME}\` + Latest: \`${IMAGE_URI}:latest\` + Platforms: \`linux/amd64\`, \`linux/arm64\` + Image-Digest: \`${IMAGE_DIGEST}\` + Build-Fingerprint: \`${IMAGE_FINGERPRINT}\` + EOF + + - name: Create GitHub release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ env.TAG_NAME }} + generate_release_notes: true + body_path: /tmp/release-highlights.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cf457db --- /dev/null +++ b/.gitignore @@ -0,0 +1,85 @@ +# Binaries +*.exe +*.exe~ +*.dll +*.so +*.dylib +/backend/bin/ +/backend/tmp/ + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool +*.out + +# Go vendor directory +/backend/vendor/ + +# Node modules +/frontend/node_modules/ +/frontend/dist/ +/frontend/dist-ssr/ +/outputs/ +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +# Environment files +.env +.env.local +.env.*.local + +# Database +*.db +*.sqlite + +# OS files +Thumbs.db + +# Temporary files +*.tmp +*.temp +/test_* +/deploy/ +/test.tar +/fix-install-plan.md +/docs/deployment-storage-remediation-plan.zh-CN.md + +# Team runtime config snapshots +/team-*.json +/agency-agents-main/ + + +.codex-logs/ +.codex-temp/ +.cache/ +.VSCodeCounter/ +.superpowers/ +dev_docs/ +devplan.md +AGENTS.md +TASK_BREAKDOWN.md +dev_progress.md +CLAUDE.md +TEAM_IMAGE_DEPLOY_STEPS.md +TEAM_PROFILES_LOCAL_TESTING.md +/team-optimization-docs/ +/clawmanager-apply.sh +/clawmanager-team-profiles-test.yaml +/clawmanager-tenant.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..15e4982 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,51 @@ +FROM --platform=$BUILDPLATFORM node:20-alpine AS frontend-builder + +WORKDIR /app/frontend + +COPY frontend/package*.json ./ +RUN npm ci + +COPY frontend/ ./ +RUN npm run build + +FROM --platform=$BUILDPLATFORM golang:1.26.1-alpine AS backend-builder + +WORKDIR /app/backend + +ARG TARGETOS +ARG TARGETARCH + +ARG GOPROXY +ARG GOSUMDB +ARG GOFLAGS +ENV GOFLAGS=${GOFLAGS} + +RUN apk add --no-cache git + +COPY backend/go.mod backend/go.sum ./ +RUN go mod download + +COPY backend/ ./ +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -trimpath -buildvcs=false -ldflags="-s -w -buildid=" -o /out/clawreef-server ./cmd/server + +FROM nginx:1.27-alpine + +# nginx-module-njs provides ngx_http_js_module.so, loaded by nginx.conf to run +# the desktop access-token verification (deployments/nginx/njs/desktop_auth.js). +RUN apk add --no-cache dumb-init openssl nginx-module-njs + +WORKDIR /app + +COPY --from=backend-builder /out/clawreef-server /usr/local/bin/clawreef-server +COPY --from=frontend-builder /app/frontend/dist /usr/share/nginx/html +COPY deployments/nginx/nginx.conf /etc/nginx/nginx.conf +COPY deployments/nginx/njs/desktop_auth.js /etc/nginx/njs/desktop_auth.js +COPY deployments/container/start.sh /app/start.sh + +RUN chmod +x /app/start.sh \ + && mkdir -p /etc/nginx/tls /var/log/clawreef + +EXPOSE 8443 + +ENTRYPOINT ["dumb-init", "--"] +CMD ["/app/start.sh"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6512716 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 ClawManager contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.de.md b/README.de.md new file mode 100644 index 0000000..072df20 --- /dev/null +++ b/README.de.md @@ -0,0 +1,293 @@ +# ClawManager + +

+ ClawManager +

+ +

+ ClawManager ist eine Kubernetes-native Control Plane fuer die Verwaltung von AI-Agent-Instanzen mit kontrolliertem AI-Zugriff, Runtime-Orchestrierung und wiederverwendbaren Ressourcen ueber mehrere Agent-Runtimes hinweg. +

+ +

+ Sprachen: + English | + 简体中文 | + 日本語 | + 한국어 | + Deutsch +

+ +

+ ClawManager Control Plane + Go 1.21+ + React 19 + Kubernetes Native + MIT License + + ClawManager Discord-Community beitreten + +

+ +

+ Produktueberblick | + Team Workspaces | + AI Gateway | + Agent Control Plane | + Runtime-Integrationen | + Ressourcenverwaltung | + Erste Schritte +

+ +

+ + Star ClawManager on GitHub + +

+ +

ClawManager in 60 Sekunden

+ +

+ClawManager Produktdemo +

+ +

+ Ein schneller Blick auf Agent-Provisionierung, Skill-Verwaltung und -Scanning sowie AI-Gateway-Governance. +

+ +## Neuigkeiten + +Wichtige aktuelle Produkt- und Dokumentations-Updates. + +- [2026-07-07] Security Protection Platform (secplane) Frontend-Konsole hinzugefuegt — umfassende Sicherheitskonsole mit Runtime-Abwehr (Eingabe-/Zustands-/Entscheidungs-/Ausgabeoberflaeche, Asset-Schutz, menschliche Freigabe), Host-Haertung und Container-Isolierung, Outbound-Vertrauens-Governance, Richtlinien-Governance, Kill-Switch/Circuit-Breaker, Full-Chain-Audit, SecureClaw-Daten- und Komponentenvertrauens-Audit, Kollaborations-Governance und Eingabeerkennung. 4 Verteidigungsschichten in einer einheitlichen Admin-UI mit vollstaendiger i18n fuer 5 Sprachen. +- [2026-06-14] Lite-/Pro-Runtime-Modi und Rollout-Support hinzugefuegt: Lite-Instanzen laufen ueber gemeinsame Gateway-Runtime-Pools, waehrend Pro-Instanzen dedizierte Desktop-Deployments fuer staerkere Isolation behalten. +- [2026-05-18] Team-Workspace-MVP mit Einfuehrung und Vorschau hinzugefuegt, inklusive One-Click-Team-Erstellung, OpenClaw-Member-Orchestrierung, Redis-Team-Bus-Injection, Shared Storage, Member-Status, Task-Dispatch sowie Event- und Ergebnisansichten. +- [2026-04-29] Hermes-Runtime-Integration hinzugefuegt, inklusive Webtop-basierter Instanzbereitstellung, Agent-Control-Plane-Registrierung, AI-Gateway-Injection, channel- und skill-Bootstrap sowie `.hermes` Import/Export. Siehe [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md). +- [2026-04-08] Skill-Verwaltung und Skill-Scanning wurden der Plattform hinzugefuegt. Details siehe [Merged PR #52](https://github.com/Yuan-lab-LLM/ClawManager/pull/52). +- [2026-03-26] Die AI-Gateway-Dokumentation wurde erweitert und deckt nun Modell-Governance, Audit und Trace, Kostenrechnung sowie Risikokontrolle genauer ab. Siehe [AI Gateway Guide](./docs/aigateway.md). +- [2026-03-20] ClawManager hat sich zu einer breiteren Control Plane fuer AI-Agent-Workspaces entwickelt, mit staerkerer Runtime-Steuerung, wiederverwendbaren Ressourcen und Security-Scanning-Workflows. + +> Wenn ClawManager fuer dein Team nuetzlich ist, gib dem Projekt gerne einen Star, damit mehr Nutzer und Entwickler es entdecken. + +

+ +Star ClawManager on GitHub + +

+ +## Community + +Tritt der ClawManager Open-Source-Community auf WeChat oder Discord bei, um Produkt-Updates zu verfolgen, Nutzungserfahrungen auszutauschen und mit Mitwirkenden ins Gespraech zu kommen. + + + + + + +
+ QR-Code zur ClawManager WeChat-Gruppe +

+ WeChat +
+ QR-Code scannen, um der WeChat-Gruppe beizutreten +
+ QR-Code zur ClawManager Discord-Einladung +

+ Discord +
+ QR-Code scannen, um unserem Discord-Server beizutreten +
+ + +## Produktueberblick + +ClawManager bringt den Betrieb von AI-Agent-Instanzen auf Kubernetes und legt darauf drei hoeherwertige Control Planes. Teams koennen damit AI-Zugriff steuern, Runtime-Verhalten ueber Agents orchestrieren und Workspace-Faehigkeiten ueber scanbare und wiederverwendbare channel- und skill-Ressourcen bereitstellen. + +Es eignet sich besonders fuer: + +- Plattformteams, die AI-Agent-Instanzen fuer mehrere Nutzer betreiben +- Betriebsteams, die Runtime-Sichtbarkeit, Command-Dispatch und Desired-State-Kontrolle benoetigen +- Entwicklungsteams, die Agent-Workspaces ueber wiederverwendbare Ressourcen statt ueber manuelle Konfiguration bereitstellen wollen + + +## Team Workspaces + +Team Workspaces bieten einen vereinfachten OpenClaw-Lite-Kollaborationsablauf: Rollenvorlage auswaehlen, Team erstellen und das Ziel im Team-Chat beschreiben. Der Leader plant, koordiniert die Mitglieder, sammelt Ergebnisse und liefert die finale Zusammenfassung. + +- feste Leader-vermittelte Zusammenarbeit ohne Runtime- oder Ressourcenprofil je Mitglied +- integrierte Vorlagen fuer Auslieferung, Produkterkundung und Softwareentwicklung +- Team-Chat fuer Plan, Zuweisung, Fortschritt, Review, Lieferung und Zusammenfassung +- Execution Kanban fuer Gesamtaufgabe und aktuelle Mitgliederlieferungen + +Siehe [Team Workspace Quick Guide](./docs/team-workspaces-guide_de.md) fuer Erstellung, Kollaborationsphasen und Ergebnisansicht. + + +## Runtime-Integrationen + +ClawManager unterstuetzt derzeit die folgenden verwalteten Runtimes: + +- OpenClaw icon `OpenClaw`: die standardmaessige OpenClaw-artige Workspace-Runtime fuer von ClawManager verwaltete Desktop-Instanzen +- Hermes icon `Hermes`: eine Webtop-basierte Runtime-Integration mit persistentem `.hermes`-Workspace und eingebettetem Hermes agent + +Runtime-Vorschau: + +**OpenClaw icon OpenClaw** + +![openclaw](./docs/images/openclaw.png) + +**Hermes icon Hermes** + +![hermes](./docs/images/hermes.png) + +Runtime-Autoren koennen dem [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md), dem [Generic Runtime Agent Integration Guide](./docs/runtime-agent-integration-guide.md) und der [Skill Content MD5 Spec](./docs/skill-content-md5-spec.md) folgen, um kompatible Agents zu bauen. + + +## Erste Schritte + +ClawManager bietet jetzt klarere Einstiegspfade sowohl fuer Standard-Kubernetes als auch fuer leichtere Cluster-Setups. Zum Evaluieren der Plattform ist es am sinnvollsten, zuerst den passenden Deployment-Pfad fuer die eigene Umgebung zu waehlen und danach dem First-Use-Flow zu folgen. + +- Standard-Kubernetes-Deployment: [deployments/k8s/clawmanager.yaml](./deployments/k8s/clawmanager.yaml) +- K3s / leichtgewichtiges Deployment: [deployments/k3s/clawmanager.yaml](./deployments/k3s/clawmanager.yaml) +- First-Login- und Schnellstart-Ablauf: [Benutzerhandbuch](./docs/use_guide_de.md) +- Deployment-Hinweise und Architekturkontext: [Deployment Guide (English)](./docs/deployment.md) + +## Drei Control Planes + + +### AI Gateway + +AI Gateway ist die Governance-Ebene fuer Modellzugriffe in ClawManager. Es stellt verwalteten Agent-Runtimes einen einheitlichen OpenAI-kompatiblen Einstiegspunkt bereit und legt Richtlinien-, Audit- und Kostenkontrollen ueber die Upstream-Provider. + +- Einheitlicher Einstiegspunkt fuer Modell-Traffic +- Sichere Modell-Routing-Logik und policy-gesteuerte Modellauswahl +- End-to-End-Audit- und Trace-Aufzeichnungen +- Integrierte Kostenrechnung und Nutzungsanalyse +- Regeln fuer Risikokontrolle mit Block- oder Umleitungslogik + +Siehe [AI Gateway Guide (English)](./docs/aigateway.md). + + +### Agent Control Plane + +Agent Control Plane ist die Runtime-Orchestrierungsschicht fuer verwaltete AI-Agent-Instanzen. Jede Instanz wird damit zu einer verwalteten Runtime, die sich registrieren, Status melden, Commands empfangen und sich am Desired State der Plattform ausrichten kann. + +- Agent-Registrierung mit sicherem Bootstrap und Session-Lifecycle +- Heartbeat-basierte Runtime-Status- und Health-Reports +- Desired-State-Synchronisierung zwischen Control Plane und Instanz +- Command-Dispatch fuer Start, Stop, Konfigurationsanwendung, Health Checks und Skill-Operationen +- Sichtbarkeit pro Instanz fuer Agent-Status, channel, skill und Command-Historie + +Siehe [Agent Control Plane Guide (English)](./docs/agent-control-plane.md). + + +### Ressourcenverwaltung + +Ressourcenverwaltung ist die wiederverwendbare Asset-Schicht fuer AI-Agent-Workspaces. Teams koennen channel und skill vorbereiten, zu bundles zusammensetzen, in Instanzen injizieren und Security-Reviews direkt in diesen Ablauf integrieren. + +- `Channel`-Verwaltung fuer Workspace-Konnektivitaet und Integrationsvorlagen +- `Skill`-Verwaltung fuer wiederverwendbare Faehigkeitspakete +- `Skill Scanner`-Workflows fuer Risikoanalyse und Scan-Jobs +- Bundle-basierte Ressourcenzusammenstellung fuer reproduzierbare Setups +- Injection-Snapshots zur Nachverfolgung der tatsaechlich angewendeten Inhalte + +Siehe [Resource Management Guide (English)](./docs/resource-management.md) und [Security / Skill Scanner Guide (English)](./docs/security-skill-scanner.md). + +## Produktgalerie + +ClawManager ist so gestaltet, dass Administration, Zugriff und AI-Governance nicht wie getrennte Werkzeuge wirken, sondern wie eine zusammenhaengende Produkterfahrung. + +### Lite-Mode-Deployment + +Lite Mode stellt Instanzen ueber einen gemeinsamen Gateway-Runtime-Pool bereit. Jeder Workspace laeuft als isolierter Gateway-Prozess in verwalteten Runtime-Pods. Das sorgt fuer schnelle Starts und reduziert dedizierte CPU-, Memory-, Storage- und GPU-Allocation, waehrend Workspace-Zugriff, Share Link / Password Access, channel- und skill-Injection sowie Admin-Sichtbarkeit erhalten bleiben. + +![](./docs/main/liteopenclaw.png) + +### Pro-Mode-Deployment + +Pro Mode stellt fuer jede Instanz eine dedizierte Desktop-Runtime bereit, gestuetzt durch ein eigenes Kubernetes Deployment, einen Service und eine PVC. Es eignet sich fuer Nutzer, die staerkere Isolation, volle Desktop-Ressourcen, Runtime Events, Instanz-Skill-Verwaltung und die vollstaendige Desktop-Management-Erfahrung benoetigen. + +![](./docs/main/proopenclaw.png) + +### Team Workspace + +Die Team-Workspace-Seite bringt Leader-Desktop, Team-Chat, Member-Tabelle und Dispatch-Workflow in eine gemeinsame Betriebsansicht, damit Nutzer den Kollaborationsfortschritt direkt in ClawManager verfolgen koennen. + +

+ ClawManager Team Workspace +

+ +### Admin Console + +Die Admin-Konsole vereint Nutzer, Quotas, Runtime-Operationen, Security-Kontrollen und plattformweite Richtlinien in einer Oberflaeche. Sie ist die zentrale Arbeitsflaeche fuer Teams, die AI-Agent-Infrastruktur im grossen Massstab betreiben. + +

+ ClawManager Admin Console +

+ +### Portal Access + +Das Portal bietet Nutzern einen klaren Einstiegspunkt in ihre Workspaces. Der Zugriff erfolgt browserbasiert, waehrend Runtime-Zustand und Plattformsicht erhalten bleiben, ohne dass Infrastrukturdetails direkt exponiert werden. + +

+ ClawManager Portal Access +

+ +### AI Gateway + +AI Gateway integriert Modell-Governance direkt in die Workspace-Erfahrung. Audit-Trails, Kostentransparenz und risikobasiertes Routing machen AI-Nutzung zu einem Teil der Plattform statt zu einer losen Einzelintegration. + +

+ ClawManager AI Gateway +

+ +## So funktioniert es + +1. Administratoren definieren Governance-Richtlinien und wiederverwendbare Ressourcen. +2. Nutzer erstellen oder betreten verwaltete AI-Agent-Workspaces auf Kubernetes. +3. Team Workspaces koennen mehrere Member-Runtimes mit Redis Team Bus und Shared-Storage-Konfiguration bereitstellen. +4. Agents verbinden sich mit der Control Plane und melden Runtime-Zustaende. +5. Channel, skill und bundle werden kompiliert und auf Instanzen angewendet. +6. AI-Traffic fliesst ueber das AI Gateway und erhaelt Audit-, Risiko- und Kostenkontrollen. + +## Entwicklerueberblick + +ClawManager ist eine Kubernetes-native Plattform mit React-Frontend, Go-Backend, MySQL fuer Zustandsdaten sowie Integrationen wie `skill-scanner` und Object Storage. Die Codebasis ist nach Produktsubsystemen organisiert, daher ist der schnellste Einstieg, mit dem passenden Guide zu beginnen und danach in den Code zu gehen. + +- Frontend fuer Admin- und Nutzeroberflaechen unter `frontend/` +- Backend-Services, Handler, Repositorys und Migrationen unter `backend/` +- Deployment-Assets unter `deployments/` +- Produktdokumentation und Medien unter `docs/` + +Siehe [Developer Guide (English)](./docs/developer-guide.md). + +## Dokumentation + +- [Benutzerhandbuch](./docs/use_guide_de.md) +- [Team Workspace Quick Guide](./docs/team-workspaces-guide_de.md) +- [Deployment Guide (English)](./docs/deployment.md) +- [Admin and User Guide (English)](./docs/admin-user-guide.md) +- [Agent Control Plane Guide (English)](./docs/agent-control-plane.md) +- [AI Gateway Guide (English)](./docs/aigateway.md) +- [Security / Skill Scanner Guide (English)](./docs/security-skill-scanner.md) +- [Resource Management Guide (English)](./docs/resource-management.md) +- [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md) +- [Generic Runtime Agent Integration Guide](./docs/runtime-agent-integration-guide.md) +- [Skill Content MD5 Spec](./docs/skill-content-md5-spec.md) +- [Developer Guide (English)](./docs/developer-guide.md) + +## Lizenz + +Dieses Projekt steht unter der MIT License. + +## Open Source + +Issues und Pull Requests sind willkommen. + +## Star History + + + + + + Star History Chart + + diff --git a/README.ja.md b/README.ja.md new file mode 100644 index 0000000..ccd8186 --- /dev/null +++ b/README.ja.md @@ -0,0 +1,293 @@ +# ClawManager + +

+ ClawManager +

+ +

+ ClawManager は、AI エージェントインスタンス管理のための Kubernetes ネイティブなコントロールプレーンです。ガバナンス付きの AI アクセス、ランタイムオーケストレーション、そして複数の Agent Runtime にまたがる再利用可能なリソース管理を提供します。 +

+ +

+ 言語: + English | + 简体中文 | + 日本語 | + 한국어 | + Deutsch +

+ +

+ ClawManager Control Plane + Go 1.21+ + React 19 + Kubernetes Native + MIT License + + ClawManager Discord コミュニティに参加 + +

+ +

+ 製品紹介 | + Team ワークスペース | + AI Gateway | + Agent Control Plane | + Runtime 連携 | + リソース管理 | + はじめに +

+ +

+ + Star ClawManager on GitHub + +

+ +

60 秒でわかる ClawManager

+ +

+ClawManager 製品デモ +

+ +

+ エージェントの高速プロビジョニング、Skill 管理とスキャン、AI Gateway ガバナンスを短時間で確認できます。 +

+ +## 最新情報 + +最近の重要な製品アップデートとドキュメント更新です。 + +- [2026-07-07] セキュリティ保護プラットフォーム(secplane)フロントエンドコンソールを追加しました。ランタイム防御(入力/状態/決定/出力サーフェス、資産改ざん防止、ヒューマン承認)、ホスト強化とコンテナ分離、アウトバウンド信頼エンドポイントガバナンス、ポリシーガバナンス、キルスイッチ/サーキットブレーカー、フルチェーン監査、SecureClaw データ・コンポーネント信頼監査、コラボレーションガバナンス、入力検出をカバーする4層防御の統合管理UIを5言語i18nで提供します。 +- [2026-06-14] Lite / Pro ランタイムモードとロールアウト対応を追加しました。Lite インスタンスは共有 gateway runtime pool で動作し、Pro インスタンスはより強い分離のため専用 desktop deployment を維持します。 +- [2026-05-18] Team ワークスペース MVP の紹介とプレビューを追加しました。ワンクリック Team 作成、OpenClaw メンバーのオーケストレーション、Redis Team Bus 注入、共有ストレージ、メンバー状態、タスク配布、イベント/結果ビューをカバーします。 +- [2026-04-29] Hermes Runtime 連携を追加しました。Webtop ベースのインスタンス作成、Agent Control Plane 登録、AI Gateway 注入、channel と skill のブートストラップ、`.hermes` のインポート/エクスポートに対応しています。詳しくは [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md) を参照してください。 +- [2026-04-08] プラットフォームに Skill 管理と Skill スキャンのワークフローを追加しました。詳細は [Merged PR #52](https://github.com/Yuan-lab-LLM/ClawManager/pull/52) を参照してください。 +- [2026-03-26] AI Gateway ドキュメントを更新し、モデルガバナンス、監査とトレース、コスト計算、リスク制御の説明を強化しました。詳しくは [AI Gateway Guide](./docs/aigateway.md) を参照してください。 +- [2026-03-20] ClawManager は、AI エージェントワークスペース向けのより広いコントロールプレーンへと進化し、ランタイム制御、再利用可能なリソース、安全スキャンのワークフローを強化しました。 + +> ClawManager があなたのチームに役立つなら、ぜひ Star を付けて、より多くのユーザーや開発者に届くよう応援してください。 + +

+ +Star ClawManager on GitHub + +

+ +## コミュニティ + +ClawManager オープンソースコミュニティに WeChat または Discord から参加してください。プロダクト更新の確認、使い方の相談、コントリビューター同士の交流にご活用ください。 + + + + + + +
+ ClawManager WeChatグループのQRコード +

+ WeChat +
+ QRコードをスキャンして WeChat グループに参加 +
+ ClawManager Discord 招待QRコード +

+ Discord +
+ QRコードをスキャンして Discord サーバーに参加 +
+ + +## 製品紹介 + +ClawManager は、AI エージェントインスタンスの運用を Kubernetes に持ち込み、そのランタイム基盤の上に 3 つの高次なコントロールプレーンを重ねます。チームはこれを使って AI アクセスを統制し、Agent を通じてランタイム動作を編成し、スキャン可能で再利用可能な channel と skill を用いてワークスペース機能を提供できます。 + +次のようなチームに向いています。 + +- 複数ユーザー向けに AI エージェントインスタンスを運用するプラットフォームチーム +- ランタイムの可観測性、コマンド配布、 desired state 管理が必要な運用チーム +- 手作業の設定ではなく、再利用可能なリソースで Agent ワークスペースを届けたい開発チーム + + +## Team ワークスペース + +Team ワークスペースは、簡素化された OpenClaw Lite の協働フローを提供します。ロールテンプレートを選び、Team を作成して、Team チャットで目標を説明するだけです。Leader が計画、メンバー調整、成果物収集、最終結果の提示を担当します。 + +- メンバーごとの Runtime やリソースプリセットを設定せず、Leader 仲介型の協働に固定 +- デリバリー、製品探索、ソフトウェア開発向けの組み込みテンプレート +- 計画、割り当て、進捗、レビュー、成果物、最終統合を表示する Team チャット +- 総合タスク状態と現在の成果物を表示する Execution Kanban + +作成、協働段階、結果の確認方法は [Team Workspace Quick Guide](./docs/team-workspaces-guide_ja.md) を参照してください。 + + +## Runtime 連携 + +ClawManager は現在、次の管理対象 Runtime をサポートします。 + +- OpenClaw icon `OpenClaw`: ClawManager が管理するデスクトップインスタンスで使われる標準の OpenClaw スタイル Runtime +- Hermes icon `Hermes`: 永続化された `.hermes` ワークスペースと内蔵 Hermes agent を備えた Webtop ベースの Runtime 連携 + +Runtime プレビュー: + +**OpenClaw icon OpenClaw** + +![openclaw](./docs/images/openclaw.png) + +**Hermes icon Hermes** + +![hermes](./docs/images/hermes.png) + +Runtime 開発者は、[Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md)、[Generic Runtime Agent Integration Guide](./docs/runtime-agent-integration-guide.md)、[Skill Content MD5 Spec](./docs/skill-content-md5-spec.md) を参照して互換 agent を実装できます。 + + +## はじめに + +ClawManager は、標準 Kubernetes と軽量クラスタの両方に対して、より明確な導入入口を提供します。まずは自分の環境に合うデプロイパスを選び、その後に初回ログインと基本操作のフローへ進むのがおすすめです。 + +- 標準 Kubernetes デプロイ: [deployments/k8s/clawmanager.yaml](./deployments/k8s/clawmanager.yaml) +- K3s / 軽量クラスタ向けデプロイ: [deployments/k3s/clawmanager.yaml](./deployments/k3s/clawmanager.yaml) +- 初回ログインと基本操作フロー: [ユーザーガイド](./docs/use_guide_ja.md) +- デプロイ説明とアーキテクチャ背景: [Deployment Guide (English)](./docs/deployment.md) + +## 3 つのコントロールプレーン + + +### AI Gateway + +AI Gateway は、ClawManager におけるモデルアクセスのガバナンスプレーンです。管理対象の Agent Runtime に統一された OpenAI 互換エントリポイントを提供し、上流プロバイダの上にポリシー、監査、コスト制御を追加します。 + +- モデルトラフィックの統一エントリポイント +- セキュアモデルのルーティングとポリシー駆動のモデル選択 +- エンドツーエンドの監査・トレース記録 +- 組み込みのコスト計算と利用分析 +- ブロックやルート変更を行えるリスク制御ルール + +[AI Gateway Guide (English)](./docs/aigateway.md) を参照してください。 + + +### Agent Control Plane + +Agent Control Plane は、管理対象 AI エージェントインスタンスのランタイム編成レイヤーです。各インスタンスを、登録・状態報告・コマンド受信・プラットフォーム側 desired state への整合が可能な管理対象ランタイムへと変えます。 + +- セキュアなブートストラップとセッションライフサイクルによる Agent 登録 +- ハートビートベースのランタイム状態とヘルス報告 +- コントロールプレーンとインスタンス間の desired state 同期 +- 起動、停止、設定適用、ヘルスチェック、Skill 操作のコマンド配布 +- インスタンス単位での Agent 状態、channel、skill、コマンド履歴の可視化 + +[Agent Control Plane Guide (English)](./docs/agent-control-plane.md) を参照してください。 + + +### リソース管理 + +リソース管理は、AI エージェントワークスペース向けの再利用可能な資産レイヤーです。チームは channel や skill を準備し、bundle として組み合わせ、インスタンスへ注入し、安全レビューをその流れに組み込むことができます。 + +- `Channel` 管理: ワークスペース接続と統合テンプレート +- `Skill` 管理: 再利用可能な機能パッケージ +- `Skill Scanner` ワークフロー: リスク確認とスキャンジョブ +- bundle ベースのリソース構成: 再現性の高いセットアップ +- 注入スナップショットによる実適用内容の追跡 + +[Resource Management Guide (English)](./docs/resource-management.md) と [Security / Skill Scanner Guide (English)](./docs/security-skill-scanner.md) を参照してください。 + +## 製品ギャラリー + +ClawManager は、管理、アクセス、AI ガバナンスを別々のツールとして扱うのではなく、ひとつの製品体験としてまとめるよう設計されています。 + +### Lite モードデプロイ + +Lite モードは共有 gateway runtime pool 経由でインスタンスをプロビジョニングします。各ワークスペースは管理された runtime Pod 内の独立した gateway プロセスとして動作するため、起動が速く、専用 CPU、メモリ、ストレージ、GPU 割り当ての負担を抑えながら、ワークスペースアクセス、Share Link / Password アクセス、channel と skill の注入、管理画面での可視性を維持します。 + +![](./docs/main/liteopenclaw.png) + +### Pro モードデプロイ + +Pro モードは各インスタンスに専用 desktop runtime をプロビジョニングし、独自の Kubernetes Deployment、Service、PVC で構成します。より強い分離、フルデスクトップリソース、runtime events、インスタンス単位の skill 管理、完全なデスクトップ管理体験が必要な場合に適しています。 + +![](./docs/main/proopenclaw.png) + +### Team ワークスペース + +Team ワークスペース画面は、Leader デスクトップ、Team チャット、メンバー表、配布ワークフローを 1 つの運用ビューにまとめ、ClawManager から離れずに協調作業の進捗を追えるようにします。 + +

+ ClawManager Team ワークスペース +

+ +### 管理コンソール + +管理コンソールでは、ユーザー、クォータ、ランタイム操作、セキュリティ制御、プラットフォームレベルのポリシーをひとつの画面に集約します。大規模な AI エージェント基盤を運用するチームの中心となる作業面です。 + +

+ ClawManager 管理コンソール +

+ +### Portal Access + +Portal は、ユーザーに一貫したワークスペース入口を提供します。ブラウザベースでアクセスしながら、コントロールプレーンと同期したランタイム状態を確認でき、インフラの細部を直接意識する必要はありません。 + +

+ ClawManager Portal Access +

+ +### AI Gateway + +AI Gateway は、モデル利用のガバナンスをワークスペース体験そのものに統合します。監査ログ、コスト可視化、リスクルーティングを通じて、AI 利用を単発の統合ではなく、プラットフォーム機能として扱えるようにします。 + +

+ ClawManager AI Gateway +

+ +## 動作の流れ + +1. 管理者がガバナンスポリシーと再利用可能なリソースを定義します。 +2. ユーザーが Kubernetes 上で管理対象の AI エージェントワークスペースを作成または利用します。 +3. Team ワークスペースは、複数のメンバー Runtime を Redis Team Bus と共有ストレージ設定付きでプロビジョニングできます。 +4. Agent がコントロールプレーンへ接続し、ランタイム状態を報告します。 +5. Channel、skill、bundle がコンパイルされ、インスタンスへ適用されます。 +6. AI トラフィックは AI Gateway を経由し、監査、リスク、コスト制御が付与されます。 + +## 開発者向け概要 + +ClawManager は、React フロントエンド、Go バックエンド、状態管理用 MySQL、そして `skill-scanner` やオブジェクトストレージ統合を含む Kubernetes ネイティブなプラットフォームです。コードベースは製品サブシステムごとに整理されているため、該当ガイドから入り、その後コードへ進むのが最も効率的です。 + +- フロントエンドの管理画面とユーザー画面は `frontend/` +- バックエンドのサービス、handler、repository、migration は `backend/` +- デプロイ資産は `deployments/` +- 製品ドキュメントと素材は `docs/` + +[Developer Guide (English)](./docs/developer-guide.md) を参照してください。 + +## ドキュメント + +- [ユーザーガイド](./docs/use_guide_ja.md) +- [Team Workspace Quick Guide](./docs/team-workspaces-guide_ja.md) +- [Deployment Guide (English)](./docs/deployment.md) +- [Admin and User Guide (English)](./docs/admin-user-guide.md) +- [Agent Control Plane Guide (English)](./docs/agent-control-plane.md) +- [AI Gateway Guide (English)](./docs/aigateway.md) +- [Security / Skill Scanner Guide (English)](./docs/security-skill-scanner.md) +- [Resource Management Guide (English)](./docs/resource-management.md) +- [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md) +- [Generic Runtime Agent Integration Guide](./docs/runtime-agent-integration-guide.md) +- [Skill Content MD5 Spec](./docs/skill-content-md5-spec.md) +- [Developer Guide (English)](./docs/developer-guide.md) + +## ライセンス + +このプロジェクトは MIT License のもとで公開されています。 + +## オープンソース + +Issue と Pull Request を歓迎します。 + +## Star History + + + + + + Star History Chart + + diff --git a/README.ko.md b/README.ko.md new file mode 100644 index 0000000..dca5f89 --- /dev/null +++ b/README.ko.md @@ -0,0 +1,293 @@ +# ClawManager + +

+ ClawManager +

+ +

+ ClawManager는 AI Agent 인스턴스 관리를 위한 Kubernetes 네이티브 컨트롤 플레인으로, 거버넌스가 적용된 AI 접근, 런타임 오케스트레이션, 그리고 여러 Agent Runtime 전반에 걸친 재사용 가능한 리소스 관리를 제공합니다. +

+ +

+ 언어: + English | + 简体中文 | + 日本語 | + 한국어 | + Deutsch +

+ +

+ ClawManager Control Plane + Go 1.21+ + React 19 + Kubernetes Native + MIT License + + ClawManager Discord 커뮤니티 참여 + +

+ +

+ 제품 소개 | + Team 워크스페이스 | + AI Gateway | + Agent Control Plane | + Runtime 연동 | + 리소스 관리 | + 시작하기 +

+ +

+ + Star ClawManager on GitHub + +

+ +

60초 안에 보는 ClawManager

+ +

+ClawManager 제품 데모 +

+ +

+ 빠른 Agent 프로비저닝, Skill 관리와 스캔, AI Gateway 거버넌스를 짧게 확인할 수 있습니다. +

+ +## 최신 업데이트 + +최근의 중요한 제품 및 문서 업데이트입니다. + +- [2026-07-07] 보안 방어 플랫폼(secplane) 프론트엔드 콘솔을 추가했습니다. 런타임 방어(입력/상태/의사결정/출력 표면, 자산 변조 방지, 휴먼 승인), 호스트 강화 및 컨테이너 격리, 아웃바운드 신뢰 엔드포인트 거버넌스, 정책 거버넌스, 킬스위치/서킷브레이커, 전체 체인 감사, SecureClaw 데이터 및 컴포넌트 신뢰 감사, 협업 거버넌스, 입력 탐지를 포괄하는 4계층 방어 통합 관리 UI를 5개 언어 i18n으로 제공합니다. +- [2026-06-14] Lite / Pro 런타임 모드와 롤아웃 지원을 추가했습니다. Lite 인스턴스는 공유 gateway runtime pool에서 실행되고, Pro 인스턴스는 더 강한 격리를 위해 전용 desktop deployment를 유지합니다. +- [2026-05-18] Team 워크스페이스 MVP 소개와 미리보기를 추가했습니다. 원클릭 Team 생성, OpenClaw 멤버 오케스트레이션, Redis Team Bus 주입, 공유 스토리지, 멤버 상태, 작업 배포, 이벤트/결과 보기를 포함합니다. +- [2026-04-29] Hermes Runtime 연동을 추가했습니다. Webtop 기반 인스턴스 생성, Agent Control Plane 등록, AI Gateway 주입, channel 및 skill 부트스트랩, `.hermes` 가져오기/내보내기 흐름을 지원합니다. 자세한 내용은 [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md)를 참고하세요. +- [2026-04-08] 플랫폼에 Skill 관리와 Skill 스캔 워크플로우가 추가되었습니다. 자세한 내용은 [Merged PR #52](https://github.com/Yuan-lab-LLM/ClawManager/pull/52)를 참고하세요. +- [2026-03-26] AI Gateway 문서를 업데이트하여 모델 거버넌스, 감사와 추적, 비용 계산, 리스크 제어 설명을 강화했습니다. 자세한 내용은 [AI Gateway Guide](./docs/aigateway.md)를 참고하세요. +- [2026-03-20] ClawManager는 AI Agent 워크스페이스를 위한 더 넓은 컨트롤 플레인으로 발전했으며, 런타임 제어, 재사용 가능한 리소스, 보안 스캔 워크플로우가 강화되었습니다. + +> ClawManager가 여러분의 팀에 도움이 된다면, 프로젝트에 Star를 남겨 더 많은 사용자와 개발자가 발견할 수 있도록 도와주세요. + +

+ +Star ClawManager on GitHub + +

+ +## 커뮤니티 + +ClawManager 오픈소스 커뮤니티에 WeChat 또는 Discord로 참여해 제품 업데이트를 확인하고, 사용 경험을 나누며, 기여자들과 함께 소통해 보세요. + + + + + + +
+ ClawManager WeChat 그룹 QR 코드 +

+ WeChat +
+ QR 코드를 스캔하여 WeChat 그룹에 참여 +
+ ClawManager Discord 초대 QR 코드 +

+ Discord +
+ QR 코드를 스캔하여 Discord 서버에 참여 +
+ + +## 제품 소개 + +ClawManager는 AI Agent 인스턴스 운영을 Kubernetes 위로 확장하고, 그 런타임 기반 위에 3개의 상위 컨트롤 플레인을 제공합니다. 팀은 이를 통해 AI 접근을 통제하고, Agent를 통해 런타임 동작을 오케스트레이션하며, 스캔 가능하고 재사용 가능한 channel 및 skill 리소스로 워크스페이스 기능을 제공할 수 있습니다. + +다음과 같은 팀에 적합합니다. + +- 여러 사용자를 대상으로 AI Agent 인스턴스를 운영하는 플랫폼 팀 +- 런타임 가시성, 명령 배포, desired state 제어가 필요한 운영 팀 +- 수동 설정 대신 재사용 가능한 리소스로 Agent 워크스페이스를 제공하고 싶은 개발 팀 + + +## Team 워크스페이스 + +Team 워크스페이스는 단순화된 OpenClaw Lite 협업 흐름을 제공합니다. 역할 템플릿을 고르고 Team을 만든 뒤 Team 채팅에 목표를 설명하면 됩니다. Leader가 계획 수립, 멤버 조율, 산출물 수집, 최종 결과 정리를 담당합니다. + +- 멤버별 Runtime 또는 리소스 프리셋 설정 없이 Leader 중개 협업으로 고정 +- 납품, 제품 탐색, 소프트웨어 엔지니어링을 위한 기본 템플릿 +- 계획, 배정, 진행, 검토, 산출물, 최종 종합을 보여 주는 Team 채팅 +- 전체 작업 상태와 현재 멤버 산출물을 보여 주는 Execution Kanban + +생성 과정, 협업 단계, 결과 확인은 [Team Workspace Quick Guide](./docs/team-workspaces-guide_ko.md)를 참고하세요. + + +## Runtime 연동 + +ClawManager는 현재 다음 관리형 Runtime을 지원합니다. + +- OpenClaw icon `OpenClaw`: ClawManager가 관리하는 데스크톱 인스턴스에서 사용하는 기본 OpenClaw 스타일 워크스페이스 Runtime +- Hermes icon `Hermes`: 영구 `.hermes` 워크스페이스와 내장 Hermes agent를 포함한 Webtop 기반 Runtime 연동 + +Runtime 미리보기: + +**OpenClaw icon OpenClaw** + +![openclaw](./docs/images/openclaw.png) + +**Hermes icon Hermes** + +![hermes](./docs/images/hermes.png) + +Runtime 개발자는 [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md), [Generic Runtime Agent Integration Guide](./docs/runtime-agent-integration-guide.md), [Skill Content MD5 Spec](./docs/skill-content-md5-spec.md)를 참고해 호환 agent를 구현할 수 있습니다. + + +## 시작하기 + +ClawManager는 이제 표준 Kubernetes 환경과 경량 클러스터 환경 모두에 대해 더 명확한 진입 경로를 제공합니다. 먼저 자신의 환경에 맞는 배포 경로를 선택한 뒤, 첫 로그인 및 기본 사용 흐름으로 이어가면 됩니다. + +- 표준 Kubernetes 배포: [deployments/k8s/clawmanager.yaml](./deployments/k8s/clawmanager.yaml) +- K3s / 경량 클러스터 배포: [deployments/k3s/clawmanager.yaml](./deployments/k3s/clawmanager.yaml) +- 첫 로그인 및 기본 사용 흐름: [사용자 가이드](./docs/use_guide_ko.md) +- 배포 설명 및 아키텍처 배경: [Deployment Guide (English)](./docs/deployment.md) + +## 세 가지 컨트롤 플레인 + + +### AI Gateway + +AI Gateway는 ClawManager에서 모델 접근을 거버넌스하는 컨트롤 플레인입니다. 관리되는 Agent Runtime에 통합된 OpenAI 호환 진입점을 제공하고, 상위 모델 제공자 위에 정책, 감사, 비용 제어를 추가합니다. + +- 모델 트래픽을 위한 통합 진입점 +- 보안 모델 라우팅과 정책 기반 모델 선택 +- 엔드투엔드 감사 및 추적 기록 +- 내장된 비용 계산과 사용량 분석 +- 차단 또는 라우팅 전환이 가능한 리스크 제어 규칙 + +[AI Gateway Guide (English)](./docs/aigateway.md)를 참고하세요. + + +### Agent Control Plane + +Agent Control Plane은 관리되는 AI Agent 인스턴스를 위한 런타임 오케스트레이션 계층입니다. 각 인스턴스를 등록, 상태 보고, 명령 수신, 그리고 플랫폼 측 desired state와의 정렬이 가능한 관리형 런타임으로 만듭니다. + +- 보안 부트스트랩과 세션 라이프사이클 기반 Agent 등록 +- 하트비트 기반 런타임 상태 및 헬스 리포팅 +- 컨트롤 플레인과 인스턴스 간 desired state 동기화 +- 시작, 중지, 설정 적용, 헬스체크, Skill 작업을 위한 명령 배포 +- 인스턴스 단위의 Agent 상태, channel, skill, 명령 이력 가시화 + +[Agent Control Plane Guide (English)](./docs/agent-control-plane.md)를 참고하세요. + + +### 리소스 관리 + +리소스 관리는 AI Agent 워크스페이스를 위한 재사용 가능한 자산 계층입니다. 팀은 channel과 skill을 준비하고, bundle로 조합하고, 인스턴스에 주입하며, 그 과정에 보안 검토를 자연스럽게 포함시킬 수 있습니다. + +- `Channel` 관리: 워크스페이스 연결과 통합 템플릿 +- `Skill` 관리: 재사용 가능한 기능 패키지 +- `Skill Scanner` 워크플로우: 리스크 검토와 스캔 작업 +- bundle 기반 리소스 조합: 반복 가능한 워크스페이스 구성 +- 주입 스냅샷을 통한 실제 적용 결과 추적 + +[Resource Management Guide (English)](./docs/resource-management.md)와 [Security / Skill Scanner Guide (English)](./docs/security-skill-scanner.md)를 참고하세요. + +## 제품 갤러리 + +ClawManager는 관리, 접근, AI 거버넌스를 서로 분리된 도구로 다루지 않고, 하나의 일관된 제품 경험으로 묶도록 설계되었습니다. + +### Lite 모드 배포 + +Lite 모드는 공유 gateway runtime pool을 통해 인스턴스를 프로비저닝합니다. 각 워크스페이스는 관리되는 runtime Pod 안의 독립 gateway 프로세스로 실행되어 빠르게 시작되고 전용 CPU, 메모리, 스토리지, GPU 할당 부담을 줄이면서도 워크스페이스 접근, Share Link / Password 접근, channel 및 skill 주입, 관리자 가시성을 유지합니다. + +![](./docs/main/liteopenclaw.png) + +### Pro 모드 배포 + +Pro 모드는 각 인스턴스에 전용 desktop runtime을 프로비저닝하며, 독립 Kubernetes Deployment, Service, PVC로 구성됩니다. 더 강한 격리, 전체 데스크톱 리소스, runtime events, 인스턴스 skill 관리, 완전한 데스크톱 관리 경험이 필요한 경우에 적합합니다. + +![](./docs/main/proopenclaw.png) + +### Team 워크스페이스 + +Team 워크스페이스 화면은 Leader 데스크톱, Team 채팅, 멤버 테이블, 배포 워크플로우를 하나의 운영 화면에 모아 ClawManager 안에서 협업 진행 상황을 따라갈 수 있게 합니다. + +

+ ClawManager Team 워크스페이스 +

+ +### 관리 콘솔 + +관리 콘솔은 사용자, 쿼터, 런타임 작업, 보안 제어, 플랫폼 수준 정책을 하나의 화면으로 묶습니다. 대규모 AI Agent 인프라를 운영하는 팀의 핵심 작업 공간입니다. + +

+ ClawManager 관리 콘솔 +

+ +### Portal Access + +Portal은 사용자에게 일관된 워크스페이스 진입점을 제공합니다. 브라우저 기반으로 접근하면서도 컨트롤 플레인과 동기화된 런타임 상태를 확인할 수 있어, 사용자가 인프라 세부 사항을 직접 다루지 않아도 됩니다. + +

+ ClawManager Portal Access +

+ +### AI Gateway + +AI Gateway는 모델 사용 거버넌스를 워크스페이스 경험 자체에 통합합니다. 감사 로그, 비용 가시성, 리스크 라우팅을 제공하여 AI 사용을 개별 통합이 아닌 플랫폼 기능으로 다룰 수 있게 합니다. + +

+ ClawManager AI Gateway +

+ +## 동작 방식 + +1. 관리자가 거버넌스 정책과 재사용 가능한 리소스를 정의합니다. +2. 사용자가 Kubernetes에서 관리되는 AI Agent 워크스페이스를 생성하거나 진입합니다. +3. Team 워크스페이스는 여러 멤버 Runtime을 Redis Team Bus와 공유 스토리지 설정과 함께 프로비저닝할 수 있습니다. +4. Agent가 컨트롤 플레인에 연결해 런타임 상태를 보고합니다. +5. Channel, skill, bundle이 컴파일되어 인스턴스에 적용됩니다. +6. AI 트래픽은 AI Gateway를 통해 전달되며, 감사, 리스크, 비용 제어가 함께 적용됩니다. + +## 개발자 개요 + +ClawManager는 React 프런트엔드, Go 백엔드, 상태 저장용 MySQL, 그리고 `skill-scanner` 및 오브젝트 스토리지 통합을 포함한 Kubernetes 네이티브 플랫폼입니다. 코드베이스는 제품 서브시스템 단위로 구성되어 있으므로, 관련 가이드에서 시작한 뒤 코드로 들어가는 방식이 가장 효율적입니다. + +- 프런트엔드의 관리자 및 사용자 화면은 `frontend/` +- 백엔드 서비스, handler, repository, migration은 `backend/` +- 배포 자산은 `deployments/` +- 제품 문서와 이미지 자산은 `docs/` + +[Developer Guide (English)](./docs/developer-guide.md)를 참고하세요. + +## 문서 + +- [사용자 가이드](./docs/use_guide_ko.md) +- [Team Workspace Quick Guide](./docs/team-workspaces-guide_ko.md) +- [Deployment Guide (English)](./docs/deployment.md) +- [Admin and User Guide (English)](./docs/admin-user-guide.md) +- [Agent Control Plane Guide (English)](./docs/agent-control-plane.md) +- [AI Gateway Guide (English)](./docs/aigateway.md) +- [Security / Skill Scanner Guide (English)](./docs/security-skill-scanner.md) +- [Resource Management Guide (English)](./docs/resource-management.md) +- [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md) +- [Generic Runtime Agent Integration Guide](./docs/runtime-agent-integration-guide.md) +- [Skill Content MD5 Spec](./docs/skill-content-md5-spec.md) +- [Developer Guide (English)](./docs/developer-guide.md) + +## 라이선스 + +이 프로젝트는 MIT License로 공개됩니다. + +## 오픈소스 + +Issue와 Pull Request를 환영합니다. + +## Star History + + + + + + Star History Chart + + diff --git a/README.md b/README.md new file mode 100644 index 0000000..440c03a --- /dev/null +++ b/README.md @@ -0,0 +1,293 @@ +# ClawManager + +

+ ClawManager +

+ +

+ A Kubernetes-native control plane for AI agent instance management, with governed AI access, runtime orchestration, and reusable resources across multiple agent runtimes. +

+ +

+ Languages: + English | + Chinese | + Japanese | + Korean | + Deutsch +

+ +

+ ClawManager Control Plane + Go 1.21+ + React 19 + Kubernetes Native + MIT License + + Join ClawManager on Discord + +

+ +

+ Explore the Product | + Team Workspaces | + AI Gateway | + Agent Control Plane | + Runtime Integrations | + Resource Management | + Get Started +

+ +

+ + Star ClawManager on GitHub + +

+ +

See ClawManager in 60 Seconds

+ +

+ClawManager product launch demo +

+ +

+ A quick look at fast agent provisioning, skill management and scanning, and AI Gateway governance. +

+ +## What's New + +Recent highlights from the latest product and documentation updates. + +- [2026-07-07] Added the Security Protection Platform (secplane) frontend — a comprehensive security console covering runtime defense (input/state/decision/output surface, asset tamper-proofing, human approval), host hardening & container isolation, outbound trusted-endpoint governance, policy governance, kill-switch/circuit-breaker, full-chain audit, SecureClaw data-and-component trust auditing, collaboration governance, and input detection. All 4 defense layers are accessible from a unified admin UI with full i18n for 5 languages. +- [2026-06-14] Added Lite / Pro runtime modes and rollout support, so Lite instances can run through shared gateway runtime pools while Pro instances keep dedicated desktop deployments for stronger isolation. +- [2026-05-18] Added the Team workspace MVP introduction and preview, covering one-click Team creation, OpenClaw member orchestration, Redis Team Bus injection, shared storage, member status, task dispatch, and event/result views. +- [2026-04-29] Added Hermes runtime integration support, including Webtop-based instance provisioning, Agent Control Plane registration, AI Gateway injection, channel and skill bootstrap, and `.hermes` import/export workflows. See the [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md). +- [2026-04-08] Added skill management and skill scanning workflows to the platform, via [Merged PR #52](https://github.com/Yuan-lab-LLM/ClawManager/pull/52). +- [2026-03-26] AI Gateway documentation was refreshed with stronger coverage for model governance, audit and trace, cost accounting, and risk control. See the [AI Gateway Guide](./docs/aigateway.md). +- [2026-03-20] ClawManager evolved into a broader control plane for AI agent workspaces, with stronger runtime control, reusable resources, and security scanning workflows. + +> If ClawManager is useful to your team, please star the project to help more users and contributors discover it. + +

+ +Star ClawManager on GitHub + +

+ + +## Community + +Join the ClawManager open source community on WeChat or Discord for product updates, usage discussion, and contributor collaboration. + + + + + + +
+ ClawManager WeChat group QR code +

+ WeChat +
+ Scan to join the WeChat community group +
+ ClawManager Discord invite QR code +

+ Discord +
+ Scan to join our Discord server +
+ +## Product Tour + +ClawManager brings AI agent instance operations to Kubernetes and layers three higher-level control planes on top of that runtime foundation. Teams use it to govern AI access, orchestrate runtime behavior through agents, and manage reusable channels and skills with scanning and bundle-based delivery. + +It is designed for: + +- platform teams running AI agent instances for multiple users +- operators who need runtime visibility, command dispatch, and desired-state control +- builders who want governed AI access and reusable resource injection instead of manual per-instance setup + + +## Team Workspaces + +Team Workspaces provide a simplified OpenClaw Lite collaboration flow: choose a role template, create the Team, and describe the goal in the Team chat. The Leader plans the work, coordinates members, collects deliveries, and publishes the final result. + +- fixed Leader-mediated collaboration, without per-member runtime or resource-preset setup +- built-in templates for focused delivery, product discovery, and software engineering work +- Team chat for plans, assignments, progress, reviews, deliveries, and final synthesis +- Execution Kanban for the root-task state and current member deliveries + +See the [Team Workspace Quick Guide](./docs/team-workspaces-guide_en.md) for the creation flow, collaboration stages, and result viewing. + + +## Runtime Integrations + +ClawManager currently supports the following managed runtimes: + +- OpenClaw icon `OpenClaw`: the default OpenClaw-style workspace runtime used by ClawManager-managed desktop instances +- Hermes icon `Hermes`: a Webtop-based runtime integration with a persistent `.hermes` workspace and embedded Hermes agent + +Runtime previews: + +**OpenClaw icon OpenClaw** + +![openclaw](./docs/images/openclaw.png) + +**Hermes icon Hermes** + +![hermes](./docs/images/hermes.png) + +Runtime authors can follow the [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md), the [Generic Runtime Agent Integration Guide](./docs/runtime-agent-integration-guide.md), and the [Skill Content MD5 Spec](./docs/skill-content-md5-spec.md) to build compatible agents. + +## Get Started + +ClawManager now separates the Kubernetes distribution from the storage profile. Choose `k3s` or `k8s` first, then choose the storage profile that matches the cluster shape: + +- k3s single-node HostPath: [deployments/k3s/single-node/clawmanager.yaml](./deployments/k3s/single-node/clawmanager.yaml) +- k3s cluster CSI/RWX example: [deployments/k3s/cluster/clawmanager.yaml](./deployments/k3s/cluster/clawmanager.yaml) +- Kubernetes single-node HostPath: [deployments/k8s/single-node/clawmanager.yaml](./deployments/k8s/single-node/clawmanager.yaml) +- Kubernetes cluster CSI/RWX example: [deployments/k8s/cluster/clawmanager.yaml](./deployments/k8s/cluster/clawmanager.yaml) +- Operations-oriented quick start and first login flow: [User Guide](./docs/use_guide_en.md) +- Deployment notes and architecture-level context: [Deployment Guide](./docs/deployment.md) + +The cluster profile is validated with Longhorn (`longhorn` for RWO data and `longhorn-rwx` for RWX workspaces), but these StorageClass names are examples. You can replace them with any CSI classes that provide the same access modes. + +## Three Control Planes + +### AI Gateway + +AI Gateway is the governance plane for model access inside ClawManager. It gives managed agent runtimes a unified OpenAI-compatible entry point while adding policy and audit controls on top of upstream providers. + +- Unified gateway entry for model traffic +- Secure model routing and policy-aware model selection +- End-to-end audit and trace records +- Built-in cost accounting and usage analysis +- Risk control rules that can block or reroute requests + +See the [AI Gateway Guide](./docs/aigateway.md). + +### Agent Control Plane + +Agent Control Plane is the runtime orchestration layer for managed AI agent instances. It turns each instance into a managed runtime that can register, report status, receive commands, and stay aligned with platform-side desired state. + +- Agent registration with secure bootstrap and session lifecycle +- Heartbeat-driven runtime status and health reporting +- Desired-state synchronization between the control plane and the instance +- Runtime command dispatch for start, stop, config apply, health checks, and skill operations +- Instance-level visibility into agent status, channels, skills, and command history + +See the [Agent Control Plane Guide](./docs/agent-control-plane.md). + +### Resource Management + +Resource Management is the reusable asset layer for AI agent workspaces. It helps teams prepare channels and skills once, organize them into bundles, inject them into instances, and keep security review in the loop. + +- Channel management for workspace connectivity and integration templates +- Skill management for reusable packaged capabilities +- Skill Scanner workflows for risk review and scan operations +- Bundle-based resource composition for repeatable workspace setup +- Injection snapshots and runtime-level visibility into what was applied + +See the [Resource Management Guide](./docs/resource-management.md) and the [Security / Skill Scanner Guide](./docs/security-skill-scanner.md). + +## Product Gallery + +The product is designed to feel coherent across administration, workspace access, and AI governance. Instead of treating these as separate tools, ClawManager brings them into one control surface. + +### Lite Mode Deployment + +Lite mode provisions instances through a shared gateway runtime pool. Each workspace runs as an isolated gateway process inside managed runtime Pods, which keeps startup fast and lowers dedicated CPU, memory, storage, and GPU allocation overhead while preserving workspace access, Share Link / Password access, channel and skill injection, and admin visibility. + +![](./docs/main/liteopenclaw.png) + +### Pro Mode Deployment + +Pro mode provisions a dedicated desktop runtime for each instance, backed by its own Kubernetes Deployment, Service, and PVC. Use it when users need stronger isolation, full desktop resources, runtime events, instance skill management, and the complete desktop management experience. + +![](./docs/main/proopenclaw.png) + +### Team Workspace + +The Team workspace page brings the leader desktop, Team chat, member table, and dispatch workflow into one operational view, so users can follow collaboration progress without leaving ClawManager. + +

+ ClawManager Team workspace +

+ +### Admin Console + +The admin console brings together users, quotas, runtime operations, security controls, and platform-level policies in one place. It is the operational center for teams running AI agent infrastructure at scale. + +

+ ClawManager admin console +

+ +### Portal Access + +The portal experience gives users a clean entry point into their workspaces, with browser-based access and runtime visibility that stays connected to the control plane instead of exposing infrastructure details directly. + +

+ ClawManager portal access +

+ +### AI Gateway + +AI Gateway extends the workspace experience with governed model access, audit trails, cost visibility, and risk-aware routing, making AI usage manageable as part of the platform rather than an isolated integration. + +

+ ClawManager AI Gateway +

+ +## How It Works + +1. Admins define governance policies and reusable resources. +2. Users create or enter managed AI agent workspaces on Kubernetes. +3. Team workspaces can provision multiple member runtimes with Redis Team Bus and shared storage configuration. +4. Agents connect back to the control plane and report runtime state. +5. Channels, skills, and bundles are compiled and applied to instances. +6. AI traffic flows through AI Gateway with audit, risk, and cost controls. + +## Developer Snapshot + +ClawManager is built as a Kubernetes-native platform with a React frontend, a Go backend, MySQL for state, and supporting services such as skill-scanner and object storage integrations. The repository is organized around product subsystems rather than a single monolith page, so the best developer experience is to start from the relevant guide and then move into the code. + +- Frontend app and admin/user surfaces live under `frontend/` +- Backend services, handlers, repositories, and migrations live under `backend/` +- Deployment assets live under `deployments/` +- Supporting product docs live under `docs/` + +See the [Developer Guide](./docs/developer-guide.md). + +## Documentation + +- [User Guide](./docs/use_guide_en.md) +- [Team Workspace Quick Guide](./docs/team-workspaces-guide_en.md) +- [Deployment Guide](./docs/deployment.md) +- [Admin and User Guide](./docs/admin-user-guide.md) +- [Agent Control Plane Guide](./docs/agent-control-plane.md) +- [AI Gateway Guide](./docs/aigateway.md) +- [Security / Skill Scanner Guide](./docs/security-skill-scanner.md) +- [Resource Management Guide](./docs/resource-management.md) +- [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md) +- [Generic Runtime Agent Integration Guide](./docs/runtime-agent-integration-guide.md) +- [Skill Content MD5 Spec](./docs/skill-content-md5-spec.md) +- [Developer Guide](./docs/developer-guide.md) + +## License + +This project is licensed under the MIT License. + +## Open Source + +Issues and pull requests are welcome. + +## Star History + + + + + + Star History Chart + + diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..591b567 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目:`Yuan-lab-LLM/ClawManager` +- 原始仓库:https://github.com/Yuan-lab-LLM/ClawManager +- 导入方式:上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源,不代表 WeHub 是原项目作者 diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 0000000..98f326e --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,297 @@ +# ClawManager + +

+ ClawManager +

+ +

+ 一个面向 AI Agent 实例管理的 Kubernetes 原生控制平面,提供受治理的 AI 访问、运行时编排,以及适用于多种 Agent Runtime 的可复用资源管理能力。 +

+ +

+ 语言: + English | + 简体中文 | + 日本語 | + 한국어 | + Deutsch +

+ +

+ ClawManager Control Plane + Go 1.21+ + React 19 + Kubernetes Native + MIT License + + 加入 ClawManager Discord 社区 + +

+ +

+ 了解产品 | + Team 协作 | + AI Gateway | + Agent Control Plane | + Runtime 接入 | + 资源管理 | + 快速开始 +

+ +

+ + Star ClawManager on GitHub + +

+ +

60 秒认识 ClawManager

+ +

+ClawManager 产品演示 +

+ +

+ 快速了解 Agent 实例创建、Skill 管理与扫描,以及 AI Gateway 治理能力。 +

+ +## 最新动态 + +这里展示最近的重要产品与文档更新。 + +- [2026-07-07] 新增安全防护平台(secplane)前端控制台——覆盖运行时防御(输入面/状态面/决策面/输出面、资产防篡改、人因审批)、主机加固与容器隔离、出站可信端点治理、策略治理、应急熔断、全链路审计、SecureClaw 数据与组件可信审计、协同接入治理及输入检测,4 层防护统一管理界面,5 语言 i18n 完整支持。 +- [2026-06-14] 新增 Lite / Pro 运行时模式与滚动升级支持,Lite 实例可通过共享 gateway 运行时池运行,Pro 实例保留专属 desktop deployment 以获得更强隔离。 +- [2026-05-18] 新增 Team 工作空间 MVP 介绍与界面预览,覆盖一键创建 Team、OpenClaw 成员编排、Redis Team Bus 配置注入、共享存储、成员状态、任务派发,以及事件和结果查看。 +- [2026-04-29] 新增 Hermes Runtime 接入支持,覆盖基于 Webtop 的实例创建、Agent Control Plane 注册、AI Gateway 注入、channel 与 skill 引导注入,以及 `.hermes` 导入导出流程。见 [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md)。 +- [2026-04-08] 平台新增了 Skill 管理与 Skill 扫描工作流,见 [Merged PR #52](https://github.com/Yuan-lab-LLM/ClawManager/pull/52)。 +- [2026-03-26] AI Gateway 文档已更新,补充了模型治理、审计追踪、成本核算与风险控制能力,见 [AI Gateway Guide](./docs/aigateway.md)。 +- [2026-03-20] ClawManager 进一步演进为面向 AI Agent 工作空间的控制平面,强化了运行时控制、可复用资源与安全扫描工作流。 + +> 如果 ClawManager 对你的团队有帮助,欢迎为项目点一个 Star,帮助更多用户和开发者发现它。 + +

+ +Star ClawManager on GitHub + +

+ +## 社区交流 + +欢迎加入 ClawManager 开源社区,可通过微信群或 Discord 获取产品更新、交流使用经验,并与贡献者一起讨论共建。 + + + + + + +
+ ClawManager 微信群二维码 +

+ 微信群 +
+ 扫描二维码加入微信群 +
+ ClawManager Discord 邀请二维码 +

+ Discord +
+ 扫描二维码加入 Discord 服务器 +
+ + +## 产品介绍 + +ClawManager 将 AI Agent 实例的运行、治理与运维能力带到 Kubernetes,并在运行时基础之上叠加三层更高阶的控制平面。团队可以用它治理 AI 访问、通过 Agent 编排运行时行为,并通过可扫描、可复用的 channel 与 skill 资源交付工作空间能力。 + +它适合以下场景: + +- 面向多用户运行 AI Agent 实例的平台团队 +- 需要运行时可观测性、命令下发与期望态控制的运维团队 +- 希望以可复用资源而不是手工配置方式交付 Agent 工作空间的开发团队 + + +## Team 工作空间 + +Team 工作空间提供简化的 OpenClaw Lite 协作流程:选择角色模板、创建 Team,然后在团队群聊中描述目标即可。Leader 会负责制定计划、协调成员、收集交付并输出最终结果。 + +- 固定为 Leader 中介协作,无需逐个配置成员运行时或资源预设 +- 内置交付、产品探索和软件工程等成员模板 +- 团队群聊展示计划、派发、进度、验收、交付和最终汇总 +- Execution Kanban 展示总任务状态及当前成员交付 + +参见 [Team 协作快速指南](./docs/team-workspaces-guide.md),了解创建、协作阶段和查看交付结果的流程。 + + +## Runtime 接入 + +ClawManager 当前支持以下受管 Runtime: + +- OpenClaw icon `OpenClaw`:ClawManager 默认支持的 OpenClaw 风格桌面工作空间 Runtime +- Hermes icon `Hermes`:基于 Webtop 的 Runtime 接入,带有持久化 `.hermes` 工作空间和内置 Hermes agent + +Runtime 预览: + +**OpenClaw icon OpenClaw** + +![openclaw](./docs/images/openclaw.png) + +**Hermes icon Hermes** + +![hermes](./docs/images/hermes.png) + +Runtime 开发方可以参考 [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md)、[通用 Runtime Agent 接入指南](./docs/runtime-agent-integration-guide.md) 与 [Skill Content MD5 规范](./docs/skill-content-md5-spec.md) 实现兼容 agent。 + + +## 快速开始 + +ClawManager 现在将 Kubernetes 发行版与存储 profile 拆开。先选择 `k3s` 或 `k8s`,再选择匹配集群形态的存储 profile: + +- k3s 单节点 HostPath: [deployments/k3s/single-node/clawmanager.yaml](./deployments/k3s/single-node/clawmanager.yaml) +- k3s 多节点 CSI/RWX 示例: [deployments/k3s/cluster/clawmanager.yaml](./deployments/k3s/cluster/clawmanager.yaml) +- Kubernetes 单节点 HostPath: [deployments/k8s/single-node/clawmanager.yaml](./deployments/k8s/single-node/clawmanager.yaml) +- Kubernetes 多节点 CSI/RWX 示例: [deployments/k8s/cluster/clawmanager.yaml](./deployments/k8s/cluster/clawmanager.yaml) +- 首次登录与操作流程: [用户指南](./docs/use_guide_cn.md) +- 部署说明与架构背景: [Deployment Guide(英文)](./docs/deployment.md) + +多节点 cluster profile 使用 Longhorn 作为官方验证示例,其中 `longhorn` 用于 RWO 数据卷,`longhorn-rwx` 用于 RWX workspace。项目不绑定 Longhorn,用户可以替换为具备相同访问模式能力的 CSI StorageClass。 + +## 三大控制平面 + + +### AI Gateway + +AI Gateway 是 ClawManager 中负责模型访问治理的控制平面。它为受管 Agent Runtime 提供统一的 OpenAI 兼容入口,同时在上游模型服务之上叠加策略、审计与成本控制能力。 + +- 统一的模型访问入口 +- 安全模型路由与策略驱动的模型选择 +- 端到端审计与追踪记录 +- 内建成本核算与使用分析 +- 可阻断或改道路由的风险控制规则 + +参见 [AI Gateway Guide(英文)](./docs/aigateway.md)。 + + +### Agent Control Plane + +Agent Control Plane 是受管 AI Agent 实例的运行时编排层。它让每一个实例都成为可注册、可汇报状态、可接收命令,并持续对齐平台期望态的受管运行时。 + +- 基于安全引导与会话生命周期的 Agent 注册 +- 依靠心跳机制进行运行时状态与健康上报 +- 控制平面与实例之间的期望态同步 +- 支持启动、停止、配置应用、健康检查与 Skill 操作的命令下发 +- 在实例维度查看 Agent 状态、channel、skill 与命令历史 + +参见 [Agent Control Plane Guide(英文)](./docs/agent-control-plane.md)。 + + +### 资源管理 + +资源管理是 AI Agent 工作空间的可复用资产层。团队可以先准备好 channel 和 skill,再通过 bundle 进行组合、注入到实例中,并把安全审查纳入整个交付流程。 + +- `Channel` 管理,用于工作空间连接与集成模板 +- `Skill` 管理,用于可复用能力包 +- `Skill Scanner` 工作流,用于风险审查与扫描任务 +- 基于 bundle 的资源组合,用于可重复交付 +- 通过注入快照追踪实际下发到实例的内容 + +参见 [Resource Management Guide(英文)](./docs/resource-management.md) 与 [Security / Skill Scanner Guide(英文)](./docs/security-skill-scanner.md)。 + +## 产品界面 + +ClawManager 的设计目标,是让管理、访问与 AI 治理体验形成统一的产品界面,而不是分散在多个孤立工具中。 + +### Lite 模式部署 + +Lite 模式通过共享 gateway 运行时池创建实例。每个工作空间作为受管 runtime Pod 中的独立 gateway 进程运行,启动更快,并减少专属 CPU、内存、存储和 GPU 配额开销,同时保留工作空间访问、Share Link / Password 访问、channel 与 skill 注入,以及管理端可见性。 + +![](./docs/main/liteopenclaw.png) + +### Pro 模式部署 + +Pro 模式为每个实例创建专属 desktop runtime,并使用独立 Kubernetes Deployment、Service 和 PVC。适用于需要更强隔离、完整桌面资源、runtime events、实例级 skill 管理和完整桌面管理体验的场景。 + +![](./docs/main/proopenclaw.png) + +### Team 工作空间 + +Team 工作空间页面把 Leader 桌面、团队群聊、成员表格和调试派发流程集中在同一个操作视图中,用户可以直接观察协作进度、成员反馈和任务结果。 + +

+ ClawManager Team 工作空间 +

+ +### 管理控制台 + +管理控制台将用户、配额、运行时操作、安全控制与平台级策略集中到一起,是团队管理 AI Agent 基础设施的核心工作台。 + +

+ ClawManager 管理控制台 +

+ +### Portal 访问 + +Portal 为用户提供统一的工作空间入口。用户可以通过浏览器访问实例,并查看与控制平面保持一致的运行时状态,而不需要直接面对底层基础设施细节。 + +

+ ClawManager Portal 访问 +

+ +### AI Gateway + +AI Gateway 将模型访问治理纳入工作空间体验本身,提供审计记录、成本可见性与风险路由能力,让 AI 使用成为平台能力的一部分,而不是零散接入。 + +

+ ClawManager AI Gateway +

+ +## 工作方式 + +1. 管理员先定义治理策略与可复用资源。 +2. 用户在 Kubernetes 上创建或进入受管 AI Agent 工作空间。 +3. Team 工作空间可以一次编排多个成员 Runtime,并注入 Redis Team Bus 与共享存储配置。 +4. Agent 回连控制平面并上报运行时状态。 +5. Channel、skill 与 bundle 被编译并应用到实例中。 +6. AI 流量通过 AI Gateway 进入上游服务,并附带审计、风险与成本控制。 + +## 开发者概览 + +ClawManager 是一个 Kubernetes 原生平台,包含 React 前端、Go 后端、MySQL 状态存储,以及 `skill-scanner` 与对象存储等支撑组件。代码库按产品子系统组织,因此更适合从对应能力的指南切入,再进入代码实现。 + +- 前端管理界面与用户界面位于 `frontend/` +- 后端服务、handler、repository 与 migration 位于 `backend/` +- 部署资产位于 `deployments/` +- 产品文档与素材位于 `docs/` + +参见 [Developer Guide(英文)](./docs/developer-guide.md)。 + +## 文档 + +- [用户指南](./docs/use_guide_cn.md) +- [Team 协作快速指南](./docs/team-workspaces-guide.md) +- [Deployment Guide(英文)](./docs/deployment.md) +- [Admin and User Guide(英文)](./docs/admin-user-guide.md) +- [Agent Control Plane Guide(英文)](./docs/agent-control-plane.md) +- [AI Gateway Guide(英文)](./docs/aigateway.md) +- [Security / Skill Scanner Guide(英文)](./docs/security-skill-scanner.md) +- [Resource Management Guide(英文)](./docs/resource-management.md) +- [Hermes Runtime Guide](./docs/hermes-runtime-agent-development.md) +- [通用 Runtime Agent 接入指南](./docs/runtime-agent-integration-guide.md) +- [Skill Content MD5 规范](./docs/skill-content-md5-spec.md) +- [Developer Guide(英文)](./docs/developer-guide.md) + +## 许可证 + +本项目基于 MIT License 开源。 + +## 开源协作 + +欢迎提交 Issue 与 Pull Request。 + +## Star History + + + + + + Star History Chart + + diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 0000000..ae376cf --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1,35 @@ +# Binaries +bin/ +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool +*.out + +# Go workspace file +go.work + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Environment variables +.env +.env.local + +# Database +*.db +*.sqlite diff --git a/backend/Makefile b/backend/Makefile new file mode 100644 index 0000000..ff39276 --- /dev/null +++ b/backend/Makefile @@ -0,0 +1,61 @@ +.PHONY: build run test clean docker-build docker-up docker-down migrate + +# Variables +APP_NAME=clawreef +BINARY=bin/server +MAIN_FILE=cmd/server/main.go + +# Build the application +build: + go build -o $(BINARY) $(MAIN_FILE) + +# Run the application +run: + go run $(MAIN_FILE) + +# Run tests +test: + go test -v ./... + +# Clean build artifacts +clean: + rm -rf bin/ + go clean + +# Download dependencies +deps: + go mod download + go mod tidy + +# Format code +fmt: + go fmt ./... + +# Lint code +lint: + golangci-lint run + +# Docker commands +docker-build: + docker build -t $(APP_NAME):latest -f deployments/docker/Dockerfile . + +docker-up: + docker-compose -f deployments/docker/docker-compose.yml up -d + +docker-down: + docker-compose -f deployments/docker/docker-compose.yml down + +# Database migration (requires mysql client) +migrate: + for file in internal/db/migrations/*.sql; do mysql -h localhost -u clawreef -pclawreef123 clawreef < $$file; done + +# Install tools +install-tools: + go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest + +# Development setup +dev-setup: deps + @echo "Development environment setup complete" + @echo "Run 'make docker-up' to start MySQL" + @echo "Run 'make migrate' to initialize database" + @echo "Run 'make run' to start the server" diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..afb4e38 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,85 @@ +# ClawReef Backend + +ClawReef virtual desktop management platform backend API. + +## Tech Stack + +- Golang 1.21+ +- Gin 1.9+ +- upper/db 4.x +- MySQL 8.0+ +- JWT Authentication + +## Quick Start + +### Prerequisites + +- Go 1.21 or higher +- MySQL 8.0+ +- Docker (optional) + +### Development Setup + +1. **Install dependencies** + ```bash + make deps + ``` + +2. **Start MySQL with Docker** + ```bash + make docker-up + ``` + +3. **Run database migration** + ```bash + make migrate + ``` + +4. **Start the server** + ```bash + make run + ``` + +### API Endpoints + +Server runs on port **9001**. + +#### Authentication +- `POST /api/v1/auth/register` - User registration +- `POST /api/v1/auth/login` - User login +- `POST /api/v1/auth/refresh` - Refresh token +- `POST /api/v1/auth/logout` - User logout +- `GET /api/v1/auth/me` - Get current user + +### Default Admin Account + +- Username: `admin` +- Password: `admin123` + +## Project Structure + +``` +backend/ +├── cmd/server/ # Application entry point +├── internal/ +│ ├── config/ # Configuration +│ ├── db/ # Database connection & migrations +│ ├── models/ # Data models +│ ├── repository/ # Data access layer +│ ├── services/ # Business logic +│ ├── handlers/ # HTTP handlers +│ ├── middleware/ # HTTP middleware +│ └── utils/ # Utilities +├── deployments/ # Docker & K8s configs +└── configs/ # Configuration files +``` + +## Make Commands + +- `make build` - Build the binary +- `make run` - Run the server +- `make test` - Run tests +- `make fmt` - Format code +- `make lint` - Run linter +- `make docker-up` - Start MySQL container +- `make migrate` - Run database migrations diff --git a/backend/cleanup_db.go b/backend/cleanup_db.go new file mode 100644 index 0000000..09a473e --- /dev/null +++ b/backend/cleanup_db.go @@ -0,0 +1,48 @@ +//go:build ignore +// +build ignore + +package main + +import ( + "fmt" + "os" + + "clawreef/internal/config" + "clawreef/internal/db" + "clawreef/internal/repository" +) + +func main() { + cfg, err := config.Load() + if err != nil { + fmt.Printf("Failed to load config: %v\n", err) + os.Exit(1) + } + + database, err := db.Initialize(cfg.Database) + if err != nil { + fmt.Printf("Failed to init DB: %v\n", err) + os.Exit(1) + } + defer db.Close() + + instanceRepo := repository.NewInstanceRepository(database) + + // Get all instances + instances, err := instanceRepo.GetByUserID(2, 0, 100) + if err != nil { + fmt.Printf("Error: %v\n", err) + os.Exit(1) + } + + fmt.Println("Current instances:") + for _, inst := range instances { + fmt.Printf("ID=%d, Name=%s, Status=%s\n", inst.ID, inst.Name, inst.Status) + if inst.Status == "creating" { + fmt.Printf(" -> Deleting failed instance %d\n", inst.ID) + instanceRepo.Delete(inst.ID) + } + } + + fmt.Println("\nCleanup complete!") +} diff --git a/backend/cmd/fixpassword/main.go b/backend/cmd/fixpassword/main.go new file mode 100644 index 0000000..3fece53 --- /dev/null +++ b/backend/cmd/fixpassword/main.go @@ -0,0 +1,44 @@ +package main + +import ( + "database/sql" + "fmt" + "log" + + _ "github.com/go-sql-driver/mysql" + "golang.org/x/crypto/bcrypt" +) + +func main() { + // 生成正确的密码哈希 + password := "admin123" + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Password: %s\n", password) + fmt.Printf("Hash: %s\n", string(hash)) + + // 验证 + err = bcrypt.CompareHashAndPassword(hash, []byte(password)) + fmt.Printf("Verify result: %v\n", err == nil) + + // 更新数据库 + dsn := "root:123456@tcp(localhost:13306)/clawreef" + db, err := sql.Open("mysql", dsn) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + _, err = db.Exec("UPDATE users SET password_hash = ? WHERE username = 'admin'", string(hash)) + if err != nil { + log.Fatal("Failed to update password:", err) + } + + fmt.Println("Password updated successfully!") + fmt.Println("\nYou can now login with:") + fmt.Println(" Username: admin") + fmt.Println(" Password: admin123") +} diff --git a/backend/cmd/initdb/main.go b/backend/cmd/initdb/main.go new file mode 100644 index 0000000..cd366ce --- /dev/null +++ b/backend/cmd/initdb/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "bufio" + "database/sql" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + _ "github.com/go-sql-driver/mysql" +) + +func main() { + // 连接MySQL(不指定数据库) + dsn := "root:123456@tcp(localhost:13306)/" + db, err := sql.Open("mysql", dsn) + if err != nil { + log.Fatal("Failed to connect to MySQL:", err) + } + defer db.Close() + + // 创建数据库 + _, err = db.Exec("CREATE DATABASE IF NOT EXISTS clawreef CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci") + if err != nil { + log.Fatal("Failed to create database:", err) + } + log.Println("Database 'clawreef' created successfully") + + // 连接到新创建的数据库 + db.Close() + dsn = "root:123456@tcp(localhost:13306)/clawreef?multiStatements=true" + db, err = sql.Open("mysql", dsn) + if err != nil { + log.Fatal("Failed to connect to clawreef database:", err) + } + defer db.Close() + + migrationFiles, err := filepath.Glob("internal/db/migrations/*.sql") + if err != nil { + log.Fatal("Failed to list migrations:", err) + } + + for _, migrationFile := range migrationFiles { + sqlBytes, readErr := os.ReadFile(migrationFile) + if readErr != nil { + log.Fatal("Failed to read SQL file:", readErr) + } + + scanner := bufio.NewScanner(strings.NewReader(string(sqlBytes))) + var statement strings.Builder + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(strings.TrimSpace(line), "--") || strings.TrimSpace(line) == "" { + continue + } + statement.WriteString(line) + statement.WriteString("\n") + + if strings.HasSuffix(strings.TrimSpace(line), ";") { + sql := statement.String() + _, err = db.Exec(sql) + if err != nil { + log.Printf("Failed to execute statement from %s: %v", migrationFile, err) + log.Printf("Statement: %s", sql) + } + statement.Reset() + } + } + } + + log.Println("Database schema initialized successfully") + fmt.Println("Admin user created: username='admin', password='admin123'") +} diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go new file mode 100644 index 0000000..919f540 --- /dev/null +++ b/backend/cmd/server/main.go @@ -0,0 +1,620 @@ +package main + +import ( + "context" + "log" + "net/http" + "os" + "os/signal" + "strings" + "sync" + "syscall" + "time" + + "clawreef/internal/aigateway" + "clawreef/internal/config" + "clawreef/internal/db" + "clawreef/internal/handlers" + "clawreef/internal/middleware" + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services" + "clawreef/internal/services/k8s" + "clawreef/internal/services/leader" + + "github.com/gin-gonic/gin" +) + +func main() { + // Load configuration + cfg, err := config.Load() + if err != nil { + log.Fatalf("Failed to load config: %v", err) + } + + // Initialize database + database, err := db.Initialize(cfg.Database) + if err != nil { + log.Fatalf("Failed to initialize database: %v", err) + } + defer db.Close() + + // Initialize Kubernetes client + log.Printf("K8s StorageClass config: %s", cfg.GetStorageClass()) + if err := k8s.Initialize(cfg); err != nil { + log.Printf("Warning: Failed to initialize Kubernetes client: %v", err) + log.Println("Instance management features will not work without K8s connectivity") + } else { + client := k8s.GetClient() + log.Printf("Kubernetes client initialized successfully (mode: %s, storageClass: %s)", + client.GetConnectionMode(), client.StorageClass) + } + + // Initialize repositories + userRepo := repository.NewUserRepository(database) + quotaRepo := repository.NewQuotaRepository(database) + instanceRepo := repository.NewInstanceRepository(database) + systemImageSettingRepo := repository.NewSystemImageSettingRepository(database) + llmModelRepo := repository.NewLLMModelRepository(database) + modelInvocationRepo := repository.NewModelInvocationRepository(database) + auditEventRepo := repository.NewAuditEventRepository(database) + costRecordRepo := repository.NewCostRecordRepository(database) + chatSessionRepo := repository.NewChatSessionRepository(database) + chatMessageRepo := repository.NewChatMessageRepository(database) + riskRuleRepo := repository.NewRiskRuleRepository(database) + riskHitRepo := repository.NewRiskHitRepository(database) + openClawConfigRepo := repository.NewOpenClawConfigRepository(database) + instanceAgentRepo := repository.NewInstanceAgentRepository(database) + instanceRuntimeStatusRepo := repository.NewInstanceRuntimeStatusRepository(database) + instanceDesiredStateRepo := repository.NewInstanceDesiredStateRepository(database) + instanceCommandRepo := repository.NewInstanceCommandRepository(database) + instanceConfigRevisionRepo := repository.NewInstanceConfigRevisionRepository(database) + runtimePodRepo := repository.NewRuntimePodRepository(database) + bindingRepo := repository.NewInstanceRuntimeBindingRepository(database) + rolloutRepo := repository.NewRuntimeRolloutRepository(database) + workspaceFileAuditRepo := repository.NewWorkspaceFileAuditRepository(database) + teamRepo := repository.NewTeamRepository(database) + skillRepo := repository.NewSkillRepository(database) + securityScanRepo := repository.NewSecurityScanRepository(database) + instanceExternalAccessRepo := repository.NewInstanceExternalAccessRepository(database) + + if repaired, repairErr := services.RepairSeededAdminPassword(userRepo); repairErr != nil { + log.Printf("Warning: failed to repair seeded admin password: %v", repairErr) + } else if repaired { + log.Printf("Repaired seeded admin password hash for default admin account") + } + + // Initialize services + authService := services.NewAuthService(userRepo, cfg.JWT) + quotaService := services.NewQuotaService(quotaRepo) + userService := services.NewUserService(userRepo, quotaRepo) + systemImageSettingService := services.NewSystemImageSettingService(systemImageSettingRepo) + llmModelService := services.NewLLMModelService(llmModelRepo) + modelInvocationService := services.NewModelInvocationService(modelInvocationRepo) + auditEventService := services.NewAuditEventService(auditEventRepo) + costRecordService := services.NewCostRecordService(costRecordRepo) + chatSessionService := services.NewChatSessionService(chatSessionRepo) + chatMessageService := services.NewChatMessageService(chatMessageRepo) + riskDetectionService := services.NewRiskDetectionService(riskRuleRepo) + riskHitService := services.NewRiskHitService(riskHitRepo) + riskRuleService := services.NewRiskRuleService(riskRuleRepo) + openClawConfigService := services.NewOpenClawConfigService(openClawConfigRepo, skillRepo) + objectStorageService, err := services.NewObjectStorageService(cfg.ObjectStorage) + if err != nil { + log.Fatalf("Failed to initialize object storage: %v", err) + } + skillScannerClient := services.NewSkillScannerClient(cfg.SkillScanner) + aiObservabilityService := services.NewAIObservabilityService(modelInvocationRepo, auditEventRepo, costRecordRepo, riskHitRepo, chatMessageRepo, llmModelRepo, instanceRepo, userRepo) + clusterResourceService := services.NewClusterResourceService(instanceRepo) + services.SetRuntimeImageSettingsProvider(systemImageSettingService) + services.SetOpenClawTransferRuntimeRepositories(instanceRepo, bindingRepo, runtimePodRepo) + runtimeAgentClient := services.NewRuntimeAgentClient(cfg.Runtime.AgentControlToken) + instanceService := services.NewInstanceService( + instanceRepo, + quotaRepo, + llmModelRepo, + openClawConfigService, + services.WithPrivilegedInstancePods(cfg.Kubernetes.Runtime.Pod.Privileged), + services.WithV2RuntimeLifecycle(runtimePodRepo, bindingRepo, runtimeAgentClient, cfg.Runtime.WorkspaceRoot), + ) + instanceAgentService := services.NewInstanceAgentService(instanceRepo, instanceAgentRepo, instanceDesiredStateRepo, instanceRuntimeStatusRepo, instanceCommandRepo) + instanceRuntimeStatusService := services.NewInstanceRuntimeStatusService(instanceRuntimeStatusRepo, instanceAgentRepo, instanceDesiredStateRepo) + instanceCommandService := services.NewInstanceCommandService(instanceCommandRepo, instanceRuntimeStatusRepo, instanceDesiredStateRepo, skillRepo) + instanceConfigRevisionService := services.NewInstanceConfigRevisionService(instanceConfigRevisionRepo) + teamService := services.NewTeamService( + teamRepo, + instanceService, + services.WithTeamRuntimeWorkspaceRoot(cfg.Runtime.WorkspaceRoot), + services.WithTeamOpenClawConfigService(openClawConfigService), + ) + var platformRedis services.PlatformRedisClient + if redisURL := strings.TrimSpace(cfg.Runtime.RedisURL); redisURL != "" { + var redisErr error + platformRedis, redisErr = services.NewPlatformRedisClient(redisURL) + if redisErr != nil { + log.Printf("platform redis disabled: %v", redisErr) + } + } else { + log.Printf("platform redis disabled: redis url is empty") + } + runtimeEvents := services.NewRuntimeEventService(platformRedis) + workspaceFileService := services.NewWorkspaceFileService(workspaceFileAuditRepo) + runtimeWorkspaceFileService := services.NewRuntimeWorkspaceFileService(workspaceFileAuditRepo) + skillService := services.NewSkillService(skillRepo, instanceRepo, instanceCommandService, objectStorageService, skillScannerClient) + securityScanService := services.NewSecurityScanService(securityScanRepo, skillRepo, objectStorageService, skillScannerClient) + externalAccessService := services.NewInstanceExternalAccessService(instanceExternalAccessRepo) + aiGatewayService := aigateway.NewService(llmModelRepo, modelInvocationService, auditEventService, costRecordService, riskDetectionService, riskHitService, chatSessionService, chatMessageService) + + // Initialize handlers + authHandler := handlers.NewAuthHandler(authService) + userHandler := handlers.NewUserHandler(userService, quotaService) + instanceHandler := handlers.NewInstanceHandler( + instanceService, + instanceAgentService, + instanceRuntimeStatusService, + instanceCommandService, + instanceConfigRevisionService, + openClawConfigService, + skillService, + externalAccessService, + services.WithInstanceProxyRuntimeRepositories(instanceRepo, runtimePodRepo, bindingRepo), + ) + systemSettingsHandler := handlers.NewSystemSettingsHandler(systemImageSettingService) + llmModelHandler := handlers.NewLLMModelHandler(llmModelService) + aiGatewayHandler := handlers.NewAIGatewayHandler(aiGatewayService) + aiObservabilityHandler := handlers.NewAIObservabilityHandler(aiObservabilityService) + riskRuleHandler := handlers.NewRiskRuleHandler(riskRuleService) + clusterResourceHandler := handlers.NewClusterResourceHandler(clusterResourceService) + egressProxyHandler := handlers.NewEgressProxyHandler() + openClawConfigHandler := handlers.NewOpenClawConfigHandler(openClawConfigService) + skillHandler := handlers.NewSkillHandler(skillService, instanceService) + securityHandler := handlers.NewSecurityHandler(securityScanService) + agentHandler := handlers.NewAgentHandler(instanceAgentService, instanceCommandService, instanceRuntimeStatusService, instanceConfigRevisionService, skillService) + teamHandler := handlers.NewTeamHandler(teamService) + workspaceFileHandler := handlers.NewWorkspaceFileHandler(instanceService, workspaceFileService, runtimeWorkspaceFileService) + workspaceFileHandler.SetSkillRepository(skillRepo) + runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, instanceRepo, runtimeEvents) + + // Initialize WebSocket hub and handler + wsHub := services.GetHub() + wsHandler := handlers.NewWebSocketHandler(wsHub) + var runtimeAdminEventBridgeCancel context.CancelFunc + if platformRedis != nil { + var bridgeCtx context.Context + bridgeCtx, runtimeAdminEventBridgeCancel = context.WithCancel(context.Background()) + services.StartRuntimeAdminEventBridge(bridgeCtx, runtimeEvents, wsHub) + } + + // Control-plane singleton background loops. The HTTP API and the in-pod + // nginx desktop data plane run on every replica, but these loops must run + // on exactly one replica. With leader election enabled they only run on the + // elected leader and migrate on failover; with it disabled (single-replica + // deployments) they run directly. + syncService := services.NewSyncService(instanceRepo, instanceRuntimeStatusService) + var runtimeSchedulerCancel context.CancelFunc + var runtimeSchedulerMu sync.Mutex + var runtimeScheduler *services.RuntimeScheduler + if cfg.Runtime.SchedulerEnabled { + k8sClient := k8s.GetClient() + if k8sClient == nil || k8sClient.Clientset == nil { + log.Printf("runtime scheduler disabled: k8s client is unavailable") + } else { + runtimeLeader := services.NewRuntimeLeaderService(k8sClient.Clientset, cfg.Runtime.Namespace, cfg.Runtime.BackendReplicaID) + runtimeDeployments := k8s.NewRuntimeDeploymentService(k8sClient.Clientset) + runtimeSchedulerOptions := []services.RuntimeSchedulerOption{ + services.WithRuntimeSchedulerWorkspaceRoot(cfg.Runtime.WorkspaceRoot), + services.WithRuntimeSchedulerNamespace(cfg.Runtime.Namespace), + services.WithRuntimeSchedulerGatewayPortRange(cfg.Runtime.GatewayPortStart, cfg.Runtime.GatewayPortEnd), + services.WithRuntimeSchedulerHeartbeatTimeout(cfg.Runtime.HeartbeatTimeout), + services.WithRuntimeSchedulerMaxGatewaysPerPod(cfg.Runtime.MaxGatewaysPerPod), + } + if gatewayEnvProvider, ok := instanceService.(interface { + BuildGatewayEnv(*models.Instance) (map[string]string, error) + }); ok { + runtimeSchedulerOptions = append(runtimeSchedulerOptions, services.WithRuntimeSchedulerGatewayEnvBuilder(gatewayEnvProvider.BuildGatewayEnv)) + } + runtimeScheduler = services.NewRuntimeScheduler( + instanceRepo, + runtimePodRepo, + bindingRepo, + rolloutRepo, + runtimeAgentClient, + runtimeEvents, + runtimeLeader, + runtimeDeployments, + cfg.Runtime.SchedulerTick, + runtimeSchedulerOptions..., + ) + log.Printf("runtime scheduler initialized") + } + } else { + log.Printf("runtime scheduler disabled by configuration") + } + runtimePoolHandler := handlers.NewRuntimePoolHandler(runtimePodRepo, bindingRepo, rolloutRepo, runtimeScheduler, runtimeEvents) + + leaderCtx, leaderCancel := context.WithCancel(context.Background()) + defer leaderCancel() + + startBackground := func(ctx context.Context) { + log.Printf("Starting leader-only background loops (identity=%s)", cfg.LeaderElection.Identity) + syncService.Start() + teamService.StartBackground(ctx) + if runtimeScheduler != nil { + runtimeSchedulerMu.Lock() + if runtimeSchedulerCancel == nil { + var schedulerCtx context.Context + schedulerCtx, runtimeSchedulerCancel = context.WithCancel(ctx) + runtimeScheduler.Start(schedulerCtx) + log.Printf("runtime scheduler started") + } + runtimeSchedulerMu.Unlock() + } + } + stopBackground := func() { + log.Printf("Stopping leader-only background loops (identity=%s)", cfg.LeaderElection.Identity) + runtimeSchedulerMu.Lock() + if runtimeSchedulerCancel != nil { + runtimeSchedulerCancel() + runtimeSchedulerCancel = nil + } + runtimeSchedulerMu.Unlock() + teamService.StopBackground() + syncService.Stop() + } + + if cfg.LeaderElection.Enabled && k8s.GetClient() != nil && k8s.GetClient().Clientset != nil { + go leader.Run(leaderCtx, k8s.GetClient().Clientset, leader.Config{ + Namespace: cfg.LeaderElection.Namespace, + LeaseName: cfg.LeaderElection.LeaseName, + Identity: cfg.LeaderElection.Identity, + LeaseDuration: time.Duration(cfg.LeaderElection.LeaseDuration) * time.Second, + RenewDeadline: time.Duration(cfg.LeaderElection.RenewDeadline) * time.Second, + RetryPeriod: time.Duration(cfg.LeaderElection.RetryPeriod) * time.Second, + }, leader.Callbacks{ + OnStartedLeading: startBackground, + OnStoppedLeading: stopBackground, + }) + } else { + log.Println("Leader election disabled or K8s unavailable; running control-plane background loops directly") + startBackground(leaderCtx) + } + + // Setup router + r := gin.Default() + + // Middleware + r.Use(middleware.CORS()) + r.Use(middleware.ErrorHandler()) + r.NoRoute(egressProxyHandler.Handle) + r.NoMethod(egressProxyHandler.Handle) + + // Routes + r.Any("/s/:code", instanceHandler.OpenShortExternalAccess) + r.Any("/s/:code/*path", instanceHandler.OpenShortExternalAccess) + + api := r.Group("/api/v1") + { + runtimeAgent := api.Group("/runtime-agent") + { + runtimeAgent.POST("/register", runtimeAgentHandler.Register) + runtimeAgent.POST("/heartbeat", runtimeAgentHandler.Heartbeat) + runtimeAgent.POST("/metrics/report", runtimeAgentHandler.ReportMetrics) + runtimeAgent.POST("/gateways/report", runtimeAgentHandler.ReportGateways) + runtimeAgent.POST("/skills/report", runtimeAgentHandler.ReportSkills) + } + + // Auth routes + auth := api.Group("/auth") + { + auth.POST("/register", authHandler.Register) + auth.POST("/login", authHandler.Login) + auth.POST("/refresh", authHandler.RefreshToken) + auth.POST("/logout", authHandler.Logout) + auth.GET("/me", middleware.Auth(), middleware.SetUserInfo(userRepo), authHandler.GetCurrentUser) + auth.POST("/change-password", middleware.Auth(), authHandler.ChangePassword) + } + + // User routes (authenticated) + users := api.Group("/users") + users.Use(middleware.Auth()) + users.Use(middleware.SetUserInfo(userRepo)) + { + // Admin only routes + adminOnly := users.Group("") + adminOnly.Use(middleware.NewAdminAuth(userRepo)) + { + adminOnly.GET("", userHandler.ListUsers) + adminOnly.POST("", userHandler.CreateUser) + adminOnly.POST("/import", userHandler.ImportUsers) + adminOnly.DELETE("/:id", userHandler.DeleteUser) + adminOnly.PUT("/:id/role", userHandler.UpdateRole) + adminOnly.PUT("/:id/quota", userHandler.UpdateUserQuota) + } + + // User or admin routes + users.GET("/:id", userHandler.GetUser) + users.PUT("/:id", userHandler.UpdateUser) + users.GET("/:id/quota", userHandler.GetUserQuota) + } + + // Instance routes (authenticated) + instances := api.Group("/instances") + instances.Use(middleware.Auth()) + instances.Use(middleware.SetUserInfo(userRepo)) + { + instances.GET("", instanceHandler.ListInstances) + instances.POST("", instanceHandler.CreateInstance) + instances.POST("/batch/lite", instanceHandler.BatchCreateLiteInstances) + instances.POST("/batch/delete", instanceHandler.BatchDeleteLiteInstances) + instances.GET("/:id", instanceHandler.GetInstance) + instances.PUT("/:id", instanceHandler.UpdateInstance) + instances.DELETE("/:id", instanceHandler.DeleteInstance) + instances.POST("/:id/start", instanceHandler.StartInstance) + instances.POST("/:id/stop", instanceHandler.StopInstance) + instances.POST("/:id/restart", instanceHandler.RestartInstance) + instances.GET("/:id/status", instanceHandler.GetInstanceStatus) + instances.GET("/:id/runtime", instanceHandler.GetRuntimeDetails) + instances.POST("/:id/runtime/:command", instanceHandler.CreateRuntimeCommand) + instances.GET("/:id/config/revisions", instanceHandler.ListConfigRevisions) + instances.POST("/:id/config/revisions/publish", instanceHandler.PublishConfigRevision) + instances.POST("/:id/access", instanceHandler.GenerateAccessToken) + instances.GET("/:id/access", instanceHandler.AccessInstance) + instances.GET("/:id/shell", instanceHandler.StreamShell) + instances.POST("/:id/sync", instanceHandler.ForceSync) + instances.GET("/:id/openclaw/export", instanceHandler.ExportOpenClaw) + instances.POST("/:id/openclaw/import", instanceHandler.ImportOpenClaw) + instances.GET("/:id/hermes/export", instanceHandler.ExportHermes) + instances.POST("/:id/hermes/import", instanceHandler.ImportHermes) + instances.GET("/:id/external-access", instanceHandler.GetExternalAccess) + instances.POST("/:id/external-access/share-link", instanceHandler.EnableShareLink) + instances.POST("/:id/external-access/password", instanceHandler.CreateExternalAccessPassword) + instances.DELETE("/:id/external-access", instanceHandler.DisableExternalAccess) + instances.GET("/:id/workspace/files", workspaceFileHandler.List) + instances.GET("/:id/workspace/preview", workspaceFileHandler.Preview) + instances.GET("/:id/workspace/download", workspaceFileHandler.Download) + instances.POST("/:id/workspace/upload", workspaceFileHandler.Upload) + instances.POST("/:id/workspace/folders", workspaceFileHandler.Mkdir) + instances.PATCH("/:id/workspace/entries", workspaceFileHandler.Rename) + instances.DELETE("/:id/workspace/entries", workspaceFileHandler.Delete) + instances.GET("/:id/skills", skillHandler.ListInstanceSkills) + instances.GET("/:id/skills/available", skillHandler.ListAvailableInstanceSkills) + instances.POST("/:id/skills", skillHandler.AttachSkillToInstance) + instances.DELETE("/:id/skills/:skillId", skillHandler.RemoveSkillFromInstance) + } + + // Admin console: cross-user instance listing. Gated by admin + // middleware — non-admin callers get 403. The workspace + // /instances endpoint above stays caller-scoped regardless of + // role; admin status only unlocks this dedicated surface. + adminInstances := api.Group("/admin/instances") + adminInstances.Use(middleware.Auth()) + adminInstances.Use(middleware.SetUserInfo(userRepo)) + adminInstances.Use(middleware.NewAdminAuth(userRepo)) + { + adminInstances.GET("", instanceHandler.ListAllInstances) + } + + adminRuntime := api.Group("/admin") + adminRuntime.Use(middleware.Auth()) + adminRuntime.Use(middleware.SetUserInfo(userRepo)) + adminRuntime.Use(middleware.NewAdminAuth(userRepo)) + { + adminRuntime.GET("/runtime-pods", runtimePoolHandler.ListPods) + adminRuntime.GET("/runtime-pods/:id/gateways", runtimePoolHandler.GetPodGateways) + adminRuntime.POST("/runtime-pods/:id/drain", runtimePoolHandler.DrainPod) + adminRuntime.POST("/runtime-rollouts", runtimePoolHandler.StartRollout) + } + + teams := api.Group("/teams") + teams.Use(middleware.Auth()) + teams.Use(middleware.SetUserInfo(userRepo)) + { + teams.GET("", teamHandler.ListTeams) + teams.POST("", teamHandler.CreateTeam) + teams.GET("/:id", teamHandler.GetTeam) + teams.DELETE("/:id", teamHandler.DeleteTeam) + teams.GET("/:id/tasks", teamHandler.ListTasks) + teams.POST("/:id/tasks", teamHandler.DispatchTask) + teams.GET("/:id/events", teamHandler.ListEvents) + teams.GET("/:id/workspace/files", teamHandler.ListWorkspaceFiles) + teams.GET("/:id/workspace/preview", teamHandler.PreviewWorkspaceFile) + teams.GET("/:id/workspace/download", teamHandler.DownloadWorkspaceFile) + teams.POST("/:id/workspace/folders", teamHandler.CreateWorkspaceFolder) + teams.POST("/:id/workspace/rename", teamHandler.RenameWorkspaceEntry) + teams.POST("/:id/workspace/upload", teamHandler.UploadWorkspaceFiles) + teams.DELETE("/:id/workspace/files", teamHandler.DeleteWorkspaceEntry) + teams.DELETE("/:id/members/:memberID", teamHandler.DeleteMember) + } + + openClawConfigs := api.Group("/openclaw-configs") + openClawConfigs.Use(middleware.Auth()) + openClawConfigs.Use(middleware.SetUserInfo(userRepo)) + { + openClawConfigs.GET("/resources", openClawConfigHandler.ListResources) + openClawConfigs.POST("/resources", openClawConfigHandler.CreateResource) + openClawConfigs.POST("/resources/validate", openClawConfigHandler.ValidateResource) + openClawConfigs.GET("/resources/:id", openClawConfigHandler.GetResource) + openClawConfigs.PUT("/resources/:id", openClawConfigHandler.UpdateResource) + openClawConfigs.DELETE("/resources/:id", openClawConfigHandler.DeleteResource) + openClawConfigs.POST("/resources/:id/clone", openClawConfigHandler.CloneResource) + + openClawConfigs.GET("/bundles", openClawConfigHandler.ListBundles) + openClawConfigs.POST("/bundles", openClawConfigHandler.CreateBundle) + openClawConfigs.GET("/bundles/:id", openClawConfigHandler.GetBundle) + openClawConfigs.PUT("/bundles/:id", openClawConfigHandler.UpdateBundle) + openClawConfigs.DELETE("/bundles/:id", openClawConfigHandler.DeleteBundle) + openClawConfigs.POST("/bundles/:id/clone", openClawConfigHandler.CloneBundle) + + openClawConfigs.POST("/compile-preview", openClawConfigHandler.CompilePreview) + openClawConfigs.GET("/injections", openClawConfigHandler.ListSnapshots) + openClawConfigs.GET("/injections/:id", openClawConfigHandler.GetSnapshot) + } + + skills := api.Group("/skills") + skills.Use(middleware.Auth()) + skills.Use(middleware.SetUserInfo(userRepo)) + { + skills.GET("", skillHandler.ListSkills) + skills.POST("/import", skillHandler.ImportSkills) + skills.GET("/:id", skillHandler.GetSkill) + skills.PUT("/:id", skillHandler.UpdateSkill) + skills.DELETE("/:id", skillHandler.DeleteSkill) + skills.GET("/:id/download", skillHandler.DownloadSkill) + skills.GET("/:id/versions", skillHandler.ListVersions) + skills.GET("/:id/scan-results", skillHandler.ListScanResults) + } + + systemSettings := api.Group("/system-settings") + systemSettings.Use(middleware.Auth()) + systemSettings.Use(middleware.SetUserInfo(userRepo)) + { + systemSettings.GET("/images", systemSettingsHandler.ListSystemImageSettings) + } + + adminSystemSettings := api.Group("/system-settings") + adminSystemSettings.Use(middleware.Auth()) + adminSystemSettings.Use(middleware.SetUserInfo(userRepo)) + adminSystemSettings.Use(middleware.NewAdminAuth(userRepo)) + { + adminSystemSettings.PUT("/images", systemSettingsHandler.UpsertSystemImageSetting) + adminSystemSettings.DELETE("/images/:instanceType", systemSettingsHandler.DeleteSystemImageSetting) + adminSystemSettings.GET("/cluster-resources", clusterResourceHandler.GetOverview) + } + + adminModels := api.Group("/admin/models") + adminModels.Use(middleware.Auth()) + adminModels.Use(middleware.SetUserInfo(userRepo)) + adminModels.Use(middleware.NewAdminAuth(userRepo)) + { + adminModels.GET("", llmModelHandler.ListModels) + adminModels.POST("/discover", llmModelHandler.DiscoverModels) + adminModels.PUT("", llmModelHandler.UpsertModel) + adminModels.DELETE("/:id", llmModelHandler.DeleteModel) + } + + adminAIAudit := api.Group("/admin/ai-audit") + adminAIAudit.Use(middleware.Auth()) + adminAIAudit.Use(middleware.SetUserInfo(userRepo)) + adminAIAudit.Use(middleware.NewAdminAuth(userRepo)) + { + adminAIAudit.GET("", aiObservabilityHandler.ListAuditItems) + adminAIAudit.GET("/:traceId", aiObservabilityHandler.GetTraceDetail) + } + + adminCosts := api.Group("/admin/costs") + adminCosts.Use(middleware.Auth()) + adminCosts.Use(middleware.SetUserInfo(userRepo)) + adminCosts.Use(middleware.NewAdminAuth(userRepo)) + { + adminCosts.GET("", aiObservabilityHandler.GetCostOverview) + } + + adminRiskRules := api.Group("/admin/risk-rules") + adminRiskRules.Use(middleware.Auth()) + adminRiskRules.Use(middleware.SetUserInfo(userRepo)) + adminRiskRules.Use(middleware.NewAdminAuth(userRepo)) + { + adminRiskRules.GET("", riskRuleHandler.ListRules) + adminRiskRules.POST("/test", riskRuleHandler.TestRules) + adminRiskRules.POST("/bulk-status", riskRuleHandler.BulkUpdateStatus) + adminRiskRules.PUT("", riskRuleHandler.UpsertRule) + adminRiskRules.DELETE("/:ruleId", riskRuleHandler.DeleteRule) + } + + adminSkills := api.Group("/admin/skills") + adminSkills.Use(middleware.Auth()) + adminSkills.Use(middleware.SetUserInfo(userRepo)) + adminSkills.Use(middleware.NewAdminAuth(userRepo)) + { + adminSkills.GET("", skillHandler.ListAllSkills) + } + + adminSecurity := api.Group("/admin/security") + adminSecurity.Use(middleware.Auth()) + adminSecurity.Use(middleware.SetUserInfo(userRepo)) + adminSecurity.Use(middleware.NewAdminAuth(userRepo)) + { + adminSecurity.GET("/config", securityHandler.GetConfig) + adminSecurity.PUT("/config", securityHandler.SaveConfig) + adminSecurity.POST("/scan-jobs", securityHandler.StartScan) + adminSecurity.POST("/skills/:id/rescan", securityHandler.RescanSkill) + adminSecurity.GET("/scan-jobs", securityHandler.ListJobs) + adminSecurity.GET("/scan-jobs/:id", securityHandler.GetJob) + } + + gatewayLLM := api.Group("/gateway/llm") + gatewayLLM.Use(middleware.GatewayAuth(instanceRepo, bindingRepo)) + { + gatewayLLM.GET("/models", aiGatewayHandler.ListModels) + gatewayLLM.POST("/chat/completions", aiGatewayHandler.ChatCompletions) + } + + agent := api.Group("/agent") + { + agent.POST("/register", agentHandler.Register) + agent.POST("/heartbeat", agentHandler.Heartbeat) + agent.GET("/commands/next", agentHandler.NextCommand) + agent.POST("/commands/:id/start", agentHandler.StartCommand) + agent.POST("/commands/:id/finish", agentHandler.FinishCommand) + agent.POST("/state/report", agentHandler.ReportState) + agent.POST("/skills/inventory", agentHandler.ReportSkillInventory) + agent.POST("/skills/upload", agentHandler.UploadSkillPackage) + agent.GET("/skills/versions/:skillVersion/download", skillHandler.DownloadSkillVersionForAgent) + agent.GET("/config/revisions/:id", agentHandler.GetConfigRevision) + } + + // Instance proxy routes (token-based auth, no session required) + // These routes proxy requests to the actual instance pods + api.Any("/instances/:id/proxy", instanceHandler.ProxyInstance) + api.Any("/instances/:id/proxy/*path", instanceHandler.ProxyInstance) + + // WebSocket routes + ws := api.Group("/ws") + ws.Use(middleware.Auth()) + ws.Use(middleware.SetUserInfo(userRepo)) + { + ws.GET("", wsHandler.HandleWebSocket) + ws.GET("/stats", wsHandler.GetConnectionCount) + } + } + + // Start server with graceful shutdown + srv := &http.Server{ + Addr: cfg.Server.Address, + Handler: r, + } + + go func() { + log.Printf("Server starting on %s", cfg.Server.Address) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("Failed to start server: %v", err) + } + }() + + // Wait for interrupt signal + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + sig := <-quit + log.Printf("Received signal %v, shutting down gracefully...", sig) + + // Give active requests up to 10 seconds to finish + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := srv.Shutdown(ctx); err != nil { + log.Printf("HTTP server forced to shutdown: %v", err) + } + + // Stop background services. Cancelling leaderCtx releases the lease (and, + // if we were leader, triggers stopBackground); the explicit stopBackground + // call is idempotent and covers the leader-election-disabled path. + leaderCancel() + stopBackground() + if runtimeAdminEventBridgeCancel != nil { + runtimeAdminEventBridgeCancel() + } + wsHub.Stop() + instanceHandler.Shutdown() + + log.Println("Server exited cleanly") +} diff --git a/backend/configs/dev.yaml b/backend/configs/dev.yaml new file mode 100644 index 0000000..30d41df --- /dev/null +++ b/backend/configs/dev.yaml @@ -0,0 +1,15 @@ +server: + address: ":9001" + mode: "debug" + +database: + host: "localhost" + port: 13306 + user: "root" + password: "123456" + database: "clawreef" + +jwt: + secret: "clawreef-dev-secret-key-change-in-production" + access_expiry: 60 + refresh_expiry: 168 diff --git a/backend/configs/k8s.yaml b/backend/configs/k8s.yaml new file mode 100644 index 0000000..8a072ad --- /dev/null +++ b/backend/configs/k8s.yaml @@ -0,0 +1,128 @@ +# ClawReef Kubernetes 配置文件 +# 用于配置 K8s 连接方式和相关参数 + +kubernetes: + # 连接模式: auto | incluster | outofcluster + # auto: 自动检测,优先尝试 in-cluster,失败则使用 kubeconfig(推荐) + # incluster: 强制使用 in-cluster 配置(适用于 Pod 内运行) + # outofcluster: 强制使用 kubeconfig(适用于本地开发) + mode: "auto" + + # Out-of-cluster 配置(当 mode 为 outofcluster 或 auto 回退时使用) + outOfCluster: + # kubeconfig 文件路径 + # 空字符串时自动查找顺序: + # 1. 环境变量 KUBECONFIG + # 2. 环境变量 K8S_KUBECONFIG + # 3. ~/.kube/config + kubeconfig: "" + + # 当前上下文(空字符串使用 kubeconfig 中的 current-context) + context: "" + + # API Server 地址(可选,覆盖 kubeconfig 中的配置) + # 示例: https://192.168.1.100:6443 + apiServer: "" + + # TLS 配置(可选) + tls: + # 是否跳过 TLS 验证(仅开发环境使用) + insecureSkipVerify: false + + # CA 证书路径(可选) + caFile: "" + + # 客户端证书路径(可选) + certFile: "" + + # 客户端私钥路径(可选) + keyFile: "" + + # In-cluster 配置(当 mode 为 incluster 时使用,通常无需修改) + inCluster: + # Service Account Token 路径(通常无需修改) + tokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + + # CA 证书路径(通常无需修改) + caPath: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + + # Namespace 文件路径(通常无需修改) + namespacePath: "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + + # 通用配置 + common: + # 默认命名空间前缀 + # 实际命名空间会加上用户ID: {namespace}-user-{userID} + namespace: "clawreef" + + # 默认 StorageClass + # 空字符串使用集群默认 StorageClass + storageClass: "external-default-sc" + + # 请求超时时间(秒) + timeout: 30 + + # 连接重试次数 + retryCount: 3 + + # 是否创建命名空间(如果不存在) + autoCreateNamespace: true + +# 运行时配置 +runtime: + # Pod 配置 + pod: + # 默认镜像仓库 + imageRegistry: "docker.io/clawreef" + + # 默认容器端口 + containerPort: 8080 + + # 默认挂载路径 + mountPath: "/home/user/data" + + # 是否启用特权模式(某些桌面环境需要) + privileged: false + + # 额外的标签(会添加到所有 Pod) + extraLabels: {} + # env: production + # team: devops + + # 节点选择器 + nodeSelector: {} + # kubernetes.io/os: linux + + # 容忍配置 + tolerations: [] + # - key: "dedicated" + # operator: "Equal" + # value: "desktop" + # effect: "NoSchedule" + + # PVC 配置 + pvc: + # 访问模式: ReadWriteOnce | ReadWriteMany | ReadOnlyMany + accessMode: "ReadWriteOnce" + + # 存储卷模式: Filesystem | Block + volumeMode: "Filesystem" + + # 是否启用卷扩展 + allowVolumeExpansion: true + + # 保留策略: Retain | Delete | Recycle + reclaimPolicy: "Delete" + + # HostPath 前缀(用于手动创建 PV 时的主机路径) + # 注意:不要使用 /tmp/ 等临时目录,节点重启可能导致数据丢失 + # 环境变量覆盖: K8S_PV_HOST_PATH_PREFIX + hostPathPrefix: "/data/clawreef" + +# 日志配置 +logging: + # 日志级别: debug | info | warn | error + level: "info" + + # 是否记录 K8s API 调用详情 + logApiCalls: false diff --git a/backend/deployments/docker/Dockerfile b/backend/deployments/docker/Dockerfile new file mode 100644 index 0000000..985d755 --- /dev/null +++ b/backend/deployments/docker/Dockerfile @@ -0,0 +1,33 @@ +# Build stage +FROM golang:1.21-alpine AS builder + +WORKDIR /app + +# Install dependencies +RUN apk add --no-cache git + +# Copy go mod files +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Build the application +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o bin/server cmd/server/main.go + +# Final stage +FROM alpine:latest + +RUN apk --no-cache add ca-certificates + +WORKDIR /root/ + +# Copy the binary from builder +COPY --from=builder /app/bin/server . + +# Expose port +EXPOSE 8080 + +# Run the binary +CMD ["./server"] diff --git a/backend/deployments/docker/Dockerfile.incluster b/backend/deployments/docker/Dockerfile.incluster new file mode 100644 index 0000000..e23dac7 --- /dev/null +++ b/backend/deployments/docker/Dockerfile.incluster @@ -0,0 +1,9 @@ +FROM scratch + +WORKDIR / + +COPY bin/server /server + +EXPOSE 9001 + +ENTRYPOINT ["/server"] diff --git a/backend/deployments/docker/docker-compose.yml b/backend/deployments/docker/docker-compose.yml new file mode 100644 index 0000000..42c3c3a --- /dev/null +++ b/backend/deployments/docker/docker-compose.yml @@ -0,0 +1,47 @@ +version: '3.8' + +services: + mysql: + image: mysql:8.0 + container_name: clawreef-mysql + environment: + MYSQL_ROOT_PASSWORD: root123 + MYSQL_DATABASE: clawreef + MYSQL_USER: clawreef + MYSQL_PASSWORD: clawreef123 + ports: + - "3306:3306" + volumes: + - mysql_data:/var/lib/mysql + - ../internal/db/migrations:/docker-entrypoint-initdb.d + command: --default-authentication-plugin=mysql_native_password + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + timeout: 20s + retries: 10 + + backend: + build: + context: ../.. + dockerfile: deployments/docker/Dockerfile + container_name: clawreef-backend + environment: + SERVER_ADDRESS: ":9001" + SERVER_MODE: "debug" + DB_HOST: "mysql" + DB_PORT: "3306" + DB_USER: "clawreef" + DB_PASSWORD: "clawreef123" + DB_NAME: "clawreef" + JWT_SECRET: "clawreef-dev-secret-key" + CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB: "500" + ports: + - "9001:9001" + depends_on: + mysql: + condition: service_healthy + volumes: + - ../../configs:/app/configs + +volumes: + mysql_data: diff --git a/backend/deployments/k8s/clawreef-incluster.yaml b/backend/deployments/k8s/clawreef-incluster.yaml new file mode 100644 index 0000000..96c2c0e --- /dev/null +++ b/backend/deployments/k8s/clawreef-incluster.yaml @@ -0,0 +1,654 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: clawreef-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: clawreef-secrets + namespace: clawreef-system +type: Opaque +stringData: + mysql-root-password: root123 + mysql-password: clawreef123 + jwt-secret: clawreef-dev-secret-key-change-in-production +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: clawreef-mysql-init + namespace: clawreef-system +data: + 001_init_schema.sql: | + CREATE DATABASE IF NOT EXISTS clawreef CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + USE clawreef; + + CREATE TABLE IF NOT EXISTS users ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(255) UNIQUE NOT NULL, + email VARCHAR(320) UNIQUE NOT NULL, + password_hash VARCHAR(255) NOT NULL, + role ENUM('admin', 'user') DEFAULT 'user', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + last_login TIMESTAMP, + INDEX idx_username (username), + INDEX idx_role (role) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instances ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT, + type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu', + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + status ENUM('creating', 'running', 'stopped', 'error', 'deleting') DEFAULT 'creating', + cpu_cores DECIMAL(10,2) NOT NULL, + memory_gb INT NOT NULL, + disk_gb INT NOT NULL, + gpu_enabled BOOLEAN DEFAULT FALSE, + gpu_type VARCHAR(100), + gpu_count INT DEFAULT 0, + os_type VARCHAR(50) NOT NULL, + os_version VARCHAR(50) NOT NULL, + image_registry VARCHAR(255), + image_tag VARCHAR(100), + storage_class VARCHAR(50) DEFAULT 'standard', + mount_path VARCHAR(255) DEFAULT '/data', + pod_name VARCHAR(255), + pod_namespace VARCHAR(255), + pod_ip VARCHAR(45), + access_url VARCHAR(500), + access_token VARCHAR(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + started_at TIMESTAMP, + stopped_at TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id), + INDEX idx_status (status), + INDEX idx_type (type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS persistent_volumes ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + pvc_name VARCHAR(255) UNIQUE NOT NULL, + pvc_namespace VARCHAR(255) NOT NULL, + storage_size_gb INT NOT NULL, + storage_class VARCHAR(50), + mount_path VARCHAR(255), + status ENUM('pending', 'bound', 'released', 'failed') DEFAULT 'pending', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + UNIQUE KEY uk_pvc_name_namespace (pvc_name, pvc_namespace) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backups ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + backup_name VARCHAR(255) NOT NULL, + backup_size_gb INT, + backup_path VARCHAR(500), + status ENUM('creating', 'completed', 'failed', 'deleted') DEFAULT 'creating', + backup_type ENUM('manual', 'scheduled') DEFAULT 'manual', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP, + expires_at TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backup_schedules ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + schedule_name VARCHAR(255), + cron_expression VARCHAR(100) NOT NULL, + retention_days INT DEFAULT 30, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS user_quotas ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL UNIQUE, + max_instances INT DEFAULT 10, + max_cpu_cores DECIMAL(10,2) DEFAULT 40, + max_memory_gb INT DEFAULT 100, + max_storage_gb INT DEFAULT 500, + max_gpu_count INT DEFAULT 2, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_usage ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + cpu_usage_percent DECIMAL(5,2), + memory_usage_gb DECIMAL(10,2), + disk_usage_gb DECIMAL(10,2), + gpu_usage_percent DECIMAL(5,2), + uptime_seconds INT, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_recorded (instance_id, recorded_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS audit_logs ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT, + action VARCHAR(100) NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_id INT, + details JSON, + ip_address VARCHAR(45), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL, + INDEX idx_user_id (user_id), + INDEX idx_action (action), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + INSERT INTO users (username, email, password_hash, role, is_active) + SELECT 'admin', 'admin@clawreef.local', '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi', 'admin', TRUE + WHERE NOT EXISTS (SELECT 1 FROM users WHERE username = 'admin'); + + INSERT INTO user_quotas (user_id, max_instances, max_cpu_cores, max_memory_gb, max_storage_gb, max_gpu_count) + SELECT id, 100, 200, 1000, 5000, 10 FROM users + WHERE username = 'admin' + AND NOT EXISTS (SELECT 1 FROM user_quotas WHERE user_id = users.id); + 002_add_webtop_instance_type.sql: | + USE clawreef; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 003_add_system_image_settings.sql: | + USE clawreef; + CREATE TABLE IF NOT EXISTS system_image_settings ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_type VARCHAR(50) NOT NULL, + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + display_name VARCHAR(255) NOT NULL, + image VARCHAR(500) NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_instance_type (instance_type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 004_fix_seeded_admin_password.sql: | + USE clawreef; + UPDATE users + SET password_hash = '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi' + WHERE username = 'admin' + AND password_hash = '$2a$10$N9qo8uLOickgx2ZMRZoMy.MqrzL9wGC3qD3Q.ZHqQH6t3q7l1L5uG'; + 005_update_openclaw_default_image.sql: | + USE clawreef; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + 006_add_openclaw_config_center.sql: | + USE clawreef; + SET @openclaw_snapshot_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'openclaw_config_snapshot_id' + ); + SET @openclaw_snapshot_column_sql = IF( + @openclaw_snapshot_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN openclaw_config_snapshot_id INT NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE openclaw_snapshot_column_stmt FROM @openclaw_snapshot_column_sql; + EXECUTE openclaw_snapshot_column_stmt; + DEALLOCATE PREPARE openclaw_snapshot_column_stmt; + + CREATE TABLE IF NOT EXISTS openclaw_config_resources ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_key VARCHAR(100) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + tags_json LONGTEXT NOT NULL, + content_json LONGTEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_resource_key (user_id, resource_type, resource_key), + INDEX idx_openclaw_resource_user_type (user_id, resource_type), + INDEX idx_openclaw_resource_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundles ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_openclaw_bundle_user (user_id), + INDEX idx_openclaw_bundle_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_items ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + resource_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (resource_id) REFERENCES openclaw_config_resources(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_resource (bundle_id, resource_id), + INDEX idx_openclaw_bundle_item_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_injection_snapshots ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NULL, + user_id INT NOT NULL, + mode VARCHAR(20) NOT NULL, + bundle_id INT NULL, + selected_resource_ids_json LONGTEXT NOT NULL, + resolved_resources_json LONGTEXT NOT NULL, + rendered_manifest_json LONGTEXT NOT NULL, + rendered_env_json LONGTEXT NOT NULL, + secret_name VARCHAR(255) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + activated_at TIMESTAMP NULL, + INDEX idx_openclaw_snapshot_user_created (user_id, created_at), + INDEX idx_openclaw_snapshot_instance (instance_id), + INDEX idx_openclaw_snapshot_bundle (bundle_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 011_add_hermes_instance_type.sql: | + USE clawreef; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 012_update_agents_runtime_default_images.sql: | + USE clawreef; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' + WHERE instance_type = 'hermes' + AND image IN ( + 'registry.example.com/hermes-webtop:latest', + 'lscr.io/linuxserver/webtop:ubuntu-xfce' + ); + 013_normalize_agents_runtime_image_names.sql: | + USE clawreef; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/openclaw:latest'; + + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/hermes:latest'; + 019_add_instance_runtime_type.sql: | + USE clawreef; + SET @instance_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'runtime_type' + ); + SET @instance_runtime_type_column_sql = IF( + @instance_runtime_type_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER type', + 'SELECT 1' + ); + PREPARE instance_runtime_type_column_stmt FROM @instance_runtime_type_column_sql; + EXECUTE instance_runtime_type_column_stmt; + DEALLOCATE PREPARE instance_runtime_type_column_stmt; + 020_add_system_image_runtime_type.sql: | + USE clawreef; + SET @system_image_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'runtime_type' + ); + SET @system_image_runtime_type_column_sql = IF( + @system_image_runtime_type_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER instance_type', + 'SELECT 1' + ); + PREPARE system_image_runtime_type_column_stmt FROM @system_image_runtime_type_column_sql; + EXECUTE system_image_runtime_type_column_stmt; + DEALLOCATE PREPARE system_image_runtime_type_column_stmt; + 021_add_openclaw_shell_runtime_image.sql: | + USE clawreef; + SET @system_image_is_enabled_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'is_enabled' + ); + SET @system_image_is_enabled_column_sql = IF( + @system_image_is_enabled_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN is_enabled BOOLEAN NOT NULL DEFAULT TRUE', + 'SELECT 1' + ); + PREPARE system_image_is_enabled_column_stmt FROM @system_image_is_enabled_column_sql; + EXECUTE system_image_is_enabled_column_stmt; + DEALLOCATE PREPARE system_image_is_enabled_column_stmt; + + SET @system_image_unique_instance_type_index_name = ( + SELECT INDEX_NAME + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'instance_type' + AND NON_UNIQUE = 0 + AND INDEX_NAME <> 'PRIMARY' + LIMIT 1 + ); + SET @system_image_unique_instance_type_index_sql = IF( + @system_image_unique_instance_type_index_name IS NOT NULL, + CONCAT( + 'ALTER TABLE system_image_settings DROP INDEX `', + REPLACE(@system_image_unique_instance_type_index_name, '`', '``'), + '`' + ), + 'SELECT 1' + ); + PREPARE system_image_unique_instance_type_index_stmt FROM @system_image_unique_instance_type_index_sql; + EXECUTE system_image_unique_instance_type_index_stmt; + DEALLOCATE PREPARE system_image_unique_instance_type_index_stmt; + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'desktop', + 'OpenClaw Desktop', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'desktop' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'shell', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + 022_add_openclaw_config_bundle_skills.sql: | + USE clawreef; + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + skill_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id), + INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 023_normalize_runtime_config_mounts.sql: | + USE clawreef; + UPDATE instances + SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP + WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); + 031_add_hermes_lite_runtime_image.sql: | + USE clawreef; + UPDATE system_image_settings + SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + 'registry.example.com/your-custom-shell-image:latest' + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'hermes', + 'shell', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND is_enabled = FALSE + ); +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-data + namespace: clawreef-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: local-path +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql + namespace: clawreef-system +spec: + replicas: 1 + selector: + matchLabels: + app: mysql + template: + metadata: + labels: + app: mysql + spec: + containers: + - name: mysql + image: mysql:8.4 + ports: + - containerPort: 3306 + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawreef-secrets + key: mysql-root-password + - name: MYSQL_DATABASE + value: clawreef + - name: MYSQL_USER + value: clawreef + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: clawreef-secrets + key: mysql-password + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + - name: mysql-init + mountPath: /docker-entrypoint-initdb.d + readinessProbe: + exec: + command: ["sh", "-c", "mysqladmin ping -h 127.0.0.1 -uroot -p$MYSQL_ROOT_PASSWORD"] + initialDelaySeconds: 20 + periodSeconds: 10 + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: mysql-data + - name: mysql-init + configMap: + name: clawreef-mysql-init +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: clawreef-system +spec: + selector: + app: mysql + ports: + - name: mysql + port: 3306 + targetPort: 3306 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: clawreef-backend + namespace: clawreef-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawreef-backend-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: clawreef-backend + namespace: clawreef-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawreef-backend + namespace: clawreef-system +spec: + replicas: 1 + selector: + matchLabels: + app: clawreef-backend + template: + metadata: + labels: + app: clawreef-backend + spec: + serviceAccountName: clawreef-backend + containers: + - name: backend + image: clawreef-backend:dev + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9001 + env: + - name: SERVER_ADDRESS + value: ":9001" + - name: SERVER_MODE + value: "debug" + - name: DB_HOST + value: "host.lima.internal" + - name: DB_PORT + value: "3306" + - name: DB_USER + value: "clawreef" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: clawreef-secrets + key: mysql-password + - name: DB_NAME + value: "clawreef" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: clawreef-secrets + key: jwt-secret + - name: K8S_MODE + value: "incluster" + - name: K8S_NAMESPACE + value: "clawreef" + - name: K8S_STORAGE_CLASS + value: "local-path" + - name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB + value: "500" + readinessProbe: + tcpSocket: + port: 9001 + initialDelaySeconds: 10 + periodSeconds: 10 +--- +apiVersion: v1 +kind: Service +metadata: + name: clawreef-backend + namespace: clawreef-system +spec: + selector: + app: clawreef-backend + ports: + - name: http + port: 9001 + targetPort: 9001 diff --git a/backend/go.mod b/backend/go.mod new file mode 100644 index 0000000..a3bd934 --- /dev/null +++ b/backend/go.mod @@ -0,0 +1,87 @@ +module clawreef + +go 1.26.1 + +require ( + github.com/gin-gonic/gin v1.12.0 + github.com/go-playground/validator/v10 v10.30.1 + github.com/go-sql-driver/mysql v1.9.0 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/gorilla/websocket v1.5.3 + github.com/upper/db/v4 v4.10.0 + golang.org/x/crypto v0.49.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.32.3 + k8s.io/apimachinery v0.32.3 + k8s.io/client-go v0.32.3 +) + +require ( + filippo.io/edwards25519 v1.1.0 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.85 // indirect + github.com/moby/spdystream v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/segmentio/fasthash v1.0.3 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/net v0.51.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/term v0.41.0 // indirect + golang.org/x/text v0.35.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/backend/go.sum b/backend/go.sum new file mode 100644 index 0000000..4e6d584 --- /dev/null +++ b/backend/go.sum @@ -0,0 +1,243 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/go-sql-driver/mysql v1.9.0 h1:Y0zIbQXhQKmQgTp44Y1dp3wTXcn804QoTptLZT1vtvo= +github.com/go-sql-driver/mysql v1.9.0/go.mod h1:pDetrLJeA3oMujJuvXc8RJoasr589B6A9fwzD3QMrqw= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.85 h1:9psTLS/NTvC3MWoyjhjXpwcKoNbkongaCSF3PNpSuXo= +github.com/minio/minio-go/v7 v7.0.85/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= +github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/upper/db/v4 v4.10.0 h1:u5fdqcFZAOwUZWtkS0ueQttecKcSpVF8qmBwZesS9nc= +github.com/upper/db/v4 v4.10.0/go.mod h1:s3qHxKIKvqZNZBG5jrAPufMUXqCBmMdIHa7buGfR+OU= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls= +k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k= +k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= +k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU= +k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/backend/internal/aigateway/service.go b/backend/internal/aigateway/service.go new file mode 100644 index 0000000..4fc84b1 --- /dev/null +++ b/backend/internal/aigateway/service.go @@ -0,0 +1,2607 @@ +package aigateway + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services" +) + +// ToolCallFunction represents a tool/function call payload. +type ToolCallFunction struct { + Name string `json:"name,omitempty"` + Arguments string `json:"arguments,omitempty"` +} + +// ToolCall represents a tool call emitted by an assistant response. +type ToolCall struct { + ID string `json:"id,omitempty"` + Type string `json:"type,omitempty"` + Function *ToolCallFunction `json:"function,omitempty"` + Index *int `json:"index,omitempty"` +} + +// ChatMessage represents an OpenAI-compatible chat message. +type ChatMessage struct { + Role string `json:"role"` + Content interface{} `json:"content"` + Name string `json:"name,omitempty"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + Refusal interface{} `json:"refusal,omitempty"` + Audio interface{} `json:"audio,omitempty"` +} + +// ChatCompletionRequest is the platform gateway request shape. +type ChatCompletionRequest struct { + RawBody []byte `json:"-"` + Model string `json:"model"` + Messages []ChatMessage `json:"messages"` + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + MaxTokens *int `json:"max_tokens,omitempty"` + Stream bool `json:"stream"` + Tools json.RawMessage `json:"tools,omitempty"` + ToolChoice json.RawMessage `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + ResponseFormat json.RawMessage `json:"response_format,omitempty"` + Stop json.RawMessage `json:"stop,omitempty"` + N *int `json:"n,omitempty"` + FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"` + PresencePenalty *float64 `json:"presence_penalty,omitempty"` + ReasoningEffort *string `json:"reasoning_effort,omitempty"` + StreamOptions json.RawMessage `json:"stream_options,omitempty"` + User *string `json:"user,omitempty"` + SessionID *string `json:"session_id,omitempty"` + InstanceID *int `json:"instance_id,omitempty"` + InstanceMode *string `json:"instance_mode,omitempty"` + RuntimeType *string `json:"runtime_type,omitempty"` + GatewayID *string `json:"gateway_id,omitempty"` + RuntimePodID *int64 `json:"runtime_pod_id,omitempty"` + TraceID *string `json:"trace_id,omitempty"` + RequestID *string `json:"request_id,omitempty"` +} + +// ChatCompletionResponse is used for audit parsing only. +type ChatCompletionResponse struct { + ID string `json:"id"` + Object string `json:"object,omitempty"` + Created int64 `json:"created,omitempty"` + Model string `json:"model"` + Choices []struct { + Index int `json:"index"` + Message struct { + Role string `json:"role"` + Content interface{} `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + Refusal interface{} `json:"refusal,omitempty"` + } `json:"message"` + FinishReason string `json:"finish_reason,omitempty"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` +} + +// ProxyResponse is the raw provider response returned to the client. +type ProxyResponse struct { + StatusCode int + Headers http.Header + Body []byte +} + +// AvailableModel represents a user-selectable active model. +type AvailableModel struct { + ID int `json:"id"` + DisplayName string `json:"display_name"` + Description *string `json:"description,omitempty"` + IsSecure bool `json:"is_secure"` + Provider string `json:"provider_type"` +} + +const autoModelID = "auto" +const maxStoredIdentifierLength = 100 +const anthropicVersionHeader = "2023-06-01" +const defaultAnthropicMaxTokens = 4096 + +var providerVersionSegmentPattern = regexp.MustCompile(`(?i)^v\d+(?:[a-z0-9._-]*)?$`) + +type preparedChatRequest struct { + traceID string + sessionID string + sessionIDPtr *string + requestID string + requestIDPtr *string + userID int + userIDPtr *int + selectedModel *models.LLMModel + resolvedModel *models.LLMModel + req ChatCompletionRequest +} + +type openAIStreamChunk struct { + ID string `json:"id"` + Object string `json:"object,omitempty"` + Created int64 `json:"created,omitempty"` + Model string `json:"model,omitempty"` + Choices []struct { + Index int `json:"index"` + Delta struct { + Role string `json:"role,omitempty"` + Content interface{} `json:"content,omitempty"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + Refusal interface{} `json:"refusal,omitempty"` + } `json:"delta"` + FinishReason *string `json:"finish_reason,omitempty"` + } `json:"choices"` + Usage *struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage,omitempty"` +} + +type openAIStreamDelta struct { + Role string `json:"role,omitempty"` + Content interface{} `json:"content,omitempty"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` +} + +type openAIStreamChoice struct { + Index int `json:"index"` + Delta openAIStreamDelta `json:"delta"` + FinishReason *string `json:"finish_reason,omitempty"` +} + +type openAIStreamResponseChunk struct { + ID string `json:"id"` + Object string `json:"object,omitempty"` + Created int64 `json:"created,omitempty"` + Model string `json:"model,omitempty"` + Choices []openAIStreamChoice `json:"choices"` + Usage *struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage,omitempty"` +} + +type anthropicRequestMessage struct { + Role string `json:"role"` + Content []anthropicContentBlock `json:"content"` +} + +type anthropicRequestPayload struct { + Model string `json:"model"` + Messages []anthropicRequestMessage `json:"messages"` + System string `json:"system,omitempty"` + MaxTokens int `json:"max_tokens"` + Stream bool `json:"stream,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + StopSeqs []string `json:"stop_sequences,omitempty"` + Tools []anthropicToolDefinition `json:"tools,omitempty"` + ToolChoice map[string]interface{} `json:"tool_choice,omitempty"` +} + +type anthropicToolDefinition struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + InputSchema interface{} `json:"input_schema"` +} + +type anthropicContentBlock struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Input interface{} `json:"input,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` + Content interface{} `json:"content,omitempty"` +} + +type anthropicMessageResponse struct { + ID string `json:"id"` + Type string `json:"type,omitempty"` + Role string `json:"role,omitempty"` + Model string `json:"model,omitempty"` + Content []anthropicContentBlock `json:"content"` + StopReason string `json:"stop_reason,omitempty"` + Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + } `json:"usage"` +} + +type anthropicStreamEvent struct { + Type string `json:"type"` + Message *anthropicMessageResponse `json:"message,omitempty"` + Index *int `json:"index,omitempty"` + ContentBlock *anthropicContentBlock `json:"content_block,omitempty"` + Delta *struct { + Type string `json:"type,omitempty"` + Text string `json:"text,omitempty"` + PartialJSON string `json:"partial_json,omitempty"` + StopReason string `json:"stop_reason,omitempty"` + StopSequence string `json:"stop_sequence,omitempty"` + } `json:"delta,omitempty"` + Usage *struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + } `json:"usage,omitempty"` + Error *struct { + Type string `json:"type,omitempty"` + Message string `json:"message,omitempty"` + } `json:"error,omitempty"` +} + +type anthropicStreamState struct { + ResponseID string + Model string + Created int64 + PromptTokens int + CompletionTokens int + AssistantText strings.Builder + ToolCalls map[int]*anthropicToolCallState + SentRole bool +} + +type anthropicToolCallState struct { + ID string + Name string + Arguments strings.Builder +} + +// Service defines gateway operations. +type Service interface { + ListAvailableModels() ([]AvailableModel, error) + ChatCompletions(ctx context.Context, userID int, req ChatCompletionRequest) (*ProxyResponse, string, error) + StreamChatCompletions(ctx context.Context, userID int, req ChatCompletionRequest, w http.ResponseWriter) (string, error) +} + +type service struct { + modelRepo repository.LLMModelRepository + invocationService services.ModelInvocationService + auditEventService services.AuditEventService + costRecordService services.CostRecordService + riskDetector services.RiskDetectionService + riskHitService services.RiskHitService + chatSessionService services.ChatSessionService + chatMessageService services.ChatMessageService + secretRefService services.SecretRefService + httpClient *http.Client +} + +// NewService creates a new AI gateway service. +func NewService( + modelRepo repository.LLMModelRepository, + invocationService services.ModelInvocationService, + auditEventService services.AuditEventService, + costRecordService services.CostRecordService, + riskDetector services.RiskDetectionService, + riskHitService services.RiskHitService, + chatSessionService services.ChatSessionService, + chatMessageService services.ChatMessageService, +) Service { + return &service{ + modelRepo: modelRepo, + invocationService: invocationService, + auditEventService: auditEventService, + costRecordService: costRecordService, + riskDetector: riskDetector, + riskHitService: riskHitService, + chatSessionService: chatSessionService, + chatMessageService: chatMessageService, + secretRefService: services.NewSecretRefService(), + httpClient: &http.Client{ + Timeout: 90 * time.Second, + }, + } +} + +func (s *service) ListAvailableModels() ([]AvailableModel, error) { + items, err := s.modelRepo.ListActive() + if err != nil { + return nil, fmt.Errorf("failed to list available models: %w", err) + } + if len(items) == 0 { + return []AvailableModel{}, nil + } + + return []AvailableModel{ + { + ID: 0, + DisplayName: "Auto", + Description: stringPtr("Automatically route requests to the best available model under current governance policy."), + IsSecure: false, + Provider: "gateway", + }, + }, nil +} + +func (s *service) ChatCompletions(ctx context.Context, userID int, req ChatCompletionRequest) (*ProxyResponse, string, error) { + prepared, err := s.prepareChatRequest(userID, req) + if err != nil { + traceID := "" + if prepared != nil { + traceID = prepared.traceID + } + return nil, traceID, err + } + + switch models.ResolveLLMProtocolTypeOrDefault(prepared.resolvedModel.ProviderType, prepared.resolvedModel.ProtocolType) { + case models.ProtocolTypeOpenAI, models.ProtocolTypeOpenAICompatible: + return s.callOpenAICompatible(ctx, prepared) + case models.ProtocolTypeAnthropic: + return s.callAnthropic(ctx, prepared) + default: + _ = s.auditEventService.RecordEvent(&models.AuditEvent{ + TraceID: prepared.traceID, + SessionID: prepared.req.SessionID, + RequestID: prepared.requestIDPtr, + UserID: prepared.userIDPtr, + InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), + EventType: "gateway.request.blocked", + TrafficClass: models.TrafficClassLLM, + Severity: models.AuditSeverityWarn, + Message: fmt.Sprintf("Provider type %s is not supported yet", prepared.resolvedModel.ProviderType), + }) + return nil, prepared.traceID, errors.New("provider type is not supported yet") + } +} + +func (s *service) StreamChatCompletions(ctx context.Context, userID int, req ChatCompletionRequest, w http.ResponseWriter) (string, error) { + prepared, err := s.prepareChatRequest(userID, req) + if err != nil { + traceID := "" + if prepared != nil { + traceID = prepared.traceID + } + return traceID, err + } + + switch models.ResolveLLMProtocolTypeOrDefault(prepared.resolvedModel.ProviderType, prepared.resolvedModel.ProtocolType) { + case models.ProtocolTypeOpenAI, models.ProtocolTypeOpenAICompatible: + return prepared.traceID, s.streamOpenAICompatible(ctx, prepared, w) + case models.ProtocolTypeAnthropic: + return prepared.traceID, s.streamAnthropic(ctx, prepared, w) + default: + _ = s.auditEventService.RecordEvent(&models.AuditEvent{ + TraceID: prepared.traceID, + SessionID: prepared.req.SessionID, + RequestID: prepared.requestIDPtr, + UserID: prepared.userIDPtr, + InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), + EventType: "gateway.request.blocked", + TrafficClass: models.TrafficClassLLM, + Severity: models.AuditSeverityWarn, + Message: fmt.Sprintf("Provider type %s is not supported yet", prepared.resolvedModel.ProviderType), + }) + return prepared.traceID, errors.New("provider type is not supported yet") + } +} + +func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*preparedChatRequest, error) { + if strings.TrimSpace(req.Model) == "" { + return nil, errors.New("model is required") + } + if len(req.Messages) == 0 { + return nil, errors.New("messages are required") + } + + requestedModel := strings.TrimSpace(req.Model) + selectedModel, err := s.resolveRequestedModel(requestedModel) + if err != nil { + return nil, err + } + + prepared := &preparedChatRequest{ + userID: userID, + userIDPtr: intPtr(userID), + selectedModel: selectedModel, + req: req, + } + prepared.sessionID = resolveSessionID(req) + prepared.traceID = s.resolveTraceID(userID, req, prepared.sessionID) + if prepared.sessionID == "" { + prepared.sessionID = normalizeSessionID(nil, prepared.traceID) + } + prepared.sessionIDPtr = stringPtr(prepared.sessionID) + prepared.req.SessionID = prepared.sessionIDPtr + prepared.requestID = normalizeOrCreateID(req.RequestID, "req") + prepared.requestIDPtr = stringPtr(prepared.requestID) + + sessionTitle := deriveSessionTitle(prepared.req.Messages) + if _, err := s.chatSessionService.EnsureSession(prepared.sessionID, prepared.userIDPtr, prepared.req.InstanceID, stringPtr(prepared.traceID), sessionTitle); err != nil { + logPersistenceError("ensure chat session", prepared.traceID, err) + } + if err := s.chatMessageService.RecordMessages( + prepared.traceID, + prepared.sessionID, + prepared.requestIDPtr, + prepared.userIDPtr, + prepared.req.InstanceID, + nil, + buildPersistedMessages(prepared.req.Messages), + ); err != nil { + logPersistenceError("record request messages", prepared.traceID, err) + } + + if err := s.auditEventService.RecordEvent(&models.AuditEvent{ + TraceID: prepared.traceID, + SessionID: prepared.req.SessionID, + RequestID: prepared.requestIDPtr, + UserID: prepared.userIDPtr, + InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), + EventType: "gateway.request.received", + TrafficClass: models.TrafficClassLLM, + Severity: models.AuditSeverityInfo, + Message: fmt.Sprintf("Received LLM request for model %s", prepared.req.Model), + }); err != nil { + logPersistenceError("record gateway.request.received", prepared.traceID, err) + } + + riskAnalysis := s.riskDetector.AnalyzeText(flattenMessages(prepared.req.Messages)) + if riskAnalysis.IsSensitive { + if err := s.auditEventService.RecordEvent(&models.AuditEvent{ + TraceID: prepared.traceID, + SessionID: prepared.req.SessionID, + RequestID: prepared.requestIDPtr, + UserID: prepared.userIDPtr, + InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), + EventType: "gateway.risk.detected", + TrafficClass: models.TrafficClassLLM, + Severity: models.AuditSeverityWarn, + Message: fmt.Sprintf("Sensitive content detected with %d hit(s)", len(riskAnalysis.Hits)), + }); err != nil { + logPersistenceError("record gateway.risk.detected", prepared.traceID, err) + } + } + + resolvedModel, riskAction, resolveErr := s.resolveTargetModel(prepared.selectedModel, riskAnalysis) + if resolveErr != nil { + blockedInvocationID := s.recordBlockedInvocation(prepared.traceID, prepared.requestID, prepared.req, prepared.userID, prepared.selectedModel, resolveErr.Error()) + if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, blockedInvocationID, riskHitAttribution(prepared.req), riskAction, riskAnalysis.Hits); err != nil { + logPersistenceError("record blocked risk hits", prepared.traceID, err) + } + if err := s.auditEventService.RecordEvent(&models.AuditEvent{ + TraceID: prepared.traceID, + SessionID: prepared.req.SessionID, + RequestID: prepared.requestIDPtr, + UserID: prepared.userIDPtr, + InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), + InvocationID: blockedInvocationID, + EventType: "gateway.request.blocked", + TrafficClass: models.TrafficClassLLM, + Severity: models.AuditSeverityWarn, + Message: resolveErr.Error(), + }); err != nil { + logPersistenceError("record gateway.request.blocked", prepared.traceID, err) + } + return prepared, resolveErr + } + + if riskAnalysis.IsSensitive { + if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, nil, riskHitAttribution(prepared.req), riskAction, riskAnalysis.Hits); err != nil { + logPersistenceError("record risk hits", prepared.traceID, err) + } + if riskAction == models.RiskActionRouteSecureModel && resolvedModel != nil && resolvedModel.ID != prepared.selectedModel.ID { + if err := s.auditEventService.RecordEvent(&models.AuditEvent{ + TraceID: prepared.traceID, + SessionID: prepared.req.SessionID, + RequestID: prepared.requestIDPtr, + UserID: prepared.userIDPtr, + InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), + EventType: "gateway.request.rerouted", + TrafficClass: models.TrafficClassLLM, + Severity: models.AuditSeverityWarn, + Message: fmt.Sprintf("Sensitive content rerouted from model %s to secure model %s", prepared.selectedModel.DisplayName, resolvedModel.DisplayName), + }); err != nil { + logPersistenceError("record gateway.request.rerouted", prepared.traceID, err) + } + } + } + + prepared.resolvedModel = resolvedModel + return prepared, nil +} + +func (s *service) callOpenAICompatible(ctx context.Context, prepared *preparedChatRequest) (*ProxyResponse, string, error) { + resolvedAPIKey, err := s.secretRefService.ResolveString(ctx, prepared.resolvedModel.APIKey, prepared.resolvedModel.APIKeySecretRef) + if err != nil { + return nil, prepared.traceID, err + } + + providerRequestBody, err := buildProviderRequestBody(prepared.req, prepared.resolvedModel) + if err != nil { + return nil, prepared.traceID, err + } + + httpRequest, err := buildProviderHTTPRequest(ctx, prepared.traceID, prepared.requestID, prepared.resolvedModel, providerRequestBody, resolvedAPIKey, false) + if err != nil { + return nil, prepared.traceID, err + } + + startedAt := time.Now() + response, err := s.httpClient.Do(httpRequest) + if err != nil { + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("provider call failed: %v", err), providerRequestBody) + return nil, prepared.traceID, fmt.Errorf("failed to call provider: %w", err) + } + defer response.Body.Close() + + responseBody, readErr := io.ReadAll(response.Body) + if readErr != nil { + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("failed to read provider response: %v", readErr), providerRequestBody) + return nil, prepared.traceID, fmt.Errorf("failed to read provider response: %w", readErr) + } + + proxyResponse := &ProxyResponse{ + StatusCode: response.StatusCode, + Headers: cloneProxyHeaders(response.Header), + Body: responseBody, + } + + if response.StatusCode < 200 || response.StatusCode >= 300 { + message := strings.TrimSpace(string(responseBody)) + if message == "" { + message = response.Status + } + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, "provider returned non-success status: "+message, providerRequestBody) + return proxyResponse, prepared.traceID, nil + } + + var providerResponse ChatCompletionResponse + if err := json.Unmarshal(responseBody, &providerResponse); err != nil { + s.recordSuccess(prepared, providerRequestBody, string(responseBody), "", 0, 0, 0, int(time.Since(startedAt).Milliseconds()), false) + return proxyResponse, prepared.traceID, nil + } + + assistantContent := extractAssistantContent(providerResponse) + s.recordSuccess(prepared, providerRequestBody, string(responseBody), assistantContent, providerResponse.Usage.PromptTokens, providerResponse.Usage.CompletionTokens, providerResponse.Usage.TotalTokens, int(time.Since(startedAt).Milliseconds()), false) + return proxyResponse, prepared.traceID, nil +} + +func (s *service) callAnthropic(ctx context.Context, prepared *preparedChatRequest) (*ProxyResponse, string, error) { + resolvedAPIKey, err := s.secretRefService.ResolveString(ctx, prepared.resolvedModel.APIKey, prepared.resolvedModel.APIKeySecretRef) + if err != nil { + return nil, prepared.traceID, err + } + + providerRequestBody, err := buildProviderRequestBody(prepared.req, prepared.resolvedModel) + if err != nil { + return nil, prepared.traceID, err + } + + httpRequest, err := buildProviderHTTPRequest(ctx, prepared.traceID, prepared.requestID, prepared.resolvedModel, providerRequestBody, resolvedAPIKey, false) + if err != nil { + return nil, prepared.traceID, err + } + + startedAt := time.Now() + response, err := s.httpClient.Do(httpRequest) + if err != nil { + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("provider call failed: %v", err), providerRequestBody) + return nil, prepared.traceID, fmt.Errorf("failed to call provider: %w", err) + } + defer response.Body.Close() + + responseBody, readErr := io.ReadAll(response.Body) + if readErr != nil { + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("failed to read provider response: %v", readErr), providerRequestBody) + return nil, prepared.traceID, fmt.Errorf("failed to read provider response: %w", readErr) + } + + proxyResponse := &ProxyResponse{ + StatusCode: response.StatusCode, + Headers: cloneProxyHeaders(response.Header), + Body: responseBody, + } + + if response.StatusCode < 200 || response.StatusCode >= 300 { + message := strings.TrimSpace(string(responseBody)) + if message == "" { + message = response.Status + } + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, "provider returned non-success status: "+message, providerRequestBody) + return proxyResponse, prepared.traceID, nil + } + + var providerResponse anthropicMessageResponse + if err := json.Unmarshal(responseBody, &providerResponse); err != nil { + s.recordSuccess(prepared, providerRequestBody, string(responseBody), "", 0, 0, 0, int(time.Since(startedAt).Milliseconds()), false) + return proxyResponse, prepared.traceID, nil + } + + normalizedBody, assistantContent, promptTokens, completionTokens, totalTokens, normalizeErr := normalizeAnthropicResponse(providerResponse) + if normalizeErr != nil { + s.recordSuccess(prepared, providerRequestBody, string(responseBody), "", providerResponse.Usage.InputTokens, providerResponse.Usage.OutputTokens, providerResponse.Usage.InputTokens+providerResponse.Usage.OutputTokens, int(time.Since(startedAt).Milliseconds()), false) + return proxyResponse, prepared.traceID, nil + } + + proxyResponse.Headers.Del("Content-Length") + proxyResponse.Body = normalizedBody + s.recordSuccess(prepared, providerRequestBody, string(normalizedBody), assistantContent, promptTokens, completionTokens, totalTokens, int(time.Since(startedAt).Milliseconds()), false) + return proxyResponse, prepared.traceID, nil +} + +func (s *service) recordFailure(traceID, requestID string, req ChatCompletionRequest, userID int, model *models.LLMModel, startedAt time.Time, failure string, providerRequestBody []byte) { + completedAt := time.Now() + latencyMs := int(time.Since(startedAt).Milliseconds()) + requestPayload := string(providerRequestBody) + if strings.TrimSpace(requestPayload) == "" { + requestPayload = rawOrJSONRequestPayload(req) + } + responsePayload := failure + invocation := &models.ModelInvocation{ + TraceID: traceID, + SessionID: req.SessionID, + RequestID: requestID, + UserID: intPtr(userID), + InstanceID: req.InstanceID, + InstanceMode: runtimeAttributionString(req.InstanceMode), + RuntimeType: runtimeAttributionString(req.RuntimeType), + GatewayID: runtimeAttributionString(req.GatewayID), + RuntimePodID: runtimeAttributionInt64(req.RuntimePodID), + ModelID: intPtr(model.ID), + ProviderType: model.ProviderType, + RequestedModel: req.Model, + ActualProviderModel: model.ProviderModelName, + TrafficClass: models.TrafficClassLLM, + RequestPayload: &requestPayload, + ResponsePayload: &responsePayload, + LatencyMs: &latencyMs, + IsStreaming: req.Stream, + Status: models.ModelInvocationStatusFailed, + ErrorMessage: stringPtr(failure), + CompletedAt: &completedAt, + } + if err := s.invocationService.RecordInvocation(invocation); err != nil { + logPersistenceError("record failed invocation", traceID, err) + } + + providerRequestPayload := requestPayload + if err := s.auditEventService.RecordEvent(&models.AuditEvent{ + TraceID: traceID, + SessionID: req.SessionID, + RequestID: stringPtr(requestID), + UserID: intPtr(userID), + InstanceID: req.InstanceID, + InstanceMode: runtimeAttributionString(req.InstanceMode), + RuntimeType: runtimeAttributionString(req.RuntimeType), + GatewayID: runtimeAttributionString(req.GatewayID), + RuntimePodID: runtimeAttributionInt64(req.RuntimePodID), + InvocationID: intPtr(invocation.ID), + EventType: "gateway.request.failed", + TrafficClass: models.TrafficClassLLM, + Severity: models.AuditSeverityError, + Message: failure, + Details: &providerRequestPayload, + }); err != nil { + logPersistenceError("record gateway.request.failed", traceID, err) + } +} + +func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBody []byte, responsePayload, assistantContent string, promptTokens, completionTokens, totalTokens, latencyMs int, isStreaming bool) { + completedAt := time.Now() + requestPayload := string(providerRequestBody) + if strings.TrimSpace(requestPayload) == "" { + requestPayload = rawOrJSONRequestPayload(prepared.req) + } + promptTokens, completionTokens, totalTokens, usageEstimated := resolveUsage(prepared.req.Messages, assistantContent, promptTokens, completionTokens, totalTokens) + invocation := &models.ModelInvocation{ + TraceID: prepared.traceID, + SessionID: prepared.sessionIDPtr, + RequestID: prepared.requestID, + UserID: prepared.userIDPtr, + InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), + ModelID: intPtr(prepared.resolvedModel.ID), + ProviderType: prepared.resolvedModel.ProviderType, + RequestedModel: prepared.req.Model, + ActualProviderModel: prepared.resolvedModel.ProviderModelName, + TrafficClass: models.TrafficClassLLM, + RequestPayload: &requestPayload, + ResponsePayload: &responsePayload, + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: totalTokens, + LatencyMs: &latencyMs, + IsStreaming: isStreaming, + Status: models.ModelInvocationStatusCompleted, + CompletedAt: &completedAt, + } + if err := s.invocationService.RecordInvocation(invocation); err != nil { + logPersistenceError("record completed invocation", prepared.traceID, err) + } + if err := s.chatMessageService.RecordMessages( + prepared.traceID, + prepared.sessionID, + prepared.requestIDPtr, + prepared.userIDPtr, + prepared.req.InstanceID, + intPtr(invocation.ID), + []services.PersistedChatMessage{ + { + Role: "assistant", + Content: assistantContent, + SequenceNo: len(prepared.req.Messages) + 1, + }, + }, + ); err != nil { + logPersistenceError("record assistant messages", prepared.traceID, err) + } + + estimatedCost := calculateEstimatedCost(prepared.resolvedModel, promptTokens, completionTokens) + internalCost := 0.0 + if prepared.resolvedModel.IsSecure { + internalCost = estimatedCost + } + if err := s.costRecordService.RecordCost(&models.CostRecord{ + TraceID: prepared.traceID, + SessionID: prepared.req.SessionID, + RequestID: prepared.requestIDPtr, + UserID: prepared.userIDPtr, + InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), + InvocationID: intPtr(invocation.ID), + ModelID: intPtr(prepared.resolvedModel.ID), + ProviderType: prepared.resolvedModel.ProviderType, + ModelName: prepared.resolvedModel.DisplayName, + Currency: fallbackCurrency(prepared.resolvedModel.Currency), + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: totalTokens, + InputUnitPrice: prepared.resolvedModel.InputPrice, + OutputUnitPrice: prepared.resolvedModel.OutputPrice, + EstimatedCost: estimatedCost, + InternalCost: internalCost, + }); err != nil { + logPersistenceError("record cost", prepared.traceID, err) + } + + if err := s.auditEventService.RecordEvent(&models.AuditEvent{ + TraceID: prepared.traceID, + SessionID: prepared.sessionIDPtr, + RequestID: prepared.requestIDPtr, + UserID: prepared.userIDPtr, + InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), + InvocationID: intPtr(invocation.ID), + EventType: "gateway.request.completed", + TrafficClass: models.TrafficClassLLM, + Severity: models.AuditSeverityInfo, + Message: fmt.Sprintf("Completed LLM request for model %s", prepared.resolvedModel.DisplayName), + }); err != nil { + logPersistenceError("record gateway.request.completed", prepared.traceID, err) + } + if usageEstimated { + if err := s.auditEventService.RecordEvent(&models.AuditEvent{ + TraceID: prepared.traceID, + SessionID: prepared.sessionIDPtr, + RequestID: prepared.requestIDPtr, + UserID: prepared.userIDPtr, + InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), + InvocationID: intPtr(invocation.ID), + EventType: "gateway.usage.estimated", + TrafficClass: models.TrafficClassLLM, + Severity: models.AuditSeverityWarn, + Message: "Provider usage was missing; token usage was estimated locally.", + }); err != nil { + logPersistenceError("record gateway.usage.estimated", prepared.traceID, err) + } + } +} + +func (s *service) streamOpenAICompatible(ctx context.Context, prepared *preparedChatRequest, w http.ResponseWriter) error { + resolvedAPIKey, err := s.secretRefService.ResolveString(ctx, prepared.resolvedModel.APIKey, prepared.resolvedModel.APIKeySecretRef) + if err != nil { + return err + } + + providerRequestBody, err := buildProviderRequestBody(prepared.req, prepared.resolvedModel) + if err != nil { + return err + } + + httpRequest, err := buildProviderHTTPRequest(ctx, prepared.traceID, prepared.requestID, prepared.resolvedModel, providerRequestBody, resolvedAPIKey, true) + if err != nil { + return err + } + + startedAt := time.Now() + response, err := s.httpClient.Do(httpRequest) + if err != nil { + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("provider call failed: %v", err), providerRequestBody) + return fmt.Errorf("failed to call provider: %w", err) + } + defer response.Body.Close() + + flusher, ok := w.(http.Flusher) + if !ok { + return errors.New("streaming response writer is not supported") + } + + copyProxyHeaders(w.Header(), response.Header) + w.Header().Set("X-Trace-ID", prepared.traceID) + w.WriteHeader(response.StatusCode) + flusher.Flush() + + if response.StatusCode < 200 || response.StatusCode >= 300 { + responseBody, _ := io.ReadAll(response.Body) + if len(responseBody) > 0 { + _, _ = w.Write(responseBody) + flusher.Flush() + } + message := strings.TrimSpace(string(responseBody)) + if message == "" { + message = response.Status + } + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, "provider returned non-success status: "+message, providerRequestBody) + return nil + } + + reader := bufio.NewReader(response.Body) + var rawStream strings.Builder + var assistantText strings.Builder + promptTokens := 0 + completionTokens := 0 + totalTokens := 0 + streamFailed := false + + for { + line, readErr := reader.ReadString('\n') + if line != "" { + done := inspectStreamLine(line, &assistantText, &promptTokens, &completionTokens, &totalTokens) + rawStream.WriteString(line) + if _, err := io.WriteString(w, line); err == nil { + flusher.Flush() + } + if done { + break + } + } + + if readErr != nil { + if errors.Is(readErr, io.EOF) { + break + } + streamFailed = true + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("failed while reading provider stream: %v", readErr), providerRequestBody) + break + } + } + + if !streamFailed { + assistantContent := assistantText.String() + if strings.TrimSpace(assistantContent) == "" { + assistantContent = rawStream.String() + } + s.recordSuccess(prepared, providerRequestBody, rawStream.String(), assistantContent, promptTokens, completionTokens, totalTokens, int(time.Since(startedAt).Milliseconds()), true) + } + return nil +} + +func (s *service) streamAnthropic(ctx context.Context, prepared *preparedChatRequest, w http.ResponseWriter) error { + resolvedAPIKey, err := s.secretRefService.ResolveString(ctx, prepared.resolvedModel.APIKey, prepared.resolvedModel.APIKeySecretRef) + if err != nil { + return err + } + + providerRequestBody, err := buildProviderRequestBody(prepared.req, prepared.resolvedModel) + if err != nil { + return err + } + + httpRequest, err := buildProviderHTTPRequest(ctx, prepared.traceID, prepared.requestID, prepared.resolvedModel, providerRequestBody, resolvedAPIKey, true) + if err != nil { + return err + } + + startedAt := time.Now() + response, err := s.httpClient.Do(httpRequest) + if err != nil { + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("provider call failed: %v", err), providerRequestBody) + return fmt.Errorf("failed to call provider: %w", err) + } + defer response.Body.Close() + + flusher, ok := w.(http.Flusher) + if !ok { + return errors.New("streaming response writer is not supported") + } + + copyProxyHeaders(w.Header(), response.Header) + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.Header().Set("X-Trace-ID", prepared.traceID) + w.WriteHeader(response.StatusCode) + flusher.Flush() + + if response.StatusCode < 200 || response.StatusCode >= 300 { + responseBody, _ := io.ReadAll(response.Body) + if len(responseBody) > 0 { + _, _ = w.Write(responseBody) + flusher.Flush() + } + message := strings.TrimSpace(string(responseBody)) + if message == "" { + message = response.Status + } + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, "provider returned non-success status: "+message, providerRequestBody) + return nil + } + + reader := bufio.NewReader(response.Body) + var rawStream strings.Builder + state := &anthropicStreamState{ + Created: time.Now().Unix(), + ToolCalls: map[int]*anthropicToolCallState{}, + } + eventType := "" + dataLines := make([]string, 0, 2) + streamFailed := false + done := false + + flushEvent := func() error { + if len(dataLines) == 0 { + eventType = "" + return nil + } + payload := strings.Join(dataLines, "\n") + dataLines = dataLines[:0] + finished, err := processAnthropicStreamEvent(payload, eventType, state, w, flusher) + eventType = "" + if err != nil { + return err + } + if finished { + done = true + } + return nil + } + + for !done { + line, readErr := reader.ReadString('\n') + if line != "" { + rawStream.WriteString(line) + trimmedLine := strings.TrimRight(line, "\r\n") + switch { + case trimmedLine == "": + if err := flushEvent(); err != nil { + streamFailed = true + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("failed while processing provider stream: %v", err), providerRequestBody) + break + } + case strings.HasPrefix(trimmedLine, "event:"): + eventType = strings.TrimSpace(strings.TrimPrefix(trimmedLine, "event:")) + case strings.HasPrefix(trimmedLine, "data:"): + dataLines = append(dataLines, strings.TrimSpace(strings.TrimPrefix(trimmedLine, "data:"))) + } + } + + if readErr != nil { + if errors.Is(readErr, io.EOF) { + if err := flushEvent(); err != nil { + streamFailed = true + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("failed while processing provider stream: %v", err), providerRequestBody) + } + break + } + streamFailed = true + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("failed while reading provider stream: %v", readErr), providerRequestBody) + break + } + } + + if !streamFailed { + finalChunk := openAIStreamResponseChunk{ + ID: defaultIfBlank(state.ResponseID, "chatcmpl-anthropic"), + Object: "chat.completion.chunk", + Created: state.Created, + Model: state.Model, + Choices: []openAIStreamChoice{ + { + Index: 0, + Delta: openAIStreamDelta{}, + }, + }, + Usage: &struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + }{ + PromptTokens: state.PromptTokens, + CompletionTokens: state.CompletionTokens, + TotalTokens: state.PromptTokens + state.CompletionTokens, + }, + } + if err := emitOpenAIStreamPayload(w, flusher, finalChunk); err != nil { + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("failed while writing normalized stream: %v", err), providerRequestBody) + return nil + } + if err := emitOpenAIStreamDone(w, flusher); err != nil { + s.recordFailure(prepared.traceID, prepared.requestID, prepared.req, userIDOrZero(prepared.userIDPtr), prepared.resolvedModel, startedAt, fmt.Sprintf("failed while closing normalized stream: %v", err), providerRequestBody) + return nil + } + + assistantContent := strings.TrimSpace(state.AssistantText.String()) + if assistantContent == "" { + assistantContent = renderAnthropicToolCalls(state) + } + if assistantContent == "" { + assistantContent = rawStream.String() + } + normalizedStream := rawStream.String() + s.recordSuccess(prepared, providerRequestBody, normalizedStream, assistantContent, state.PromptTokens, state.CompletionTokens, state.PromptTokens+state.CompletionTokens, int(time.Since(startedAt).Milliseconds()), true) + } + return nil +} + +func inspectStreamLine(line string, assistantText *strings.Builder, promptTokens, completionTokens, totalTokens *int) bool { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "data:") { + return false + } + + payload := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:")) + if payload == "" { + return false + } + if payload == "[DONE]" { + return true + } + + var chunk openAIStreamChunk + if err := json.Unmarshal([]byte(payload), &chunk); err != nil { + return false + } + + if chunk.Usage != nil { + *promptTokens = chunk.Usage.PromptTokens + *completionTokens = chunk.Usage.CompletionTokens + *totalTokens = chunk.Usage.TotalTokens + } + + for _, choice := range chunk.Choices { + content := flattenMessageContent(choice.Delta.Content) + if content != "" { + assistantText.WriteString(content) + } + } + return false +} + +func buildProviderRequestBody(req ChatCompletionRequest, model *models.LLMModel) ([]byte, error) { + if model == nil { + return nil, errors.New("model is not active or does not exist") + } + + switch models.ResolveLLMProtocolTypeOrDefault(model.ProviderType, model.ProtocolType) { + case models.ProtocolTypeAnthropic: + return buildAnthropicRequestBody(req, model) + default: + return buildOpenAICompatibleRequestBody(req, model) + } +} + +func processAnthropicStreamEvent(payload, eventType string, state *anthropicStreamState, w io.Writer, flusher http.Flusher) (bool, error) { + if strings.TrimSpace(payload) == "" { + return false, nil + } + + var event anthropicStreamEvent + if err := json.Unmarshal([]byte(payload), &event); err != nil { + return false, nil + } + if strings.TrimSpace(eventType) == "" { + eventType = event.Type + } + + switch eventType { + case "message_start": + if event.Message != nil { + state.ResponseID = event.Message.ID + state.Model = event.Message.Model + state.PromptTokens = event.Message.Usage.InputTokens + } + case "content_block_start": + if event.ContentBlock == nil || event.Index == nil { + return false, nil + } + index := *event.Index + switch event.ContentBlock.Type { + case "text": + chunk := openAIStreamResponseChunk{ + ID: defaultIfBlank(state.ResponseID, "chatcmpl-anthropic"), + Object: "chat.completion.chunk", + Created: state.Created, + Model: state.Model, + Choices: []openAIStreamChoice{ + { + Index: 0, + Delta: openAIStreamDelta{ + Role: conditionalAssistantRole(state), + }, + }, + }, + } + if err := emitOpenAIStreamPayload(w, flusher, chunk); err != nil { + return false, err + } + case "tool_use": + toolState := &anthropicToolCallState{ + ID: defaultIfBlank(event.ContentBlock.ID, fmt.Sprintf("call_%d", index)), + Name: strings.TrimSpace(event.ContentBlock.Name), + } + state.ToolCalls[index] = toolState + toolCallIndex := index + chunk := openAIStreamResponseChunk{ + ID: defaultIfBlank(state.ResponseID, "chatcmpl-anthropic"), + Object: "chat.completion.chunk", + Created: state.Created, + Model: state.Model, + Choices: []openAIStreamChoice{ + { + Index: 0, + Delta: openAIStreamDelta{ + Role: conditionalAssistantRole(state), + ToolCalls: []ToolCall{ + { + ID: toolState.ID, + Type: "function", + Index: &toolCallIndex, + Function: &ToolCallFunction{ + Name: toolState.Name, + Arguments: "", + }, + }, + }, + }, + }, + }, + } + if err := emitOpenAIStreamPayload(w, flusher, chunk); err != nil { + return false, err + } + } + case "content_block_delta": + if event.Index == nil || event.Delta == nil { + return false, nil + } + index := *event.Index + switch event.Delta.Type { + case "text_delta": + if event.Delta.Text == "" { + return false, nil + } + state.AssistantText.WriteString(event.Delta.Text) + chunk := openAIStreamResponseChunk{ + ID: defaultIfBlank(state.ResponseID, "chatcmpl-anthropic"), + Object: "chat.completion.chunk", + Created: state.Created, + Model: state.Model, + Choices: []openAIStreamChoice{ + { + Index: 0, + Delta: openAIStreamDelta{ + Content: event.Delta.Text, + }, + }, + }, + } + if err := emitOpenAIStreamPayload(w, flusher, chunk); err != nil { + return false, err + } + case "input_json_delta": + toolState := state.ToolCalls[index] + if toolState == nil { + return false, nil + } + toolState.Arguments.WriteString(event.Delta.PartialJSON) + toolCallIndex := index + chunk := openAIStreamResponseChunk{ + ID: defaultIfBlank(state.ResponseID, "chatcmpl-anthropic"), + Object: "chat.completion.chunk", + Created: state.Created, + Model: state.Model, + Choices: []openAIStreamChoice{ + { + Index: 0, + Delta: openAIStreamDelta{ + ToolCalls: []ToolCall{ + { + ID: toolState.ID, + Type: "function", + Index: &toolCallIndex, + Function: &ToolCallFunction{ + Arguments: event.Delta.PartialJSON, + }, + }, + }, + }, + }, + }, + } + if err := emitOpenAIStreamPayload(w, flusher, chunk); err != nil { + return false, err + } + } + case "message_delta": + if event.Usage != nil { + state.CompletionTokens = event.Usage.OutputTokens + } + if event.Delta != nil { + finishReason := normalizeAnthropicStopReason(event.Delta.StopReason) + if finishReason != "" { + chunk := openAIStreamResponseChunk{ + ID: defaultIfBlank(state.ResponseID, "chatcmpl-anthropic"), + Object: "chat.completion.chunk", + Created: state.Created, + Model: state.Model, + Choices: []openAIStreamChoice{ + { + Index: 0, + Delta: openAIStreamDelta{}, + FinishReason: &finishReason, + }, + }, + } + if err := emitOpenAIStreamPayload(w, flusher, chunk); err != nil { + return false, err + } + } + } + case "message_stop": + return true, nil + case "error": + if event.Error != nil && strings.TrimSpace(event.Error.Message) != "" { + return false, errors.New(event.Error.Message) + } + } + + return false, nil +} + +func emitOpenAIStreamPayload(w io.Writer, flusher http.Flusher, payload interface{}) error { + data, err := json.Marshal(payload) + if err != nil { + return err + } + if _, err := fmt.Fprintf(w, "data: %s\n\n", data); err != nil { + return err + } + flusher.Flush() + return nil +} + +func emitOpenAIStreamDone(w io.Writer, flusher http.Flusher) error { + if _, err := io.WriteString(w, "data: [DONE]\n\n"); err != nil { + return err + } + flusher.Flush() + return nil +} + +func conditionalAssistantRole(state *anthropicStreamState) string { + if state.SentRole { + return "" + } + state.SentRole = true + return "assistant" +} + +func buildOpenAICompatibleRequestBody(req ChatCompletionRequest, model *models.LLMModel) ([]byte, error) { + if model == nil { + return nil, errors.New("model is not active or does not exist") + } + payload := map[string]json.RawMessage{} + if len(req.RawBody) > 0 { + if err := json.Unmarshal(req.RawBody, &payload); err != nil { + return nil, fmt.Errorf("failed to decode provider request: %w", err) + } + } + if payload == nil { + payload = map[string]json.RawMessage{} + } + delete(payload, "session_id") + delete(payload, "instance_id") + delete(payload, "instance_mode") + delete(payload, "runtime_type") + delete(payload, "gateway_id") + delete(payload, "runtime_pod_id") + delete(payload, "trace_id") + delete(payload, "request_id") + modelPayload, err := json.Marshal(model.ProviderModelName) + if err != nil { + return nil, fmt.Errorf("failed to encode provider model name: %w", err) + } + payload["model"] = json.RawMessage(modelPayload) + body, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("failed to encode provider request: %w", err) + } + return body, nil +} + +func buildAnthropicRequestBody(req ChatCompletionRequest, model *models.LLMModel) ([]byte, error) { + systemPrompt, messages := convertChatMessagesToAnthropic(req.Messages) + tools, err := convertToolsToAnthropic(req.Tools) + if err != nil { + return nil, err + } + toolChoice, err := convertToolChoiceToAnthropic(req.ToolChoice) + if err != nil { + return nil, err + } + + stopSequences, err := convertStopSequences(req.Stop) + if err != nil { + return nil, err + } + + maxTokens := defaultAnthropicMaxTokens + if req.MaxTokens != nil && *req.MaxTokens > 0 { + maxTokens = *req.MaxTokens + } + + payload := anthropicRequestPayload{ + Model: model.ProviderModelName, + Messages: messages, + System: systemPrompt, + MaxTokens: maxTokens, + Stream: req.Stream, + Temperature: req.Temperature, + TopP: req.TopP, + StopSeqs: stopSequences, + } + if len(tools) > 0 { + payload.Tools = tools + if toolChoice != nil { + payload.ToolChoice = toolChoice + } + } + + body, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("failed to encode anthropic request: %w", err) + } + return body, nil +} + +func buildProviderHTTPRequest(ctx context.Context, traceID, requestID string, model *models.LLMModel, providerRequestBody []byte, resolvedAPIKey *string, acceptStream bool) (*http.Request, error) { + if model == nil { + return nil, errors.New("model is not active or does not exist") + } + + switch models.ResolveLLMProtocolTypeOrDefault(model.ProviderType, model.ProtocolType) { + case models.ProtocolTypeAnthropic: + return buildAnthropicProviderHTTPRequest(ctx, traceID, requestID, model, providerRequestBody, resolvedAPIKey, acceptStream) + default: + return buildOpenAICompatibleProviderHTTPRequest(ctx, traceID, requestID, model, providerRequestBody, resolvedAPIKey, acceptStream) + } +} + +func buildOpenAICompatibleProviderHTTPRequest(ctx context.Context, traceID, requestID string, model *models.LLMModel, providerRequestBody []byte, resolvedAPIKey *string, acceptStream bool) (*http.Request, error) { + endpoint := strings.TrimRight(strings.TrimSpace(model.BaseURL), "/") + "/chat/completions" + httpRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(providerRequestBody)) + if err != nil { + return nil, fmt.Errorf("failed to build provider request: %w", err) + } + httpRequest.Header.Set("Content-Type", "application/json") + if acceptStream { + httpRequest.Header.Set("Accept", "text/event-stream") + } else { + httpRequest.Header.Set("Accept", "application/json") + } + httpRequest.Header.Set("X-Trace-ID", traceID) + httpRequest.Header.Set("X-Request-ID", requestID) + if resolvedAPIKey != nil && strings.TrimSpace(*resolvedAPIKey) != "" { + httpRequest.Header.Set("Authorization", "Bearer "+strings.TrimSpace(*resolvedAPIKey)) + } + return httpRequest, nil +} + +func buildAnthropicProviderHTTPRequest(ctx context.Context, traceID, requestID string, model *models.LLMModel, providerRequestBody []byte, resolvedAPIKey *string, acceptStream bool) (*http.Request, error) { + endpoint, err := buildProviderAPIEndpoint(model.BaseURL, "v1", "messages") + if err != nil { + return nil, err + } + + httpRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(providerRequestBody)) + if err != nil { + return nil, fmt.Errorf("failed to build provider request: %w", err) + } + httpRequest.Header.Set("Content-Type", "application/json") + httpRequest.Header.Set("anthropic-version", anthropicVersionHeader) + if acceptStream { + httpRequest.Header.Set("Accept", "text/event-stream") + } else { + httpRequest.Header.Set("Accept", "application/json") + } + httpRequest.Header.Set("X-Trace-ID", traceID) + httpRequest.Header.Set("X-Request-ID", requestID) + if resolvedAPIKey != nil && strings.TrimSpace(*resolvedAPIKey) != "" { + httpRequest.Header.Set("x-api-key", strings.TrimSpace(*resolvedAPIKey)) + } + return httpRequest, nil +} + +func buildProviderAPIEndpoint(baseURL, versionPrefix, resource string) (string, error) { + trimmed := strings.TrimSpace(strings.TrimRight(baseURL, "/")) + if trimmed == "" { + return "", errors.New("base URL is required") + } + + parsed, err := url.Parse(trimmed) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", errors.New("base URL is invalid") + } + + versionPath := "/" + strings.Trim(versionPrefix, "/") + resourcePath := strings.Trim(resource, "/") + if strings.HasSuffix(strings.ToLower(parsed.Path), strings.ToLower(versionPath)) { + return trimmed + "/" + resourcePath, nil + } + + pathSegments := strings.Split(strings.Trim(parsed.Path, "/"), "/") + lastSegment := "" + if len(pathSegments) > 0 { + lastSegment = pathSegments[len(pathSegments)-1] + } + if providerVersionSegmentPattern.MatchString(lastSegment) { + return trimmed + "/" + resourcePath, nil + } + + return trimmed + versionPath + "/" + resourcePath, nil +} + +func convertChatMessagesToAnthropic(messages []ChatMessage) (string, []anthropicRequestMessage) { + systemParts := make([]string, 0) + converted := make([]anthropicRequestMessage, 0, len(messages)) + + appendMessage := func(role string, blocks []anthropicContentBlock) { + if len(blocks) == 0 { + return + } + if len(converted) > 0 && converted[len(converted)-1].Role == role { + converted[len(converted)-1].Content = append(converted[len(converted)-1].Content, blocks...) + return + } + converted = append(converted, anthropicRequestMessage{ + Role: role, + Content: blocks, + }) + } + + for _, message := range messages { + role := strings.TrimSpace(strings.ToLower(message.Role)) + switch role { + case "system": + if text := strings.TrimSpace(flattenChatMessage(message)); text != "" { + systemParts = append(systemParts, text) + } + case "assistant": + appendMessage("assistant", convertAssistantMessageToAnthropicBlocks(message)) + case "tool": + appendMessage("user", convertToolMessageToAnthropicBlocks(message)) + default: + appendMessage("user", convertUserMessageToAnthropicBlocks(message)) + } + } + + return strings.Join(systemParts, "\n\n"), converted +} + +func convertUserMessageToAnthropicBlocks(message ChatMessage) []anthropicContentBlock { + blocks := convertContentToAnthropicTextBlocks(message.Content) + if len(blocks) > 0 { + return blocks + } + fallback := strings.TrimSpace(flattenChatMessage(message)) + if fallback == "" { + return nil + } + return []anthropicContentBlock{{Type: "text", Text: fallback}} +} + +func convertAssistantMessageToAnthropicBlocks(message ChatMessage) []anthropicContentBlock { + blocks := convertContentToAnthropicTextBlocks(message.Content) + for _, toolCall := range message.ToolCalls { + if toolCall.Function == nil { + continue + } + blocks = append(blocks, anthropicContentBlock{ + Type: "tool_use", + ID: defaultIfBlank(toolCall.ID, "tool_call"), + Name: strings.TrimSpace(toolCall.Function.Name), + Input: parseToolCallArguments(toolCall.Function.Arguments), + }) + } + return blocks +} + +func convertToolMessageToAnthropicBlocks(message ChatMessage) []anthropicContentBlock { + toolUseID := strings.TrimSpace(message.ToolCallID) + content := strings.TrimSpace(flattenMessageContent(message.Content)) + if content == "" { + content = mustJSONString(message.Content) + } + if toolUseID == "" && content == "" { + return nil + } + return []anthropicContentBlock{ + { + Type: "tool_result", + ToolUseID: toolUseID, + Content: defaultIfBlank(content, "{}"), + }, + } +} + +func convertContentToAnthropicTextBlocks(content interface{}) []anthropicContentBlock { + switch value := content.(type) { + case nil: + return nil + case string: + if strings.TrimSpace(value) == "" { + return nil + } + return []anthropicContentBlock{{Type: "text", Text: value}} + case []interface{}: + blocks := make([]anthropicContentBlock, 0, len(value)) + for _, item := range value { + text := strings.TrimSpace(flattenStructuredContentPart(item)) + if text == "" { + continue + } + blocks = append(blocks, anthropicContentBlock{Type: "text", Text: text}) + } + if len(blocks) > 0 { + return blocks + } + case map[string]interface{}: + if text := strings.TrimSpace(flattenStructuredContentPart(value)); text != "" { + return []anthropicContentBlock{{Type: "text", Text: text}} + } + } + + flattened := strings.TrimSpace(flattenMessageContent(content)) + if flattened == "" { + return nil + } + return []anthropicContentBlock{{Type: "text", Text: flattened}} +} + +func convertToolsToAnthropic(raw json.RawMessage) ([]anthropicToolDefinition, error) { + if len(bytes.TrimSpace(raw)) == 0 { + return nil, nil + } + + var items []map[string]interface{} + if err := json.Unmarshal(raw, &items); err != nil { + return nil, fmt.Errorf("failed to decode provider tools: %w", err) + } + + tools := make([]anthropicToolDefinition, 0, len(items)) + for _, item := range items { + itemType, _ := item["type"].(string) + if strings.TrimSpace(itemType) != "" && !strings.EqualFold(strings.TrimSpace(itemType), "function") { + continue + } + functionValue, ok := item["function"].(map[string]interface{}) + if !ok { + continue + } + name, _ := functionValue["name"].(string) + if strings.TrimSpace(name) == "" { + continue + } + description, _ := functionValue["description"].(string) + inputSchema := functionValue["parameters"] + if inputSchema == nil { + inputSchema = map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{}, + } + } + tools = append(tools, anthropicToolDefinition{ + Name: strings.TrimSpace(name), + Description: strings.TrimSpace(description), + InputSchema: inputSchema, + }) + } + + return tools, nil +} + +func convertToolChoiceToAnthropic(raw json.RawMessage) (map[string]interface{}, error) { + if len(bytes.TrimSpace(raw)) == 0 { + return nil, nil + } + + var parsed interface{} + if err := json.Unmarshal(raw, &parsed); err != nil { + return nil, fmt.Errorf("failed to decode tool choice: %w", err) + } + + switch value := parsed.(type) { + case string: + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "none": + return nil, nil + case "required": + return map[string]interface{}{"type": "any"}, nil + case "auto", "any": + return map[string]interface{}{"type": strings.ToLower(strings.TrimSpace(value))}, nil + } + case map[string]interface{}: + choiceType, _ := value["type"].(string) + switch strings.ToLower(strings.TrimSpace(choiceType)) { + case "none", "": + return nil, nil + case "function": + functionValue, _ := value["function"].(map[string]interface{}) + name, _ := functionValue["name"].(string) + if strings.TrimSpace(name) == "" { + return nil, nil + } + return map[string]interface{}{"type": "tool", "name": strings.TrimSpace(name)}, nil + case "tool": + return value, nil + case "auto", "any": + return map[string]interface{}{"type": strings.ToLower(strings.TrimSpace(choiceType))}, nil + } + } + + return nil, nil +} + +func convertStopSequences(raw json.RawMessage) ([]string, error) { + if len(bytes.TrimSpace(raw)) == 0 { + return nil, nil + } + + var single string + if err := json.Unmarshal(raw, &single); err == nil { + single = strings.TrimSpace(single) + if single == "" { + return nil, nil + } + return []string{single}, nil + } + + var many []string + if err := json.Unmarshal(raw, &many); err != nil { + return nil, fmt.Errorf("failed to decode stop sequences: %w", err) + } + + filtered := make([]string, 0, len(many)) + for _, item := range many { + if trimmed := strings.TrimSpace(item); trimmed != "" { + filtered = append(filtered, trimmed) + } + } + return filtered, nil +} + +func parseToolCallArguments(raw string) interface{} { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return map[string]interface{}{} + } + + var parsed interface{} + if err := json.Unmarshal([]byte(trimmed), &parsed); err != nil { + return map[string]interface{}{ + "raw": trimmed, + } + } + return parsed +} + +func normalizeAnthropicResponse(response anthropicMessageResponse) ([]byte, string, int, int, int, error) { + normalized := ChatCompletionResponse{ + ID: defaultIfBlank(response.ID, "chatcmpl-anthropic"), + Object: "chat.completion", + Created: time.Now().Unix(), + Model: response.Model, + Usage: struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + }{ + PromptTokens: response.Usage.InputTokens, + CompletionTokens: response.Usage.OutputTokens, + TotalTokens: response.Usage.InputTokens + response.Usage.OutputTokens, + }, + } + + contentValue, toolCalls := convertAnthropicBlocksToOpenAIMessage(response.Content) + normalized.Choices = []struct { + Index int `json:"index"` + Message struct { + Role string `json:"role"` + Content interface{} `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + Refusal interface{} `json:"refusal,omitempty"` + } `json:"message"` + FinishReason string `json:"finish_reason,omitempty"` + }{ + { + Index: 0, + Message: struct { + Role string `json:"role"` + Content interface{} `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + Refusal interface{} `json:"refusal,omitempty"` + }{ + Role: defaultIfBlank(response.Role, "assistant"), + Content: contentValue, + ToolCalls: toolCalls, + }, + FinishReason: normalizeAnthropicStopReason(response.StopReason), + }, + } + + body, err := json.Marshal(normalized) + if err != nil { + return nil, "", 0, 0, 0, err + } + + return body, extractAssistantContent(normalized), normalized.Usage.PromptTokens, normalized.Usage.CompletionTokens, normalized.Usage.TotalTokens, nil +} + +func convertAnthropicBlocksToOpenAIMessage(blocks []anthropicContentBlock) (interface{}, []ToolCall) { + textParts := make([]string, 0) + toolCalls := make([]ToolCall, 0) + for _, block := range blocks { + switch block.Type { + case "text": + if strings.TrimSpace(block.Text) != "" { + textParts = append(textParts, block.Text) + } + case "tool_use": + inputPayload := "{}" + if block.Input != nil { + if encoded, err := json.Marshal(block.Input); err == nil { + inputPayload = string(encoded) + } + } + toolCalls = append(toolCalls, ToolCall{ + ID: defaultIfBlank(block.ID, "tool_call"), + Type: "function", + Function: &ToolCallFunction{ + Name: strings.TrimSpace(block.Name), + Arguments: inputPayload, + }, + }) + } + } + + if len(textParts) > 0 { + return strings.Join(textParts, "\n"), toolCalls + } + if len(toolCalls) > 0 { + return nil, toolCalls + } + return "", nil +} + +func normalizeAnthropicStopReason(reason string) string { + switch strings.TrimSpace(reason) { + case "end_turn", "stop_sequence": + return "stop" + case "max_tokens": + return "length" + case "tool_use": + return "tool_calls" + default: + return "" + } +} + +func renderAnthropicToolCalls(state *anthropicStreamState) string { + if len(state.ToolCalls) == 0 { + return "" + } + + indexes := make([]int, 0, len(state.ToolCalls)) + for index := range state.ToolCalls { + indexes = append(indexes, index) + } + sort.Ints(indexes) + + parts := make([]string, 0, len(state.ToolCalls)) + for _, index := range indexes { + toolState := state.ToolCalls[index] + if toolState == nil { + continue + } + args := strings.TrimSpace(toolState.Arguments.String()) + switch { + case toolState.Name != "" && args != "": + parts = append(parts, fmt.Sprintf("tool_call %s(%s)", toolState.Name, args)) + case toolState.Name != "": + parts = append(parts, "tool_call "+toolState.Name) + } + } + return strings.Join(parts, "\n") +} + +func defaultIfBlank(value, fallback string) string { + if strings.TrimSpace(value) == "" { + return fallback + } + return value +} + +func (s *service) recordBlockedInvocation(traceID, requestID string, req ChatCompletionRequest, userID int, model *models.LLMModel, reason string) *int { + requestPayload := mustJSONString(req) + responsePayload := reason + invocation := &models.ModelInvocation{ + TraceID: traceID, + SessionID: req.SessionID, + RequestID: requestID, + UserID: intPtr(userID), + InstanceID: req.InstanceID, + InstanceMode: runtimeAttributionString(req.InstanceMode), + RuntimeType: runtimeAttributionString(req.RuntimeType), + GatewayID: runtimeAttributionString(req.GatewayID), + RuntimePodID: runtimeAttributionInt64(req.RuntimePodID), + ModelID: intPtr(model.ID), + ProviderType: model.ProviderType, + RequestedModel: req.Model, + ActualProviderModel: model.ProviderModelName, + TrafficClass: models.TrafficClassLLM, + RequestPayload: &requestPayload, + ResponsePayload: &responsePayload, + IsStreaming: false, + Status: models.ModelInvocationStatusBlocked, + ErrorMessage: stringPtr(reason), + } + if err := s.invocationService.RecordInvocation(invocation); err != nil { + return nil + } + return intPtr(invocation.ID) +} + +func (s *service) resolveTargetModel(selectedModel *models.LLMModel, analysis services.RiskAnalysis) (*models.LLMModel, string, error) { + if selectedModel == nil { + return nil, models.RiskActionAllow, errors.New("model is not active or does not exist") + } + if !analysis.IsSensitive { + return selectedModel, models.RiskActionAllow, nil + } + if analysis.HighestAction == "require_approval" || analysis.HighestAction == models.RiskActionBlock { + return nil, models.RiskActionBlock, errors.New("request was blocked by risk policy") + } + if selectedModel.IsSecure { + return selectedModel, models.RiskActionAllow, nil + } + + activeModels, err := s.modelRepo.ListActive() + if err != nil { + return nil, models.RiskActionBlock, fmt.Errorf("failed to list active secure models: %w", err) + } + for _, item := range activeModels { + if item.IsSecure { + resolved := item + return &resolved, models.RiskActionRouteSecureModel, nil + } + } + return nil, models.RiskActionBlock, errors.New("sensitive content requires an active secure model") +} + +func (s *service) resolveRequestedModel(requestedModel string) (*models.LLMModel, error) { + if isAutoModelRequest(requestedModel) { + return s.selectAutoModel() + } + + selectedModel, err := s.modelRepo.GetByDisplayName(requestedModel) + if err != nil { + return nil, fmt.Errorf("failed to get model: %w", err) + } + if selectedModel == nil || !selectedModel.IsActive { + return nil, errors.New("model is not active or does not exist") + } + return selectedModel, nil +} + +func (s *service) selectAutoModel() (*models.LLMModel, error) { + items, err := s.modelRepo.ListActive() + if err != nil { + return nil, fmt.Errorf("failed to list active models: %w", err) + } + if len(items) == 0 { + return nil, errors.New("no active models are configured") + } + + for _, item := range items { + if !item.IsSecure { + selected := item + return &selected, nil + } + } + + selected := items[0] + return &selected, nil +} + +func isAutoModelRequest(requestedModel string) bool { + return strings.EqualFold(strings.TrimSpace(requestedModel), autoModelID) +} + +func calculateEstimatedCost(model *models.LLMModel, promptTokens, completionTokens int) float64 { + return (float64(promptTokens) * model.InputPrice / 1_000_000.0) + (float64(completionTokens) * model.OutputPrice / 1_000_000.0) +} + +func normalizeOrCreateID(value *string, prefix string) string { + if value != nil { + if normalized := normalizeExistingIdentifier(*value, prefix); normalized != "" { + return normalized + } + } + return prefix + "_" + randomHex(8) +} + +func normalizeExistingIdentifier(rawValue string, prefix string) string { + trimmed := strings.TrimSpace(rawValue) + if trimmed == "" { + return "" + } + if len(trimmed) <= maxStoredIdentifierLength { + return trimmed + } + + sum := sha256.Sum256([]byte(trimmed)) + normalized := prefix + "_" + hex.EncodeToString(sum[:16]) + if len(normalized) > maxStoredIdentifierLength { + return normalized[:maxStoredIdentifierLength] + } + return normalized +} + +func (s *service) resolveTraceID(userID int, req ChatCompletionRequest, sessionID string) string { + if req.TraceID != nil { + if normalized := normalizeExistingIdentifier(*req.TraceID, "trc"); normalized != "" { + return normalized + } + } + + toolReferenceIDs := extractToolReferenceIDs(req.Messages) + if len(toolReferenceIDs) == 0 { + return normalizeOrCreateID(nil, "trc") + } + + normalizedSessionID := strings.TrimSpace(sessionID) + if normalizedSessionID != "" { + if session, err := s.chatSessionService.GetSession(normalizedSessionID); err == nil && session != nil && session.LastTraceID != nil { + lastTraceID := normalizeExistingIdentifier(*session.LastTraceID, "trc") + if lastTraceID != "" { + if instanceIDsCompatible(req.InstanceID, session.InstanceID) { + return lastTraceID + } + items, listErr := s.invocationService.ListInvocationsByTraceID(lastTraceID) + if listErr != nil { + logPersistenceError("load session last trace invocations", lastTraceID, listErr) + } else if invocationsContainAnyToolReference(items, toolReferenceIDs) { + return lastTraceID + } + } + } else if err != nil { + logPersistenceError("load chat session for trace reuse", normalizedSessionID, err) + } + + items, err := s.invocationService.ListInvocationsBySessionID(normalizedSessionID, 100) + if err != nil { + logPersistenceError("list invocations by session for trace reuse", normalizedSessionID, err) + } else { + for _, invocation := range items { + if strings.TrimSpace(invocation.TraceID) == "" || invocation.ResponsePayload == nil { + continue + } + if req.InstanceID != nil && invocation.InstanceID != nil && *req.InstanceID != *invocation.InstanceID { + continue + } + if payloadContainsAny(*invocation.ResponsePayload, toolReferenceIDs) { + return normalizeExistingIdentifier(invocation.TraceID, "trc") + } + } + } + } + + items, err := s.invocationService.ListInvocationsByUserID(userID, 200) + if err != nil { + logPersistenceError("list invocations for trace reuse", "", err) + return normalizeOrCreateID(nil, "trc") + } + + for _, invocation := range items { + if strings.TrimSpace(invocation.TraceID) == "" || invocation.ResponsePayload == nil { + continue + } + if req.InstanceID != nil && invocation.InstanceID != nil && *req.InstanceID != *invocation.InstanceID { + continue + } + if payloadContainsAny(*invocation.ResponsePayload, toolReferenceIDs) { + return normalizeExistingIdentifier(invocation.TraceID, "trc") + } + } + + return normalizeOrCreateID(nil, "trc") +} + +func randomHex(size int) string { + bytes := make([]byte, size) + if _, err := rand.Read(bytes); err != nil { + return fmt.Sprintf("%d", time.Now().UnixNano()) + } + return hex.EncodeToString(bytes) +} + +func intPtr(value int) *int { + return &value +} + +func runtimeAttributionString(value *string) *string { + if value == nil { + return nil + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return nil + } + return &trimmed +} + +func runtimeAttributionInt64(value *int64) *int64 { + if value == nil || *value <= 0 { + return nil + } + copied := *value + return &copied +} + +func applyInvocationRuntimeAttribution(invocation *models.ModelInvocation, req ChatCompletionRequest) { + if invocation == nil { + return + } + invocation.InstanceMode = runtimeAttributionString(req.InstanceMode) + invocation.RuntimeType = runtimeAttributionString(req.RuntimeType) + invocation.GatewayID = runtimeAttributionString(req.GatewayID) + invocation.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID) +} + +func applyAuditRuntimeAttribution(event *models.AuditEvent, req ChatCompletionRequest) { + if event == nil { + return + } + event.InstanceMode = runtimeAttributionString(req.InstanceMode) + event.RuntimeType = runtimeAttributionString(req.RuntimeType) + event.GatewayID = runtimeAttributionString(req.GatewayID) + event.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID) +} + +func applyCostRuntimeAttribution(record *models.CostRecord, req ChatCompletionRequest) { + if record == nil { + return + } + record.InstanceMode = runtimeAttributionString(req.InstanceMode) + record.RuntimeType = runtimeAttributionString(req.RuntimeType) + record.GatewayID = runtimeAttributionString(req.GatewayID) + record.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID) +} + +func riskHitAttribution(req ChatCompletionRequest) *services.RiskHitAttribution { + attribution := &services.RiskHitAttribution{ + InstanceMode: runtimeAttributionString(req.InstanceMode), + RuntimeType: runtimeAttributionString(req.RuntimeType), + GatewayID: runtimeAttributionString(req.GatewayID), + RuntimePodID: runtimeAttributionInt64(req.RuntimePodID), + } + if attribution.InstanceMode == nil && attribution.RuntimeType == nil && attribution.GatewayID == nil && attribution.RuntimePodID == nil { + return nil + } + return attribution +} + +func userIDOrZero(value *int) int { + if value == nil { + return 0 + } + return *value +} + +func instanceIDsCompatible(requestInstanceID, sessionInstanceID *int) bool { + if requestInstanceID == nil || sessionInstanceID == nil { + return true + } + return *requestInstanceID == *sessionInstanceID +} + +func stringPtr(value string) *string { + return &value +} + +func mustJSONString(value interface{}) string { + data, err := json.Marshal(value) + if err != nil { + return "{}" + } + return string(data) +} + +func rawOrJSONRequestPayload(req ChatCompletionRequest) string { + if len(req.RawBody) > 0 { + return string(req.RawBody) + } + return mustJSONString(req) +} + +func cloneProxyHeaders(headers http.Header) http.Header { + cloned := make(http.Header) + copyProxyHeaders(cloned, headers) + return cloned +} + +func copyProxyHeaders(dst, src http.Header) { + for key, values := range src { + if isHopByHopHeader(key) || strings.EqualFold(key, "Content-Length") { + continue + } + for _, value := range values { + dst.Add(key, value) + } + } +} + +func isHopByHopHeader(header string) bool { + switch strings.ToLower(strings.TrimSpace(header)) { + case "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade": + return true + default: + return false + } +} + +func fallbackCurrency(currency string) string { + if trimmed := strings.TrimSpace(currency); trimmed != "" { + return trimmed + } + return "USD" +} + +func resolveSessionID(req ChatCompletionRequest) string { + if normalized := normalizeOptionalString(req.SessionID); normalized != "" { + return normalizeExistingIdentifier(normalized, "sess") + } + if normalized := normalizeOptionalString(req.User); normalized != "" { + return normalizeExistingIdentifier(normalized, "sess") + } + return "" +} + +func normalizeSessionID(value *string, traceID string) string { + if value != nil { + if normalized := normalizeExistingIdentifier(*value, "sess"); normalized != "" { + return normalized + } + } + return normalizeExistingIdentifier("sess_"+traceID, "sess") +} + +func deriveSessionTitle(messages []ChatMessage) *string { + for _, message := range messages { + if strings.TrimSpace(message.Role) != "user" { + continue + } + content := strings.TrimSpace(flattenChatMessage(message)) + if content == "" { + continue + } + runes := []rune(content) + if len(runes) > 72 { + content = string(runes[:72]) + } + return &content + } + return nil +} + +func buildPersistedMessages(messages []ChatMessage) []services.PersistedChatMessage { + currentTurn := currentTurnMessages(messages) + items := make([]services.PersistedChatMessage, 0, len(currentTurn)) + for index, message := range currentTurn { + content := strings.TrimSpace(flattenChatMessage(message)) + role := strings.TrimSpace(message.Role) + if role == "" || content == "" { + continue + } + items = append(items, services.PersistedChatMessage{ + Role: role, + Content: content, + SequenceNo: index + 1, + }) + } + return items +} + +func resolveUsage(messages []ChatMessage, responsePayload string, promptTokens, completionTokens, totalTokens int) (int, int, int, bool) { + if totalTokens > 0 || promptTokens > 0 || completionTokens > 0 { + if totalTokens == 0 { + totalTokens = promptTokens + completionTokens + } + return promptTokens, completionTokens, totalTokens, false + } + + promptEstimate := estimateTokens(flattenMessages(messages)) + completionEstimate := estimateTokens(responsePayload) + return promptEstimate, completionEstimate, promptEstimate + completionEstimate, true +} + +func estimateTokens(text string) int { + trimmed := strings.TrimSpace(text) + if trimmed == "" { + return 0 + } + runes := len([]rune(trimmed)) + tokens := runes / 4 + if runes%4 != 0 { + tokens++ + } + if tokens == 0 { + return 1 + } + return tokens +} + +func flattenMessages(messages []ChatMessage) string { + var parts []string + for _, message := range messages { + content := flattenChatMessage(message) + if content == "" { + continue + } + if message.Role != "" { + parts = append(parts, message.Role+": "+content) + continue + } + parts = append(parts, content) + } + return strings.Join(parts, "\n") +} + +func extractToolReferenceIDs(messages []ChatMessage) []string { + currentTurn := currentTurnMessages(messages) + if len(currentTurn) > 0 && strings.EqualFold(strings.TrimSpace(currentTurn[0].Role), "user") { + currentTurn = currentTurn[1:] + } + + seen := make(map[string]struct{}) + ids := make([]string, 0) + for _, message := range currentTurn { + if normalized := normalizeToolReferenceID(message.ToolCallID); normalized != "" { + if _, exists := seen[normalized]; !exists { + seen[normalized] = struct{}{} + ids = append(ids, normalized) + } + } + for _, toolCall := range message.ToolCalls { + if normalized := normalizeToolReferenceID(toolCall.ID); normalized != "" { + if _, exists := seen[normalized]; !exists { + seen[normalized] = struct{}{} + ids = append(ids, normalized) + } + } + } + } + return ids +} + +func normalizeToolReferenceID(value string) string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return "" + } + + var builder strings.Builder + builder.Grow(len(trimmed)) + for _, char := range trimmed { + if unicode.IsLetter(char) || unicode.IsDigit(char) { + builder.WriteRune(unicode.ToLower(char)) + } + } + return builder.String() +} + +func extractToolReferenceIDsFromPayload(payload string) map[string]struct{} { + ids := make(map[string]struct{}) + addID := func(value string) { + if normalized := normalizeToolReferenceID(value); normalized != "" { + ids[normalized] = struct{}{} + } + } + + trimmed := strings.TrimSpace(payload) + if trimmed == "" { + return ids + } + + if strings.HasPrefix(trimmed, "data:") { + for _, line := range strings.Split(trimmed, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "data:") { + continue + } + + chunkPayload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if chunkPayload == "" || chunkPayload == "[DONE]" { + continue + } + + var chunk openAIStreamChunk + if err := json.Unmarshal([]byte(chunkPayload), &chunk); err != nil { + continue + } + + for _, choice := range chunk.Choices { + for _, toolCall := range choice.Delta.ToolCalls { + addID(toolCall.ID) + } + } + } + return ids + } + + var response ChatCompletionResponse + if err := json.Unmarshal([]byte(trimmed), &response); err == nil { + for _, choice := range response.Choices { + for _, toolCall := range choice.Message.ToolCalls { + addID(toolCall.ID) + } + } + } + + return ids +} + +func normalizedToolReferenceSet(candidates []string) map[string]struct{} { + items := make(map[string]struct{}, len(candidates)) + for _, candidate := range candidates { + if normalized := normalizeToolReferenceID(candidate); normalized != "" { + items[normalized] = struct{}{} + } + } + return items +} + +func payloadContainsAny(payload string, candidates []string) bool { + if payload == "" || len(candidates) == 0 { + return false + } + + normalizedCandidates := normalizedToolReferenceSet(candidates) + if len(normalizedCandidates) == 0 { + return false + } + + for payloadID := range extractToolReferenceIDsFromPayload(payload) { + if _, exists := normalizedCandidates[payloadID]; exists { + return true + } + } + + normalizedPayload := normalizeToolReferenceID(payload) + for candidate := range normalizedCandidates { + if candidate != "" && strings.Contains(normalizedPayload, candidate) { + return true + } + } + return false +} + +func currentTurnMessages(messages []ChatMessage) []ChatMessage { + startIndex := 0 + for index := len(messages) - 1; index >= 0; index-- { + if strings.EqualFold(strings.TrimSpace(messages[index].Role), "user") { + startIndex = index + break + } + } + return messages[startIndex:] +} + +func normalizeOptionalString(value *string) string { + if value == nil { + return "" + } + return strings.TrimSpace(*value) +} + +func invocationsContainAnyToolReference(items []models.ModelInvocation, toolReferenceIDs []string) bool { + for _, invocation := range items { + if invocation.ResponsePayload == nil { + continue + } + if payloadContainsAny(*invocation.ResponsePayload, toolReferenceIDs) { + return true + } + } + return false +} + +func logPersistenceError(action, traceID string, err error) { + if err == nil { + return + } + if strings.TrimSpace(traceID) == "" { + log.Printf("ai gateway: %s: %v", action, err) + return + } + log.Printf("ai gateway: %s for trace %s: %v", action, traceID, err) +} + +func flattenMessageContent(content interface{}) string { + switch value := content.(type) { + case nil: + return "" + case string: + return value + case []interface{}: + parts := make([]string, 0, len(value)) + for _, item := range value { + if text := flattenStructuredContentPart(item); text != "" { + parts = append(parts, text) + } + } + if len(parts) > 0 { + return strings.Join(parts, "\n") + } + return mustJSONString(value) + case map[string]interface{}: + if text := flattenStructuredContentPart(value); text != "" { + return text + } + return mustJSONString(value) + default: + return mustJSONString(value) + } +} + +func flattenStructuredContentPart(content interface{}) string { + part, ok := content.(map[string]interface{}) + if !ok { + return "" + } + + if text, ok := part["text"].(string); ok { + return strings.TrimSpace(text) + } + if text, ok := part["input_text"].(string); ok { + return strings.TrimSpace(text) + } + if text, ok := part["output_text"].(string); ok { + return strings.TrimSpace(text) + } + + return "" +} + +func flattenChatMessage(message ChatMessage) string { + parts := []string{} + + content := strings.TrimSpace(flattenMessageContent(message.Content)) + if content != "" { + parts = append(parts, content) + } + + toolCalls := strings.TrimSpace(flattenToolCalls(message.ToolCalls)) + if toolCalls != "" { + parts = append(parts, toolCalls) + } + + return strings.Join(parts, "\n") +} + +func flattenToolCalls(toolCalls []ToolCall) string { + if len(toolCalls) == 0 { + return "" + } + + parts := make([]string, 0, len(toolCalls)) + for _, toolCall := range toolCalls { + if toolCall.Function == nil { + parts = append(parts, mustJSONString(toolCall)) + continue + } + + name := strings.TrimSpace(toolCall.Function.Name) + args := strings.TrimSpace(toolCall.Function.Arguments) + switch { + case name != "" && args != "": + parts = append(parts, fmt.Sprintf("tool_call %s(%s)", name, args)) + case name != "": + parts = append(parts, "tool_call "+name) + default: + parts = append(parts, mustJSONString(toolCall)) + } + } + + return strings.Join(parts, "\n") +} + +func extractAssistantContent(response ChatCompletionResponse) string { + var parts []string + for _, choice := range response.Choices { + content := strings.TrimSpace(flattenMessageContent(choice.Message.Content)) + if content != "" { + parts = append(parts, content) + continue + } + + toolCalls := strings.TrimSpace(flattenToolCalls(choice.Message.ToolCalls)) + if toolCalls != "" { + parts = append(parts, toolCalls) + } + } + return strings.Join(parts, "\n") +} diff --git a/backend/internal/aigateway/service_test.go b/backend/internal/aigateway/service_test.go new file mode 100644 index 0000000..18772a7 --- /dev/null +++ b/backend/internal/aigateway/service_test.go @@ -0,0 +1,552 @@ +package aigateway + +import ( + "encoding/json" + "strings" + "testing" + + "clawreef/internal/models" +) + +type stubModelInvocationService struct { + items []models.ModelInvocation +} + +func (s *stubModelInvocationService) RecordInvocation(invocation *models.ModelInvocation) error { + return nil +} + +func (s *stubModelInvocationService) GetInvocationByID(id int) (*models.ModelInvocation, error) { + return nil, nil +} + +func (s *stubModelInvocationService) ListInvocationsByTraceID(traceID string) ([]models.ModelInvocation, error) { + return s.items, nil +} + +func (s *stubModelInvocationService) ListInvocationsBySessionID(sessionID string, limit int) ([]models.ModelInvocation, error) { + return s.items, nil +} + +func (s *stubModelInvocationService) ListInvocationsByUserID(userID, limit int) ([]models.ModelInvocation, error) { + return s.items, nil +} + +type stubChatSessionService struct { + session *models.ChatSession +} + +func (s *stubChatSessionService) GetSession(sessionID string) (*models.ChatSession, error) { + return s.session, nil +} + +func (s *stubChatSessionService) EnsureSession(sessionID string, userID, instanceID *int, traceID *string, title *string) (*models.ChatSession, error) { + return s.session, nil +} + +func TestBuildProviderRequestPreservesToolConfiguration(t *testing.T) { + req := ChatCompletionRequest{ + Model: "gateway-model", + RawBody: []byte(`{"model":"gateway-model","messages":[{"role":"user","content":[{"type":"text","text":"weather in shanghai"}]}],"tools":[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"get_weather"}},"stream":true,"stream_options":{"include_usage":false,"custom":"value"},"custom_field":{"nested":[1,2,3]},"session_id":"sess_123","request_id":"req_123","trace_id":"trc_123","instance_id":99,"instance_mode":"lite","runtime_type":"gateway","gateway_id":"gw_123","runtime_pod_id":5}`), + } + + model := &models.LLMModel{ + ProviderModelName: "provider-model", + } + + providerRequestBody, err := buildProviderRequestBody(req, model) + if err != nil { + t.Fatalf("buildProviderRequestBody returned error: %v", err) + } + + var payload map[string]json.RawMessage + if err := json.Unmarshal(providerRequestBody, &payload); err != nil { + t.Fatalf("failed to decode provider request body: %v", err) + } + + var modelName string + if err := json.Unmarshal(payload["model"], &modelName); err != nil { + t.Fatalf("failed to decode model name: %v", err) + } + if modelName != "provider-model" { + t.Fatalf("expected provider model name to be replaced, got %q", modelName) + } + if _, ok := payload["messages"]; !ok { + t.Fatalf("expected messages to be forwarded") + } + if _, ok := payload["tools"]; !ok { + t.Fatalf("expected tools to be forwarded") + } + if _, ok := payload["tool_choice"]; !ok { + t.Fatalf("expected tool_choice to be forwarded") + } + if _, ok := payload["custom_field"]; !ok { + t.Fatalf("expected unknown provider fields to survive") + } + if _, ok := payload["session_id"]; ok { + t.Fatalf("expected internal session_id to be stripped") + } + if _, ok := payload["request_id"]; ok { + t.Fatalf("expected internal request_id to be stripped") + } + if _, ok := payload["trace_id"]; ok { + t.Fatalf("expected internal trace_id to be stripped") + } + if _, ok := payload["instance_id"]; ok { + t.Fatalf("expected internal instance_id to be stripped") + } + if _, ok := payload["instance_mode"]; ok { + t.Fatalf("expected internal instance_mode to be stripped") + } + if _, ok := payload["runtime_type"]; ok { + t.Fatalf("expected internal runtime_type to be stripped") + } + if _, ok := payload["gateway_id"]; ok { + t.Fatalf("expected internal gateway_id to be stripped") + } + if _, ok := payload["runtime_pod_id"]; ok { + t.Fatalf("expected internal runtime_pod_id to be stripped") + } + if string(payload["stream_options"]) != `{"include_usage":false,"custom":"value"}` { + t.Fatalf("expected stream_options to pass through unchanged, got %s", string(payload["stream_options"])) + } +} + +func TestBuildProviderRequestUsesAnthropicProtocolForLocalModel(t *testing.T) { + req := ChatCompletionRequest{ + Model: "gateway-model", + Messages: []ChatMessage{ + { + Role: "user", + Content: "hello from local anthropic", + }, + }, + } + + model := &models.LLMModel{ + ProviderType: models.ProviderTypeLocal, + ProtocolType: models.ProtocolTypeAnthropic, + ProviderModelName: "claude-local", + } + + providerRequestBody, err := buildProviderRequestBody(req, model) + if err != nil { + t.Fatalf("buildProviderRequestBody returned error: %v", err) + } + + var payload anthropicRequestPayload + if err := json.Unmarshal(providerRequestBody, &payload); err != nil { + t.Fatalf("expected anthropic payload, got decode error: %v", err) + } + + if payload.Model != "claude-local" { + t.Fatalf("expected provider model name to be replaced, got %q", payload.Model) + } + if len(payload.Messages) != 1 || payload.Messages[0].Role != "user" { + t.Fatalf("expected anthropic user message, got %#v", payload.Messages) + } +} + +func TestRuntimeAttributionHelpersPopulateRecords(t *testing.T) { + mode := "lite" + runtimeType := "gateway" + gatewayID := "gw_instance_1" + runtimePodID := int64(17) + req := ChatCompletionRequest{ + InstanceMode: &mode, + RuntimeType: &runtimeType, + GatewayID: &gatewayID, + RuntimePodID: &runtimePodID, + } + + invocation := &models.ModelInvocation{} + applyInvocationRuntimeAttribution(invocation, req) + if invocation.InstanceMode == nil || *invocation.InstanceMode != mode { + t.Fatalf("invocation instance mode not populated") + } + if invocation.GatewayID == nil || *invocation.GatewayID != gatewayID { + t.Fatalf("invocation gateway ID not populated") + } + + audit := &models.AuditEvent{} + applyAuditRuntimeAttribution(audit, req) + if audit.RuntimeType == nil || *audit.RuntimeType != runtimeType { + t.Fatalf("audit runtime type not populated") + } + if audit.RuntimePodID == nil || *audit.RuntimePodID != runtimePodID { + t.Fatalf("audit runtime pod ID not populated") + } + + cost := &models.CostRecord{} + applyCostRuntimeAttribution(cost, req) + if cost.GatewayID == nil || *cost.GatewayID != gatewayID { + t.Fatalf("cost gateway ID not populated") + } + + risk := riskHitAttribution(req) + if risk == nil || risk.GatewayID == nil || *risk.GatewayID != gatewayID { + t.Fatalf("risk attribution gateway ID not populated") + } +} + +func TestRewriteStreamLineKeepsToolCalls(t *testing.T) { + line := "data: {\"id\":\"chatcmpl-1\",\"model\":\"provider-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Shanghai\\\"}\"}}]},\"finish_reason\":null}]}\n" + + var assistantText strings.Builder + var promptTokens int + var completionTokens int + var totalTokens int + + done := inspectStreamLine(line, &assistantText, &promptTokens, &completionTokens, &totalTokens) + if done { + t.Fatalf("expected tool chunk not to finish the stream") + } + if assistantText.String() != "" { + t.Fatalf("expected tool chunks not to be appended as assistant text, got %q", assistantText.String()) + } + + payload := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "data:")) + var chunk openAIStreamChunk + if err := json.Unmarshal([]byte(payload), &chunk); err != nil { + t.Fatalf("failed to decode chunk: %v", err) + } + + if len(chunk.Choices) != 1 || len(chunk.Choices[0].Delta.ToolCalls) != 1 { + t.Fatalf("expected tool_calls to be preserved, got %#v", chunk.Choices) + } + if chunk.Choices[0].Delta.ToolCalls[0].Function == nil || chunk.Choices[0].Delta.ToolCalls[0].Function.Name != "get_weather" { + t.Fatalf("expected function metadata to survive, got %#v", chunk.Choices[0].Delta.ToolCalls[0]) + } +} + +func TestExtractAssistantContentFallsBackToToolCalls(t *testing.T) { + response := ChatCompletionResponse{ + Choices: []struct { + Index int `json:"index"` + Message struct { + Role string `json:"role"` + Content interface{} `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + Refusal interface{} `json:"refusal,omitempty"` + } `json:"message"` + FinishReason string `json:"finish_reason,omitempty"` + }{ + { + Index: 0, + Message: struct { + Role string `json:"role"` + Content interface{} `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + Refusal interface{} `json:"refusal,omitempty"` + }{ + Role: "assistant", + Content: nil, + ToolCalls: []ToolCall{ + { + ID: "call_1", + Type: "function", + Function: &ToolCallFunction{ + Name: "get_weather", + Arguments: `{"city":"Shanghai"}`, + }, + }, + }, + }, + FinishReason: "tool_calls", + }, + }, + } + + content := extractAssistantContent(response) + if !strings.Contains(content, "get_weather") { + t.Fatalf("expected tool call content to be included, got %q", content) + } + if strings.Contains(content, "null") { + t.Fatalf("expected nil content not to become \"null\", got %q", content) + } +} + +func TestResolveTraceIDReusesRecentTraceForToolLoop(t *testing.T) { + toolCallID := "call_weather_123" + svc := &service{ + chatSessionService: &stubChatSessionService{ + session: &models.ChatSession{ + SessionID: "agent:openclaw:main", + LastTraceID: stringPtr("trc_existing"), + }, + }, + invocationService: &stubModelInvocationService{ + items: []models.ModelInvocation{ + { + TraceID: "trc_existing", + InstanceID: intPtr(9), + ResponsePayload: stringPtr(`{"choices":[{"message":{"role":"assistant","tool_calls":[{"id":"call_weather_123","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Beijing\"}"}}]}}]}`), + }, + }, + }, + } + + traceID := svc.resolveTraceID(7, ChatCompletionRequest{ + InstanceID: intPtr(9), + Messages: []ChatMessage{ + { + Role: "tool", + ToolCallID: toolCallID, + Content: `{"temp_c":25}`, + }, + }, + }, "agent:openclaw:main") + + if traceID != "trc_existing" { + t.Fatalf("expected tool continuation to reuse trace trc_existing, got %q", traceID) + } +} + +func TestResolveTraceIDReusesSessionLastTraceBeforeInvocationIsPersisted(t *testing.T) { + toolCallID := "call_weather_123" + svc := &service{ + chatSessionService: &stubChatSessionService{ + session: &models.ChatSession{ + SessionID: "agent:openclaw:main", + LastTraceID: stringPtr("trc_existing"), + }, + }, + invocationService: &stubModelInvocationService{}, + } + + traceID := svc.resolveTraceID(7, ChatCompletionRequest{ + InstanceID: intPtr(9), + Messages: []ChatMessage{ + { + Role: "user", + Content: "What is the weather in Beijing now?", + }, + { + Role: "assistant", + ToolCalls: []ToolCall{ + { + ID: toolCallID, + Function: &ToolCallFunction{ + Name: "get_weather", + Arguments: `{"city":"Beijing"}`, + }, + }, + }, + }, + { + Role: "tool", + ToolCallID: toolCallID, + Content: `{"temp_c":25}`, + }, + }, + }, "agent:openclaw:main") + + if traceID != "trc_existing" { + t.Fatalf("expected session last trace to be reused before invocation persistence, got %q", traceID) + } +} + +func TestResolveTraceIDReusesTraceWhenOpenClawStripsToolCallUnderscore(t *testing.T) { + svc := &service{ + invocationService: &stubModelInvocationService{ + items: []models.ModelInvocation{ + { + TraceID: "trc_existing", + UserID: intPtr(7), + InstanceID: intPtr(19), + ResponsePayload: stringPtr("data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"id\":\"call_d1cad4719db94d1594f93456\",\"index\":0,\"type\":\"function\",\"function\":{\"name\":\"exec\",\"arguments\":\"\"}}]}}]}\n\ndata: [DONE]\n"), + }, + }, + }, + } + + traceID := svc.resolveTraceID(7, ChatCompletionRequest{ + InstanceID: intPtr(19), + Messages: []ChatMessage{ + { + Role: "user", + Content: "What is the weather in Changchun?", + }, + { + Role: "assistant", + ToolCalls: []ToolCall{ + { + ID: "calld1cad4719db94d1594f93456", + Function: &ToolCallFunction{ + Name: "exec", + Arguments: `{"command":"curl -s \"wttr.in/Changchun\""}`, + }, + }, + }, + }, + { + Role: "tool", + ToolCallID: "calld1cad4719db94d1594f93456", + Content: "changchun: +9C", + }, + }, + }, "") + + if traceID != "trc_existing" { + t.Fatalf("expected trace reuse when tool id separators differ, got %q", traceID) + } +} + +func TestResolveTraceIDDoesNotReuseHistoricalToolLoopAcrossNewUserTurn(t *testing.T) { + toolCallID := "call_weather_123" + svc := &service{ + chatSessionService: &stubChatSessionService{ + session: &models.ChatSession{ + SessionID: "agent:openclaw:main", + LastTraceID: stringPtr("trc_existing"), + }, + }, + invocationService: &stubModelInvocationService{ + items: []models.ModelInvocation{ + { + TraceID: "trc_existing", + InstanceID: intPtr(9), + ResponsePayload: stringPtr(`{"choices":[{"message":{"role":"assistant","tool_calls":[{"id":"call_weather_123","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Beijing\"}"}}]}}]}`), + }, + }, + }, + } + + traceID := svc.resolveTraceID(7, ChatCompletionRequest{ + InstanceID: intPtr(9), + Messages: []ChatMessage{ + { + Role: "user", + Content: "What is the weather in Beijing now?", + }, + { + Role: "assistant", + ToolCalls: []ToolCall{ + { + ID: toolCallID, + Function: &ToolCallFunction{ + Name: "get_weather", + Arguments: `{"city":"Beijing"}`, + }, + }, + }, + }, + { + Role: "tool", + ToolCallID: toolCallID, + Content: `{"temp_c":25}`, + }, + { + Role: "assistant", + Content: "Beijing is cloudy and 25C.", + }, + { + Role: "user", + Content: "How about Shanghai?", + }, + }, + }, "agent:openclaw:main") + + if traceID == "trc_existing" { + t.Fatalf("expected a new trace for the next user turn, got %q", traceID) + } + if !strings.HasPrefix(traceID, "trc_") { + t.Fatalf("expected generated trace id to keep trc_ prefix, got %q", traceID) + } +} + +func TestBuildPersistedMessagesKeepsOnlyCurrentTurnMessages(t *testing.T) { + items := buildPersistedMessages([]ChatMessage{ + { + Role: "system", + Content: "system prompt", + }, + { + Role: "user", + Content: "Previous turn question", + }, + { + Role: "assistant", + Content: "Previous turn answer", + }, + { + Role: "user", + Content: "Current turn question", + }, + { + Role: "assistant", + ToolCalls: []ToolCall{ + { + ID: "call_current", + Function: &ToolCallFunction{ + Name: "get_weather", + Arguments: `{"city":"Shanghai"}`, + }, + }, + }, + }, + { + Role: "tool", + ToolCallID: "call_current", + Content: `{"temp_c":16}`, + }, + }) + + if len(items) != 3 { + t.Fatalf("expected only current turn messages to be persisted, got %d", len(items)) + } + if items[0].Role != "user" || items[0].Content != "Current turn question" { + t.Fatalf("expected current turn user message first, got %#v", items[0]) + } + if items[1].Role != "assistant" || !strings.Contains(items[1].Content, "get_weather") { + t.Fatalf("expected current turn assistant tool call second, got %#v", items[1]) + } + if items[2].Role != "tool" || items[2].Content != `{"temp_c":16}` { + t.Fatalf("expected current turn tool output third, got %#v", items[2]) + } + if items[0].SequenceNo != 1 || items[1].SequenceNo != 2 || items[2].SequenceNo != 3 { + t.Fatalf("expected current turn sequence numbers to restart at 1, got %#v", items) + } +} + +func TestResolveTraceIDKeepsExplicitTraceID(t *testing.T) { + explicit := "trc_explicit" + svc := &service{} + + traceID := svc.resolveTraceID(7, ChatCompletionRequest{ + TraceID: &explicit, + Messages: []ChatMessage{ + { + Role: "user", + Content: "hello", + }, + }, + }, "") + + if traceID != explicit { + t.Fatalf("expected explicit trace id %q, got %q", explicit, traceID) + } +} + +func TestResolveSessionIDPrefersExplicitSessionThenOpenAIUser(t *testing.T) { + explicitSession := "agent:main:direct:alice" + if got := resolveSessionID(ChatCompletionRequest{SessionID: &explicitSession}); got != explicitSession { + t.Fatalf("expected explicit session id %q, got %q", explicitSession, got) + } + + openAIUser := "agent:main:direct:bob" + if got := resolveSessionID(ChatCompletionRequest{User: &openAIUser}); got != openAIUser { + t.Fatalf("expected OpenAI user to become session id %q, got %q", openAIUser, got) + } +} + +func TestNormalizeExistingIdentifierHashesLongOpenClawSessionKeys(t *testing.T) { + longKey := "agent:very-long-openclaw-agent-id:discord:default:direct:12345678901234567890123456789012345678901234567890" + normalized := normalizeExistingIdentifier(longKey, "sess") + if normalized == longKey { + t.Fatalf("expected long session key to be normalized") + } + if len(normalized) > maxStoredIdentifierLength { + t.Fatalf("expected normalized id length <= %d, got %d", maxStoredIdentifierLength, len(normalized)) + } +} diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go new file mode 100644 index 0000000..ddb093a --- /dev/null +++ b/backend/internal/config/config.go @@ -0,0 +1,677 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +// Config holds all application configuration +type Config struct { + Server ServerConfig `yaml:"server"` + Database DatabaseConfig `yaml:"database"` + JWT JWTConfig `yaml:"jwt"` + Kubernetes KubernetesConfig `yaml:"kubernetes"` + Storage StorageConfig `yaml:"storage"` + Runtime RuntimePoolConfig `yaml:"runtime"` + ObjectStorage ObjectStorageConfig `yaml:"objectStorage"` + SkillScanner SkillScannerConfig `yaml:"skillScanner"` + LeaderElection LeaderElectionConfig `yaml:"leaderElection"` +} + +// LeaderElectionConfig controls the control-plane leader election that gates +// singleton background loops (SyncService + Team event consumers + stale-task +// monitor) so they run on exactly one replica when clawmanager-app is scaled +// horizontally. The HTTP API and the in-pod nginx data plane run on every +// replica regardless. +type LeaderElectionConfig struct { + Enabled bool `yaml:"enabled"` + Namespace string `yaml:"namespace"` + LeaseName string `yaml:"leaseName"` + Identity string `yaml:"identity"` + LeaseDuration int `yaml:"leaseDuration"` // seconds + RenewDeadline int `yaml:"renewDeadline"` // seconds + RetryPeriod int `yaml:"retryPeriod"` // seconds +} + +const ( + StorageProfileCluster = "cluster" + StorageProfileSingle = "single-node" + StorageProfileLegacyNFS = "legacy-nfs" + DefaultPVCBindTimeout = 2 * time.Minute +) + +// ServerConfig holds server-related configuration +type ServerConfig struct { + Address string `yaml:"address"` + Mode string `yaml:"mode"` +} + +// DatabaseConfig holds database-related configuration +type DatabaseConfig struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + User string `yaml:"user"` + Password string `yaml:"password"` + Database string `yaml:"database"` +} + +// JWTConfig holds JWT-related configuration +type JWTConfig struct { + Secret string `yaml:"secret"` + AccessExpiry int `yaml:"access_expiry"` // minutes + RefreshExpiry int `yaml:"refresh_expiry"` // hours +} + +// KubernetesConfig holds Kubernetes-related configuration +type KubernetesConfig struct { + Mode string `yaml:"mode"` // 连接模式: auto, incluster, outofcluster + OutOfCluster OutOfClusterConfig `yaml:"outOfCluster"` + InCluster InClusterConfig `yaml:"inCluster"` + Common CommonKubernetesConfig `yaml:"common"` + Runtime KubernetesRuntimeConfig `yaml:"runtime"` + Logging LoggingConfig `yaml:"logging"` +} + +// OutOfClusterConfig holds out-of-cluster Kubernetes configuration +type OutOfClusterConfig struct { + Kubeconfig string `yaml:"kubeconfig"` + Context string `yaml:"context"` + APIServer string `yaml:"apiServer"` + TLS TLSConfig `yaml:"tls"` +} + +// TLSConfig holds TLS configuration +type TLSConfig struct { + InsecureSkipVerify bool `yaml:"insecureSkipVerify"` + CAFile string `yaml:"caFile"` + CertFile string `yaml:"certFile"` + KeyFile string `yaml:"keyFile"` +} + +// InClusterConfig holds in-cluster Kubernetes configuration +type InClusterConfig struct { + TokenPath string `yaml:"tokenPath"` + CAPath string `yaml:"caPath"` + NamespacePath string `yaml:"namespacePath"` +} + +// CommonKubernetesConfig holds common Kubernetes configuration +type CommonKubernetesConfig struct { + Namespace string `yaml:"namespace"` + StorageClass string `yaml:"storageClass"` + Timeout int `yaml:"timeout"` + RetryCount int `yaml:"retryCount"` + AutoCreateNamespace bool `yaml:"autoCreateNamespace"` +} + +// KubernetesRuntimeConfig holds Kubernetes runtime configuration +type KubernetesRuntimeConfig struct { + Pod RuntimePodConfig `yaml:"pod"` + PVC RuntimePVCConfig `yaml:"pvc"` +} + +// RuntimePodConfig holds pod runtime configuration +type RuntimePodConfig struct { + ImageRegistry string `yaml:"imageRegistry"` + ContainerPort int32 `yaml:"containerPort"` + MountPath string `yaml:"mountPath"` + Privileged bool `yaml:"privileged"` + ExtraLabels map[string]string `yaml:"extraLabels"` + NodeSelector map[string]string `yaml:"nodeSelector"` + Tolerations []Toleration `yaml:"tolerations"` +} + +// Toleration represents a Kubernetes toleration +type Toleration struct { + Key string `yaml:"key"` + Operator string `yaml:"operator"` + Value string `yaml:"value"` + Effect string `yaml:"effect"` +} + +// RuntimePVCConfig holds PVC runtime configuration +type RuntimePVCConfig struct { + AccessMode string `yaml:"accessMode"` + VolumeMode string `yaml:"volumeMode"` + AllowVolumeExpansion bool `yaml:"allowVolumeExpansion"` + ReclaimPolicy string `yaml:"reclaimPolicy"` + HostPathPrefix string `yaml:"hostPathPrefix"` +} + +// StorageConfig controls the validated installation storage profile. +type StorageConfig struct { + Profile string `yaml:"profile"` + HostPathFallbackEnabled bool `yaml:"hostPathFallbackEnabled"` + PVCBindTimeout time.Duration `yaml:"pvcBindTimeout"` + ControlPlaneStorageClass string `yaml:"controlPlaneStorageClass"` + InstanceStorageClass string `yaml:"instanceStorageClass"` + WorkspaceStorageClass string `yaml:"workspaceStorageClass"` + WorkspaceAccessMode string `yaml:"workspaceAccessMode"` +} + +// RuntimePoolConfig holds shared V2 runtime pool configuration. +type RuntimePoolConfig struct { + Namespace string `yaml:"namespace"` + WorkspaceRoot string `yaml:"workspaceRoot"` + WorkspacePVCClaimName string `yaml:"workspacePvcClaimName"` + WorkspaceNFSServer string `yaml:"workspaceNfsServer"` + WorkspaceNFSPath string `yaml:"workspaceNfsPath"` + AgentControlToken string `yaml:"agentControlToken"` + AgentReportToken string `yaml:"agentReportToken"` + BackendReplicaID string `yaml:"backendReplicaId"` + RedisURL string `yaml:"redisUrl"` + SchedulerEnabled bool `yaml:"schedulerEnabled"` + HeartbeatTimeout time.Duration `yaml:"heartbeatTimeout"` + SchedulerTick time.Duration `yaml:"schedulerTick"` + OpenClawImage string `yaml:"openClawImage"` + HermesImage string `yaml:"hermesImage"` + MaxGatewaysPerPod int `yaml:"maxGatewaysPerPod"` + GatewayPortStart int `yaml:"gatewayPortStart"` + GatewayPortEnd int `yaml:"gatewayPortEnd"` +} + +// LoggingConfig holds logging configuration +type LoggingConfig struct { + Level string `yaml:"level"` + LogAPICalls bool `yaml:"logApiCalls"` +} + +type ObjectStorageConfig struct { + Endpoint string `yaml:"endpoint"` + Region string `yaml:"region"` + AccessKey string `yaml:"accessKey"` + SecretKey string `yaml:"secretKey"` + Bucket string `yaml:"bucket"` + UseSSL bool `yaml:"useSSL"` + BasePath string `yaml:"basePath"` + ForcePathStyle bool `yaml:"forcePathStyle"` + LocalFallback string `yaml:"localFallback"` +} + +type SkillScannerConfig struct { + BaseURL string `yaml:"baseUrl"` + APIKey string `yaml:"apiKey"` + TimeoutSeconds int `yaml:"timeoutSeconds"` + Enabled bool `yaml:"enabled"` +} + +// Load loads configuration from file and environment variables +func Load() (*Config, error) { + runtimeNamespace := getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", "clawmanager-system")) + defaultStorageClass := getEnv("K8S_STORAGE_CLASS", "standard") + config := &Config{ + Server: ServerConfig{ + Address: ":9001", + Mode: "debug", + }, + Database: DatabaseConfig{ + Host: "localhost", + Port: 3306, + User: "clawreef", + Password: "clawreef123", + Database: "clawreef", + }, + JWT: JWTConfig{ + Secret: getEnv("JWT_SECRET", "clawreef-secret-key-change-in-production"), + AccessExpiry: 60, // 60 minutes + RefreshExpiry: 168, // 7 days + }, + Kubernetes: KubernetesConfig{ + Mode: getEnv("K8S_MODE", "auto"), + OutOfCluster: OutOfClusterConfig{ + Kubeconfig: getEnv("KUBECONFIG", getEnv("K8S_KUBECONFIG", "")), + }, + InCluster: InClusterConfig{ + TokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token", + CAPath: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + NamespacePath: "/var/run/secrets/kubernetes.io/serviceaccount/namespace", + }, + Common: CommonKubernetesConfig{ + Namespace: getEnv("K8S_NAMESPACE", "clawreef"), + StorageClass: defaultStorageClass, + Timeout: 30, + RetryCount: 3, + AutoCreateNamespace: true, + }, + Runtime: KubernetesRuntimeConfig{ + Pod: RuntimePodConfig{ + ImageRegistry: "docker.io/clawreef", + ContainerPort: 3001, + MountPath: "/home/user/data", + Privileged: false, + ExtraLabels: make(map[string]string), + NodeSelector: make(map[string]string), + }, + PVC: RuntimePVCConfig{ + AccessMode: "ReadWriteOnce", + VolumeMode: "Filesystem", + AllowVolumeExpansion: true, + ReclaimPolicy: "Delete", + HostPathPrefix: getEnv("K8S_PV_HOST_PATH_PREFIX", "/data/clawreef"), + }, + }, + Logging: LoggingConfig{ + Level: "info", + LogAPICalls: false, + }, + }, + Storage: StorageConfig{ + Profile: getEnv("CLAWMANAGER_STORAGE_PROFILE", StorageProfileCluster), + HostPathFallbackEnabled: getEnvBool("K8S_HOSTPATH_FALLBACK_ENABLED", false), + PVCBindTimeout: getEnvDuration("K8S_PVC_BIND_TIMEOUT", DefaultPVCBindTimeout), + ControlPlaneStorageClass: getEnv("K8S_CONTROL_PLANE_STORAGE_CLASS", defaultStorageClass), + InstanceStorageClass: getEnv("K8S_INSTANCE_STORAGE_CLASS", defaultStorageClass), + WorkspaceStorageClass: getEnv("K8S_WORKSPACE_STORAGE_CLASS", defaultStorageClass), + WorkspaceAccessMode: getEnv("K8S_WORKSPACE_ACCESS_MODE", "ReadWriteMany"), + }, + Runtime: RuntimePoolConfig{ + Namespace: runtimeNamespace, + WorkspaceRoot: getEnv("RUNTIME_WORKSPACE_ROOT", "/workspaces"), + WorkspacePVCClaimName: getEnv("RUNTIME_WORKSPACE_PVC_CLAIM", ""), + WorkspaceNFSServer: getEnv("RUNTIME_WORKSPACE_NFS_SERVER", ""), + WorkspaceNFSPath: getEnv("RUNTIME_WORKSPACE_NFS_PATH", "/"), + AgentControlToken: getEnv("RUNTIME_AGENT_CONTROL_TOKEN", ""), + AgentReportToken: getEnv("RUNTIME_AGENT_REPORT_TOKEN", ""), + BackendReplicaID: getEnv("HOSTNAME", "clawmanager-backend-local"), + RedisURL: getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", "")), + SchedulerEnabled: getEnvBool("RUNTIME_SCHEDULER_ENABLED", true), + HeartbeatTimeout: getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", 10*time.Second), + SchedulerTick: getEnvDuration("RUNTIME_SCHEDULER_TICK", 2*time.Second), + OpenClawImage: getEnv("OPENCLAW_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest"), + HermesImage: getEnv("HERMES_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"), + MaxGatewaysPerPod: getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", 100), + GatewayPortStart: getEnvInt("RUNTIME_GATEWAY_PORT_START", 20000), + GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", 20099), + }, + ObjectStorage: ObjectStorageConfig{ + Endpoint: getEnv("OBJECT_STORAGE_ENDPOINT", ""), + Region: getEnv("OBJECT_STORAGE_REGION", ""), + AccessKey: getEnv("OBJECT_STORAGE_ACCESS_KEY", ""), + SecretKey: getEnv("OBJECT_STORAGE_SECRET_KEY", ""), + Bucket: getEnv("OBJECT_STORAGE_BUCKET", "clawmanager-skills"), + UseSSL: strings.EqualFold(getEnv("OBJECT_STORAGE_USE_SSL", "false"), "true"), + BasePath: getEnv("OBJECT_STORAGE_BASE_PATH", "skills"), + ForcePathStyle: strings.EqualFold(getEnv("OBJECT_STORAGE_FORCE_PATH_STYLE", "true"), "true"), + LocalFallback: getEnv("OBJECT_STORAGE_LOCAL_FALLBACK", ".data/object-storage"), + }, + SkillScanner: SkillScannerConfig{ + BaseURL: getEnv("SKILL_SCANNER_BASE_URL", ""), + APIKey: getEnv("SKILL_SCANNER_API_KEY", ""), + TimeoutSeconds: 30, + Enabled: strings.EqualFold(getEnv("SKILL_SCANNER_ENABLED", "false"), "true"), + }, + LeaderElection: LeaderElectionConfig{ + Enabled: strings.EqualFold(getEnv("CLAWMANAGER_LEADER_ELECTION", "true"), "true"), + Namespace: getEnv("POD_NAMESPACE", "clawmanager-system"), + LeaseName: getEnv("CLAWMANAGER_LEADER_LEASE_NAME", "clawmanager-controlplane-leader"), + Identity: getEnv("POD_NAME", ""), + LeaseDuration: 15, + RenewDeadline: 10, + RetryPeriod: 2, + }, + } + + // Try to load from k8s config file + configPaths := []string{ + "configs/k8s.yaml", + "../configs/k8s.yaml", + "/app/configs/k8s.yaml", + } + + for _, path := range configPaths { + if _, err := os.Stat(path); err == nil { + fmt.Printf("Loading k8s config from: %s\n", path) + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read k8s config file: %w", err) + } + if err := yaml.Unmarshal(data, config); err != nil { + return nil, fmt.Errorf("failed to parse k8s config file: %w", err) + } + break + } + } + + // Try to load from legacy dev.yaml config file + if _, err := os.Stat("configs/dev.yaml"); err == nil { + data, err := os.ReadFile("configs/dev.yaml") + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + if err := yaml.Unmarshal(data, config); err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + } + + // Override with environment variables + applyEnvOverrides(config) + normalizeStorageConfig(config) + + return config, nil +} + +// applyEnvOverrides applies environment variable overrides +func applyEnvOverrides(config *Config) { + // Server config + if addr := os.Getenv("SERVER_ADDRESS"); addr != "" { + config.Server.Address = addr + } + if mode := os.Getenv("SERVER_MODE"); mode != "" { + config.Server.Mode = mode + } + + // Database config + if host := os.Getenv("DB_HOST"); host != "" { + config.Database.Host = host + } + if port := os.Getenv("DB_PORT"); port != "" { + fmt.Sscanf(port, "%d", &config.Database.Port) + } + if user := os.Getenv("DB_USER"); user != "" { + config.Database.User = user + } + if pass := os.Getenv("DB_PASSWORD"); pass != "" { + config.Database.Password = pass + } + if db := os.Getenv("DB_NAME"); db != "" { + config.Database.Database = db + } + + // JWT config + if secret := os.Getenv("JWT_SECRET"); secret != "" { + config.JWT.Secret = secret + } + + // Kubernetes config + if mode := os.Getenv("K8S_MODE"); mode != "" { + config.Kubernetes.Mode = mode + } + if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" { + config.Kubernetes.OutOfCluster.Kubeconfig = kubeconfig + } + if kubeconfig := os.Getenv("K8S_KUBECONFIG"); kubeconfig != "" { + config.Kubernetes.OutOfCluster.Kubeconfig = kubeconfig + } + if namespace := os.Getenv("K8S_NAMESPACE"); namespace != "" { + config.Kubernetes.Common.Namespace = namespace + } + if storageClass := os.Getenv("K8S_STORAGE_CLASS"); storageClass != "" { + config.Kubernetes.Common.StorageClass = storageClass + } + if hostPathPrefix := os.Getenv("K8S_PV_HOST_PATH_PREFIX"); hostPathPrefix != "" { + config.Kubernetes.Runtime.PVC.HostPathPrefix = hostPathPrefix + } + if profile := os.Getenv("CLAWMANAGER_STORAGE_PROFILE"); profile != "" { + config.Storage.Profile = profile + } + if fallback := os.Getenv("K8S_HOSTPATH_FALLBACK_ENABLED"); fallback != "" { + config.Storage.HostPathFallbackEnabled = strings.EqualFold(fallback, "true") + } + if timeout := os.Getenv("K8S_PVC_BIND_TIMEOUT"); timeout != "" { + if parsed, err := time.ParseDuration(timeout); err == nil { + config.Storage.PVCBindTimeout = parsed + } + } + if storageClass := os.Getenv("K8S_CONTROL_PLANE_STORAGE_CLASS"); storageClass != "" { + config.Storage.ControlPlaneStorageClass = storageClass + } + if storageClass := os.Getenv("K8S_INSTANCE_STORAGE_CLASS"); storageClass != "" { + config.Storage.InstanceStorageClass = storageClass + } + if storageClass := os.Getenv("K8S_WORKSPACE_STORAGE_CLASS"); storageClass != "" { + config.Storage.WorkspaceStorageClass = storageClass + } + if accessMode := os.Getenv("K8S_WORKSPACE_ACCESS_MODE"); accessMode != "" { + config.Storage.WorkspaceAccessMode = accessMode + } + + config.Runtime.Namespace = getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", config.Runtime.Namespace)) + config.Runtime.WorkspaceRoot = getEnv("RUNTIME_WORKSPACE_ROOT", config.Runtime.WorkspaceRoot) + config.Runtime.WorkspacePVCClaimName = getEnv("RUNTIME_WORKSPACE_PVC_CLAIM", config.Runtime.WorkspacePVCClaimName) + config.Runtime.WorkspaceNFSServer = getEnv("RUNTIME_WORKSPACE_NFS_SERVER", config.Runtime.WorkspaceNFSServer) + config.Runtime.WorkspaceNFSPath = getEnv("RUNTIME_WORKSPACE_NFS_PATH", config.Runtime.WorkspaceNFSPath) + if strings.TrimSpace(config.Runtime.WorkspaceNFSPath) == "" { + config.Runtime.WorkspaceNFSPath = "/" + } + config.Runtime.AgentControlToken = getEnv("RUNTIME_AGENT_CONTROL_TOKEN", config.Runtime.AgentControlToken) + config.Runtime.AgentReportToken = getEnv("RUNTIME_AGENT_REPORT_TOKEN", config.Runtime.AgentReportToken) + config.Runtime.BackendReplicaID = getEnv("HOSTNAME", config.Runtime.BackendReplicaID) + config.Runtime.RedisURL = getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", config.Runtime.RedisURL)) + config.Runtime.SchedulerEnabled = getEnvBool("RUNTIME_SCHEDULER_ENABLED", config.Runtime.SchedulerEnabled) + config.Runtime.HeartbeatTimeout = getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", config.Runtime.HeartbeatTimeout) + config.Runtime.SchedulerTick = getEnvDuration("RUNTIME_SCHEDULER_TICK", config.Runtime.SchedulerTick) + config.Runtime.OpenClawImage = getEnv("OPENCLAW_RUNTIME_IMAGE", config.Runtime.OpenClawImage) + config.Runtime.HermesImage = getEnv("HERMES_RUNTIME_IMAGE", config.Runtime.HermesImage) + config.Runtime.MaxGatewaysPerPod = getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", config.Runtime.MaxGatewaysPerPod) + config.Runtime.GatewayPortStart = getEnvInt("RUNTIME_GATEWAY_PORT_START", config.Runtime.GatewayPortStart) + config.Runtime.GatewayPortEnd = getEnvInt("RUNTIME_GATEWAY_PORT_END", config.Runtime.GatewayPortEnd) + config.LeaderElection.Enabled = getEnvBool("CLAWMANAGER_LEADER_ELECTION", config.LeaderElection.Enabled) + config.LeaderElection.Namespace = getEnv("POD_NAMESPACE", config.LeaderElection.Namespace) + config.LeaderElection.LeaseName = getEnv("CLAWMANAGER_LEADER_LEASE_NAME", config.LeaderElection.LeaseName) + config.LeaderElection.Identity = getEnv("POD_NAME", config.LeaderElection.Identity) + config.LeaderElection.LeaseDuration = getEnvInt("CLAWMANAGER_LEADER_LEASE_DURATION", config.LeaderElection.LeaseDuration) + config.LeaderElection.RenewDeadline = getEnvInt("CLAWMANAGER_LEADER_RENEW_DEADLINE", config.LeaderElection.RenewDeadline) + config.LeaderElection.RetryPeriod = getEnvInt("CLAWMANAGER_LEADER_RETRY_PERIOD", config.LeaderElection.RetryPeriod) + + if endpoint := os.Getenv("OBJECT_STORAGE_ENDPOINT"); endpoint != "" { + config.ObjectStorage.Endpoint = endpoint + } + if region := os.Getenv("OBJECT_STORAGE_REGION"); region != "" { + config.ObjectStorage.Region = region + } + if accessKey := os.Getenv("OBJECT_STORAGE_ACCESS_KEY"); accessKey != "" { + config.ObjectStorage.AccessKey = accessKey + } + if secretKey := os.Getenv("OBJECT_STORAGE_SECRET_KEY"); secretKey != "" { + config.ObjectStorage.SecretKey = secretKey + } + if bucket := os.Getenv("OBJECT_STORAGE_BUCKET"); bucket != "" { + config.ObjectStorage.Bucket = bucket + } + if useSSL := os.Getenv("OBJECT_STORAGE_USE_SSL"); useSSL != "" { + config.ObjectStorage.UseSSL = strings.EqualFold(useSSL, "true") + } + if basePath := os.Getenv("OBJECT_STORAGE_BASE_PATH"); basePath != "" { + config.ObjectStorage.BasePath = basePath + } + if forcePathStyle := os.Getenv("OBJECT_STORAGE_FORCE_PATH_STYLE"); forcePathStyle != "" { + config.ObjectStorage.ForcePathStyle = strings.EqualFold(forcePathStyle, "true") + } + if localFallback := os.Getenv("OBJECT_STORAGE_LOCAL_FALLBACK"); localFallback != "" { + config.ObjectStorage.LocalFallback = localFallback + } + if baseURL := os.Getenv("SKILL_SCANNER_BASE_URL"); baseURL != "" { + config.SkillScanner.BaseURL = baseURL + } + if apiKey := os.Getenv("SKILL_SCANNER_API_KEY"); apiKey != "" { + config.SkillScanner.APIKey = apiKey + } + if enabled := os.Getenv("SKILL_SCANNER_ENABLED"); enabled != "" { + config.SkillScanner.Enabled = strings.EqualFold(enabled, "true") + } + if timeoutSeconds := os.Getenv("SKILL_SCANNER_TIMEOUT_SECONDS"); timeoutSeconds != "" { + fmt.Sscanf(timeoutSeconds, "%d", &config.SkillScanner.TimeoutSeconds) + } +} + +func normalizeStorageConfig(config *Config) { + if config == nil { + return + } + config.Storage.Profile = normalizeStorageProfile(config.Storage.Profile) + if config.Storage.PVCBindTimeout <= 0 { + config.Storage.PVCBindTimeout = DefaultPVCBindTimeout + } + if strings.TrimSpace(config.Kubernetes.Common.StorageClass) == "" { + config.Kubernetes.Common.StorageClass = "standard" + } + if strings.TrimSpace(config.Storage.ControlPlaneStorageClass) == "" { + config.Storage.ControlPlaneStorageClass = config.Kubernetes.Common.StorageClass + } + if strings.TrimSpace(config.Storage.InstanceStorageClass) == "" { + config.Storage.InstanceStorageClass = config.Kubernetes.Common.StorageClass + } + if strings.TrimSpace(config.Storage.WorkspaceStorageClass) == "" { + config.Storage.WorkspaceStorageClass = config.Kubernetes.Common.StorageClass + } + if strings.TrimSpace(config.Storage.WorkspaceAccessMode) == "" { + config.Storage.WorkspaceAccessMode = "ReadWriteMany" + } + if config.Storage.Profile == StorageProfileLegacyNFS && strings.TrimSpace(config.Runtime.WorkspaceNFSServer) == "" { + config.Runtime.WorkspaceNFSServer = defaultWorkspaceNFSServer(config.Runtime.Namespace) + } + if strings.TrimSpace(config.Runtime.WorkspaceNFSPath) == "" { + config.Runtime.WorkspaceNFSPath = "/" + } +} + +func normalizeStorageProfile(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "cluster", "cluster-csi", "csi": + return StorageProfileCluster + case "single", "single-node", "single_node", "hostpath": + return StorageProfileSingle + case "legacy", "legacy-nfs", "legacy_nfs", "nfs": + return StorageProfileLegacyNFS + default: + return strings.ToLower(strings.TrimSpace(value)) + } +} + +func getEnv(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +func defaultWorkspaceNFSServer(namespace string) string { + namespace = strings.TrimSpace(namespace) + if namespace == "" { + namespace = "clawmanager-system" + } + return fmt.Sprintf("workspace-store.%s.svc.cluster.local", namespace) +} + +func getEnvInt(key string, defaultValue int) int { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + parsed, err := strconv.Atoi(value) + if err != nil { + return defaultValue + } + return parsed +} + +func getEnvBool(key string, defaultValue bool) bool { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return defaultValue + } + return parsed +} + +func getEnvDuration(key string, defaultValue time.Duration) time.Duration { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + parsed, err := time.ParseDuration(value) + if err != nil { + return defaultValue + } + return parsed +} + +// GetKubeconfigPath returns the kubeconfig path for out-of-cluster mode +func (c *Config) GetKubeconfigPath() string { + return c.Kubernetes.OutOfCluster.Kubeconfig +} + +// GetNamespace returns the namespace prefix +func (c *Config) GetNamespace() string { + return c.Kubernetes.Common.Namespace +} + +// GetStorageClass returns the storage class +func (c *Config) GetStorageClass() string { + return c.Kubernetes.Common.StorageClass +} + +func (c *Config) GetStorageProfile() string { + if c == nil { + return StorageProfileCluster + } + return normalizeStorageProfile(c.Storage.Profile) +} + +func (c *Config) GetPVCBindTimeout() time.Duration { + if c == nil || c.Storage.PVCBindTimeout <= 0 { + return DefaultPVCBindTimeout + } + return c.Storage.PVCBindTimeout +} + +func (c *Config) GetControlPlaneStorageClass() string { + if c == nil { + return "standard" + } + if value := strings.TrimSpace(c.Storage.ControlPlaneStorageClass); value != "" { + return value + } + return c.GetStorageClass() +} + +func (c *Config) GetInstanceStorageClass() string { + if c == nil { + return "standard" + } + if value := strings.TrimSpace(c.Storage.InstanceStorageClass); value != "" { + return value + } + return c.GetStorageClass() +} + +func (c *Config) GetWorkspaceStorageClass() string { + if c == nil { + return "standard" + } + if value := strings.TrimSpace(c.Storage.WorkspaceStorageClass); value != "" { + return value + } + return c.GetStorageClass() +} + +func (c *Config) GetWorkspaceAccessMode() string { + if c == nil { + return "ReadWriteMany" + } + if value := strings.TrimSpace(c.Storage.WorkspaceAccessMode); value != "" { + return value + } + return "ReadWriteMany" +} + +// GetHostPathPrefix returns the host path prefix for PV creation +func (c *Config) GetHostPathPrefix() string { + if c.Kubernetes.Runtime.PVC.HostPathPrefix != "" { + return c.Kubernetes.Runtime.PVC.HostPathPrefix + } + return "/data/clawreef" +} + +// GetMode returns the K8s connection mode +func (c *Config) GetMode() string { + return c.Kubernetes.Mode +} diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go new file mode 100644 index 0000000..63d4c01 --- /dev/null +++ b/backend/internal/config/config_test.go @@ -0,0 +1,120 @@ +package config + +import ( + "testing" + "time" +) + +func TestLoadRuntimeDefaults(t *testing.T) { + for _, key := range []string{ + "HERMES_RUNTIME_IMAGE", + "OPENCLAW_RUNTIME_IMAGE", + "RUNTIME_NAMESPACE", + "K8S_NAMESPACE", + "HOSTNAME", + "PLATFORM_REDIS_URL", + "TEAM_REDIS_URL", + "RUNTIME_WORKSPACE_NFS_SERVER", + } { + t.Setenv(key, "") + } + + cfg, err := Load() + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if got, want := cfg.Runtime.HermesImage, "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"; got != want { + t.Fatalf("expected Hermes default image %q, got %q", want, got) + } + if got, want := cfg.Runtime.OpenClawImage, "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest"; got != want { + t.Fatalf("expected OpenClaw default image %q, got %q", want, got) + } +} + +func TestLoadStorageProfileDefaultsDisableHostPathFallback(t *testing.T) { + for _, key := range []string{ + "CLAWMANAGER_STORAGE_PROFILE", + "K8S_HOSTPATH_FALLBACK_ENABLED", + "K8S_PVC_BIND_TIMEOUT", + "K8S_CONTROL_PLANE_STORAGE_CLASS", + "K8S_INSTANCE_STORAGE_CLASS", + "K8S_WORKSPACE_STORAGE_CLASS", + "K8S_WORKSPACE_ACCESS_MODE", + "K8S_STORAGE_CLASS", + "RUNTIME_WORKSPACE_NFS_SERVER", + } { + t.Setenv(key, "") + } + + cfg, err := Load() + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if got, want := cfg.Storage.Profile, "cluster"; got != want { + t.Fatalf("storage profile = %q, want %q", got, want) + } + if cfg.Storage.HostPathFallbackEnabled { + t.Fatalf("hostPath fallback must default to disabled for cluster installs") + } + if got, want := cfg.GetPVCBindTimeout(), 2*time.Minute; got != want { + t.Fatalf("PVC bind timeout = %s, want %s", got, want) + } + if got, want := cfg.GetControlPlaneStorageClass(), "standard"; got != want { + t.Fatalf("control-plane storage class = %q, want %q", got, want) + } + if got, want := cfg.GetInstanceStorageClass(), "standard"; got != want { + t.Fatalf("instance storage class = %q, want %q", got, want) + } + if got, want := cfg.GetWorkspaceStorageClass(), "standard"; got != want { + t.Fatalf("workspace storage class = %q, want %q", got, want) + } + if got, want := cfg.GetWorkspaceAccessMode(), "ReadWriteMany"; got != want { + t.Fatalf("workspace access mode = %q, want %q", got, want) + } + if got := cfg.Runtime.WorkspaceNFSServer; got != "" { + t.Fatalf("workspace NFS server must not default to in-cluster service DNS, got %q", got) + } +} + +func TestLoadStorageProfileEnvOverrides(t *testing.T) { + t.Setenv("CLAWMANAGER_STORAGE_PROFILE", "single-node") + t.Setenv("K8S_HOSTPATH_FALLBACK_ENABLED", "true") + t.Setenv("K8S_PVC_BIND_TIMEOUT", "45s") + t.Setenv("K8S_CONTROL_PLANE_STORAGE_CLASS", "manual") + t.Setenv("K8S_INSTANCE_STORAGE_CLASS", "manual") + t.Setenv("K8S_WORKSPACE_STORAGE_CLASS", "manual-workspace") + t.Setenv("K8S_WORKSPACE_ACCESS_MODE", "ReadWriteMany") + t.Setenv("RUNTIME_WORKSPACE_PVC_CLAIM", "clawmanager-workspaces") + + cfg, err := Load() + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if got, want := cfg.Storage.Profile, "single-node"; got != want { + t.Fatalf("storage profile = %q, want %q", got, want) + } + if !cfg.Storage.HostPathFallbackEnabled { + t.Fatalf("hostPath fallback should be explicitly enabled") + } + if got, want := cfg.GetPVCBindTimeout(), 45*time.Second; got != want { + t.Fatalf("PVC bind timeout = %s, want %s", got, want) + } + if got, want := cfg.GetControlPlaneStorageClass(), "manual"; got != want { + t.Fatalf("control-plane storage class = %q, want %q", got, want) + } + if got, want := cfg.GetInstanceStorageClass(), "manual"; got != want { + t.Fatalf("instance storage class = %q, want %q", got, want) + } + if got, want := cfg.GetWorkspaceStorageClass(), "manual-workspace"; got != want { + t.Fatalf("workspace storage class = %q, want %q", got, want) + } + if got, want := cfg.GetWorkspaceAccessMode(), "ReadWriteMany"; got != want { + t.Fatalf("workspace access mode = %q, want %q", got, want) + } + if got, want := cfg.Runtime.WorkspacePVCClaimName, "clawmanager-workspaces"; got != want { + t.Fatalf("runtime workspace PVC claim = %q, want %q", got, want) + } +} diff --git a/backend/internal/db/db.go b/backend/internal/db/db.go new file mode 100644 index 0000000..48d33c0 --- /dev/null +++ b/backend/internal/db/db.go @@ -0,0 +1,59 @@ +package db + +import ( + "fmt" + "log" + + "clawreef/internal/config" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/mysql" +) + +// Session holds the database session +var Session db.Session + +// Initialize initializes the database connection +func Initialize(cfg config.DatabaseConfig) (db.Session, error) { + hostPort := fmt.Sprintf("%s:%d", cfg.Host, cfg.Port) + settings := mysql.ConnectionURL{ + Host: hostPort, + User: cfg.User, + Password: cfg.Password, + Database: cfg.Database, + Options: map[string]string{ + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "parseTime": "true", + }, + } + + session, err := mysql.Open(settings) + if err != nil { + return nil, fmt.Errorf("failed to connect to database: %w", err) + } + if _, err := session.SQL().Exec("SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci"); err != nil { + _ = session.Close() + return nil, fmt.Errorf("failed to configure database connection charset: %w", err) + } + if err := applyEmbeddedMigrations(session); err != nil { + _ = session.Close() + return nil, fmt.Errorf("failed to apply database migrations: %w", err) + } + + Session = session + log.Println("Database connected successfully") + return session, nil +} + +// Close closes the database connection +func Close() error { + if Session != nil { + return Session.Close() + } + return nil +} + +// GetSession returns the current database session +func GetSession() db.Session { + return Session +} diff --git a/backend/internal/db/migrations.go b/backend/internal/db/migrations.go new file mode 100644 index 0000000..7df40da --- /dev/null +++ b/backend/internal/db/migrations.go @@ -0,0 +1,229 @@ +package db + +import ( + "embed" + "fmt" + "log" + "path" + "sort" + "strings" + "time" + + "github.com/upper/db/v4" +) + +//go:embed migrations/*.sql +var embeddedMigrations embed.FS + +const schemaMigrationsTable = "schema_migrations" + +func applyEmbeddedMigrations(session db.Session) error { + if err := ensureSchemaMigrationsTable(session); err != nil { + return err + } + + applied, err := listAppliedMigrations(session) + if err != nil { + return err + } + + entries, err := embeddedMigrations.ReadDir("migrations") + if err != nil { + return fmt.Errorf("failed to list embedded migrations: %w", err) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].Name() < entries[j].Name() + }) + + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".sql") { + continue + } + if _, ok := applied[entry.Name()]; ok { + continue + } + + rawSQL, err := embeddedMigrations.ReadFile(path.Join("migrations", entry.Name())) + if err != nil { + return fmt.Errorf("failed to read embedded migration %s: %w", entry.Name(), err) + } + + statements := splitSQLStatements(string(rawSQL)) + for idx, statement := range statements { + if _, err := session.SQL().Exec(statement); err != nil { + return fmt.Errorf("failed to execute migration %s statement %d: %w", entry.Name(), idx+1, err) + } + } + + if _, err := session.SQL().Exec( + fmt.Sprintf("INSERT INTO %s (filename, applied_at) VALUES (?, ?)", schemaMigrationsTable), + entry.Name(), + time.Now().UTC(), + ); err != nil { + return fmt.Errorf("failed to record applied migration %s: %w", entry.Name(), err) + } + + log.Printf("Applied database migration %s", entry.Name()) + } + + return nil +} + +func ensureSchemaMigrationsTable(session db.Session) error { + _, err := session.SQL().Exec(fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id INT AUTO_INCREMENT PRIMARY KEY, + filename VARCHAR(255) NOT NULL, + applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE KEY uk_schema_migrations_filename (filename) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + `, schemaMigrationsTable)) + if err != nil { + return fmt.Errorf("failed to ensure schema migrations table: %w", err) + } + return nil +} + +func listAppliedMigrations(session db.Session) (map[string]struct{}, error) { + rows, err := session.SQL().Query(fmt.Sprintf("SELECT filename FROM %s", schemaMigrationsTable)) + if err != nil { + return nil, fmt.Errorf("failed to query applied migrations: %w", err) + } + defer rows.Close() + + applied := make(map[string]struct{}) + for rows.Next() { + var filename string + if err := rows.Scan(&filename); err != nil { + return nil, fmt.Errorf("failed to scan applied migration: %w", err) + } + applied[filename] = struct{}{} + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to iterate applied migrations: %w", err) + } + + return applied, nil +} + +func splitSQLStatements(input string) []string { + statements := make([]string, 0) + var current strings.Builder + + inSingleQuote := false + inDoubleQuote := false + inBacktick := false + inLineComment := false + inBlockComment := false + + for i := 0; i < len(input); i++ { + ch := input[i] + var next byte + if i+1 < len(input) { + next = input[i+1] + } + + if inLineComment { + if ch == '\n' { + inLineComment = false + } + continue + } + + if inBlockComment { + if ch == '*' && next == '/' { + inBlockComment = false + i++ + } + continue + } + + if inSingleQuote { + current.WriteByte(ch) + if ch == '\\' && next != 0 { + i++ + current.WriteByte(next) + continue + } + if ch == '\'' { + if next == '\'' { + i++ + current.WriteByte(next) + continue + } + inSingleQuote = false + } + continue + } + + if inDoubleQuote { + current.WriteByte(ch) + if ch == '\\' && next != 0 { + i++ + current.WriteByte(next) + continue + } + if ch == '"' { + if next == '"' { + i++ + current.WriteByte(next) + continue + } + inDoubleQuote = false + } + continue + } + + if inBacktick { + current.WriteByte(ch) + if ch == '`' { + inBacktick = false + } + continue + } + + if ch == '-' && next == '-' { + var afterNext byte + if i+2 < len(input) { + afterNext = input[i+2] + } + if afterNext == 0 || afterNext == ' ' || afterNext == '\t' || afterNext == '\r' || afterNext == '\n' { + inLineComment = true + i++ + continue + } + } + + if ch == '/' && next == '*' { + inBlockComment = true + i++ + continue + } + + switch ch { + case '\'': + inSingleQuote = true + current.WriteByte(ch) + case '"': + inDoubleQuote = true + current.WriteByte(ch) + case '`': + inBacktick = true + current.WriteByte(ch) + case ';': + statement := strings.TrimSpace(current.String()) + if statement != "" { + statements = append(statements, statement) + } + current.Reset() + default: + current.WriteByte(ch) + } + } + + if statement := strings.TrimSpace(current.String()); statement != "" { + statements = append(statements, statement) + } + + return statements +} diff --git a/backend/internal/db/migrations/001_init_schema.sql b/backend/internal/db/migrations/001_init_schema.sql new file mode 100644 index 0000000..f0d33e9 --- /dev/null +++ b/backend/internal/db/migrations/001_init_schema.sql @@ -0,0 +1,169 @@ +-- Initial schema for ClawReef +-- Created: 2026-03-15 + +-- Users table +CREATE TABLE IF NOT EXISTS users ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(255) UNIQUE NOT NULL, + email VARCHAR(320) UNIQUE NOT NULL, + password_hash VARCHAR(255) NOT NULL, + role ENUM('admin', 'user') DEFAULT 'user', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + last_login TIMESTAMP, + INDEX idx_username (username), + INDEX idx_role (role) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- Instances table +CREATE TABLE IF NOT EXISTS instances ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT, + type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu', + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + status ENUM('creating', 'running', 'stopped', 'error', 'deleting') DEFAULT 'creating', + cpu_cores INT NOT NULL, + memory_gb INT NOT NULL, + disk_gb INT NOT NULL, + gpu_enabled BOOLEAN DEFAULT FALSE, + gpu_type VARCHAR(100), + gpu_count INT DEFAULT 0, + os_type VARCHAR(50) NOT NULL, + os_version VARCHAR(50) NOT NULL, + image_registry VARCHAR(255), + image_tag VARCHAR(100), + environment_overrides_json LONGTEXT, + storage_class VARCHAR(50) DEFAULT 'standard', + mount_path VARCHAR(255) DEFAULT '/data', + pod_name VARCHAR(255), + pod_namespace VARCHAR(255), + pod_ip VARCHAR(45), + access_url VARCHAR(500), + access_token VARCHAR(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + started_at TIMESTAMP, + stopped_at TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id), + INDEX idx_status (status), + INDEX idx_type (type), + UNIQUE KEY uk_user_instance_name (user_id, name) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- Persistent volumes table +CREATE TABLE IF NOT EXISTS persistent_volumes ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + pvc_name VARCHAR(255) UNIQUE NOT NULL, + pvc_namespace VARCHAR(255) NOT NULL, + storage_size_gb INT NOT NULL, + storage_class VARCHAR(50), + mount_path VARCHAR(255), + status ENUM('pending', 'bound', 'released', 'failed') DEFAULT 'pending', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + UNIQUE KEY uk_pvc_name_namespace (pvc_name, pvc_namespace) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- Backups table +CREATE TABLE IF NOT EXISTS backups ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + backup_name VARCHAR(255) NOT NULL, + backup_size_gb INT, + backup_path VARCHAR(500), + status ENUM('creating', 'completed', 'failed', 'deleted') DEFAULT 'creating', + backup_type ENUM('manual', 'scheduled') DEFAULT 'manual', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP, + expires_at TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + INDEX idx_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- Backup schedules table +CREATE TABLE IF NOT EXISTS backup_schedules ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + schedule_name VARCHAR(255), + cron_expression VARCHAR(100) NOT NULL, + retention_days INT DEFAULT 30, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- User quotas table +CREATE TABLE IF NOT EXISTS user_quotas ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL UNIQUE, + max_instances INT DEFAULT 10, + max_cpu_cores INT DEFAULT 40, + max_memory_gb INT DEFAULT 100, + max_storage_gb INT DEFAULT 500, + max_gpu_count INT DEFAULT 2, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- Instance usage table +CREATE TABLE IF NOT EXISTS instance_usage ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + cpu_usage_percent DECIMAL(5,2), + memory_usage_gb DECIMAL(10,2), + disk_usage_gb DECIMAL(10,2), + gpu_usage_percent DECIMAL(5,2), + uptime_seconds INT, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_recorded (instance_id, recorded_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- Audit logs table +CREATE TABLE IF NOT EXISTS audit_logs ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT, + action VARCHAR(100) NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_id INT, + details JSON, + ip_address VARCHAR(45), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL, + INDEX idx_user_id (user_id), + INDEX idx_action (action), + INDEX idx_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- Insert default admin user (password: admin123) +INSERT INTO users (username, email, password_hash, role, is_active) +SELECT 'admin', 'admin@clawreef.local', '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi', 'admin', TRUE +FROM DUAL +WHERE NOT EXISTS ( + SELECT 1 + FROM users + WHERE username = 'admin' OR email = 'admin@clawreef.local' +); + +-- Insert default quota for admin +INSERT INTO user_quotas (user_id, max_instances, max_cpu_cores, max_memory_gb, max_storage_gb, max_gpu_count) +SELECT users.id, 100, 200, 1000, 5000, 10 +FROM users +WHERE username = 'admin' + AND NOT EXISTS ( + SELECT 1 + FROM user_quotas + WHERE user_quotas.user_id = users.id + ); diff --git a/backend/internal/db/migrations/002_add_webtop_instance_type.sql b/backend/internal/db/migrations/002_add_webtop_instance_type.sql new file mode 100644 index 0000000..11c8663 --- /dev/null +++ b/backend/internal/db/migrations/002_add_webtop_instance_type.sql @@ -0,0 +1,2 @@ +ALTER TABLE instances +MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; diff --git a/backend/internal/db/migrations/003_add_system_image_settings.sql b/backend/internal/db/migrations/003_add_system_image_settings.sql new file mode 100644 index 0000000..6908e65 --- /dev/null +++ b/backend/internal/db/migrations/003_add_system_image_settings.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS system_image_settings ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_type VARCHAR(50) NOT NULL, + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + display_name VARCHAR(255) NOT NULL, + image VARCHAR(500) NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_instance_type (instance_type) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/004_fix_seeded_admin_password.sql b/backend/internal/db/migrations/004_fix_seeded_admin_password.sql new file mode 100644 index 0000000..d003aa0 --- /dev/null +++ b/backend/internal/db/migrations/004_fix_seeded_admin_password.sql @@ -0,0 +1,6 @@ +-- Repair previously shipped broken admin seed hashes without touching +-- user-changed passwords. +UPDATE users +SET password_hash = '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi' +WHERE username = 'admin' + AND password_hash = '$2a$10$N9qo8uLOickgx2ZMRZoMy.MqrzL9wGC3qD3Q.ZHqQH6t3q7l1L5uG'; diff --git a/backend/internal/db/migrations/005_update_openclaw_default_image.sql b/backend/internal/db/migrations/005_update_openclaw_default_image.sql new file mode 100644 index 0000000..9bb3298 --- /dev/null +++ b/backend/internal/db/migrations/005_update_openclaw_default_image.sql @@ -0,0 +1,7 @@ +UPDATE system_image_settings +SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' +WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); diff --git a/backend/internal/db/migrations/006_add_openclaw_config_center.sql b/backend/internal/db/migrations/006_add_openclaw_config_center.sql new file mode 100644 index 0000000..0318f26 --- /dev/null +++ b/backend/internal/db/migrations/006_add_openclaw_config_center.sql @@ -0,0 +1,82 @@ +SET @openclaw_snapshot_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'openclaw_config_snapshot_id' +); +SET @openclaw_snapshot_column_sql = IF( + @openclaw_snapshot_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN openclaw_config_snapshot_id INT NULL AFTER access_token', + 'SELECT 1' +); +PREPARE openclaw_snapshot_column_stmt FROM @openclaw_snapshot_column_sql; +EXECUTE openclaw_snapshot_column_stmt; +DEALLOCATE PREPARE openclaw_snapshot_column_stmt; + +CREATE TABLE IF NOT EXISTS openclaw_config_resources ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_key VARCHAR(100) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + tags_json LONGTEXT NOT NULL, + content_json LONGTEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_resource_key (user_id, resource_type, resource_key), + INDEX idx_openclaw_resource_user_type (user_id, resource_type), + INDEX idx_openclaw_resource_user_enabled (user_id, enabled) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS openclaw_config_bundles ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_openclaw_bundle_user (user_id), + INDEX idx_openclaw_bundle_user_enabled (user_id, enabled) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS openclaw_config_bundle_items ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + resource_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (resource_id) REFERENCES openclaw_config_resources(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_resource (bundle_id, resource_id), + INDEX idx_openclaw_bundle_item_bundle (bundle_id, sort_order) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS openclaw_injection_snapshots ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NULL, + user_id INT NOT NULL, + mode VARCHAR(20) NOT NULL, + bundle_id INT NULL, + selected_resource_ids_json LONGTEXT NOT NULL, + resolved_resources_json LONGTEXT NOT NULL, + rendered_manifest_json LONGTEXT NOT NULL, + rendered_env_json LONGTEXT NOT NULL, + secret_name VARCHAR(255) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + activated_at TIMESTAMP NULL, + INDEX idx_openclaw_snapshot_user_created (user_id, created_at), + INDEX idx_openclaw_snapshot_instance (instance_id), + INDEX idx_openclaw_snapshot_bundle (bundle_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/007_add_instance_agent_control_plane.sql b/backend/internal/db/migrations/007_add_instance_agent_control_plane.sql new file mode 100644 index 0000000..a18e7e2 --- /dev/null +++ b/backend/internal/db/migrations/007_add_instance_agent_control_plane.sql @@ -0,0 +1,123 @@ +SET @instance_agent_bootstrap_token_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'agent_bootstrap_token' +); +SET @instance_agent_bootstrap_token_column_sql = IF( + @instance_agent_bootstrap_token_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN agent_bootstrap_token VARCHAR(255) NULL AFTER access_token', + 'SELECT 1' +); +PREPARE instance_agent_bootstrap_token_column_stmt FROM @instance_agent_bootstrap_token_column_sql; +EXECUTE instance_agent_bootstrap_token_column_stmt; +DEALLOCATE PREPARE instance_agent_bootstrap_token_column_stmt; + +CREATE TABLE IF NOT EXISTS instance_agents ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NOT NULL, + agent_version VARCHAR(50) NOT NULL, + protocol_version VARCHAR(50) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'online', + capabilities_json LONGTEXT NOT NULL, + host_info_json LONGTEXT NULL, + session_token VARCHAR(255) NULL, + session_expires_at TIMESTAMP NULL, + last_heartbeat_at TIMESTAMP NULL, + last_reported_at TIMESTAMP NULL, + last_seen_ip VARCHAR(45) NULL, + registered_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_agents_instance (instance_id), + UNIQUE KEY uk_instance_agents_session_token (session_token), + INDEX idx_instance_agents_agent_id (agent_id), + INDEX idx_instance_agents_status (status), + INDEX idx_instance_agents_last_heartbeat (last_heartbeat_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS instance_runtime_status ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + infra_status VARCHAR(30) NOT NULL DEFAULT 'creating', + agent_status VARCHAR(30) NOT NULL DEFAULT 'offline', + openclaw_status VARCHAR(30) NOT NULL DEFAULT 'unknown', + openclaw_pid INT NULL, + openclaw_version VARCHAR(100) NULL, + current_config_revision_id INT NULL, + desired_config_revision_id INT NULL, + summary_json LONGTEXT NULL, + system_info_json LONGTEXT NULL, + health_json LONGTEXT NULL, + last_reported_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_runtime_status_instance (instance_id), + INDEX idx_instance_runtime_status_agent_status (agent_status), + INDEX idx_instance_runtime_status_openclaw_status (openclaw_status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS instance_desired_state ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + desired_power_state VARCHAR(30) NOT NULL DEFAULT 'running', + desired_config_revision_id INT NULL, + desired_runtime_action VARCHAR(50) NULL, + updated_by INT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_desired_state_instance (instance_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS instance_commands ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NULL, + command_type VARCHAR(50) NOT NULL, + payload_json LONGTEXT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + idempotency_key VARCHAR(255) NOT NULL, + issued_by INT NULL, + issued_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + dispatched_at TIMESTAMP NULL, + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + timeout_seconds INT NOT NULL DEFAULT 300, + result_json LONGTEXT NULL, + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (issued_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_commands_idempotency (instance_id, idempotency_key), + INDEX idx_instance_commands_instance_status (instance_id, status), + INDEX idx_instance_commands_agent_status (agent_id, status), + INDEX idx_instance_commands_issued_at (issued_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS instance_config_revisions ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + source_snapshot_id INT NULL, + source_bundle_id INT NULL, + revision_no INT NOT NULL, + content_json LONGTEXT NOT NULL, + checksum VARCHAR(255) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'published', + published_by INT NULL, + published_at TIMESTAMP NULL, + activated_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (published_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_config_revision_unique (instance_id, revision_no), + INDEX idx_instance_config_revision_instance (instance_id, revision_no), + INDEX idx_instance_config_revision_status (status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/008_add_skill_management.sql b/backend/internal/db/migrations/008_add_skill_management.sql new file mode 100644 index 0000000..14d93f6 --- /dev/null +++ b/backend/internal/db/migrations/008_add_skill_management.sql @@ -0,0 +1,91 @@ +CREATE TABLE IF NOT EXISTS skill_blobs ( + id INT AUTO_INCREMENT PRIMARY KEY, + content_hash VARCHAR(128) NOT NULL, + archive_hash VARCHAR(128) NOT NULL, + object_key VARCHAR(512) NOT NULL, + file_name VARCHAR(255) NOT NULL, + media_type VARCHAR(100) NOT NULL DEFAULT 'application/gzip', + size_bytes BIGINT NOT NULL DEFAULT 0, + scan_status VARCHAR(30) NOT NULL DEFAULT 'pending', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_skill_blobs_content_hash (content_hash), + INDEX idx_skill_blobs_scan_status (scan_status), + INDEX idx_skill_blobs_risk_level (risk_level) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + skill_key VARCHAR(120) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + current_version_id INT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + status VARCHAR(30) NOT NULL DEFAULT 'active', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_skills_user_key (user_id, skill_key), + INDEX idx_skills_user_status (user_id, status), + INDEX idx_skills_risk_level (risk_level) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS skill_versions ( + id INT AUTO_INCREMENT PRIMARY KEY, + skill_id INT NOT NULL, + blob_id INT NOT NULL, + version_no INT NOT NULL, + manifest_json LONGTEXT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE RESTRICT, + UNIQUE KEY uk_skill_versions_skill_version (skill_id, version_no), + UNIQUE KEY uk_skill_versions_skill_blob (skill_id, blob_id), + INDEX idx_skill_versions_skill_id (skill_id, version_no) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS instance_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + skill_id INT NOT NULL, + skill_version_id INT NULL, + source_type VARCHAR(40) NOT NULL DEFAULT 'discovered_in_instance', + install_path VARCHAR(1024) NULL, + observed_hash VARCHAR(128) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'active', + last_seen_at TIMESTAMP NULL, + removed_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (skill_version_id) REFERENCES skill_versions(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_skills_instance_skill (instance_id, skill_id), + INDEX idx_instance_skills_instance (instance_id, status), + INDEX idx_instance_skills_skill (skill_id, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS skill_scan_results ( + id INT AUTO_INCREMENT PRIMARY KEY, + blob_id INT NOT NULL, + engine VARCHAR(60) NOT NULL, + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + status VARCHAR(30) NOT NULL DEFAULT 'completed', + summary TEXT NULL, + findings_json LONGTEXT NULL, + scanned_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE CASCADE, + INDEX idx_skill_scan_results_blob (blob_id, scanned_at), + INDEX idx_skill_scan_results_risk (risk_level, scanned_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/009_cpu_cores_decimal.sql b/backend/internal/db/migrations/009_cpu_cores_decimal.sql new file mode 100644 index 0000000..6a09d82 --- /dev/null +++ b/backend/internal/db/migrations/009_cpu_cores_decimal.sql @@ -0,0 +1,5 @@ +-- Allow fractional CPU cores for instances and quotas +-- Created: 2026-04-14 + +ALTER TABLE instances MODIFY COLUMN cpu_cores DECIMAL(10,2) NOT NULL; +ALTER TABLE user_quotas MODIFY COLUMN max_cpu_cores DECIMAL(10,2) DEFAULT 40; diff --git a/backend/internal/db/migrations/010_add_instance_environment_overrides.sql b/backend/internal/db/migrations/010_add_instance_environment_overrides.sql new file mode 100644 index 0000000..6160f49 --- /dev/null +++ b/backend/internal/db/migrations/010_add_instance_environment_overrides.sql @@ -0,0 +1,15 @@ +SET @instance_environment_overrides_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'environment_overrides_json' +); +SET @instance_environment_overrides_column_sql = IF( + @instance_environment_overrides_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN environment_overrides_json LONGTEXT NULL AFTER image_tag', + 'SELECT 1' +); +PREPARE instance_environment_overrides_column_stmt FROM @instance_environment_overrides_column_sql; +EXECUTE instance_environment_overrides_column_stmt; +DEALLOCATE PREPARE instance_environment_overrides_column_stmt; diff --git a/backend/internal/db/migrations/011_add_hermes_instance_type.sql b/backend/internal/db/migrations/011_add_hermes_instance_type.sql new file mode 100644 index 0000000..11c8663 --- /dev/null +++ b/backend/internal/db/migrations/011_add_hermes_instance_type.sql @@ -0,0 +1,2 @@ +ALTER TABLE instances +MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; diff --git a/backend/internal/db/migrations/012_update_agents_runtime_default_images.sql b/backend/internal/db/migrations/012_update_agents_runtime_default_images.sql new file mode 100644 index 0000000..285705c --- /dev/null +++ b/backend/internal/db/migrations/012_update_agents_runtime_default_images.sql @@ -0,0 +1,15 @@ +UPDATE system_image_settings +SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' +WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + +UPDATE system_image_settings +SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' +WHERE instance_type = 'hermes' + AND image IN ( + 'registry.example.com/hermes-webtop:latest', + 'lscr.io/linuxserver/webtop:ubuntu-xfce' + ); diff --git a/backend/internal/db/migrations/013_normalize_agents_runtime_image_names.sql b/backend/internal/db/migrations/013_normalize_agents_runtime_image_names.sql new file mode 100644 index 0000000..9840845 --- /dev/null +++ b/backend/internal/db/migrations/013_normalize_agents_runtime_image_names.sql @@ -0,0 +1,7 @@ +UPDATE system_image_settings +SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' +WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/openclaw:latest'; + +UPDATE system_image_settings +SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' +WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/hermes:latest'; diff --git a/backend/internal/db/migrations/014_add_team_mvp.sql b/backend/internal/db/migrations/014_add_team_mvp.sql new file mode 100644 index 0000000..6bcd530 --- /dev/null +++ b/backend/internal/db/migrations/014_add_team_mvp.sql @@ -0,0 +1,90 @@ +CREATE TABLE IF NOT EXISTS teams ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'creating', + communication_mode VARCHAR(50) NOT NULL DEFAULT 'leader_only', + redis_url_secret_name VARCHAR(255) NULL, + redis_url_secret_key VARCHAR(255) NULL, + team_token_secret_name VARCHAR(255) NULL, + team_token_secret_key VARCHAR(255) NULL, + redis_events_last_id VARCHAR(128) NOT NULL DEFAULT '0-0', + shared_pvc_name VARCHAR(255) NULL, + shared_pvc_namespace VARCHAR(255) NULL, + shared_mount_path VARCHAR(255) NOT NULL DEFAULT '/team', + storage_class VARCHAR(100) NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_teams_user_name (user_id, name), + INDEX idx_teams_user_status (user_id, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS team_members ( + id INT AUTO_INCREMENT PRIMARY KEY, + team_id INT NOT NULL, + instance_id INT NULL, + user_id INT NOT NULL, + member_key VARCHAR(100) NOT NULL, + display_name VARCHAR(255) NOT NULL, + role VARCHAR(100) NOT NULL, + description TEXT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'creating', + current_task_id INT NULL, + progress INT NOT NULL DEFAULT 0, + last_seen_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (team_id) REFERENCES teams(id) ON DELETE CASCADE, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE SET NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_team_members_key (team_id, member_key), + UNIQUE KEY uk_team_members_instance (team_id, instance_id), + INDEX idx_team_members_team_status (team_id, status), + INDEX idx_team_members_instance (instance_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS team_tasks ( + id INT AUTO_INCREMENT PRIMARY KEY, + team_id INT NOT NULL, + target_member_id INT NOT NULL, + created_by INT NULL, + message_id VARCHAR(255) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + payload_json LONGTEXT NOT NULL, + result_json LONGTEXT NULL, + error_message TEXT NULL, + redis_stream_id VARCHAR(128) NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + dispatched_at TIMESTAMP NULL, + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (team_id) REFERENCES teams(id) ON DELETE CASCADE, + FOREIGN KEY (target_member_id) REFERENCES team_members(id) ON DELETE CASCADE, + FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_team_tasks_message (team_id, message_id), + INDEX idx_team_tasks_team_status (team_id, status), + INDEX idx_team_tasks_member_status (target_member_id, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS team_events ( + id INT AUTO_INCREMENT PRIMARY KEY, + team_id INT NOT NULL, + member_id INT NULL, + task_id INT NULL, + message_id VARCHAR(255) NULL, + event_type VARCHAR(100) NOT NULL, + payload_json LONGTEXT NULL, + redis_stream_id VARCHAR(128) NULL, + occurred_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (team_id) REFERENCES teams(id) ON DELETE CASCADE, + FOREIGN KEY (member_id) REFERENCES team_members(id) ON DELETE SET NULL, + FOREIGN KEY (task_id) REFERENCES team_tasks(id) ON DELETE SET NULL, + UNIQUE KEY uk_team_events_stream_id (team_id, redis_stream_id), + INDEX idx_team_events_team_created (team_id, created_at), + INDEX idx_team_events_task (task_id), + INDEX idx_team_events_member (member_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/015_add_team_member_description.sql b/backend/internal/db/migrations/015_add_team_member_description.sql new file mode 100644 index 0000000..fe941bd --- /dev/null +++ b/backend/internal/db/migrations/015_add_team_member_description.sql @@ -0,0 +1,15 @@ +SET @team_member_description_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'team_members' + AND COLUMN_NAME = 'description' +); +SET @team_member_description_column_sql = IF( + @team_member_description_column_exists = 0, + 'ALTER TABLE team_members ADD COLUMN description TEXT NULL AFTER role', + 'SELECT 1' +); +PREPARE team_member_description_column_stmt FROM @team_member_description_column_sql; +EXECUTE team_member_description_column_stmt; +DEALLOCATE PREPARE team_member_description_column_stmt; diff --git a/backend/internal/db/migrations/016_release_deleted_team_names.sql b/backend/internal/db/migrations/016_release_deleted_team_names.sql new file mode 100644 index 0000000..925ef20 --- /dev/null +++ b/backend/internal/db/migrations/016_release_deleted_team_names.sql @@ -0,0 +1,8 @@ +UPDATE teams +SET name = CONCAT( + LEFT(name, GREATEST(1, 255 - CHAR_LENGTH(CONCAT('__deleted_', id)))), + '__deleted_', + id +) +WHERE status = 'deleted' + AND name NOT REGEXP '__deleted_[0-9]+$'; diff --git a/backend/internal/db/migrations/017_add_team_member_runtime_state.sql b/backend/internal/db/migrations/017_add_team_member_runtime_state.sql new file mode 100644 index 0000000..d05cf25 --- /dev/null +++ b/backend/internal/db/migrations/017_add_team_member_runtime_state.sql @@ -0,0 +1,95 @@ +SET @team_member_availability_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'team_members' + AND COLUMN_NAME = 'availability' +); +SET @team_member_availability_column_sql = IF( + @team_member_availability_column_exists = 0, + 'ALTER TABLE team_members ADD COLUMN availability VARCHAR(30) NOT NULL DEFAULT ''unknown'' AFTER last_seen_at', + 'SELECT 1' +); +PREPARE team_member_availability_column_stmt FROM @team_member_availability_column_sql; +EXECUTE team_member_availability_column_stmt; +DEALLOCATE PREPARE team_member_availability_column_stmt; + +SET @team_member_runtime_status_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'team_members' + AND COLUMN_NAME = 'runtime_status' +); +SET @team_member_runtime_status_column_sql = IF( + @team_member_runtime_status_column_exists = 0, + 'ALTER TABLE team_members ADD COLUMN runtime_status VARCHAR(50) NULL AFTER availability', + 'SELECT 1' +); +PREPARE team_member_runtime_status_column_stmt FROM @team_member_runtime_status_column_sql; +EXECUTE team_member_runtime_status_column_stmt; +DEALLOCATE PREPARE team_member_runtime_status_column_stmt; + +SET @team_member_runtime_task_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'team_members' + AND COLUMN_NAME = 'runtime_task_id' +); +SET @team_member_runtime_task_column_sql = IF( + @team_member_runtime_task_column_exists = 0, + 'ALTER TABLE team_members ADD COLUMN runtime_task_id VARCHAR(255) NULL AFTER runtime_status', + 'SELECT 1' +); +PREPARE team_member_runtime_task_column_stmt FROM @team_member_runtime_task_column_sql; +EXECUTE team_member_runtime_task_column_stmt; +DEALLOCATE PREPARE team_member_runtime_task_column_stmt; + +SET @team_member_runtime_intent_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'team_members' + AND COLUMN_NAME = 'runtime_intent' +); +SET @team_member_runtime_intent_column_sql = IF( + @team_member_runtime_intent_column_exists = 0, + 'ALTER TABLE team_members ADD COLUMN runtime_intent VARCHAR(100) NULL AFTER runtime_task_id', + 'SELECT 1' +); +PREPARE team_member_runtime_intent_column_stmt FROM @team_member_runtime_intent_column_sql; +EXECUTE team_member_runtime_intent_column_stmt; +DEALLOCATE PREPARE team_member_runtime_intent_column_stmt; + +SET @team_member_blocked_reason_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'team_members' + AND COLUMN_NAME = 'blocked_reason' +); +SET @team_member_blocked_reason_column_sql = IF( + @team_member_blocked_reason_column_exists = 0, + 'ALTER TABLE team_members ADD COLUMN blocked_reason TEXT NULL AFTER runtime_intent', + 'SELECT 1' +); +PREPARE team_member_blocked_reason_column_stmt FROM @team_member_blocked_reason_column_sql; +EXECUTE team_member_blocked_reason_column_stmt; +DEALLOCATE PREPARE team_member_blocked_reason_column_stmt; + +SET @team_member_last_summary_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'team_members' + AND COLUMN_NAME = 'last_summary' +); +SET @team_member_last_summary_column_sql = IF( + @team_member_last_summary_column_exists = 0, + 'ALTER TABLE team_members ADD COLUMN last_summary TEXT NULL AFTER blocked_reason', + 'SELECT 1' +); +PREPARE team_member_last_summary_column_stmt FROM @team_member_last_summary_column_sql; +EXECUTE team_member_last_summary_column_stmt; +DEALLOCATE PREPARE team_member_last_summary_column_stmt; diff --git a/backend/internal/db/migrations/018_add_team_member_runtime_type.sql b/backend/internal/db/migrations/018_add_team_member_runtime_type.sql new file mode 100644 index 0000000..8ed59e0 --- /dev/null +++ b/backend/internal/db/migrations/018_add_team_member_runtime_type.sql @@ -0,0 +1,15 @@ +SET @team_member_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'team_members' + AND COLUMN_NAME = 'runtime_type' +); +SET @team_member_runtime_type_column_sql = IF( + @team_member_runtime_type_column_exists = 0, + 'ALTER TABLE team_members ADD COLUMN runtime_type VARCHAR(30) NOT NULL DEFAULT ''openclaw'' AFTER role', + 'SELECT 1' +); +PREPARE team_member_runtime_type_column_stmt FROM @team_member_runtime_type_column_sql; +EXECUTE team_member_runtime_type_column_stmt; +DEALLOCATE PREPARE team_member_runtime_type_column_stmt; diff --git a/backend/internal/db/migrations/019_add_instance_runtime_type.sql b/backend/internal/db/migrations/019_add_instance_runtime_type.sql new file mode 100644 index 0000000..1f87fa9 --- /dev/null +++ b/backend/internal/db/migrations/019_add_instance_runtime_type.sql @@ -0,0 +1,15 @@ +SET @instance_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'runtime_type' +); +SET @instance_runtime_type_column_sql = IF( + @instance_runtime_type_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER type', + 'SELECT 1' +); +PREPARE instance_runtime_type_column_stmt FROM @instance_runtime_type_column_sql; +EXECUTE instance_runtime_type_column_stmt; +DEALLOCATE PREPARE instance_runtime_type_column_stmt; diff --git a/backend/internal/db/migrations/020_add_system_image_runtime_type.sql b/backend/internal/db/migrations/020_add_system_image_runtime_type.sql new file mode 100644 index 0000000..f5bf54e --- /dev/null +++ b/backend/internal/db/migrations/020_add_system_image_runtime_type.sql @@ -0,0 +1,15 @@ +SET @system_image_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'runtime_type' +); +SET @system_image_runtime_type_column_sql = IF( + @system_image_runtime_type_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER instance_type', + 'SELECT 1' +); +PREPARE system_image_runtime_type_column_stmt FROM @system_image_runtime_type_column_sql; +EXECUTE system_image_runtime_type_column_stmt; +DEALLOCATE PREPARE system_image_runtime_type_column_stmt; diff --git a/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql b/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql new file mode 100644 index 0000000..eddf004 --- /dev/null +++ b/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql @@ -0,0 +1,78 @@ +SET @system_image_is_enabled_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'is_enabled' +); +SET @system_image_is_enabled_column_sql = IF( + @system_image_is_enabled_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN is_enabled BOOLEAN NOT NULL DEFAULT TRUE', + 'SELECT 1' +); +PREPARE system_image_is_enabled_column_stmt FROM @system_image_is_enabled_column_sql; +EXECUTE system_image_is_enabled_column_stmt; +DEALLOCATE PREPARE system_image_is_enabled_column_stmt; + +SET @system_image_unique_instance_type_index_name = ( + SELECT INDEX_NAME + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'instance_type' + AND NON_UNIQUE = 0 + AND INDEX_NAME <> 'PRIMARY' + LIMIT 1 +); +SET @system_image_unique_instance_type_index_sql = IF( + @system_image_unique_instance_type_index_name IS NOT NULL, + CONCAT( + 'ALTER TABLE system_image_settings DROP INDEX `', + REPLACE(@system_image_unique_instance_type_index_name, '`', '``'), + '`' + ), + 'SELECT 1' +); +PREPARE system_image_unique_instance_type_index_stmt FROM @system_image_unique_instance_type_index_sql; +EXECUTE system_image_unique_instance_type_index_stmt; +DEALLOCATE PREPARE system_image_unique_instance_type_index_stmt; + +INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) +SELECT + 'openclaw', + 'desktop', + 'OpenClaw Desktop', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest', + TRUE +WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'desktop' +) +AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE +); + +INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) +SELECT + 'openclaw', + 'shell', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', + TRUE +WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'shell' +) +AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE +); diff --git a/backend/internal/db/migrations/022_add_openclaw_config_bundle_skills.sql b/backend/internal/db/migrations/022_add_openclaw_config_bundle_skills.sql new file mode 100644 index 0000000..93dc00c --- /dev/null +++ b/backend/internal/db/migrations/022_add_openclaw_config_bundle_skills.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS openclaw_config_bundle_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + skill_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id), + INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/023_add_runtime_pool_v2.sql b/backend/internal/db/migrations/023_add_runtime_pool_v2.sql new file mode 100644 index 0000000..76838d0 --- /dev/null +++ b/backend/internal/db/migrations/023_add_runtime_pool_v2.sql @@ -0,0 +1,121 @@ +SET @stmt = IF( + (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'workspace_path') = 0, + 'ALTER TABLE instances ADD COLUMN workspace_path VARCHAR(1024) NULL AFTER mount_path', + 'SELECT 1' +); +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @stmt = IF( + (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'workspace_usage_bytes') = 0, + 'ALTER TABLE instances ADD COLUMN workspace_usage_bytes BIGINT NOT NULL DEFAULT 0 AFTER workspace_path', + 'SELECT 1' +); +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @stmt = IF( + (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'runtime_generation') = 0, + 'ALTER TABLE instances ADD COLUMN runtime_generation INT NOT NULL DEFAULT 1 AFTER workspace_usage_bytes', + 'SELECT 1' +); +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @stmt = IF( + (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'runtime_error_message') = 0, + 'ALTER TABLE instances ADD COLUMN runtime_error_message TEXT NULL AFTER runtime_generation', + 'SELECT 1' +); +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +ALTER TABLE instances + MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop'; + +CREATE TABLE IF NOT EXISTS runtime_pods ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + runtime_type VARCHAR(32) NOT NULL, + namespace VARCHAR(128) NOT NULL, + pod_name VARCHAR(255) NOT NULL, + pod_uid VARCHAR(128) NULL, + pod_ip VARCHAR(64) NULL, + node_name VARCHAR(255) NULL, + deployment_name VARCHAR(255) NOT NULL, + image_ref VARCHAR(512) NOT NULL, + agent_endpoint VARCHAR(255) NULL, + state VARCHAR(32) NOT NULL DEFAULT 'pending', + capacity INT NOT NULL DEFAULT 100, + used_slots INT NOT NULL DEFAULT 0, + draining TINYINT(1) NOT NULL DEFAULT 0, + cpu_millis_used BIGINT NOT NULL DEFAULT 0, + memory_bytes_used BIGINT NOT NULL DEFAULT 0, + disk_bytes_used BIGINT NOT NULL DEFAULT 0, + network_rx_bytes BIGINT NOT NULL DEFAULT 0, + network_tx_bytes BIGINT NOT NULL DEFAULT 0, + metrics_json JSON NULL, + last_seen_at DATETIME NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_runtime_pod_namespace_name (namespace, pod_name), + KEY idx_runtime_pods_schedulable (runtime_type, state, draining, used_slots, capacity), + KEY idx_runtime_pods_last_seen (last_seen_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS instance_runtime_bindings ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + instance_id INT NOT NULL, + runtime_pod_id BIGINT NOT NULL, + runtime_type VARCHAR(32) NOT NULL, + gateway_id VARCHAR(128) NOT NULL, + gateway_port INT NOT NULL, + gateway_pid INT NULL, + workspace_path VARCHAR(1024) NOT NULL, + state VARCHAR(32) NOT NULL DEFAULT 'creating', + generation INT NOT NULL DEFAULT 1, + last_health_at DATETIME NULL, + error_message TEXT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_instance_runtime_binding_instance (instance_id), + UNIQUE KEY uk_instance_runtime_binding_gateway (runtime_pod_id, gateway_port), + KEY idx_instance_runtime_binding_pod_state (runtime_pod_id, state), + CONSTRAINT fk_instance_runtime_binding_instance + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + CONSTRAINT fk_instance_runtime_binding_pod + FOREIGN KEY (runtime_pod_id) REFERENCES runtime_pods(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS runtime_rollouts ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + runtime_type VARCHAR(32) NOT NULL, + target_image_ref VARCHAR(512) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + batch_size INT NOT NULL DEFAULT 1, + max_unavailable INT NOT NULL DEFAULT 1, + started_by INT NULL, + started_at DATETIME NULL, + finished_at DATETIME NULL, + error_message TEXT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + KEY idx_runtime_rollouts_type_status (runtime_type, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS workspace_file_audits ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + instance_id INT NOT NULL, + user_id INT NOT NULL, + action VARCHAR(32) NOT NULL, + relative_path VARCHAR(1024) NOT NULL, + bytes BIGINT NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + KEY idx_workspace_file_audits_instance_time (instance_id, created_at), + KEY idx_workspace_file_audits_user_time (user_id, created_at), + CONSTRAINT fk_workspace_file_audits_instance + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/023_normalize_runtime_config_mounts.sql b/backend/internal/db/migrations/023_normalize_runtime_config_mounts.sql new file mode 100644 index 0000000..bf5ce0b --- /dev/null +++ b/backend/internal/db/migrations/023_normalize_runtime_config_mounts.sql @@ -0,0 +1,5 @@ +UPDATE instances +SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP +WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); diff --git a/backend/internal/db/migrations/024_add_gateway_runtime_type.sql b/backend/internal/db/migrations/024_add_gateway_runtime_type.sql new file mode 100644 index 0000000..82f5f5d --- /dev/null +++ b/backend/internal/db/migrations/024_add_gateway_runtime_type.sql @@ -0,0 +1,2 @@ +ALTER TABLE instances + MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop'; diff --git a/backend/internal/db/migrations/025_add_instance_mode.sql b/backend/internal/db/migrations/025_add_instance_mode.sql new file mode 100644 index 0000000..8a24cf6 --- /dev/null +++ b/backend/internal/db/migrations/025_add_instance_mode.sql @@ -0,0 +1,24 @@ +SET @instance_mode_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'instance_mode' +); +SET @instance_mode_column_sql = IF( + @instance_mode_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN instance_mode ENUM(''lite'', ''pro'') NOT NULL DEFAULT ''lite'' AFTER runtime_type', + 'SELECT 1' +); +PREPARE instance_mode_column_stmt FROM @instance_mode_column_sql; +EXECUTE instance_mode_column_stmt; +DEALLOCATE PREPARE instance_mode_column_stmt; + +UPDATE instances +SET instance_mode = CASE + WHEN runtime_type = 'gateway' THEN 'lite' + ELSE 'pro' +END +WHERE instance_mode IS NULL + OR instance_mode = '' + OR (runtime_type <> 'gateway' AND instance_mode = 'lite'); diff --git a/backend/internal/db/migrations/027_add_instance_external_access.sql b/backend/internal/db/migrations/027_add_instance_external_access.sql new file mode 100644 index 0000000..18208c0 --- /dev/null +++ b/backend/internal/db/migrations/027_add_instance_external_access.sql @@ -0,0 +1,23 @@ +CREATE TABLE IF NOT EXISTS instance_external_access ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + instance_id INT NOT NULL, + enabled TINYINT(1) NOT NULL DEFAULT 0, + auth_mode VARCHAR(32) NOT NULL DEFAULT 'public_link', + public_slug VARCHAR(64) NULL, + public_token_hash VARCHAR(128) NULL, + short_code_hash VARCHAR(128) NULL, + api_key_hash VARCHAR(128) NULL, + password_value VARCHAR(128) NULL, + api_key_prefix VARCHAR(32) NULL, + expires_at DATETIME NULL, + created_by INT NULL, + last_used_at DATETIME NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_instance_external_access_instance (instance_id), + UNIQUE KEY uk_instance_external_access_slug (public_slug), + UNIQUE KEY uk_instance_external_access_short_code (short_code_hash), + KEY idx_instance_external_access_enabled (enabled, auth_mode), + CONSTRAINT fk_instance_external_access_instance + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql b/backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql new file mode 100644 index 0000000..1f5b6cc --- /dev/null +++ b/backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql @@ -0,0 +1,266 @@ +CREATE TABLE IF NOT EXISTS model_invocations ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NOT NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + model_id INT NULL, + provider_type VARCHAR(100) NOT NULL, + requested_model VARCHAR(255) NOT NULL, + actual_provider_model VARCHAR(255) NOT NULL, + traffic_class VARCHAR(50) NOT NULL, + request_payload LONGTEXT NULL, + response_payload LONGTEXT NULL, + prompt_tokens INT NOT NULL DEFAULT 0, + completion_tokens INT NOT NULL DEFAULT 0, + total_tokens INT NOT NULL DEFAULT 0, + cached_tokens INT NULL, + reasoning_tokens INT NULL, + latency_ms INT NULL, + is_streaming BOOLEAN NOT NULL DEFAULT FALSE, + status VARCHAR(50) NOT NULL, + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP NULL, + INDEX idx_model_invocations_trace_id (trace_id), + INDEX idx_model_invocations_session_id (session_id), + INDEX idx_model_invocations_request_id (request_id), + INDEX idx_model_invocations_user_id (user_id), + INDEX idx_model_invocations_instance_id (instance_id), + INDEX idx_model_invocations_gateway_id (gateway_id), + INDEX idx_model_invocations_model_id (model_id), + INDEX idx_model_invocations_status (status), + INDEX idx_model_invocations_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS audit_events ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + invocation_id INT NULL, + event_type VARCHAR(100) NOT NULL, + traffic_class VARCHAR(50) NOT NULL, + severity VARCHAR(20) NOT NULL, + message VARCHAR(500) NOT NULL, + details LONGTEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_audit_events_trace_id (trace_id), + INDEX idx_audit_events_request_id (request_id), + INDEX idx_audit_events_user_id (user_id), + INDEX idx_audit_events_gateway_id (gateway_id), + INDEX idx_audit_events_invocation_id (invocation_id), + INDEX idx_audit_events_event_type (event_type), + INDEX idx_audit_events_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS cost_records ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + invocation_id INT NULL, + model_id INT NULL, + provider_type VARCHAR(100) NOT NULL, + model_name VARCHAR(255) NOT NULL, + currency VARCHAR(16) NOT NULL DEFAULT 'USD', + prompt_tokens INT NOT NULL DEFAULT 0, + completion_tokens INT NOT NULL DEFAULT 0, + total_tokens INT NOT NULL DEFAULT 0, + input_unit_price DECIMAL(18,8) NOT NULL DEFAULT 0, + output_unit_price DECIMAL(18,8) NOT NULL DEFAULT 0, + estimated_cost DECIMAL(18,8) NOT NULL DEFAULT 0, + actual_cost DECIMAL(18,8) NULL, + internal_cost DECIMAL(18,8) NOT NULL DEFAULT 0, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_cost_records_trace_id (trace_id), + INDEX idx_cost_records_user_id (user_id), + INDEX idx_cost_records_gateway_id (gateway_id), + INDEX idx_cost_records_model_id (model_id), + INDEX idx_cost_records_provider_type (provider_type), + INDEX idx_cost_records_recorded_at (recorded_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS risk_hits ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + invocation_id INT NULL, + rule_id VARCHAR(100) NOT NULL, + rule_name VARCHAR(255) NOT NULL, + severity VARCHAR(20) NOT NULL, + action VARCHAR(50) NOT NULL, + match_summary VARCHAR(500) NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_risk_hits_trace_id (trace_id), + INDEX idx_risk_hits_request_id (request_id), + INDEX idx_risk_hits_user_id (user_id), + INDEX idx_risk_hits_gateway_id (gateway_id), + INDEX idx_risk_hits_invocation_id (invocation_id), + INDEX idx_risk_hits_severity (severity), + INDEX idx_risk_hits_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'instance_mode' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'runtime_type' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'gateway_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'runtime_pod_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'instance_mode' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'runtime_type' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'gateway_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'runtime_pod_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'instance_mode' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'runtime_type' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'gateway_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'runtime_pod_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'instance_mode' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'runtime_type' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'gateway_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'runtime_pod_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @index_exists = ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND INDEX_NAME = 'idx_model_invocations_gateway_id' +); +SET @ddl = IF(@index_exists = 0, 'ALTER TABLE model_invocations ADD INDEX idx_model_invocations_gateway_id (gateway_id)', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @index_exists = ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND INDEX_NAME = 'idx_audit_events_gateway_id' +); +SET @ddl = IF(@index_exists = 0, 'ALTER TABLE audit_events ADD INDEX idx_audit_events_gateway_id (gateway_id)', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @index_exists = ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND INDEX_NAME = 'idx_cost_records_gateway_id' +); +SET @ddl = IF(@index_exists = 0, 'ALTER TABLE cost_records ADD INDEX idx_cost_records_gateway_id (gateway_id)', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @index_exists = ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND INDEX_NAME = 'idx_risk_hits_gateway_id' +); +SET @ddl = IF(@index_exists = 0, 'ALTER TABLE risk_hits ADD INDEX idx_risk_hits_gateway_id (gateway_id)', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/backend/internal/db/migrations/029_add_short_external_access_links.sql b/backend/internal/db/migrations/029_add_short_external_access_links.sql new file mode 100644 index 0000000..291932e --- /dev/null +++ b/backend/internal/db/migrations/029_add_short_external_access_links.sql @@ -0,0 +1,47 @@ +SET @column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instance_external_access' + AND COLUMN_NAME = 'short_code_hash' +); +SET @ddl = IF( + @column_exists = 0, + 'ALTER TABLE instance_external_access ADD COLUMN short_code_hash VARCHAR(128) NULL AFTER public_token_hash', + 'SELECT 1' +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @public_slug_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instance_external_access' + AND COLUMN_NAME = 'public_slug' +); +SET @ddl = IF( + @public_slug_exists > 0, + 'ALTER TABLE instance_external_access MODIFY COLUMN public_slug VARCHAR(64) NULL', + 'SELECT 1' +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @index_exists = ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instance_external_access' + AND INDEX_NAME = 'uk_instance_external_access_short_code' +); +SET @ddl = IF( + @index_exists = 0, + 'ALTER TABLE instance_external_access ADD UNIQUE KEY uk_instance_external_access_short_code (short_code_hash)', + 'SELECT 1' +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend/internal/db/migrations/030_add_external_access_password_value.sql b/backend/internal/db/migrations/030_add_external_access_password_value.sql new file mode 100644 index 0000000..ee56933 --- /dev/null +++ b/backend/internal/db/migrations/030_add_external_access_password_value.sql @@ -0,0 +1,15 @@ +SET @column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instance_external_access' + AND COLUMN_NAME = 'password_value' +); +SET @ddl = IF( + @column_exists = 0, + 'ALTER TABLE instance_external_access ADD COLUMN password_value VARCHAR(128) NULL AFTER api_key_hash', + 'SELECT 1' +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend/internal/db/migrations/031_add_hermes_lite_runtime_image.sql b/backend/internal/db/migrations/031_add_hermes_lite_runtime_image.sql new file mode 100644 index 0000000..fa7855f --- /dev/null +++ b/backend/internal/db/migrations/031_add_hermes_lite_runtime_image.sql @@ -0,0 +1,31 @@ +UPDATE system_image_settings +SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE +WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + 'registry.example.com/your-custom-shell-image:latest' + ); + +INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) +SELECT + 'hermes', + 'shell', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE +WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' +) +AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND is_enabled = FALSE +); diff --git a/backend/internal/db/migrations/032_add_team_member_instance_mode.sql b/backend/internal/db/migrations/032_add_team_member_instance_mode.sql new file mode 100644 index 0000000..57d9dec --- /dev/null +++ b/backend/internal/db/migrations/032_add_team_member_instance_mode.sql @@ -0,0 +1,27 @@ +SET @team_member_instance_mode_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'team_members' + AND COLUMN_NAME = 'instance_mode' +); +SET @team_member_instance_mode_column_sql = IF( + @team_member_instance_mode_column_exists = 0, + 'ALTER TABLE team_members ADD COLUMN instance_mode ENUM(''lite'', ''pro'') NOT NULL DEFAULT ''lite'' AFTER runtime_type', + 'SELECT 1' +); +PREPARE team_member_instance_mode_column_stmt FROM @team_member_instance_mode_column_sql; +EXECUTE team_member_instance_mode_column_stmt; +DEALLOCATE PREPARE team_member_instance_mode_column_stmt; + +UPDATE team_members tm +LEFT JOIN instances i ON i.id = tm.instance_id +SET tm.instance_mode = CASE + WHEN i.instance_mode IN ('lite', 'pro') THEN i.instance_mode + WHEN i.runtime_type = 'gateway' THEN 'lite' + WHEN i.id IS NOT NULL THEN 'pro' + ELSE tm.instance_mode +END +WHERE tm.instance_mode IS NULL + OR tm.instance_mode = '' + OR (i.id IS NOT NULL AND tm.instance_mode <> COALESCE(i.instance_mode, CASE WHEN i.runtime_type = 'gateway' THEN 'lite' ELSE 'pro' END)); diff --git a/backend/internal/db/migrations/033_system_image_gateway_runtime_type.sql b/backend/internal/db/migrations/033_system_image_gateway_runtime_type.sql new file mode 100644 index 0000000..674c9f5 --- /dev/null +++ b/backend/internal/db/migrations/033_system_image_gateway_runtime_type.sql @@ -0,0 +1,26 @@ +ALTER TABLE system_image_settings + MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop'; + +UPDATE system_image_settings +SET runtime_type = 'gateway', + display_name = 'OpenClaw Lite' +WHERE instance_type = 'openclaw' + AND runtime_type = 'shell'; + +UPDATE system_image_settings +SET runtime_type = 'gateway', + display_name = 'Hermes Lite' +WHERE instance_type = 'hermes' + AND runtime_type = 'shell'; + +UPDATE system_image_settings +SET display_name = 'OpenClaw Pro' +WHERE instance_type = 'openclaw' + AND runtime_type = 'desktop' + AND display_name IN ('OpenClaw Desktop', 'OpenClaw Runtime'); + +UPDATE system_image_settings +SET display_name = 'Hermes Pro' +WHERE instance_type = 'hermes' + AND runtime_type = 'desktop' + AND display_name IN ('Hermes Runtime', 'Hermes Desktop'); diff --git a/backend/internal/db/migrations/034_update_lite_default_images.sql b/backend/internal/db/migrations/034_update_lite_default_images.sql new file mode 100644 index 0000000..e819b76 --- /dev/null +++ b/backend/internal/db/migrations/034_update_lite_default_images.sql @@ -0,0 +1,54 @@ +UPDATE system_image_settings +SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', + display_name = 'OpenClaw Lite', + is_enabled = TRUE +WHERE instance_type = 'openclaw' + AND runtime_type IN ('shell', 'gateway') + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest', + '172.16.1.12:5010/openclaw-shell:local5', + '172.16.1.12:5010/openclaw:v2dev' + ); + +UPDATE system_image_settings +SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE +WHERE instance_type = 'hermes' + AND runtime_type IN ('shell', 'gateway') + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + '172.16.1.12:5010/hermes:codex-shared-agent-tui-bundle-20260609153019', + '172.16.1.12:5010/hermes:team-lite-tui-dist-20260612174205', + 'registry.example.com/your-custom-shell-image:latest' + ); + +INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) +SELECT + 'openclaw', + 'gateway', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', + TRUE +WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type IN ('shell', 'gateway') +); + +INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) +SELECT + 'hermes', + 'gateway', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE +WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type IN ('shell', 'gateway') +); diff --git a/backend/internal/db/migrations/035_harden_team_event_protocol.sql b/backend/internal/db/migrations/035_harden_team_event_protocol.sql new file mode 100644 index 0000000..cc4728c --- /dev/null +++ b/backend/internal/db/migrations/035_harden_team_event_protocol.sql @@ -0,0 +1,34 @@ +ALTER TABLE team_events + ADD COLUMN event_id VARCHAR(128) NULL AFTER id, + ADD COLUMN completion_id VARCHAR(255) NULL AFTER event_id, + ADD COLUMN sequence_no BIGINT NOT NULL DEFAULT 0 AFTER completion_id; + +UPDATE team_events SET sequence_no = id WHERE sequence_no = 0; + +ALTER TABLE team_events + ADD UNIQUE KEY uk_team_events_event_id (team_id, event_id), + ADD UNIQUE KEY uk_team_events_completion_id (team_id, completion_id), + ADD INDEX idx_team_events_team_sequence (team_id, sequence_no); + +CREATE TABLE IF NOT EXISTS team_work_items ( + id INT AUTO_INCREMENT PRIMARY KEY, + team_id INT NOT NULL, + root_task_id INT NOT NULL, + work_id VARCHAR(255) NOT NULL, + owner_member_id INT NULL, + title VARCHAR(500) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + depends_on_json LONGTEXT NULL, + result_json LONGTEXT NULL, + artifact_refs_json LONGTEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (team_id) REFERENCES teams(id) ON DELETE CASCADE, + FOREIGN KEY (root_task_id) REFERENCES team_tasks(id) ON DELETE CASCADE, + FOREIGN KEY (owner_member_id) REFERENCES team_members(id) ON DELETE SET NULL, + UNIQUE KEY uk_team_work_items_work (team_id, root_task_id, work_id), + INDEX idx_team_work_items_root_status (root_task_id, status), + INDEX idx_team_work_items_owner_status (owner_member_id, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/036_add_team_event_outbox.sql b/backend/internal/db/migrations/036_add_team_event_outbox.sql new file mode 100644 index 0000000..d130317 --- /dev/null +++ b/backend/internal/db/migrations/036_add_team_event_outbox.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS team_event_outbox ( + id INT AUTO_INCREMENT PRIMARY KEY, + team_id INT NOT NULL, + source_event_id VARCHAR(255) NOT NULL, + destination VARCHAR(255) NOT NULL, + message_id VARCHAR(255) NOT NULL, + payload_json LONGTEXT NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + attempts INT NOT NULL DEFAULT 0, + available_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_error TEXT NULL, + delivered_at TIMESTAMP NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (team_id) REFERENCES teams(id) ON DELETE CASCADE, + UNIQUE KEY uk_team_event_outbox_message (team_id, destination, message_id), + INDEX idx_team_event_outbox_pending (status, available_at), + INDEX idx_team_event_outbox_team (team_id, created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/037_add_team_workflow_ledger.sql b/backend/internal/db/migrations/037_add_team_workflow_ledger.sql new file mode 100644 index 0000000..91546a7 --- /dev/null +++ b/backend/internal/db/migrations/037_add_team_workflow_ledger.sql @@ -0,0 +1,48 @@ +ALTER TABLE team_tasks + ADD COLUMN workflow_state VARCHAR(40) NOT NULL DEFAULT 'open' AFTER status, + ADD COLUMN plan_version BIGINT NOT NULL DEFAULT 0 AFTER workflow_state, + ADD COLUMN ledger_version BIGINT NOT NULL DEFAULT 0 AFTER plan_version, + ADD COLUMN current_phase_id VARCHAR(255) NULL AFTER ledger_version, + ADD COLUMN accepted_completion_id VARCHAR(255) NULL AFTER current_phase_id, + ADD UNIQUE KEY uk_team_tasks_accepted_completion (team_id, accepted_completion_id), + ADD INDEX idx_team_tasks_workflow (team_id, workflow_state, status); + +ALTER TABLE team_work_items + ADD COLUMN assignment_id VARCHAR(255) NULL AFTER work_id, + ADD COLUMN canonical_work_id VARCHAR(255) NULL AFTER assignment_id, + ADD COLUMN phase_id VARCHAR(255) NULL AFTER canonical_work_id, + ADD COLUMN revision INT NOT NULL DEFAULT 1 AFTER phase_id, + ADD COLUMN required_for_root BOOLEAN NOT NULL DEFAULT TRUE AFTER revision, + ADD COLUMN superseded_by VARCHAR(255) NULL AFTER required_for_root, + ADD COLUMN review_required BOOLEAN NOT NULL DEFAULT FALSE AFTER superseded_by, + ADD COLUMN validated_revision INT NULL AFTER review_required, + ADD INDEX idx_team_work_items_assignment (root_task_id, assignment_id, revision), + ADD INDEX idx_team_work_items_phase (root_task_id, phase_id, status); + +UPDATE team_work_items +SET assignment_id = work_id, + canonical_work_id = work_id, + phase_id = 'legacy' +WHERE assignment_id IS NULL OR canonical_work_id IS NULL OR phase_id IS NULL; + +CREATE TABLE IF NOT EXISTS team_workflow_phases ( + id INT AUTO_INCREMENT PRIMARY KEY, + team_id INT NOT NULL, + root_task_id INT NOT NULL, + phase_id VARCHAR(255) NOT NULL, + plan_version BIGINT NOT NULL DEFAULT 1, + sequence_no INT NOT NULL DEFAULT 0, + status VARCHAR(40) NOT NULL DEFAULT 'planned', + required_for_root BOOLEAN NOT NULL DEFAULT TRUE, + decision_required BOOLEAN NOT NULL DEFAULT FALSE, + depends_on_json LONGTEXT NULL, + next_phase_id VARCHAR(255) NULL, + completion_policy VARCHAR(80) NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP NULL, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (team_id) REFERENCES teams(id) ON DELETE CASCADE, + FOREIGN KEY (root_task_id) REFERENCES team_tasks(id) ON DELETE CASCADE, + UNIQUE KEY uk_team_workflow_phase (root_task_id, phase_id, plan_version), + INDEX idx_team_workflow_phase_state (root_task_id, plan_version, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations_test.go b/backend/internal/db/migrations_test.go new file mode 100644 index 0000000..f12b39b --- /dev/null +++ b/backend/internal/db/migrations_test.go @@ -0,0 +1,162 @@ +package db + +import ( + "reflect" + "strings" + "testing" +) + +func TestSplitSQLStatements(t *testing.T) { + input := ` +-- create table comment +CREATE TABLE demo ( + id INT PRIMARY KEY, + note VARCHAR(255) DEFAULT 'hello;world' +); + +/* block comment */ +INSERT INTO demo (id, note) VALUES (1, 'value'); +UPDATE demo SET note = "a;quoted" WHERE id = 1; +` + + got := splitSQLStatements(input) + want := []string{ + "CREATE TABLE demo (\n id INT PRIMARY KEY,\n note VARCHAR(255) DEFAULT 'hello;world'\n)", + "INSERT INTO demo (id, note) VALUES (1, 'value')", + `UPDATE demo SET note = "a;quoted" WHERE id = 1`, + } + + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected statements:\nwant: %#v\ngot: %#v", want, got) + } +} + +func TestMigration023IsEmbedded(t *testing.T) { + files, err := embeddedMigrations.ReadDir("migrations") + if err != nil { + t.Fatalf("read embedded migrations: %v", err) + } + + found := false + for _, file := range files { + if file.Name() == "023_add_runtime_pool_v2.sql" { + found = true + break + } + } + if !found { + t.Fatalf("migration 023_add_runtime_pool_v2.sql is not embedded") + } +} + +func TestMigration034UpdatesLiteDefaultImages(t *testing.T) { + raw, err := embeddedMigrations.ReadFile("migrations/034_update_lite_default_images.sql") + if err != nil { + t.Fatalf("read migration 034: %v", err) + } + sql := string(raw) + for _, image := range []string{ + "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest", + "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest", + } { + if !strings.Contains(sql, image) { + t.Fatalf("migration 034 must update lite image %s", image) + } + } +} + +func TestMigration023IsRetrySafe(t *testing.T) { + raw, err := embeddedMigrations.ReadFile("migrations/023_add_runtime_pool_v2.sql") + if err != nil { + t.Fatalf("read migration 023: %v", err) + } + + sql := string(raw) + if !strings.Contains(sql, "information_schema.COLUMNS") { + t.Fatalf("migration 023 must guard instance column additions with information_schema.COLUMNS") + } + for _, column := range []string{ + "workspace_path", + "workspace_usage_bytes", + "runtime_generation", + "runtime_error_message", + } { + if !strings.Contains(sql, "COLUMN_NAME = '"+column+"'") { + t.Fatalf("migration 023 must guard %s column addition", column) + } + } + for _, table := range []string{ + "runtime_pods", + "instance_runtime_bindings", + "runtime_rollouts", + "workspace_file_audits", + } { + if !strings.Contains(sql, "CREATE TABLE IF NOT EXISTS "+table) { + t.Fatalf("migration 023 must create %s idempotently", table) + } + } +} + +func TestMigration035HardensTeamEventProtocol(t *testing.T) { + raw, err := embeddedMigrations.ReadFile("migrations/035_harden_team_event_protocol.sql") + if err != nil { + t.Fatalf("read migration 035: %v", err) + } + sql := string(raw) + for _, required := range []string{ + "event_id", + "completion_id", + "sequence_no", + "uk_team_events_event_id", + "uk_team_events_completion_id", + "CREATE TABLE IF NOT EXISTS team_work_items", + "uk_team_work_items_work", + } { + if !strings.Contains(sql, required) { + t.Fatalf("migration 035 must contain %s", required) + } + } +} + +func TestMigration036AddsReliableTeamEventOutbox(t *testing.T) { + raw, err := embeddedMigrations.ReadFile("migrations/036_add_team_event_outbox.sql") + if err != nil { + t.Fatalf("read migration 036: %v", err) + } + sql := string(raw) + for _, required := range []string{ + "CREATE TABLE IF NOT EXISTS team_event_outbox", + "uk_team_event_outbox_message", + "idx_team_event_outbox_pending", + "source_event_id", + "available_at", + } { + if !strings.Contains(sql, required) { + t.Fatalf("migration 036 must contain %s", required) + } + } +} + +func TestMigration037AddsTeamWorkflowLedger(t *testing.T) { + raw, err := embeddedMigrations.ReadFile("migrations/037_add_team_workflow_ledger.sql") + if err != nil { + t.Fatalf("read migration 037: %v", err) + } + sql := string(raw) + for _, required := range []string{ + "workflow_state", + "plan_version", + "ledger_version", + "accepted_completion_id", + "assignment_id", + "canonical_work_id", + "phase_id", + "required_for_root", + "CREATE TABLE IF NOT EXISTS team_workflow_phases", + "uk_team_workflow_phase", + } { + if !strings.Contains(sql, required) { + t.Fatalf("migration 037 must contain %s", required) + } + } +} diff --git a/backend/internal/db/runtime_manifest_test.go b/backend/internal/db/runtime_manifest_test.go new file mode 100644 index 0000000..5663436 --- /dev/null +++ b/backend/internal/db/runtime_manifest_test.go @@ -0,0 +1,78 @@ +package db + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func TestRuntimeManifestsStartHermesRuntime(t *testing.T) { + repoRoot := filepath.Clean(filepath.Join("..", "..", "..")) + for _, manifest := range deploymentRuntimeManifests(repoRoot) { + t.Run(manifest, func(t *testing.T) { + raw, err := os.ReadFile(manifest) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + pattern := regexp.MustCompile(`(?s)name:\s+hermes-runtime.*?spec:\s+replicas:\s+([0-9]+)`) + matches := pattern.FindSubmatch(raw) + if len(matches) != 2 { + t.Fatalf("could not find hermes-runtime replicas in %s", manifest) + } + if string(matches[1]) != "1" { + t.Fatalf("expected hermes-runtime replicas 1 in %s, got %s", manifest, matches[1]) + } + }) + } +} + +func TestRuntimeManifestsSeedLiteDefaultImages(t *testing.T) { + repoRoot := filepath.Clean(filepath.Join("..", "..", "..")) + for _, manifest := range append(deploymentRuntimeManifests(repoRoot), filepath.Join(repoRoot, "backend", "deployments", "k8s", "clawreef-incluster.yaml")) { + t.Run(manifest, func(t *testing.T) { + raw, err := os.ReadFile(manifest) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + for _, image := range []string{ + "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest", + "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest", + } { + if !strings.Contains(string(raw), image) { + t.Fatalf("manifest %s must seed lite image %s", manifest, image) + } + } + }) + } +} + +func TestRuntimeManifestsExposeOpenClawGatewayOnPodIP(t *testing.T) { + repoRoot := filepath.Clean(filepath.Join("..", "..", "..")) + for _, manifest := range deploymentRuntimeManifests(repoRoot) { + t.Run(manifest, func(t *testing.T) { + raw, err := os.ReadFile(manifest) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + text := string(raw) + want := "/usr/local/bin/openclaw gateway run --allow-unconfigured --auth token --bind lan --force" + if !strings.Contains(text, want) { + t.Fatalf("manifest %s must expose OpenClaw gateway on the pod network with %q", manifest, want) + } + if strings.Contains(text, "--auth token --bind auto --force") { + t.Fatalf("manifest %s must not use OpenClaw --bind auto because it can bind to loopback inside runtime pods", manifest) + } + }) + } +} + +func deploymentRuntimeManifests(repoRoot string) []string { + return []string{ + filepath.Join(repoRoot, "deployments", "k8s", "cluster", "clawmanager.yaml"), + filepath.Join(repoRoot, "deployments", "k8s", "single-node", "clawmanager.yaml"), + filepath.Join(repoRoot, "deployments", "k3s", "cluster", "clawmanager.yaml"), + filepath.Join(repoRoot, "deployments", "k3s", "single-node", "clawmanager.yaml"), + } +} diff --git a/backend/internal/handlers/agent_handler.go b/backend/internal/handlers/agent_handler.go new file mode 100644 index 0000000..b2d1c6f --- /dev/null +++ b/backend/internal/handlers/agent_handler.go @@ -0,0 +1,258 @@ +package handlers + +import ( + "net/http" + "strconv" + "strings" + "time" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type AgentHandler struct { + agentService services.InstanceAgentService + commandService services.InstanceCommandService + runtimeStatusService services.InstanceRuntimeStatusService + configRevisionService services.InstanceConfigRevisionService + skillService services.SkillService +} + +func NewAgentHandler(agentService services.InstanceAgentService, commandService services.InstanceCommandService, runtimeStatusService services.InstanceRuntimeStatusService, configRevisionService services.InstanceConfigRevisionService, skillService services.SkillService) *AgentHandler { + return &AgentHandler{ + agentService: agentService, + commandService: commandService, + runtimeStatusService: runtimeStatusService, + configRevisionService: configRevisionService, + skillService: skillService, + } +} + +func (h *AgentHandler) Register(c *gin.Context) { + bootstrapToken := extractBearerToken(c.GetHeader("Authorization")) + if bootstrapToken == "" { + utils.Error(c, http.StatusUnauthorized, "Agent bootstrap token is required") + return + } + + var req services.AgentRegisterRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + resp, err := h.agentService.Register(bootstrapToken, req, c.ClientIP()) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Agent registered successfully", resp) +} + +func (h *AgentHandler) Heartbeat(c *gin.Context) { + session, ok := h.authenticateAgentSession(c) + if !ok { + return + } + + var req services.AgentHeartbeatRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + if req.Timestamp.IsZero() { + req.Timestamp = time.Now().UTC() + } + + resp, err := h.agentService.Heartbeat(session, req, c.ClientIP()) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Agent heartbeat accepted", resp) +} + +func (h *AgentHandler) NextCommand(c *gin.Context) { + session, ok := h.authenticateAgentSession(c) + if !ok { + return + } + + command, err := h.commandService.GetNextForAgent(session) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Agent next command retrieved successfully", gin.H{"command": command}) +} + +func (h *AgentHandler) StartCommand(c *gin.Context) { + session, ok := h.authenticateAgentSession(c) + if !ok { + return + } + commandID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid command ID") + return + } + var req struct { + AgentID string `json:"agent_id" binding:"required"` + StartedAt *time.Time `json:"started_at,omitempty"` + } + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + if req.AgentID != session.Agent.AgentID { + utils.Error(c, http.StatusForbidden, "Agent ID does not match session") + return + } + if err := h.commandService.MarkStarted(session, commandID, req.StartedAt); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Agent command marked as started", nil) +} + +func (h *AgentHandler) FinishCommand(c *gin.Context) { + session, ok := h.authenticateAgentSession(c) + if !ok { + return + } + commandID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid command ID") + return + } + + var req services.AgentCommandFinishRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + if err := h.commandService.MarkFinished(session, commandID, req); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Agent command result accepted", nil) +} + +func (h *AgentHandler) ReportState(c *gin.Context) { + session, ok := h.authenticateAgentSession(c) + if !ok { + return + } + + var req services.AgentStateReportRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + if err := h.runtimeStatusService.Report(session, req, c.ClientIP()); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Agent state reported successfully", nil) +} + +func (h *AgentHandler) GetConfigRevision(c *gin.Context) { + session, ok := h.authenticateAgentSession(c) + if !ok { + return + } + revisionID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid config revision ID") + return + } + revision, err := h.configRevisionService.GetByID(revisionID) + if err != nil { + utils.HandleError(c, err) + return + } + if revision.InstanceID != session.Instance.ID { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + utils.Success(c, http.StatusOK, "Config revision retrieved successfully", gin.H{"revision": revision}) +} + +func (h *AgentHandler) ReportSkillInventory(c *gin.Context) { + session, ok := h.authenticateAgentSession(c) + if !ok { + return + } + var req services.AgentSkillInventoryReportRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + if strings.TrimSpace(req.AgentID) != "" && strings.TrimSpace(req.AgentID) != session.Agent.AgentID { + utils.Error(c, http.StatusForbidden, "Agent ID does not match session") + return + } + if err := h.skillService.SyncAgentSkills(session.Instance.ID, req); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Agent skill inventory reported successfully", nil) +} + +func (h *AgentHandler) UploadSkillPackage(c *gin.Context) { + session, ok := h.authenticateAgentSession(c) + if !ok { + return + } + fileHeader, err := c.FormFile("file") + if err != nil { + utils.Error(c, http.StatusBadRequest, "file is required") + return + } + req := services.AgentSkillPackageUploadRequest{ + AgentID: strings.TrimSpace(c.PostForm("agent_id")), + SkillID: strings.TrimSpace(c.PostForm("skill_id")), + SkillVersion: strings.TrimSpace(c.PostForm("skill_version")), + Identifier: strings.TrimSpace(c.PostForm("identifier")), + ContentMD5: strings.TrimSpace(c.PostForm("content_md5")), + Source: strings.TrimSpace(c.PostForm("source")), + } + if req.AgentID != "" && req.AgentID != session.Agent.AgentID { + utils.Error(c, http.StatusForbidden, "Agent ID does not match session") + return + } + item, err := h.skillService.UploadAgentSkillPackage(c.Request.Context(), session.Instance.ID, req, fileHeader) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Agent skill package uploaded successfully", item) +} + +func (h *AgentHandler) authenticateAgentSession(c *gin.Context) (*services.AgentSession, bool) { + sessionToken := extractBearerToken(c.GetHeader("Authorization")) + if sessionToken == "" { + utils.Error(c, http.StatusUnauthorized, "Agent session token is required") + return nil, false + } + session, err := h.agentService.AuthenticateSession(sessionToken) + if err != nil { + utils.HandleError(c, err) + return nil, false + } + return session, true +} + +func extractBearerToken(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + parts := strings.SplitN(raw, " ", 2) + if len(parts) != 2 || !strings.EqualFold(parts[0], "bearer") { + return "" + } + return strings.TrimSpace(parts[1]) +} diff --git a/backend/internal/handlers/ai_gateway_handler.go b/backend/internal/handlers/ai_gateway_handler.go new file mode 100644 index 0000000..28e77d0 --- /dev/null +++ b/backend/internal/handlers/ai_gateway_handler.go @@ -0,0 +1,160 @@ +package handlers + +import ( + "encoding/json" + "io" + "net/http" + "strings" + + "clawreef/internal/aigateway" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +// AIGatewayHandler exposes AI gateway endpoints. +type AIGatewayHandler struct { + service aigateway.Service +} + +// NewAIGatewayHandler creates a new AI gateway handler. +func NewAIGatewayHandler(service aigateway.Service) *AIGatewayHandler { + return &AIGatewayHandler{service: service} +} + +// ListModels returns active models available to the current user. +func (h *AIGatewayHandler) ListModels(c *gin.Context) { + items, err := h.service.ListAvailableModels() + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Available gateway models retrieved successfully", gin.H{ + "items": items, + }) +} + +// ChatCompletions proxies a governed chat completion request. +func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) { + rawBody, err := io.ReadAll(c.Request.Body) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Failed to read request body") + return + } + + var req aigateway.ChatCompletionRequest + if err := json.Unmarshal(rawBody, &req); err != nil { + utils.ValidationError(c, err) + return + } + req.RawBody = rawBody + if req.SessionID == nil { + if sessionKey := strings.TrimSpace(c.GetHeader("x-openclaw-session-key")); sessionKey != "" { + req.SessionID = &sessionKey + } + } + if req.TraceID == nil { + if runID := strings.TrimSpace(c.GetHeader("x-openclaw-run-id")); runID != "" { + req.TraceID = &runID + } + } + + userID, exists := c.Get("userID") + if !exists { + utils.Error(c, http.StatusUnauthorized, "Unauthorized") + return + } + + if instanceID, exists := c.Get("instanceID"); exists { + switch value := instanceID.(type) { + case int: + if req.InstanceID == nil { + req.InstanceID = &value + } else if *req.InstanceID != value { + utils.Error(c, http.StatusForbidden, "Gateway token does not match requested instance") + return + } + } + } + if !setStringMetadata(c, &req.InstanceMode, "instanceMode") || + !setStringMetadata(c, &req.RuntimeType, "runtimeType") || + !setStringMetadata(c, &req.GatewayID, "gatewayID") || + !setInt64Metadata(c, &req.RuntimePodID, "runtimePodID") { + utils.Error(c, http.StatusForbidden, "Gateway token metadata does not match request") + return + } + + if req.Stream { + traceID, err := h.service.StreamChatCompletions(c.Request.Context(), userID.(int), req, c.Writer) + if traceID != "" { + c.Header("X-Trace-ID", traceID) + } + if err != nil { + if !c.Writer.Written() { + utils.HandleError(c, err) + } + } + return + } + + response, traceID, err := h.service.ChatCompletions(c.Request.Context(), userID.(int), req) + if err != nil { + c.Header("X-Trace-ID", traceID) + utils.HandleError(c, err) + return + } + + c.Header("X-Trace-ID", traceID) + for key, values := range response.Headers { + for _, value := range values { + c.Writer.Header().Add(key, value) + } + } + c.Status(response.StatusCode) + _, _ = c.Writer.Write(response.Body) +} + +func setStringMetadata(c *gin.Context, field **string, key string) bool { + raw, exists := c.Get(key) + if !exists { + return true + } + value, ok := raw.(string) + if !ok { + return true + } + value = strings.TrimSpace(value) + if value == "" { + return true + } + if *field == nil { + *field = &value + return true + } + return strings.TrimSpace(**field) == value +} + +func setInt64Metadata(c *gin.Context, field **int64, key string) bool { + raw, exists := c.Get(key) + if !exists { + return true + } + var value int64 + switch typed := raw.(type) { + case int64: + value = typed + case int: + value = int64(typed) + default: + return true + } + if value <= 0 { + return true + } + if *field == nil { + *field = &value + return true + } + return **field == value +} diff --git a/backend/internal/handlers/ai_observability_handler.go b/backend/internal/handlers/ai_observability_handler.go new file mode 100644 index 0000000..733cec7 --- /dev/null +++ b/backend/internal/handlers/ai_observability_handler.go @@ -0,0 +1,92 @@ +package handlers + +import ( + "net/http" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +// AIObservabilityHandler exposes admin reporting endpoints for audit and cost data. +type AIObservabilityHandler struct { + service services.AIObservabilityService +} + +// AuditQueryRequest binds supported audit list filters. +type AuditQueryRequest struct { + Page int `form:"page,default=1"` + Limit int `form:"limit,default=100"` + Search string `form:"search"` + Status string `form:"status"` + Model string `form:"model"` +} + +// CostQueryRequest binds supported cost query filters. +type CostQueryRequest struct { + Page int `form:"page,default=1"` + Limit int `form:"limit,default=100"` + Search string `form:"search"` +} + +// NewAIObservabilityHandler creates a new observability handler. +func NewAIObservabilityHandler(service services.AIObservabilityService) *AIObservabilityHandler { + return &AIObservabilityHandler{service: service} +} + +// ListAuditItems returns recent audit entries for AI model invocations. +func (h *AIObservabilityHandler) ListAuditItems(c *gin.Context) { + var req AuditQueryRequest + if err := c.ShouldBindQuery(&req); err != nil { + utils.ValidationError(c, err) + return + } + + items, err := h.service.ListAuditItems(services.AuditQuery{ + Page: req.Page, + Limit: req.Limit, + Search: req.Search, + Status: req.Status, + Model: req.Model, + }) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "AI audit items retrieved successfully", items) +} + +// GetTraceDetail returns full trace detail for a model invocation trace. +func (h *AIObservabilityHandler) GetTraceDetail(c *gin.Context) { + traceID := c.Param("traceId") + detail, err := h.service.GetTraceDetail(traceID) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "AI trace detail retrieved successfully", detail) +} + +// GetCostOverview returns token and money overview data for admin reporting. +func (h *AIObservabilityHandler) GetCostOverview(c *gin.Context) { + var req CostQueryRequest + if err := c.ShouldBindQuery(&req); err != nil { + utils.ValidationError(c, err) + return + } + + overview, err := h.service.GetCostOverview(services.CostQuery{ + Page: req.Page, + Limit: req.Limit, + Search: req.Search, + }) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "AI cost overview retrieved successfully", overview) +} diff --git a/backend/internal/handlers/auth_handler.go b/backend/internal/handlers/auth_handler.go new file mode 100644 index 0000000..9878ad5 --- /dev/null +++ b/backend/internal/handlers/auth_handler.go @@ -0,0 +1,140 @@ +package handlers + +import ( + "net/http" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +// AuthHandler handles authentication-related requests +type AuthHandler struct { + authService services.AuthService +} + +// NewAuthHandler creates a new auth handler +func NewAuthHandler(authService services.AuthService) *AuthHandler { + return &AuthHandler{authService: authService} +} + +// RegisterRequest represents a registration request +type RegisterRequest struct { + Username string `json:"username" binding:"required,min=3,max=32,alphanum"` + Email string `json:"email" binding:"required,email"` + Password string `json:"password" binding:"required,min=8"` +} + +// LoginRequest represents a login request +type LoginRequest struct { + Username string `json:"username" binding:"required"` + Password string `json:"password" binding:"required"` +} + +// RefreshTokenRequest represents a refresh token request +type RefreshTokenRequest struct { + RefreshToken string `json:"refresh_token" binding:"required"` +} + +type ChangePasswordRequest struct { + CurrentPassword string `json:"current_password" binding:"required"` + NewPassword string `json:"new_password" binding:"required,min=8"` +} + +// Register handles user registration +func (h *AuthHandler) Register(c *gin.Context) { + var req RegisterRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + user, err := h.authService.Register(req.Username, req.Email, req.Password) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusCreated, "User registered successfully", user) +} + +// Login handles user login +func (h *AuthHandler) Login(c *gin.Context) { + var req LoginRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + tokenPair, err := h.authService.Login(req.Username, req.Password) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Login successful", tokenPair) +} + +// RefreshToken handles token refresh +func (h *AuthHandler) RefreshToken(c *gin.Context) { + var req RefreshTokenRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + tokenPair, err := h.authService.RefreshToken(req.RefreshToken) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Token refreshed successfully", tokenPair) +} + +// Logout handles user logout +func (h *AuthHandler) Logout(c *gin.Context) { + // In a stateless JWT system, logout is handled client-side + // by removing the token from storage + utils.Success(c, http.StatusOK, "Logout successful", nil) +} + +// GetCurrentUser gets the current authenticated user +func (h *AuthHandler) GetCurrentUser(c *gin.Context) { + userID, exists := c.Get("userID") + if !exists { + utils.Error(c, http.StatusUnauthorized, "Unauthorized") + return + } + + user, err := h.authService.GetCurrentUser(userID.(int)) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "User retrieved successfully", user) +} + +// ChangePassword changes the current user's password +func (h *AuthHandler) ChangePassword(c *gin.Context) { + userID, exists := c.Get("userID") + if !exists { + utils.Error(c, http.StatusUnauthorized, "Unauthorized") + return + } + + var req ChangePasswordRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + if err := h.authService.ChangePassword(userID.(int), req.CurrentPassword, req.NewPassword); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Password changed successfully", nil) +} diff --git a/backend/internal/handlers/cluster_resource_handler.go b/backend/internal/handlers/cluster_resource_handler.go new file mode 100644 index 0000000..13875dd --- /dev/null +++ b/backend/internal/handlers/cluster_resource_handler.go @@ -0,0 +1,28 @@ +package handlers + +import ( + "net/http" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type ClusterResourceHandler struct { + clusterResourceService services.ClusterResourceService +} + +func NewClusterResourceHandler(clusterResourceService services.ClusterResourceService) *ClusterResourceHandler { + return &ClusterResourceHandler{clusterResourceService: clusterResourceService} +} + +func (h *ClusterResourceHandler) GetOverview(c *gin.Context) { + overview, err := h.clusterResourceService.GetOverview(c.Request.Context()) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Cluster resource overview retrieved successfully", overview) +} diff --git a/backend/internal/handlers/egress_proxy_handler.go b/backend/internal/handlers/egress_proxy_handler.go new file mode 100644 index 0000000..602e474 --- /dev/null +++ b/backend/internal/handlers/egress_proxy_handler.go @@ -0,0 +1,122 @@ +package handlers + +import ( + "io" + "net" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" +) + +// EgressProxyHandler provides a minimal forward proxy for ordinary HTTP/HTTPS traffic. +type EgressProxyHandler struct { + transport *http.Transport +} + +// NewEgressProxyHandler creates a new egress proxy handler. +func NewEgressProxyHandler() *EgressProxyHandler { + return &EgressProxyHandler{ + transport: &http.Transport{ + Proxy: nil, + DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + ForceAttemptHTTP2: false, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + } +} + +// Handle proxies ordinary HTTP or HTTPS CONNECT traffic. +func (h *EgressProxyHandler) Handle(c *gin.Context) { + if strings.EqualFold(c.Request.Method, http.MethodConnect) { + h.handleConnect(c) + return + } + + if c.Request.URL == nil || c.Request.URL.Scheme == "" || c.Request.URL.Host == "" { + c.Status(http.StatusNotFound) + return + } + + outReq := c.Request.Clone(c.Request.Context()) + outReq.RequestURI = "" + removeHopHeaders(outReq.Header) + + resp, err := h.transport.RoundTrip(outReq) + if err != nil { + c.String(http.StatusBadGateway, "proxy upstream error: %v", err) + return + } + defer resp.Body.Close() + + removeHopHeaders(resp.Header) + copyHeaders(c.Writer.Header(), resp.Header) + c.Status(resp.StatusCode) + _, _ = io.Copy(c.Writer, resp.Body) +} + +func (h *EgressProxyHandler) handleConnect(c *gin.Context) { + target := strings.TrimSpace(c.Request.Host) + if target == "" { + c.String(http.StatusBadRequest, "missing CONNECT target") + return + } + + upstreamConn, err := net.DialTimeout("tcp", target, 30*time.Second) + if err != nil { + c.String(http.StatusBadGateway, "proxy connect error: %v", err) + return + } + + hijacker, ok := c.Writer.(http.Hijacker) + if !ok { + _ = upstreamConn.Close() + c.String(http.StatusInternalServerError, "proxy hijacking not supported") + return + } + + clientConn, _, err := hijacker.Hijack() + if err != nil { + _ = upstreamConn.Close() + return + } + + _, _ = clientConn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")) + + go tunnelConns(upstreamConn, clientConn) + go tunnelConns(clientConn, upstreamConn) +} + +func tunnelConns(dst net.Conn, src net.Conn) { + defer dst.Close() + defer src.Close() + _, _ = io.Copy(dst, src) +} + +func copyHeaders(dst, src http.Header) { + for key, values := range src { + for _, value := range values { + dst.Add(key, value) + } + } +} + +func removeHopHeaders(headers http.Header) { + for _, key := range []string{ + "Connection", + "Proxy-Connection", + "Keep-Alive", + "Proxy-Authenticate", + "Proxy-Authorization", + "Te", + "Trailer", + "Transfer-Encoding", + "Upgrade", + } { + headers.Del(key) + } +} diff --git a/backend/internal/handlers/instance_handler.go b/backend/internal/handlers/instance_handler.go new file mode 100644 index 0000000..879ebd0 --- /dev/null +++ b/backend/internal/handlers/instance_handler.go @@ -0,0 +1,2108 @@ +package handlers + +import ( + "errors" + "fmt" + "html" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +// openclawMinArchiveBytes is the minimum acceptable size of an .openclaw +// export archive. A correctly-compressed tar.gz of a single empty file is +// already ~125 bytes, so anything smaller indicates a malformed or empty +// stream from the exec pipeline rather than a real workspace dump. +const openclawMinArchiveBytes = 100 + +const ( + defaultWorkspaceArchiveMaxMiB = int64(500) + workspaceArchiveMaxMiBEnv = "CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB" + maxLiteBatchCreateCount = 20 + maxLiteBatchDeleteCount = 100 +) + +// desktopDirectProxyEnv toggles embedding the instance Service "host:port" into +// the access token so the edge gateway can proxy desktop traffic directly to the +// instance instead of relaying it through this control-plane process. +const desktopDirectProxyEnv = "CLAWMANAGER_DESKTOP_DIRECT_PROXY" + +// desktopDirectProxyEnabled reports whether direct desktop proxying is enabled. +func desktopDirectProxyEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(desktopDirectProxyEnv))) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} + +func desktopProxyMode(directEnabled bool, upstream string) string { + if !directEnabled { + return "control-plane" + } + if strings.TrimSpace(upstream) == "" { + return "fallback" + } + return "direct" +} + +func (h *InstanceHandler) desktopAccessUpstream(c *gin.Context, instance *models.Instance, targetPort int32) (string, bool) { + directProxyEnabled := desktopDirectProxyEnabled() + if !directProxyEnabled { + return "", false + } + if usesRuntimeGateway(instance) { + return "", false + } + if !h.proxyService.IsWebtopInstanceType(instance.Type) { + fmt.Printf("Desktop direct proxy fallback: unsupported desktop instance type instance=%d user=%d type=%s target_port=%d\n", + instance.ID, instance.UserID, instance.Type, targetPort) + return "", true + } + + resolved, resolveErr := h.proxyService.ResolveUpstreamHostPort( + c.Request.Context(), + instance.UserID, + instance.ID, + targetPort, + ) + if resolveErr != nil { + fmt.Printf("Desktop direct proxy fallback: failed to resolve upstream instance=%d user=%d type=%s target_port=%d error=%v\n", + instance.ID, instance.UserID, instance.Type, targetPort, resolveErr) + return "", true + } + if strings.TrimSpace(resolved) == "" { + fmt.Printf("Desktop direct proxy fallback: resolved empty upstream instance=%d user=%d type=%s target_port=%d\n", + instance.ID, instance.UserID, instance.Type, targetPort) + return "", true + } + + fmt.Printf("Desktop direct proxy resolved: instance=%d user=%d type=%s target_port=%d upstream=%s\n", + instance.ID, instance.UserID, instance.Type, targetPort, resolved) + return resolved, true +} + +func usesRuntimeGateway(instance *models.Instance) bool { + if instance == nil { + return false + } + if strings.EqualFold(strings.TrimSpace(instance.RuntimeType), services.RuntimeBackendGateway) { + return true + } + if mode, ok := services.NormalizeInstanceMode(instance.InstanceMode); ok && mode == services.InstanceModeLite { + return true + } + return false +} + +func workspaceArchiveMaxMiB() int64 { + value := strings.TrimSpace(os.Getenv(workspaceArchiveMaxMiBEnv)) + if value == "" { + return defaultWorkspaceArchiveMaxMiB + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed <= 0 { + return defaultWorkspaceArchiveMaxMiB + } + return parsed +} + +func workspaceArchiveMaxBytes() int64 { + return workspaceArchiveMaxMiB() << 20 +} + +// InstanceHandler handles instance management requests +type InstanceHandler struct { + instanceService services.InstanceService + instanceAgentService services.InstanceAgentService + runtimeStatusService services.InstanceRuntimeStatusService + instanceCommandService services.InstanceCommandService + instanceConfigRevisionService services.InstanceConfigRevisionService + accessService *services.InstanceAccessService + proxyService *services.InstanceProxyService + shellService *services.InstanceShellService + openClawTransferService services.OpenClawTransferService + openClawConfigService services.OpenClawConfigService + skillService services.SkillService + externalAccessService services.InstanceExternalAccessService +} + +// NewInstanceHandler creates a new instance handler +func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService, externalAccessService services.InstanceExternalAccessService, proxyOptions ...services.InstanceProxyServiceOption) *InstanceHandler { + accessService := services.NewInstanceAccessService() + return &InstanceHandler{ + instanceService: instanceService, + instanceAgentService: instanceAgentService, + runtimeStatusService: runtimeStatusService, + instanceCommandService: instanceCommandService, + instanceConfigRevisionService: instanceConfigRevisionService, + accessService: accessService, + proxyService: services.NewInstanceProxyService(accessService, proxyOptions...), + shellService: services.NewInstanceShellService(), + openClawTransferService: services.NewOpenClawTransferService(), + openClawConfigService: openClawConfigService, + skillService: skillService, + externalAccessService: externalAccessService, + } +} + +// Shutdown releases resources held by the handler (e.g. background goroutines). +func (h *InstanceHandler) Shutdown() { + if h.accessService != nil { + h.accessService.Stop() + } +} + +type InstanceRuntimeDetailsResponse struct { + Runtime *services.InstanceRuntimeStatusPayload `json:"runtime,omitempty"` + Agent *services.InstanceAgentPayload `json:"agent,omitempty"` + Commands []services.InstanceCommandPayload `json:"commands,omitempty"` +} + +type CreateRuntimeCommandRequest struct { + IdempotencyKey string `json:"idempotency_key,omitempty"` +} + +type PublishConfigRevisionRequest struct { + SnapshotID int `json:"snapshot_id" binding:"required,min=1"` +} + +type ExternalAccessRequest struct { + ExpiresMode string `json:"expires_mode,omitempty"` + ExpiresPreset string `json:"expires_preset,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// CreateInstanceRequest represents a create instance request +type CreateInstanceRequest struct { + Name string `json:"name" binding:"required,min=3,max=50"` + Description *string `json:"description,omitempty"` + Type string `json:"type" binding:"required,oneof=openclaw ubuntu debian centos custom webtop hermes"` + Mode string `json:"mode" binding:"omitempty,oneof=lite pro"` + InstanceMode string `json:"instance_mode" binding:"omitempty,oneof=lite pro"` + RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=gateway desktop shell"` + DesktopStreamProfile string `json:"desktop_stream_profile,omitempty" binding:"omitempty,oneof=low standard high"` + CPUCores float64 `json:"cpu_cores" binding:"required,min=0.1,max=32"` + MemoryGB int `json:"memory_gb" binding:"required,min=1,max=128"` + DiskGB int `json:"disk_gb" binding:"required,min=10,max=1000"` + GPUEnabled bool `json:"gpu_enabled"` + GPUCount int `json:"gpu_count" binding:"min=0,max=4"` + OSType string `json:"os_type" binding:"required"` + OSVersion string `json:"os_version" binding:"required"` + ImageRegistry *string `json:"image_registry,omitempty"` + ImageTag *string `json:"image_tag,omitempty"` + EnvironmentOverrides map[string]string `json:"environment_overrides,omitempty"` + StorageClass string `json:"storage_class"` + OpenClawConfigPlan *services.OpenClawConfigPlan `json:"openclaw_config_plan,omitempty"` + SkillIDs []int `json:"skill_ids,omitempty"` +} + +type BatchCreateLiteInstanceTemplate struct { + Type string `json:"type,omitempty"` + CPUCores float64 `json:"cpu_cores,omitempty"` + MemoryGB int `json:"memory_gb,omitempty"` + DiskGB int `json:"disk_gb,omitempty"` + OSType string `json:"os_type,omitempty"` + OSVersion string `json:"os_version,omitempty"` + ImageRegistry *string `json:"image_registry,omitempty"` + ImageTag *string `json:"image_tag,omitempty"` + EnvironmentOverrides map[string]string `json:"environment_overrides,omitempty"` + StorageClass string `json:"storage_class,omitempty"` + OpenClawConfigPlan *services.OpenClawConfigPlan `json:"openclaw_config_plan,omitempty"` + SkillIDs []int `json:"skill_ids,omitempty"` +} + +// BatchCreateLiteInstancesRequest represents a lite batch create request +type BatchCreateLiteInstancesRequest struct { + NamePrefix string `json:"name_prefix" binding:"required,min=1,max=40"` + Count int `json:"count" binding:"required,min=1,max=20"` + StartIndex int `json:"start_index" binding:"omitempty,min=0,max=9999"` + Template *BatchCreateLiteInstanceTemplate `json:"template,omitempty"` +} + +type BatchCreateLiteInstanceResult struct { + Name string `json:"name"` + Status string `json:"status"` + Instance *models.Instance `json:"instance,omitempty"` + Error string `json:"error,omitempty"` +} + +type BatchCreateLiteInstancesResponse struct { + Requested int `json:"requested"` + Created int `json:"created"` + Failed int `json:"failed"` + Results []BatchCreateLiteInstanceResult `json:"results"` +} + +type BatchDeleteLiteInstancesRequest struct { + InstanceIDs []int `json:"instance_ids" binding:"required,min=1,max=100,dive,min=1"` +} + +type BatchDeleteLiteInstanceResult struct { + InstanceID int `json:"instance_id"` + Name string `json:"name,omitempty"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} + +type BatchDeleteLiteInstancesResponse struct { + Requested int `json:"requested"` + Deleted int `json:"deleted"` + Failed int `json:"failed"` + Results []BatchDeleteLiteInstanceResult `json:"results"` +} + +// UpdateInstanceRequest represents an update instance request +type UpdateInstanceRequest struct { + Name *string `json:"name,omitempty" binding:"omitempty,min=3,max=50"` + Description *string `json:"description,omitempty"` + DesktopStreamProfile *string `json:"desktop_stream_profile,omitempty" binding:"omitempty,oneof=low standard high"` +} + +// ListInstancesRequest represents a list instances request +type ListInstancesRequest struct { + Page int `form:"page,default=1"` + Limit int `form:"limit,default=20"` + Status string `form:"status,omitempty"` +} + +// ListInstances lists instances owned by the current user (workspace view). +// +// This endpoint is always caller-scoped — the caller's role is intentionally +// not consulted. An admin using /instances sees only instances they personally +// own. Admin-scoped cross-user listing lives on /admin/instances and is gated +// by the admin middleware; see ListAllInstances below. +func (h *InstanceHandler) ListInstances(c *gin.Context) { + userID, _ := c.Get("userID") + + var req ListInstancesRequest + if err := c.ShouldBindQuery(&req); err != nil { + utils.ValidationError(c, err) + return + } + + // Calculate offset + offset := (req.Page - 1) * req.Limit + + instances, total, err := h.instanceService.GetByUserID(userID.(int), offset, req.Limit) + if err != nil { + utils.HandleError(c, err) + return + } + + response := map[string]interface{}{ + "instances": instances, + "total": total, + "page": req.Page, + "limit": req.Limit, + } + + utils.Success(c, http.StatusOK, "Instances retrieved successfully", response) +} + +// ListAllInstances lists every instance across all users (admin console view). +// +// Gated by the admin middleware on the /admin/instances route group. The +// admin role badge only controls which API surface is reachable — it does +// not widen the caller-scoped /instances endpoint. +func (h *InstanceHandler) ListAllInstances(c *gin.Context) { + var req ListInstancesRequest + if err := c.ShouldBindQuery(&req); err != nil { + utils.ValidationError(c, err) + return + } + + offset := (req.Page - 1) * req.Limit + + instances, total, err := h.instanceService.GetAllInstances(offset, req.Limit) + if err != nil { + utils.HandleError(c, err) + return + } + + response := map[string]interface{}{ + "instances": instances, + "total": total, + "page": req.Page, + "limit": req.Limit, + } + + utils.Success(c, http.StatusOK, "Instances retrieved successfully", response) +} + +// CreateInstance creates a new instance +func (h *InstanceHandler) CreateInstance(c *gin.Context) { + userID, _ := c.Get("userID") + + var req CreateInstanceRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + createReq := instanceCreateRequestToService(req) + + instance, err := h.instanceService.Create(userID.(int), createReq) + if err != nil { + utils.HandleError(c, err) + return + } + + skillIDs, err := h.resolveCreateInstanceSkillIDs(userID.(int), req) + if err != nil { + utils.HandleError(c, err) + return + } + + for _, skillID := range skillIDs { + if _, err := h.skillService.AttachSkillToInstance(instance.ID, skillID); err != nil { + utils.HandleError(c, err) + return + } + } + + utils.Success(c, http.StatusCreated, "Instance created successfully", instance) +} + +func instanceCreateRequestToService(req CreateInstanceRequest) services.CreateInstanceRequest { + return services.CreateInstanceRequest{ + Name: req.Name, + Description: req.Description, + Type: req.Type, + Mode: req.Mode, + InstanceMode: req.InstanceMode, + RuntimeType: req.RuntimeType, + DesktopStreamProfile: req.DesktopStreamProfile, + CPUCores: req.CPUCores, + MemoryGB: req.MemoryGB, + DiskGB: req.DiskGB, + GPUEnabled: req.GPUEnabled, + GPUCount: req.GPUCount, + OSType: req.OSType, + OSVersion: req.OSVersion, + ImageRegistry: req.ImageRegistry, + ImageTag: req.ImageTag, + EnvironmentOverrides: req.EnvironmentOverrides, + StorageClass: req.StorageClass, + OpenClawConfigPlan: req.OpenClawConfigPlan, + } +} + +func (h *InstanceHandler) BatchCreateLiteInstances(c *gin.Context) { + userID, _ := c.Get("userID") + + var req BatchCreateLiteInstancesRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + createRequests, handlerRequests, err := buildLiteBatchCreateRequests(req) + if err != nil { + utils.Error(c, http.StatusBadRequest, err.Error()) + return + } + + if err := h.instanceService.ValidateCreateRequests(userID.(int), createRequests); err != nil { + utils.HandleError(c, err) + return + } + + response := BatchCreateLiteInstancesResponse{ + Requested: len(createRequests), + Results: make([]BatchCreateLiteInstanceResult, 0, len(createRequests)), + } + + for idx, createReq := range createRequests { + handlerReq := handlerRequests[idx] + result := BatchCreateLiteInstanceResult{ + Name: createReq.Name, + Status: "created", + } + instance, err := h.instanceService.Create(userID.(int), createReq) + if err != nil { + result.Status = "failed" + result.Error = err.Error() + response.Failed++ + response.Results = append(response.Results, result) + continue + } + result.Instance = instance + + skillIDs, err := h.resolveCreateInstanceSkillIDs(userID.(int), handlerReq) + if err != nil { + result.Status = "failed" + result.Error = err.Error() + response.Failed++ + response.Results = append(response.Results, result) + continue + } + attachFailed := false + for _, skillID := range skillIDs { + if _, err := h.skillService.AttachSkillToInstance(instance.ID, skillID); err != nil { + result.Status = "failed" + result.Error = err.Error() + response.Failed++ + response.Results = append(response.Results, result) + attachFailed = true + break + } + } + if attachFailed { + continue + } + + response.Created++ + response.Results = append(response.Results, result) + } + + status := http.StatusCreated + if response.Failed > 0 { + status = http.StatusMultiStatus + } + utils.Success(c, status, "Lite instances batch create completed", response) +} + +func buildLiteBatchCreateRequests(req BatchCreateLiteInstancesRequest) ([]services.CreateInstanceRequest, []CreateInstanceRequest, error) { + count := req.Count + if count <= 0 { + return nil, nil, fmt.Errorf("count is required") + } + if count > maxLiteBatchCreateCount { + return nil, nil, fmt.Errorf("count must be at most %d", maxLiteBatchCreateCount) + } + + prefix := strings.TrimSpace(req.NamePrefix) + if prefix == "" { + return nil, nil, fmt.Errorf("name prefix is required") + } + + startIndex := req.StartIndex + if startIndex <= 0 { + startIndex = 1 + } + + template := CreateInstanceRequest{} + if req.Template != nil { + template.Type = req.Template.Type + template.CPUCores = req.Template.CPUCores + template.MemoryGB = req.Template.MemoryGB + template.DiskGB = req.Template.DiskGB + template.OSType = req.Template.OSType + template.OSVersion = req.Template.OSVersion + template.ImageRegistry = req.Template.ImageRegistry + template.ImageTag = req.Template.ImageTag + template.EnvironmentOverrides = req.Template.EnvironmentOverrides + template.StorageClass = req.Template.StorageClass + template.OpenClawConfigPlan = req.Template.OpenClawConfigPlan + template.SkillIDs = req.Template.SkillIDs + } + template.Type = strings.ToLower(strings.TrimSpace(template.Type)) + if template.Type == "" { + template.Type = "openclaw" + } + if template.Type != "openclaw" && template.Type != "hermes" { + return nil, nil, fmt.Errorf("lite batch create supports openclaw or hermes instances") + } + if template.CPUCores <= 0 { + template.CPUCores = 2 + } + if template.MemoryGB <= 0 { + template.MemoryGB = 4 + } + if template.DiskGB <= 0 { + template.DiskGB = 20 + } + if strings.TrimSpace(template.OSType) == "" { + template.OSType = template.Type + } + if strings.TrimSpace(template.OSVersion) == "" { + template.OSVersion = "latest" + } + template.Mode = services.InstanceModeLite + template.InstanceMode = services.InstanceModeLite + template.RuntimeType = services.RuntimeBackendGateway + template.GPUEnabled = false + template.GPUCount = 0 + template.DesktopStreamProfile = "" + + createRequests := make([]services.CreateInstanceRequest, 0, count) + handlerRequests := make([]CreateInstanceRequest, 0, count) + for offset := 0; offset < count; offset++ { + index := startIndex + offset + item := template + item.Name = fmt.Sprintf("%s-%03d", prefix, index) + if len(item.Name) > 50 { + return nil, nil, fmt.Errorf("generated instance name %q is too long", item.Name) + } + handlerRequests = append(handlerRequests, item) + createRequests = append(createRequests, instanceCreateRequestToService(item)) + } + + return createRequests, handlerRequests, nil +} +func (h *InstanceHandler) resolveCreateInstanceSkillIDs(userID int, req CreateInstanceRequest) ([]int, error) { + seen := map[int]struct{}{} + result := make([]int, 0, len(req.SkillIDs)) + for _, skillID := range req.SkillIDs { + if skillID <= 0 { + continue + } + if _, exists := seen[skillID]; exists { + continue + } + seen[skillID] = struct{}{} + result = append(result, skillID) + } + + if h.openClawConfigService != nil { + bundleSkillIDs, err := h.openClawConfigService.ResolveBundleSkillIDs(userID, req.OpenClawConfigPlan) + if err != nil { + return nil, err + } + for _, skillID := range bundleSkillIDs { + if skillID <= 0 { + continue + } + if _, exists := seen[skillID]; exists { + continue + } + seen[skillID] = struct{}{} + result = append(result, skillID) + } + } + + return result, nil +} + +func (h *InstanceHandler) BatchDeleteLiteInstances(c *gin.Context) { + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + + var req BatchDeleteLiteInstancesRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + if len(req.InstanceIDs) > maxLiteBatchDeleteCount { + utils.Error(c, http.StatusBadRequest, fmt.Sprintf("instance_ids must include at most %d items", maxLiteBatchDeleteCount)) + return + } + + seen := map[int]struct{}{} + instances := make([]*models.Instance, 0, len(req.InstanceIDs)) + for _, id := range req.InstanceIDs { + if _, exists := seen[id]; exists { + continue + } + seen[id] = struct{}{} + + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + if instance == nil { + utils.Error(c, http.StatusNotFound, fmt.Sprintf("instance %d not found", id)) + return + } + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + if !isLiteInstance(instance) { + utils.Error(c, http.StatusBadRequest, fmt.Sprintf("instance %d is not a lite instance", id)) + return + } + instances = append(instances, instance) + } + + response := BatchDeleteLiteInstancesResponse{ + Requested: len(instances), + Results: make([]BatchDeleteLiteInstanceResult, 0, len(instances)), + } + for _, instance := range instances { + result := BatchDeleteLiteInstanceResult{ + InstanceID: instance.ID, + Name: instance.Name, + Status: "deleting", + } + if err := h.instanceService.Delete(instance.ID); err != nil { + result.Status = "failed" + result.Error = err.Error() + response.Failed++ + } else { + response.Deleted++ + } + response.Results = append(response.Results, result) + } + + status := http.StatusAccepted + if response.Failed > 0 { + status = http.StatusMultiStatus + } + utils.Success(c, status, "Lite instances batch delete started", response) +} + +func isLiteInstance(instance *models.Instance) bool { + if instance == nil { + return false + } + if mode, ok := services.NormalizeInstanceMode(instance.InstanceMode); ok && mode == services.InstanceModeLite { + return true + } + return strings.EqualFold(strings.TrimSpace(instance.RuntimeType), services.RuntimeBackendGateway) +} + +// GetInstance gets an instance by ID +func (h *InstanceHandler) GetInstance(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return + } + + // Check ownership (only admin or owner can view) + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + runtime, _ := h.runtimeStatusService.GetByInstanceID(instance.ID) + agent, _ := h.instanceAgentService.GetPayloadByInstanceID(instance.ID) + + utils.Success(c, http.StatusOK, "Instance retrieved successfully", gin.H{ + "instance": instance, + "runtime": runtime, + "agent": agent, + }) +} + +// UpdateInstance updates an instance +func (h *InstanceHandler) UpdateInstance(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + // Get instance first to check ownership + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return + } + + // Check ownership (only admin or owner can update) + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + var req UpdateInstanceRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + updateReq := services.UpdateInstanceRequest{ + Name: req.Name, + Description: req.Description, + DesktopStreamProfile: req.DesktopStreamProfile, + } + + if err := h.instanceService.Update(id, updateReq); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Instance updated successfully", nil) +} + +// DeleteInstance deletes an instance +func (h *InstanceHandler) DeleteInstance(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + // Get instance first to check ownership + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return + } + + // Check ownership (only admin or owner can delete) + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + if err := h.instanceService.Delete(id); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusAccepted, "Instance deletion started", nil) +} + +// StartInstance starts an instance +func (h *InstanceHandler) StartInstance(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + // Get instance first to check ownership + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return + } + + // Check ownership (only admin or owner can start) + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + if err := h.instanceService.Start(id); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Instance started successfully", nil) +} + +// StopInstance stops an instance +func (h *InstanceHandler) StopInstance(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + // Get instance first to check ownership + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return + } + + // Check ownership (only admin or owner can stop) + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + if err := h.instanceService.Stop(id); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Instance stopped successfully", nil) +} + +// RestartInstance restarts an instance +func (h *InstanceHandler) RestartInstance(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + // Get instance first to check ownership + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return + } + + // Check ownership (only admin or owner can restart) + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + if err := h.instanceService.Restart(id); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Instance restarted successfully", nil) +} + +// GetInstanceStatus gets the detailed status of an instance +func (h *InstanceHandler) GetInstanceStatus(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + // Get instance first to check ownership + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return + } + + // Check ownership (only admin or owner can view status) + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + status, err := h.instanceService.GetInstanceStatus(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if userRole != "admin" { + utils.Success(c, http.StatusOK, "Instance status retrieved successfully", gin.H{ + "instance_status": buildUserSafeInstanceStatus(instance, status), + }) + return + } + + runtime, _ := h.runtimeStatusService.GetByInstanceID(id) + agent, _ := h.instanceAgentService.GetPayloadByInstanceID(id) + + utils.Success(c, http.StatusOK, "Instance status retrieved successfully", gin.H{ + "instance_status": status, + "runtime": runtime, + "agent": agent, + }) +} + +func buildUserSafeInstanceStatus(instance *models.Instance, status *services.InstanceStatus) map[string]interface{} { + if status == nil { + return map[string]interface{}{} + } + payload := map[string]interface{}{ + "instance_id": status.InstanceID, + "status": status.Status, + "availability": status.Availability, + "created_at": status.CreatedAt, + } + if payload["availability"] == "" { + payload["availability"] = availabilityForInstanceStatus(status.Status) + } + if status.StartedAt != nil { + payload["started_at"] = status.StartedAt + } + if instance == nil { + return payload + } + if agentType, ok := services.NormalizeV2RuntimeType(instance.Type); ok && (strings.EqualFold(strings.TrimSpace(instance.RuntimeType), "gateway") || strings.EqualFold(strings.TrimSpace(instance.InstanceMode), "lite")) { + payload["agent_type"] = agentType + payload["workspace_usage_bytes"] = instance.WorkspaceUsageBytes + } + return payload +} + +func availabilityForInstanceStatus(status string) string { + switch strings.ToLower(strings.TrimSpace(status)) { + case "running": + return "available" + case "creating": + return "starting" + default: + return "unavailable" + } +} + +func (h *InstanceHandler) GetRuntimeDetails(c *gin.Context) { + id, _, ok := h.resolveOwnedInstance(c) + if !ok { + return + } + + runtime, err := h.runtimeStatusService.GetByInstanceID(id) + if err != nil { + utils.HandleError(c, err) + return + } + agent, err := h.instanceAgentService.GetPayloadByInstanceID(id) + if err != nil { + utils.HandleError(c, err) + return + } + commands, err := h.instanceCommandService.ListByInstanceID(id, 20) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Instance runtime details retrieved successfully", InstanceRuntimeDetailsResponse{ + Runtime: runtime, + Agent: agent, + Commands: commands, + }) +} + +func (h *InstanceHandler) CreateRuntimeCommand(c *gin.Context) { + id, _, ok := h.resolveOwnedInstance(c) + if !ok { + return + } + + commandKey := strings.TrimSpace(c.Param("command")) + commandType := "" + switch commandKey { + case "start": + commandType = services.InstanceCommandTypeStartOpenClaw + case "stop": + commandType = services.InstanceCommandTypeStopOpenClaw + case "restart": + commandType = services.InstanceCommandTypeRestartOpenClaw + case "collect-system-info": + commandType = services.InstanceCommandTypeCollectSystemInfo + case "health-check": + commandType = services.InstanceCommandTypeHealthCheck + default: + utils.Error(c, http.StatusBadRequest, "Invalid runtime command") + return + } + + var req CreateRuntimeCommandRequest + if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF { + utils.ValidationError(c, err) + return + } + + userID, _ := c.Get("userID") + issuedBy := userID.(int) + command, err := h.instanceCommandService.Create(id, &issuedBy, services.CreateInstanceCommandRequest{ + CommandType: commandType, + IdempotencyKey: req.IdempotencyKey, + }) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusCreated, "Instance runtime command created successfully", command) +} + +func (h *InstanceHandler) ListConfigRevisions(c *gin.Context) { + id, _, ok := h.resolveOwnedInstance(c) + if !ok { + return + } + + limit, _ := strconv.Atoi(c.DefaultQuery("limit", "20")) + items, err := h.instanceConfigRevisionService.ListByInstanceID(id, limit) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Instance config revisions retrieved successfully", items) +} + +func (h *InstanceHandler) PublishConfigRevision(c *gin.Context) { + id, instance, ok := h.resolveOwnedInstance(c) + if !ok { + return + } + if !strings.EqualFold(instance.Type, "openclaw") && !strings.EqualFold(instance.Type, "hermes") { + utils.Error(c, http.StatusBadRequest, "Only managed runtime instances support config revisions") + return + } + + var req PublishConfigRevisionRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + userID, _ := c.Get("userID") + snapshot, err := h.openClawConfigService.GetSnapshot(userID.(int), req.SnapshotID) + if err != nil { + utils.HandleError(c, err) + return + } + if snapshot.InstanceID != nil && *snapshot.InstanceID != id { + utils.Error(c, http.StatusBadRequest, "Snapshot does not belong to this instance") + return + } + + modelSnapshot := &models.OpenClawInjectionSnapshot{ + ID: snapshot.ID, + InstanceID: snapshot.InstanceID, + UserID: snapshot.UserID, + BundleID: snapshot.BundleID, + Mode: snapshot.Mode, + RenderedManifestJSON: string(snapshot.Manifest), + } + + issuedBy := userID.(int) + revision, err := h.instanceConfigRevisionService.CreateFromSnapshot(id, modelSnapshot, &issuedBy) + if err != nil { + utils.HandleError(c, err) + return + } + + command, err := h.instanceCommandService.Create(id, &issuedBy, services.CreateInstanceCommandRequest{ + CommandType: services.InstanceCommandTypeApplyConfigRevision, + IdempotencyKey: fmt.Sprintf("apply-config-revision-%d", revision.ID), + Payload: map[string]interface{}{ + "revision_id": revision.ID, + "snapshot_id": snapshot.ID, + }, + TimeoutSeconds: 300, + }) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusCreated, "Instance config revision published successfully", gin.H{ + "revision": revision, + "command": command, + }) +} + +func (h *InstanceHandler) resolveOwnedInstance(c *gin.Context) (int, *models.Instance, bool) { + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return 0, nil, false + } + + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return 0, nil, false + } + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return 0, nil, false + } + + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return 0, nil, false + } + + return id, instance, true +} + +// GenerateAccessToken generates an access token for an instance +func (h *InstanceHandler) GenerateAccessToken(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + // Get instance first to check ownership + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return + } + + // Check ownership (only admin or owner can generate access token) + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + // Check if instance is running + if instance.Status != "running" { + utils.Error(c, http.StatusBadRequest, "Instance is not running") + return + } + + if strings.EqualFold(strings.TrimSpace(instance.RuntimeType), "shell") { + utils.Error(c, http.StatusBadRequest, "Desktop access is not available for shell runtime instances") + return + } + + // Generate proxy entry URL. The actual Service remains internal-only. + accessURL := h.proxyService.GetProxyURLForInstance(instance, "") + + if accessURL == "" { + utils.Error(c, http.StatusServiceUnavailable, "Unable to generate access URL") + return + } + + targetPort := h.proxyService.GetTargetPortForInstance(instance) + + // When direct desktop proxying is enabled, embed the instance Service + // "host:port" into the token so the edge gateway can dial the instance + // directly. On any failure we fall back to an empty upstream, which keeps + // the request flowing through the in-process control-plane proxy. + upstream, directProxyEnabled := h.desktopAccessUpstream(c, instance, targetPort) + + // Generate access token (valid for 1 hour) + maxAgeSeconds := int(time.Hour.Seconds()) + token, err := h.accessService.GenerateToken( + userID.(int), + instance.ID, + instance.Type, + accessURL, + upstream, + targetPort, + 1*time.Hour, + ) + if err != nil { + utils.HandleError(c, err) + return + } + + // Store the short-lived access token in an HttpOnly cookie so iframe subresources + // and websocket requests can reuse it without leaking the token in URLs. + c.SetCookie( + fmt.Sprintf("instance_access_%d", instance.ID), + token.Token, + maxAgeSeconds, + fmt.Sprintf("/api/v1/instances/%d/proxy", instance.ID), + "", + false, + true, + ) + + // Return token and URLs + response := map[string]interface{}{ + "token": token.Token, + "access_url": accessURL, + "proxy_url": h.proxyService.GetProxyURLForInstance(instance, token.Token), + "expires_at": token.ExpiresAt, + "desktop_proxy_mode": desktopProxyMode(directProxyEnabled, upstream), + "desktop_upstream_present": upstream != "", + } + + utils.Success(c, http.StatusOK, "Access token generated successfully", response) +} + +// AccessInstance handles instance access via token +func (h *InstanceHandler) AccessInstance(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + // Validate access token + token := c.Query("token") + if token == "" { + utils.Error(c, http.StatusBadRequest, "Access token required") + return + } + + accessToken, err := h.accessService.ValidateToken(token) + if err != nil { + utils.Error(c, http.StatusUnauthorized, err.Error()) + return + } + + // Verify instance ID matches + if accessToken.InstanceID != id { + utils.Error(c, http.StatusForbidden, "Invalid access token for this instance") + return + } + + // Redirect to actual access URL + c.Redirect(http.StatusTemporaryRedirect, accessToken.AccessURL) +} + +func (h *InstanceHandler) StreamShell(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + + if !strings.EqualFold(strings.TrimSpace(instance.RuntimeType), "shell") { + utils.Error(c, http.StatusBadRequest, "Shell access is only available for shell runtime instances") + return + } + + if instance.Status != "running" { + utils.Error(c, http.StatusBadRequest, "Instance is not running") + return + } + + if err := h.shellService.Stream(c.Request.Context(), instance.UserID, instance.ID, c.Writer, c.Request); err != nil { + if !c.Writer.Written() { + utils.HandleError(c, err) + return + } + fmt.Printf("Shell stream for instance %d closed with error: %v\n", instance.ID, err) + } +} + +// ForceSync manually triggers a status sync +func (h *InstanceHandler) ForceSync(c *gin.Context) { + // Get instance ID from URL + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + // Get instance first to check ownership + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return + } + + // Check ownership + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + // Force sync + if err := h.instanceService.ForceSyncInstance(id); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Instance status synced", nil) +} + +// ProxyInstance proxies requests to an instance +func (h *InstanceHandler) ProxyInstance(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return + } + + token, ok := h.proxyAccessToken(c, id) + if !ok { + return + } + + h.proxyInstanceWithToken(c, id, token) +} + +func (h *InstanceHandler) proxyAccessToken(c *gin.Context, id int) (string, bool) { + cookieName := fmt.Sprintf("instance_access_%d", id) + if cookieToken, err := c.Cookie(cookieName); err == nil && strings.TrimSpace(cookieToken) != "" { + if accessToken, validateErr := h.accessService.ValidateToken(cookieToken); validateErr == nil && accessToken.InstanceID == id { + return cookieToken, true + } + } + + queryToken := strings.TrimSpace(c.Query("token")) + if queryToken == "" { + utils.Error(c, http.StatusBadRequest, "Access token required") + return "", false + } + accessToken, err := h.accessService.ValidateToken(queryToken) + if err != nil || accessToken.InstanceID != id { + utils.Error(c, http.StatusUnauthorized, "Access token expired or invalid") + return "", false + } + + // Promote only a validated ClawManager access token. Runtime applications may + // also use a token query parameter for their own websocket/session protocol. + c.SetCookie( + cookieName, + queryToken, + int(time.Hour.Seconds()), + fmt.Sprintf("/api/v1/instances/%d/proxy", id), + "", + false, + true, + ) + return queryToken, true +} + +func (h *InstanceHandler) proxyInstanceWithToken(c *gin.Context, id int, token string) { + // Check if it's a WebSocket upgrade request + if strings.EqualFold(c.GetHeader("Upgrade"), "websocket") { + if err := h.proxyService.ProxyWebSocket(c.Request.Context(), id, token, c.Writer, c.Request); err != nil { + if errors.Is(err, services.ErrInstanceGatewayUnavailable) { + http.Error(c.Writer, "Instance gateway is not available", http.StatusServiceUnavailable) + } else { + http.Error(c.Writer, err.Error(), http.StatusBadGateway) + } + } + return + } + + // Proxy regular HTTP request + if err := h.proxyService.ProxyRequest(c.Request.Context(), id, token, c.Writer, c.Request); err != nil { + // Log the error + fmt.Printf("Proxy error for instance %d: %v\n", id, err) + + // Return appropriate error response + if err.Error() == "invalid token: token expired" || + err.Error() == "invalid token: invalid token" { + http.Error(c.Writer, "Access token expired or invalid", http.StatusUnauthorized) + } else if err.Error() == "token does not match instance" { + http.Error(c.Writer, "Token does not match instance", http.StatusForbidden) + } else if errors.Is(err, services.ErrInstanceGatewayUnavailable) { + http.Error(c.Writer, "Instance gateway is not available", http.StatusServiceUnavailable) + } else { + http.Error(c.Writer, fmt.Sprintf("Failed to proxy request: %v", err), http.StatusBadGateway) + } + } +} + +func (h *InstanceHandler) ExportOpenClaw(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + + if instance.Type != "openclaw" { + utils.Error(c, http.StatusBadRequest, "openclaw import/export is only available for openclaw instances") + return + } + + if instance.Status != "running" { + utils.Error(c, http.StatusBadRequest, "instance must be running to export .openclaw") + return + } + + archive, err := h.openClawTransferService.Export(c.Request.Context(), instance.UserID, instance.ID) + if err != nil { + if errors.Is(err, services.ErrOpenClawWorkspaceMissing) { + utils.Error(c, http.StatusNotFound, "openclaw workspace is empty or missing") + return + } + utils.HandleError(c, err) + return + } + + if len(archive) < openclawMinArchiveBytes { + utils.Error(c, http.StatusInternalServerError, "export produced an empty archive") + return + } + maxArchiveBytes := workspaceArchiveMaxBytes() + if int64(len(archive)) > maxArchiveBytes { + utils.Error(c, http.StatusRequestEntityTooLarge, + fmt.Sprintf("archive too large; maximum archive size is %d MiB", maxArchiveBytes>>20)) + return + } + + filename := fmt.Sprintf("%s.openclaw.tar.gz", sanitizeDownloadName(instance.Name)) + c.Header("Content-Type", "application/gzip") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + c.Header("Content-Length", strconv.Itoa(len(archive))) + c.Data(http.StatusOK, "application/gzip", archive) +} + +func (h *InstanceHandler) ExportHermes(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + + if instance.Type != "hermes" { + utils.Error(c, http.StatusBadRequest, "hermes import/export is only available for hermes instances") + return + } + + if instance.Status != "running" { + utils.Error(c, http.StatusBadRequest, "instance must be running to export .hermes") + return + } + + archive, err := h.openClawTransferService.ExportHermes(c.Request.Context(), instance.UserID, instance.ID) + if err != nil { + if errors.Is(err, services.ErrHermesWorkspaceMissing) { + utils.Error(c, http.StatusNotFound, "hermes workspace is empty or missing") + return + } + utils.HandleError(c, err) + return + } + + if len(archive) < openclawMinArchiveBytes { + utils.Error(c, http.StatusInternalServerError, "export produced an empty archive") + return + } + maxArchiveBytes := workspaceArchiveMaxBytes() + if int64(len(archive)) > maxArchiveBytes { + utils.Error(c, http.StatusRequestEntityTooLarge, + fmt.Sprintf("archive too large; maximum archive size is %d MiB", maxArchiveBytes>>20)) + return + } + + filename := fmt.Sprintf("%s.hermes.tar.gz", sanitizeDownloadName(instance.Name, "hermes-workspace")) + c.Header("Content-Type", "application/gzip") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + c.Header("Content-Length", strconv.Itoa(len(archive))) + c.Data(http.StatusOK, "application/gzip", archive) +} + +func (h *InstanceHandler) ImportOpenClaw(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + + if instance.Type != "openclaw" { + utils.Error(c, http.StatusBadRequest, "openclaw import/export is only available for openclaw instances") + return + } + + if instance.Status != "running" { + utils.Error(c, http.StatusBadRequest, "instance must be running to import .openclaw") + return + } + + // Cap the request body at the same deployment limit used by nginx. When + // the request reaches the backend, MaxBytesReader trips ParseMultipartForm + // (invoked by c.FormFile) with a typed *http.MaxBytesError. + maxArchiveBytes := workspaceArchiveMaxBytes() + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxArchiveBytes) + + fileHeader, err := c.FormFile("file") + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + utils.Error(c, http.StatusRequestEntityTooLarge, + fmt.Sprintf("archive too large; maximum upload size is %d MiB", maxArchiveBytes>>20)) + return + } + utils.Error(c, http.StatusBadRequest, "file is required") + return + } + + if fileHeader.Size > maxArchiveBytes { + utils.Error(c, http.StatusRequestEntityTooLarge, + fmt.Sprintf("archive too large; maximum upload size is %d MiB", maxArchiveBytes>>20)) + return + } + + file, err := fileHeader.Open() + if err != nil { + utils.HandleError(c, err) + return + } + defer file.Close() + + if err := h.openClawTransferService.Import(c.Request.Context(), instance.UserID, instance.ID, io.LimitReader(file, maxArchiveBytes)); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "OpenClaw workspace imported successfully", nil) +} + +func (h *InstanceHandler) ImportHermes(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + + if instance.Type != "hermes" { + utils.Error(c, http.StatusBadRequest, "hermes import/export is only available for hermes instances") + return + } + + if instance.Status != "running" { + utils.Error(c, http.StatusBadRequest, "instance must be running to import .hermes") + return + } + + maxArchiveBytes := workspaceArchiveMaxBytes() + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxArchiveBytes) + + fileHeader, err := c.FormFile("file") + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + utils.Error(c, http.StatusRequestEntityTooLarge, + fmt.Sprintf("archive too large; maximum upload size is %d MiB", maxArchiveBytes>>20)) + return + } + utils.Error(c, http.StatusBadRequest, "file is required") + return + } + + if fileHeader.Size > maxArchiveBytes { + utils.Error(c, http.StatusRequestEntityTooLarge, + fmt.Sprintf("archive too large; maximum upload size is %d MiB", maxArchiveBytes>>20)) + return + } + + file, err := fileHeader.Open() + if err != nil { + utils.HandleError(c, err) + return + } + defer file.Close() + + if err := h.openClawTransferService.ImportHermes(c.Request.Context(), instance.UserID, instance.ID, io.LimitReader(file, maxArchiveBytes)); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Hermes workspace imported successfully", nil) +} + +func (h *InstanceHandler) requireOwnedInstance(c *gin.Context) (*models.Instance, bool) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return nil, false + } + + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return nil, false + } + + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return nil, false + } + + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return nil, false + } + + return instance, true +} + +func (h *InstanceHandler) GetExternalAccess(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + if h.externalAccessService == nil { + utils.Error(c, http.StatusServiceUnavailable, "External access is not configured") + return + } + access, err := h.externalAccessService.Get(c.Request.Context(), instance.ID) + if err != nil { + utils.HandleError(c, err) + return + } + payload := gin.H{"external_access": access} + if shareURL := services.ExternalAccessShareURL(access); shareURL != "" { + payload["share_url"] = shareURL + } + if password := services.ExternalAccessPassword(access); password != "" { + payload["password"] = password + } + utils.Success(c, http.StatusOK, "External access retrieved successfully", payload) +} + +func (h *InstanceHandler) EnableShareLink(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + if h.externalAccessService == nil { + utils.Error(c, http.StatusServiceUnavailable, "External access is not configured") + return + } + var req ExternalAccessRequest + if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF { + utils.ValidationError(c, err) + return + } + userID, _ := c.Get("userID") + result, err := h.externalAccessService.EnableShareLink(c.Request.Context(), instance.ID, userID.(int), externalAccessExpirationRequest(req)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Share link enabled successfully", result) +} + +func (h *InstanceHandler) CreateExternalAccessPassword(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + if h.externalAccessService == nil { + utils.Error(c, http.StatusServiceUnavailable, "External access is not configured") + return + } + var req ExternalAccessRequest + if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF { + utils.ValidationError(c, err) + return + } + userID, _ := c.Get("userID") + result, err := h.externalAccessService.CreatePassword(c.Request.Context(), instance.ID, userID.(int), externalAccessExpirationRequest(req)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Share link password created successfully", result) +} + +func (h *InstanceHandler) DisableExternalAccess(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + if h.externalAccessService == nil { + utils.Error(c, http.StatusServiceUnavailable, "External access is not configured") + return + } + if err := h.externalAccessService.Disable(c.Request.Context(), instance.ID); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "External access disabled successfully", nil) +} + +func (h *InstanceHandler) OpenShortExternalAccess(c *gin.Context) { + if h.externalAccessService == nil { + utils.Error(c, http.StatusServiceUnavailable, "External access is not configured") + return + } + code := strings.TrimSpace(c.Param("code")) + access, err := h.externalAccessService.ResolveShortLink(c.Request.Context(), code) + if err != nil { + utils.Error(c, http.StatusUnauthorized, err.Error()) + return + } + + var instance *models.Instance + var token string + canonicalAccessURL := "" + switch access.AuthMode { + case services.ExternalAccessModeShareLink: + if _, err := h.externalAccessService.ValidateShortLink(c.Request.Context(), code, ""); err != nil { + utils.Error(c, http.StatusUnauthorized, err.Error()) + return + } + case services.ExternalAccessModePassword: + token = h.validShortLinkAccessToken(c, code, access.InstanceID) + if token == "" { + password := externalPassword(c) + isPasswordFormPost := c.Request.Method == http.MethodPost && password == "" + if isPasswordFormPost { + password = c.PostForm("password") + } + if strings.TrimSpace(password) == "" { + if c.Request.Method == http.MethodGet || c.Request.Method == http.MethodHead { + renderShortLinkPasswordForm(c, code, "") + return + } + utils.Error(c, http.StatusUnauthorized, "share link password is required") + return + } + if _, err := h.externalAccessService.ValidateShortLink(c.Request.Context(), code, password); err != nil { + if c.Request.Method == http.MethodPost || shortExternalAccessWantsHTML(c) { + renderShortLinkPasswordForm(c, code, "Invalid password") + return + } + utils.Error(c, http.StatusUnauthorized, err.Error()) + return + } + var ok bool + instance, ok = h.requireExternalAccessInstance(c, access) + if !ok { + return + } + instanceToken, ok := h.issueShortExternalAccessToken(c, instance, code) + if !ok { + return + } + token = instanceToken.Token + canonicalAccessURL = instanceToken.AccessURL + if c.Request.Method == http.MethodPost { + c.Redirect(http.StatusSeeOther, shortExternalAccessEntryPath(code)) + return + } + } + default: + utils.Error(c, http.StatusBadRequest, "Unsupported share link mode") + return + } + + if instance == nil { + var ok bool + instance, ok = h.requireExternalAccessInstance(c, access) + if !ok { + return + } + } + if token == "" { + instanceToken, ok := h.issueShortExternalAccessToken(c, instance, code) + if !ok { + return + } + token = instanceToken.Token + canonicalAccessURL = instanceToken.AccessURL + } else { + setShortExternalAccessCookies(c, instance.ID, code, token, int(time.Hour.Seconds())) + canonicalAccessURL = h.proxyService.GetProxyURLForInstance(instance, "") + } + + originalPath := c.Request.URL.Path + if redirectTarget := shortExternalAccessEntryRedirectTarget(c.Request.Method, originalPath, code, canonicalAccessURL); redirectTarget != "" { + c.Redirect(http.StatusSeeOther, redirectTarget) + return + } + + originalRawPath := c.Request.URL.RawPath + originalAuthorization := c.Request.Header.Get("Authorization") + originalPassword := c.Request.Header.Get("X-Password") + c.Request.URL.Path = shortExternalAccessProxyPath(originalPath, code, instance.ID) + c.Request.URL.RawPath = "" + c.Request.Header.Del("Authorization") + c.Request.Header.Del("X-Password") + defer func() { + c.Request.URL.Path = originalPath + c.Request.URL.RawPath = originalRawPath + if originalAuthorization != "" { + c.Request.Header.Set("Authorization", originalAuthorization) + } + if originalPassword != "" { + c.Request.Header.Set("X-Password", originalPassword) + } + }() + + h.proxyInstanceWithToken(c, instance.ID, token) +} + +func bearerToken(header string) string { + value := strings.TrimSpace(header) + if value == "" { + return "" + } + if strings.HasPrefix(strings.ToLower(value), "bearer ") { + return strings.TrimSpace(value[len("bearer "):]) + } + return value +} + +func externalPassword(c *gin.Context) string { + if password := strings.TrimSpace(c.GetHeader("X-Password")); password != "" { + return password + } + return bearerToken(c.GetHeader("Authorization")) +} + +func externalAccessExpirationRequest(req ExternalAccessRequest) services.ExternalAccessExpirationRequest { + return services.ExternalAccessExpirationRequest{ + Mode: strings.TrimSpace(req.ExpiresMode), + Preset: strings.TrimSpace(req.ExpiresPreset), + ExpiresAt: req.ExpiresAt, + } +} + +func (h *InstanceHandler) requireExternalAccessInstance(c *gin.Context, access *models.InstanceExternalAccess) (*models.Instance, bool) { + if access == nil { + utils.Error(c, http.StatusUnauthorized, "External access is not enabled") + return nil, false + } + instance, err := h.instanceService.GetByID(access.InstanceID) + if err != nil { + utils.HandleError(c, err) + return nil, false + } + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return nil, false + } + if instance.Status != "running" { + utils.Error(c, http.StatusServiceUnavailable, "Instance is not running") + return nil, false + } + if strings.EqualFold(strings.TrimSpace(instance.RuntimeType), "shell") { + utils.Error(c, http.StatusBadRequest, "External desktop access is not available for shell runtime instances") + return nil, false + } + return instance, true +} + +func (h *InstanceHandler) issueShortExternalAccessToken(c *gin.Context, instance *models.Instance, code string) (*services.AccessToken, bool) { + accessURL := h.proxyService.GetProxyURLForInstance(instance, "") + if accessURL == "" { + utils.Error(c, http.StatusServiceUnavailable, "Unable to generate access URL") + return nil, false + } + targetPort := h.proxyService.GetTargetPortForInstance(instance) + upstream, _ := h.desktopAccessUpstream(c, instance, targetPort) + instanceToken, err := h.accessService.GenerateToken( + instance.UserID, + instance.ID, + instance.Type, + accessURL, + upstream, + targetPort, + 1*time.Hour, + ) + if err != nil { + utils.HandleError(c, err) + return nil, false + } + setShortExternalAccessCookies(c, instance.ID, code, instanceToken.Token, int(time.Hour.Seconds())) + return instanceToken, true +} + +func (h *InstanceHandler) validShortLinkAccessToken(c *gin.Context, code string, instanceID int) string { + if h == nil || h.accessService == nil { + return "" + } + token, err := c.Cookie(shortExternalAccessCookieName(code)) + if err != nil || strings.TrimSpace(token) == "" { + return "" + } + accessToken, err := h.accessService.ValidateToken(token) + if err != nil || accessToken.InstanceID != instanceID { + return "" + } + return token +} + +func setShortExternalAccessCookies(c *gin.Context, instanceID int, code, token string, maxAge int) { + c.SetCookie( + fmt.Sprintf("instance_access_%d", instanceID), + token, + maxAge, + fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID), + "", + false, + true, + ) + c.SetCookie( + shortExternalAccessCookieName(code), + token, + maxAge, + shortExternalAccessCookiePath(code), + "", + false, + true, + ) +} + +func shortExternalAccessCookieName(code string) string { + code = strings.Trim(strings.TrimSpace(code), "/") + var builder strings.Builder + for _, r := range code { + switch { + case r >= 'a' && r <= 'z': + builder.WriteRune(r) + case r >= 'A' && r <= 'Z': + builder.WriteRune(r) + case r >= '0' && r <= '9': + builder.WriteRune(r) + case r == '_' || r == '-': + builder.WriteRune(r) + default: + builder.WriteByte('_') + } + } + if builder.Len() == 0 { + return "share_link_access" + } + return "share_link_access_" + builder.String() +} + +func shortExternalAccessCookiePath(code string) string { + code = strings.Trim(strings.TrimSpace(code), "/") + if code == "" { + return "/s" + } + return "/s/" + code +} + +func shortExternalAccessEntryPath(code string) string { + return shortExternalAccessCookiePath(code) + "/" +} + +func shortExternalAccessEntryRedirectTarget(method, requestPath, code, canonicalPath string) string { + if method != http.MethodGet && method != http.MethodHead { + return "" + } + path := strings.TrimSpace(requestPath) + entryPath := shortExternalAccessEntryPath(code) + if path != entryPath && path != strings.TrimSuffix(entryPath, "/") { + return "" + } + target := strings.TrimSpace(canonicalPath) + if target == "" { + return "" + } + parsed, err := url.Parse(target) + if err != nil || parsed.Path == "" { + return "" + } + if !strings.HasPrefix(parsed.Path, "/api/v1/instances/") { + return "" + } + return parsed.Path +} + +func shortExternalAccessWantsHTML(c *gin.Context) bool { + accept := strings.ToLower(c.GetHeader("Accept")) + return strings.Contains(accept, "text/html") +} + +func renderShortLinkPasswordForm(c *gin.Context, code, errorMessage string) { + action := html.EscapeString(shortExternalAccessEntryPath(code)) + errorHTML := "" + if strings.TrimSpace(errorMessage) != "" { + errorHTML = fmt.Sprintf(`
%s
`, html.EscapeString(errorMessage)) + } + body := fmt.Sprintf(` + + + + + Share link password + + + +
+

Share link password

+ %s +
+ + + +
+
+ +`, errorHTML, action) + c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(body)) +} + +func shortExternalAccessProxyPath(requestPath, code string, instanceID int) string { + internalPrefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + shortPrefix := fmt.Sprintf("/s/%s", strings.Trim(strings.TrimSpace(code), "/")) + path := strings.TrimSpace(requestPath) + if path == "" || path == shortPrefix || path == shortPrefix+"/" { + return internalPrefix + "/" + } + if strings.HasPrefix(path, shortPrefix+"/") { + return internalPrefix + strings.TrimPrefix(path, shortPrefix) + } + return internalPrefix + "/" +} + +func sanitizeDownloadName(name string, fallback ...string) string { + name = strings.TrimSpace(name) + if name == "" { + if len(fallback) > 0 && strings.TrimSpace(fallback[0]) != "" { + return strings.TrimSpace(fallback[0]) + } + return "openclaw-workspace" + } + + replacer := strings.NewReplacer("\\", "-", "/", "-", ":", "-", "*", "-", "?", "-", "\"", "-", "<", "-", ">", "-", "|", "-") + name = replacer.Replace(name) + name = strings.ReplaceAll(name, " ", "-") + return name +} diff --git a/backend/internal/handlers/instance_handler_test.go b/backend/internal/handlers/instance_handler_test.go new file mode 100644 index 0000000..ab194cb --- /dev/null +++ b/backend/internal/handlers/instance_handler_test.go @@ -0,0 +1,325 @@ +package handlers + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/services" + + "github.com/gin-gonic/gin" +) + +func TestWorkspaceArchiveMaxMiB(t *testing.T) { + t.Setenv(workspaceArchiveMaxMiBEnv, "") + if got := workspaceArchiveMaxMiB(); got != defaultWorkspaceArchiveMaxMiB { + t.Fatalf("expected default archive limit %d MiB, got %d", defaultWorkspaceArchiveMaxMiB, got) + } + + t.Setenv(workspaceArchiveMaxMiBEnv, "750") + if got := workspaceArchiveMaxMiB(); got != 750 { + t.Fatalf("expected env archive limit 750 MiB, got %d", got) + } + + t.Setenv(workspaceArchiveMaxMiBEnv, "0") + if got := workspaceArchiveMaxMiB(); got != defaultWorkspaceArchiveMaxMiB { + t.Fatalf("expected invalid archive limit to fall back to %d MiB, got %d", defaultWorkspaceArchiveMaxMiB, got) + } + + t.Setenv(workspaceArchiveMaxMiBEnv, "not-a-number") + if got := workspaceArchiveMaxMiB(); got != defaultWorkspaceArchiveMaxMiB { + t.Fatalf("expected unparsable archive limit to fall back to %d MiB, got %d", defaultWorkspaceArchiveMaxMiB, got) + } +} + +func TestDesktopAccessUpstreamSkipsDirectProxyForRuntimeGateway(t *testing.T) { + t.Setenv(desktopDirectProxyEnv, "true") + gin.SetMode(gin.TestMode) + + tests := []struct { + name string + instance *models.Instance + }{ + { + name: "gateway runtime type", + instance: &models.Instance{ + ID: 50, + UserID: 1, + Type: "openclaw", + RuntimeType: "gateway", + }, + }, + { + name: "lite instance mode", + instance: &models.Instance{ + ID: 51, + UserID: 1, + Type: "openclaw", + InstanceMode: services.InstanceModeLite, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/50/access", nil) + + upstream, directEnabled := (&InstanceHandler{}).desktopAccessUpstream(c, tt.instance, 3001) + if upstream != "" || directEnabled { + t.Fatalf("desktopAccessUpstream() = upstream %q direct %t, want control-plane fallback", upstream, directEnabled) + } + }) + } +} + +func TestBuildUserSafeInstanceStatusHidesRuntimeSchedulingDetails(t *testing.T) { + podName := "runtime-openclaw-abc" + podNamespace := "clawmanager-system" + podIP := "10.42.0.12" + startedAt := time.Now().UTC() + workspacePath := "/workspaces/openclaw/user-45/instance-123" + instance := &models.Instance{ + ID: 123, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + WorkspaceUsageBytes: 123456, + } + status := &services.InstanceStatus{ + InstanceID: 123, + Status: "running", + PodName: &podName, + PodNamespace: &podNamespace, + PodIP: &podIP, + PodStatus: "Running", + StartedAt: &startedAt, + } + + payload := buildUserSafeInstanceStatus(instance, status) + + for _, forbiddenKey := range []string{"pod_name", "pod_namespace", "pod_ip", "pod_status", "gateway_port", "capacity", "node_name"} { + if _, exists := payload[forbiddenKey]; exists { + t.Fatalf("user-safe status exposed %q: %#v", forbiddenKey, payload) + } + } + if payload["availability"] != "available" { + t.Fatalf("availability = %v, want available", payload["availability"]) + } + if payload["agent_type"] != "openclaw" { + t.Fatalf("agent_type = %v, want openclaw", payload["agent_type"]) + } + if payload["workspace_usage_bytes"] != int64(123456) { + t.Fatalf("workspace_usage_bytes = %v, want 123456", payload["workspace_usage_bytes"]) + } +} + +func TestShortExternalAccessProxyPath(t *testing.T) { + tests := []struct { + name string + requestPath string + want string + }{ + {name: "entry without slash", requestPath: "/s/abc123", want: "/api/v1/instances/71/proxy/"}, + {name: "entry with slash", requestPath: "/s/abc123/", want: "/api/v1/instances/71/proxy/"}, + {name: "asset", requestPath: "/s/abc123/assets/index.js", want: "/api/v1/instances/71/proxy/assets/index.js"}, + {name: "nested", requestPath: "/s/abc123/apps/openclaw/settings", want: "/api/v1/instances/71/proxy/apps/openclaw/settings"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := shortExternalAccessProxyPath(tt.requestPath, "abc123", 71); got != tt.want { + t.Fatalf("shortExternalAccessProxyPath(%q) = %q, want %q", tt.requestPath, got, tt.want) + } + }) + } +} + +func TestShortExternalAccessEntryRedirectTarget(t *testing.T) { + tests := []struct { + name string + method string + requestPath string + code string + canonicalPath string + want string + }{ + { + name: "html entry redirects to canonical proxy path", + method: http.MethodGet, + requestPath: "/s/sl_abc123/", + code: "sl_abc123", + canonicalPath: "/api/v1/instances/71/proxy/chat/", + want: "/api/v1/instances/71/proxy/chat/", + }, + { + name: "entry without trailing slash redirects", + method: http.MethodGet, + requestPath: "/s/sl_abc123", + code: "sl_abc123", + canonicalPath: "/api/v1/instances/71/proxy/chat/", + want: "/api/v1/instances/71/proxy/chat/", + }, + { + name: "asset path keeps short proxy handling", + method: http.MethodGet, + requestPath: "/s/sl_abc123/assets/index.js", + code: "sl_abc123", + canonicalPath: "/api/v1/instances/71/proxy/chat/", + want: "", + }, + { + name: "post keeps password form handling", + method: http.MethodPost, + requestPath: "/s/sl_abc123/", + code: "sl_abc123", + canonicalPath: "/api/v1/instances/71/proxy/chat/", + want: "", + }, + { + name: "target strips accidental token query", + method: http.MethodGet, + requestPath: "/s/sl_abc123/", + code: "sl_abc123", + canonicalPath: "/api/v1/instances/71/proxy/chat/?token=secret", + want: "/api/v1/instances/71/proxy/chat/", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := shortExternalAccessEntryRedirectTarget(tt.method, tt.requestPath, tt.code, tt.canonicalPath); got != tt.want { + t.Fatalf("shortExternalAccessEntryRedirectTarget() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestRenderShortLinkPasswordForm(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/s/sl_abc123/", nil) + + renderShortLinkPasswordForm(c, "sl_abc123", "") + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusOK) + } + body := recorder.Body.String() + for _, want := range []string{ + "Share link password", + `name="password"`, + `type="password"`, + `action="/s/sl_abc123/"`, + } { + if !strings.Contains(body, want) { + t.Fatalf("password form missing %q in body:\n%s", want, body) + } + } + if contentType := recorder.Header().Get("Content-Type"); !strings.Contains(contentType, "text/html") { + t.Fatalf("content type = %q, want text/html", contentType) + } +} + +func TestProxyAccessTokenPrefersCookieOverRuntimeQueryToken(t *testing.T) { + gin.SetMode(gin.TestMode) + accessService := services.NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(1, 76, "hermes", "/api/v1/instances/76/proxy/chat/", "", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/76/proxy/api/ws?token=hermes-session-token", nil) + c.Request.AddCookie(&http.Cookie{Name: "instance_access_76", Value: token.Token}) + + handler := &InstanceHandler{accessService: accessService} + got, ok := handler.proxyAccessToken(c, 76) + if !ok { + t.Fatal("proxyAccessToken rejected valid cookie") + } + if got != token.Token { + t.Fatalf("proxyAccessToken = %q, want cookie token", got) + } + if recorder.Code != http.StatusOK { + t.Fatalf("unexpected response status %d", recorder.Code) + } +} + +func TestProxyAccessTokenRejectsRuntimeQueryTokenWithoutCookie(t *testing.T) { + gin.SetMode(gin.TestMode) + accessService := services.NewInstanceAccessService() + defer accessService.Stop() + + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/76/proxy/api/ws?token=hermes-session-token", nil) + + handler := &InstanceHandler{accessService: accessService} + if got, ok := handler.proxyAccessToken(c, 76); ok || got != "" { + t.Fatalf("proxyAccessToken = %q/%v, want rejected", got, ok) + } + if recorder.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusUnauthorized) + } +} + +func TestBuildLiteBatchCreateRequestsDefaultsToGatewayLite(t *testing.T) { + requests, handlerRequests, err := buildLiteBatchCreateRequests(BatchCreateLiteInstancesRequest{ + NamePrefix: "batch-lite", + Count: 2, + }) + if err != nil { + t.Fatalf("buildLiteBatchCreateRequests returned error: %v", err) + } + if len(requests) != 2 || len(handlerRequests) != 2 { + t.Fatalf("request count = %d/%d, want 2/2", len(requests), len(handlerRequests)) + } + for idx, req := range requests { + if req.Name == "" || req.Mode != services.InstanceModeLite || req.InstanceMode != services.InstanceModeLite || req.RuntimeType != services.RuntimeBackendGateway { + t.Fatalf("request %d was not normalized to lite gateway: %#v", idx, req) + } + if req.Type != "openclaw" || req.OSType != "openclaw" || req.OSVersion != "latest" { + t.Fatalf("request %d defaults = type %q os %q version %q", idx, req.Type, req.OSType, req.OSVersion) + } + } +} + +func TestBatchDeleteLiteInstancesRejectsProInstance(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/instances/batch/delete", strings.NewReader(`{"instance_ids":[42]}`)) + c.Request.Header.Set("Content-Type", "application/json") + c.Set("userID", 7) + c.Set("userRole", "user") + + handler := &InstanceHandler{ + instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 42: { + ID: 42, + UserID: 7, + Name: "pro-openclaw", + RuntimeType: services.RuntimeBackendDesktop, + InstanceMode: services.InstanceModePro, + }, + }}, + } + + handler.BatchDeleteLiteInstances(c) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String()) + } + if !strings.Contains(recorder.Body.String(), "not a lite instance") { + t.Fatalf("response did not explain lite-only rejection: %s", recorder.Body.String()) + } +} diff --git a/backend/internal/handlers/llm_model_handler.go b/backend/internal/handlers/llm_model_handler.go new file mode 100644 index 0000000..aca2672 --- /dev/null +++ b/backend/internal/handlers/llm_model_handler.go @@ -0,0 +1,134 @@ +package handlers + +import ( + "net/http" + "strconv" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +// LLMModelHandler handles admin model catalog requests. +type LLMModelHandler struct { + service services.LLMModelService +} + +// UpsertLLMModelRequest defines editable fields for model catalog entries. +type UpsertLLMModelRequest struct { + ID int `json:"id,omitempty"` + DisplayName string `json:"display_name" binding:"required"` + Description *string `json:"description,omitempty"` + ProviderType string `json:"provider_type" binding:"required"` + ProtocolType string `json:"protocol_type,omitempty"` + BaseURL string `json:"base_url" binding:"required"` + ProviderModelName string `json:"provider_model_name" binding:"required"` + APIKey *string `json:"api_key,omitempty"` + APIKeySecretRef *string `json:"api_key_secret_ref,omitempty"` + IsSecure bool `json:"is_secure"` + IsActive bool `json:"is_active"` + InputPrice float64 `json:"input_price"` + OutputPrice float64 `json:"output_price"` + Currency string `json:"currency,omitempty"` +} + +// DiscoverLLMModelsRequest defines fields needed to fetch provider models. +type DiscoverLLMModelsRequest struct { + ProviderType string `json:"provider_type" binding:"required"` + ProtocolType string `json:"protocol_type,omitempty"` + BaseURL string `json:"base_url" binding:"required"` + APIKey *string `json:"api_key,omitempty"` + APIKeySecretRef *string `json:"api_key_secret_ref,omitempty"` +} + +// NewLLMModelHandler creates a new model catalog handler. +func NewLLMModelHandler(service services.LLMModelService) *LLMModelHandler { + return &LLMModelHandler{service: service} +} + +// ListModels returns all configured models for admin management. +func (h *LLMModelHandler) ListModels(c *gin.Context) { + items, err := h.service.ListModels() + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "LLM models retrieved successfully", gin.H{ + "items": items, + }) +} + +// UpsertModel creates or updates a model catalog entry. +func (h *LLMModelHandler) UpsertModel(c *gin.Context) { + var req UpsertLLMModelRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + item, err := h.service.SaveModel(services.SaveLLMModelRequest{ + ID: req.ID, + DisplayName: req.DisplayName, + Description: req.Description, + ProviderType: req.ProviderType, + ProtocolType: req.ProtocolType, + BaseURL: req.BaseURL, + ProviderModelName: req.ProviderModelName, + APIKey: req.APIKey, + APIKeySecretRef: req.APIKeySecretRef, + IsSecure: req.IsSecure, + IsActive: req.IsActive, + InputPrice: req.InputPrice, + OutputPrice: req.OutputPrice, + Currency: req.Currency, + }) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "LLM model saved successfully", item) +} + +// DiscoverModels fetches available models from the configured provider. +func (h *LLMModelHandler) DiscoverModels(c *gin.Context) { + var req DiscoverLLMModelsRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + items, err := h.service.DiscoverProviderModels(services.DiscoverLLMModelsRequest{ + ProviderType: req.ProviderType, + ProtocolType: req.ProtocolType, + BaseURL: req.BaseURL, + APIKey: req.APIKey, + APIKeySecretRef: req.APIKeySecretRef, + }) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "LLM provider models retrieved successfully", gin.H{ + "items": items, + }) +} + +// DeleteModel removes a configured model. +func (h *LLMModelHandler) DeleteModel(c *gin.Context) { + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid model ID") + return + } + + if err := h.service.DeleteModel(id); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "LLM model deleted successfully", nil) +} diff --git a/backend/internal/handlers/openclaw_config_handler.go b/backend/internal/handlers/openclaw_config_handler.go new file mode 100644 index 0000000..9bcbf64 --- /dev/null +++ b/backend/internal/handlers/openclaw_config_handler.go @@ -0,0 +1,271 @@ +package handlers + +import ( + "net/http" + "strconv" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +// OpenClawConfigHandler handles OpenClaw config center APIs. +type OpenClawConfigHandler struct { + service services.OpenClawConfigService +} + +// NewOpenClawConfigHandler creates a new OpenClaw config handler. +func NewOpenClawConfigHandler(service services.OpenClawConfigService) *OpenClawConfigHandler { + return &OpenClawConfigHandler{service: service} +} + +func (h *OpenClawConfigHandler) ListResources(c *gin.Context) { + userID, _ := c.Get("userID") + resourceType := c.Query("resource_type") + + items, err := h.service.ListResources(userID.(int), resourceType) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw config resources retrieved successfully", items) +} + +func (h *OpenClawConfigHandler) GetResource(c *gin.Context) { + userID, _ := c.Get("userID") + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid resource ID") + return + } + + item, err := h.service.GetResource(userID.(int), id) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw config resource retrieved successfully", item) +} + +func (h *OpenClawConfigHandler) CreateResource(c *gin.Context) { + userID, _ := c.Get("userID") + var req services.UpsertOpenClawConfigResourceRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + item, err := h.service.CreateResource(userID.(int), req) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "OpenClaw config resource created successfully", item) +} + +func (h *OpenClawConfigHandler) UpdateResource(c *gin.Context) { + userID, _ := c.Get("userID") + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid resource ID") + return + } + + var req services.UpsertOpenClawConfigResourceRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + item, err := h.service.UpdateResource(userID.(int), id, req) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw config resource updated successfully", item) +} + +func (h *OpenClawConfigHandler) DeleteResource(c *gin.Context) { + userID, _ := c.Get("userID") + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid resource ID") + return + } + + if err := h.service.DeleteResource(userID.(int), id); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw config resource deleted successfully", nil) +} + +func (h *OpenClawConfigHandler) CloneResource(c *gin.Context) { + userID, _ := c.Get("userID") + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid resource ID") + return + } + + item, err := h.service.CloneResource(userID.(int), id) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "OpenClaw config resource cloned successfully", item) +} + +func (h *OpenClawConfigHandler) ValidateResource(c *gin.Context) { + var req services.UpsertOpenClawConfigResourceRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + if err := h.service.ValidateResource(req); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw config resource validated successfully", gin.H{"valid": true}) +} + +func (h *OpenClawConfigHandler) ListBundles(c *gin.Context) { + userID, _ := c.Get("userID") + items, err := h.service.ListBundles(userID.(int)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw config bundles retrieved successfully", items) +} + +func (h *OpenClawConfigHandler) GetBundle(c *gin.Context) { + userID, _ := c.Get("userID") + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid bundle ID") + return + } + + item, err := h.service.GetBundle(userID.(int), id) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw config bundle retrieved successfully", item) +} + +func (h *OpenClawConfigHandler) CreateBundle(c *gin.Context) { + userID, _ := c.Get("userID") + var req services.UpsertOpenClawConfigBundleRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + item, err := h.service.CreateBundle(userID.(int), req) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "OpenClaw config bundle created successfully", item) +} + +func (h *OpenClawConfigHandler) UpdateBundle(c *gin.Context) { + userID, _ := c.Get("userID") + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid bundle ID") + return + } + + var req services.UpsertOpenClawConfigBundleRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + item, err := h.service.UpdateBundle(userID.(int), id, req) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw config bundle updated successfully", item) +} + +func (h *OpenClawConfigHandler) DeleteBundle(c *gin.Context) { + userID, _ := c.Get("userID") + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid bundle ID") + return + } + + if err := h.service.DeleteBundle(userID.(int), id); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw config bundle deleted successfully", nil) +} + +func (h *OpenClawConfigHandler) CloneBundle(c *gin.Context) { + userID, _ := c.Get("userID") + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid bundle ID") + return + } + + item, err := h.service.CloneBundle(userID.(int), id) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "OpenClaw config bundle cloned successfully", item) +} + +func (h *OpenClawConfigHandler) CompilePreview(c *gin.Context) { + userID, _ := c.Get("userID") + var req services.OpenClawConfigPlan + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + result, err := h.service.CompilePreview(userID.(int), req) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw config preview compiled successfully", result) +} + +func (h *OpenClawConfigHandler) ListSnapshots(c *gin.Context) { + userID, _ := c.Get("userID") + limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50")) + + items, err := h.service.ListSnapshots(userID.(int), limit) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw injection snapshots retrieved successfully", items) +} + +func (h *OpenClawConfigHandler) GetSnapshot(c *gin.Context) { + userID, _ := c.Get("userID") + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid snapshot ID") + return + } + + item, err := h.service.GetSnapshot(userID.(int), id) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "OpenClaw injection snapshot retrieved successfully", item) +} diff --git a/backend/internal/handlers/risk_rule_handler.go b/backend/internal/handlers/risk_rule_handler.go new file mode 100644 index 0000000..15490ca --- /dev/null +++ b/backend/internal/handlers/risk_rule_handler.go @@ -0,0 +1,142 @@ +package handlers + +import ( + "net/http" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +// RiskRuleHandler handles admin risk rule requests. +type RiskRuleHandler struct { + service services.RiskRuleService +} + +// UpsertRiskRuleRequest defines editable fields for risk rules. +type UpsertRiskRuleRequest struct { + RuleID string `json:"rule_id" binding:"required"` + DisplayName string `json:"display_name" binding:"required"` + Description *string `json:"description,omitempty"` + Pattern string `json:"pattern" binding:"required"` + Severity string `json:"severity" binding:"required"` + Action string `json:"action" binding:"required"` + IsEnabled bool `json:"is_enabled"` + SortOrder int `json:"sort_order"` +} + +type TestRiskRuleRequest struct { + Text string `json:"text" binding:"required"` + Rule *UpsertRiskRuleRequest `json:"rule,omitempty"` +} + +type BulkRiskRuleStatusRequest struct { + RuleIDs []string `json:"rule_ids" binding:"required"` + IsEnabled bool `json:"is_enabled"` +} + +// NewRiskRuleHandler creates a new risk rule handler. +func NewRiskRuleHandler(service services.RiskRuleService) *RiskRuleHandler { + return &RiskRuleHandler{service: service} +} + +// ListRules returns all risk rules for admin management. +func (h *RiskRuleHandler) ListRules(c *gin.Context) { + items, err := h.service.ListRules() + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Risk rules retrieved successfully", gin.H{ + "items": items, + }) +} + +// UpsertRule creates or updates a risk rule. +func (h *RiskRuleHandler) UpsertRule(c *gin.Context) { + var req UpsertRiskRuleRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + item, err := h.service.SaveRule(services.SaveRiskRuleRequest{ + RuleID: req.RuleID, + DisplayName: req.DisplayName, + Description: req.Description, + Pattern: req.Pattern, + Severity: req.Severity, + Action: req.Action, + IsEnabled: req.IsEnabled, + SortOrder: req.SortOrder, + }) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Risk rule saved successfully", item) +} + +// TestRules evaluates enabled rules or a draft rule against sample text. +func (h *RiskRuleHandler) TestRules(c *gin.Context) { + var req TestRiskRuleRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + var draft *services.SaveRiskRuleRequest + if req.Rule != nil { + draft = &services.SaveRiskRuleRequest{ + RuleID: req.Rule.RuleID, + DisplayName: req.Rule.DisplayName, + Description: req.Rule.Description, + Pattern: req.Rule.Pattern, + Severity: req.Rule.Severity, + Action: req.Rule.Action, + IsEnabled: req.Rule.IsEnabled, + SortOrder: req.Rule.SortOrder, + } + } + + result, err := h.service.TestRules(services.TestRiskRulesRequest{ + Text: req.Text, + Rule: draft, + }) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Risk rule test completed successfully", result) +} + +// DeleteRule disables a risk rule. +func (h *RiskRuleHandler) DeleteRule(c *gin.Context) { + ruleID := c.Param("ruleId") + if err := h.service.DeleteRule(ruleID); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Risk rule disabled successfully", nil) +} + +// BulkUpdateStatus enables or disables multiple risk rules. +func (h *RiskRuleHandler) BulkUpdateStatus(c *gin.Context) { + var req BulkRiskRuleStatusRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + if err := h.service.BulkSetEnabled(req.RuleIDs, req.IsEnabled); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Risk rule status updated successfully", nil) +} diff --git a/backend/internal/handlers/runtime_agent_handler.go b/backend/internal/handlers/runtime_agent_handler.go new file mode 100644 index 0000000..b671899 --- /dev/null +++ b/backend/internal/handlers/runtime_agent_handler.go @@ -0,0 +1,410 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "time" + + "clawreef/internal/config" + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +const runtimeAgentTokenHeader = "X-ClawManager-Agent-Token" + +type runtimeEventPublisher interface { + Publish(ctx context.Context, eventType string, payload any) error +} + +type RuntimeAgentHandler struct { + cfg config.RuntimePoolConfig + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + instanceRepo repository.InstanceRepository + events runtimeEventPublisher +} + +type runtimeAgentPodIdentity struct { + PodID int64 `json:"pod_id"` + Namespace string `json:"namespace"` + PodName string `json:"pod_name"` +} + +type runtimeAgentRegisterRequest struct { + RuntimeType string `json:"runtime_type" binding:"required"` + Namespace string `json:"namespace" binding:"required"` + PodName string `json:"pod_name" binding:"required"` + PodUID *string `json:"pod_uid,omitempty"` + PodIP *string `json:"pod_ip,omitempty"` + NodeName *string `json:"node_name,omitempty"` + DeploymentName string `json:"deployment_name" binding:"required"` + ImageRef string `json:"image_ref" binding:"required"` + AgentEndpoint *string `json:"agent_endpoint,omitempty"` + State string `json:"state"` + Capacity int `json:"capacity"` + UsedSlots int `json:"used_slots"` + Draining bool `json:"draining"` + Metrics json.RawMessage `json:"metrics,omitempty"` + ReportedAt *time.Time `json:"reported_at,omitempty"` +} + +type runtimeAgentHeartbeatRequest struct { + runtimeAgentPodIdentity + State string `json:"state" binding:"required"` + UsedSlots int `json:"used_slots"` + Draining bool `json:"draining"` + ReportedAt *time.Time `json:"reported_at,omitempty"` +} + +type runtimeAgentMetricsRequest struct { + runtimeAgentPodIdentity + CPUMillisUsed int64 `json:"cpu_millis_used"` + MemoryBytesUsed int64 `json:"memory_bytes_used"` + DiskBytesUsed int64 `json:"disk_bytes_used"` + NetworkRXBytes int64 `json:"network_rx_bytes"` + NetworkTXBytes int64 `json:"network_tx_bytes"` + Metrics json.RawMessage `json:"metrics,omitempty"` + ReportedAt *time.Time `json:"reported_at,omitempty"` +} + +type runtimeAgentGatewaysRequest struct { + runtimeAgentPodIdentity + Gateways []runtimeAgentGatewayReport `json:"gateways" binding:"required"` +} + +type runtimeAgentGatewayReport struct { + InstanceID int `json:"instance_id" binding:"required"` + GatewayID string `json:"gateway_id"` + GatewayPort int `json:"gateway_port"` + GatewayPID *int `json:"gateway_pid,omitempty"` + State string `json:"state" binding:"required"` + Generation int `json:"generation" binding:"required"` + ErrorMessage *string `json:"error_message,omitempty"` + HealthAt *time.Time `json:"health_at,omitempty"` +} + +func NewRuntimeAgentHandler(cfg config.RuntimePoolConfig, podRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, instanceRepo repository.InstanceRepository, events runtimeEventPublisher) *RuntimeAgentHandler { + return &RuntimeAgentHandler{ + cfg: cfg, + podRepo: podRepo, + bindingRepo: bindingRepo, + instanceRepo: instanceRepo, + events: events, + } +} + +func (h *RuntimeAgentHandler) Register(c *gin.Context) { + if !h.requireAgentToken(c) { + return + } + var req runtimeAgentRegisterRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + runtimeType, ok := services.NormalizeV2RuntimeType(req.RuntimeType) + if !ok { + utils.Error(c, http.StatusBadRequest, "unsupported runtime type") + return + } + state := strings.TrimSpace(req.State) + if state == "" { + state = "ready" + } + capacity := runtimePodCapacityFromReport(req.Capacity, h.cfg.MaxGatewaysPerPod) + lastSeen := time.Now().UTC() + if req.ReportedAt != nil && !req.ReportedAt.IsZero() { + lastSeen = req.ReportedAt.UTC() + } + var metricsJSON *string + if len(req.Metrics) > 0 { + if !json.Valid(req.Metrics) { + utils.Error(c, http.StatusBadRequest, "metrics must be valid JSON") + return + } + raw := string(req.Metrics) + metricsJSON = &raw + } + pod := &models.RuntimePod{ + RuntimeType: runtimeType, + Namespace: strings.TrimSpace(req.Namespace), + PodName: strings.TrimSpace(req.PodName), + PodUID: trimStringPtr(req.PodUID), + PodIP: trimStringPtr(req.PodIP), + NodeName: trimStringPtr(req.NodeName), + DeploymentName: strings.TrimSpace(req.DeploymentName), + ImageRef: strings.TrimSpace(req.ImageRef), + AgentEndpoint: trimStringPtr(req.AgentEndpoint), + State: state, + Capacity: capacity, + UsedSlots: req.UsedSlots, + Draining: req.Draining, + MetricsJSON: metricsJSON, + LastSeenAt: &lastSeen, + CPUMillisUsed: 0, + MemoryBytesUsed: 0, + DiskBytesUsed: 0, + NetworkRXBytes: 0, + NetworkTXBytes: 0, + } + if err := h.podRepo.UpsertFromAgent(c.Request.Context(), pod); err != nil { + utils.HandleError(c, err) + return + } + h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{ + "pod_id": pod.ID, + "runtime_type": runtimeType, + "namespace": pod.Namespace, + "pod_name": pod.PodName, + "state": state, + "used_slots": req.UsedSlots, + "capacity": capacity, + "draining": req.Draining, + "last_seen_at": lastSeen, + }) + utils.Success(c, http.StatusOK, "Runtime pod registered successfully", gin.H{"pod": pod}) +} + +func (h *RuntimeAgentHandler) Heartbeat(c *gin.Context) { + if !h.requireAgentToken(c) { + return + } + var req runtimeAgentHeartbeatRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity) + if !ok { + return + } + lastSeen := time.Now().UTC() + if req.ReportedAt != nil && !req.ReportedAt.IsZero() { + lastSeen = req.ReportedAt.UTC() + } + capacity := runtimePodCapacityFromReport(0, h.cfg.MaxGatewaysPerPod) + if err := h.podRepo.UpdateHeartbeat(c.Request.Context(), podID, strings.TrimSpace(req.State), req.UsedSlots, capacity, req.Draining, lastSeen); err != nil { + utils.HandleError(c, err) + return + } + h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{ + "pod_id": podID, + "state": strings.TrimSpace(req.State), + "used_slots": req.UsedSlots, + "capacity": capacity, + "draining": req.Draining, + "last_seen_at": lastSeen, + }) + utils.Success(c, http.StatusOK, "Runtime pod heartbeat accepted", nil) +} + +func (h *RuntimeAgentHandler) ReportMetrics(c *gin.Context) { + if !h.requireAgentToken(c) { + return + } + var req runtimeAgentMetricsRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity) + if !ok { + return + } + var metricsJSON *string + if len(req.Metrics) > 0 { + if !json.Valid(req.Metrics) { + utils.Error(c, http.StatusBadRequest, "metrics must be valid JSON") + return + } + raw := string(req.Metrics) + metricsJSON = &raw + } + lastSeen := time.Now().UTC() + if req.ReportedAt != nil && !req.ReportedAt.IsZero() { + lastSeen = req.ReportedAt.UTC() + } + update := repository.RuntimePodMetricsUpdate{ + CPUMillisUsed: req.CPUMillisUsed, + MemoryBytesUsed: req.MemoryBytesUsed, + DiskBytesUsed: req.DiskBytesUsed, + NetworkRXBytes: req.NetworkRXBytes, + NetworkTXBytes: req.NetworkTXBytes, + MetricsJSON: metricsJSON, + LastSeenAt: &lastSeen, + } + if err := h.podRepo.UpdateMetrics(c.Request.Context(), podID, update); err != nil { + utils.HandleError(c, err) + return + } + payload := map[string]any{ + "pod_id": podID, + "cpu_millis_used": req.CPUMillisUsed, + "memory_bytes_used": req.MemoryBytesUsed, + "disk_bytes_used": req.DiskBytesUsed, + "network_rx_bytes": req.NetworkRXBytes, + "network_tx_bytes": req.NetworkTXBytes, + "last_seen_at": lastSeen, + } + if metricsJSON != nil { + payload["metrics"] = json.RawMessage(*metricsJSON) + } + h.publish(c.Request.Context(), "runtime_pod_metrics", payload) + utils.Success(c, http.StatusOK, "Runtime pod metrics accepted", nil) +} + +func (h *RuntimeAgentHandler) ReportGateways(c *gin.Context) { + if !h.requireAgentToken(c) { + return + } + var req runtimeAgentGatewaysRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity) + if !ok { + return + } + for _, gateway := range req.Gateways { + binding, err := h.bindingRepo.GetByInstanceID(c.Request.Context(), gateway.InstanceID) + if err != nil { + utils.HandleError(c, err) + return + } + if binding == nil || binding.RuntimePodID != podID || binding.Generation != gateway.Generation { + continue + } + state := strings.ToLower(strings.TrimSpace(gateway.State)) + switch state { + case "running", "ready", "healthy": + if err := h.bindingRepo.UpdateRunning(c.Request.Context(), gateway.InstanceID, gateway.Generation, strings.TrimSpace(gateway.GatewayID), gateway.GatewayPort, gateway.GatewayPID); err != nil { + utils.HandleError(c, err) + return + } + default: + if err := h.bindingRepo.UpdateState(c.Request.Context(), gateway.InstanceID, gateway.Generation, state, gateway.ErrorMessage); err != nil { + utils.HandleError(c, err) + return + } + } + if err := h.syncInstanceRuntimeState(c.Request.Context(), gateway, state); err != nil { + utils.HandleError(c, err) + return + } + } + h.publish(c.Request.Context(), "runtime_pod_gateways_reported", map[string]any{ + "pod_id": podID, + "gateway_count": len(req.Gateways), + }) + utils.Success(c, http.StatusOK, "Runtime gateway report accepted", nil) +} + +func (h *RuntimeAgentHandler) syncInstanceRuntimeState(ctx context.Context, gateway runtimeAgentGatewayReport, state string) error { + if h.instanceRepo == nil { + return nil + } + instanceState, message := instanceRuntimeStateFromGatewayReport(state, gateway.ErrorMessage) + return h.instanceRepo.UpdateRuntimeState(ctx, gateway.InstanceID, instanceState, gateway.Generation, message) +} + +func instanceRuntimeStateFromGatewayReport(state string, errorMessage *string) (string, *string) { + normalized := strings.ToLower(strings.TrimSpace(state)) + switch normalized { + case "running", "ready", "healthy": + return "running", nil + case "error", "failed", "failure", "errored": + if errorMessage != nil && strings.TrimSpace(*errorMessage) != "" { + msg := strings.TrimSpace(*errorMessage) + return "error", &msg + } + msg := "runtime gateway reported " + normalized + return "error", &msg + case "stopped", "deleted": + return "stopped", nil + default: + msg := "runtime gateway starting" + if errorMessage != nil && strings.TrimSpace(*errorMessage) != "" { + msg = strings.TrimSpace(*errorMessage) + } + return "creating", &msg + } +} + +func (h *RuntimeAgentHandler) ReportSkills(c *gin.Context) { + if !h.requireAgentToken(c) { + return + } + var payload map[string]any + if err := c.ShouldBindJSON(&payload); err != nil { + utils.ValidationError(c, err) + return + } + h.publish(c.Request.Context(), "runtime_agent_skills_reported", payload) + utils.Success(c, http.StatusOK, "Runtime agent skills report accepted", nil) +} + +func (h *RuntimeAgentHandler) requireAgentToken(c *gin.Context) bool { + if strings.TrimSpace(h.cfg.AgentReportToken) == "" || c.GetHeader(runtimeAgentTokenHeader) != h.cfg.AgentReportToken { + utils.Error(c, http.StatusUnauthorized, "invalid runtime agent token") + return false + } + return true +} + +func (h *RuntimeAgentHandler) resolvePodID(c *gin.Context, identity runtimeAgentPodIdentity) (int64, bool) { + if identity.PodID > 0 { + return identity.PodID, true + } + namespace := strings.TrimSpace(identity.Namespace) + podName := strings.TrimSpace(identity.PodName) + if namespace == "" || podName == "" { + utils.Error(c, http.StatusBadRequest, "pod_id or namespace and pod_name are required") + return 0, false + } + pod, err := h.podRepo.GetByNamespaceName(c.Request.Context(), namespace, podName) + if err != nil { + utils.HandleError(c, err) + return 0, false + } + if pod == nil { + utils.Error(c, http.StatusNotFound, "runtime pod not found") + return 0, false + } + return pod.ID, true +} + +func (h *RuntimeAgentHandler) publish(ctx context.Context, eventType string, payload any) { + if h.events == nil { + return + } + _ = h.events.Publish(ctx, eventType, payload) +} + +func runtimePodCapacityFromReport(reported, configured int) int { + if configured > 0 { + return configured + } + capacity := reported + if capacity <= 0 { + capacity = services.RuntimePodCapacity + } + return capacity +} + +func trimStringPtr(value *string) *string { + if value == nil { + return nil + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return nil + } + return &trimmed +} diff --git a/backend/internal/handlers/runtime_agent_handler_test.go b/backend/internal/handlers/runtime_agent_handler_test.go new file mode 100644 index 0000000..718467c --- /dev/null +++ b/backend/internal/handlers/runtime_agent_handler_test.go @@ -0,0 +1,479 @@ +package handlers + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "clawreef/internal/config" + "clawreef/internal/models" + "clawreef/internal/repository" + + "github.com/gin-gonic/gin" +) + +func TestRuntimeAgentHandlerRejectsInvalidToken(t *testing.T) { + gin.SetMode(gin.TestMode) + podRepo := &runtimeAgentHandlerPodRepo{} + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, &runtimeAgentHandlerEvents{}) + + router := gin.New() + router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/metrics/report", bytes.NewBufferString(`{"pod_id":9}`)) + req.Header.Set("Content-Type", "application/json") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, body = %s, want 401", rec.Code, rec.Body.String()) + } + if podRepo.updateMetricsCalls != 0 { + t.Fatalf("UpdateMetrics called %d times, want 0", podRepo.updateMetricsCalls) + } +} + +func TestRuntimeAgentHandlerRegisterUsesConfiguredCapacity(t *testing.T) { + gin.SetMode(gin.TestMode) + podRepo := &runtimeAgentHandlerPodRepo{} + events := &runtimeAgentHandlerEvents{} + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{ + AgentReportToken: "secret", + MaxGatewaysPerPod: 33, + }, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events) + + router := gin.New() + router.POST("/api/v1/runtime-agent/register", handler.Register) + + body := `{ + "runtime_type": "openclaw", + "namespace": "clawmanager-system", + "pod_name": "openclaw-runtime-test", + "deployment_name": "openclaw-runtime", + "image_ref": "registry/openclaw:v1", + "agent_endpoint": "http://10.0.0.1:19090", + "state": "ready", + "capacity": 10, + "used_slots": 7 + }` + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/register", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-ClawManager-Agent-Token", "secret") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + pod := podRepo.podsByID[1] + if pod == nil { + t.Fatal("runtime pod was not persisted") + } + if pod.Capacity != 33 { + t.Fatalf("stored capacity = %d, want configured capacity 33", pod.Capacity) + } + if events.lastType != "runtime_pod_state" { + t.Fatalf("event type = %q, want runtime_pod_state", events.lastType) + } + payload, ok := events.lastPayload.(map[string]any) + if !ok { + t.Fatalf("event payload = %#v, want map", events.lastPayload) + } + if payload["capacity"] != 33 { + t.Fatalf("event capacity = %#v, want configured capacity 33", payload["capacity"]) + } +} + +func TestRuntimeAgentHandlerHeartbeatUsesConfiguredCapacity(t *testing.T) { + gin.SetMode(gin.TestMode) + podRepo := &runtimeAgentHandlerPodRepo{} + events := &runtimeAgentHandlerEvents{} + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{ + AgentReportToken: "secret", + MaxGatewaysPerPod: 44, + }, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events) + + router := gin.New() + router.POST("/api/v1/runtime-agent/heartbeat", handler.Heartbeat) + + body := `{ + "pod_id": 9, + "state": "ready", + "used_slots": 8, + "draining": false + }` + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/heartbeat", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-ClawManager-Agent-Token", "secret") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if podRepo.updatedPodID != 9 { + t.Fatalf("updated pod id = %d, want 9", podRepo.updatedPodID) + } + if podRepo.lastHeartbeatCapacity != 44 { + t.Fatalf("heartbeat capacity = %d, want configured capacity 44", podRepo.lastHeartbeatCapacity) + } + payload, ok := events.lastPayload.(map[string]any) + if !ok { + t.Fatalf("event payload = %#v, want map", events.lastPayload) + } + if payload["capacity"] != 44 { + t.Fatalf("event capacity = %#v, want configured capacity 44", payload["capacity"]) + } +} + +func TestRuntimeAgentHandlerMetricsReportUpdatesPodAndPublishesEvent(t *testing.T) { + gin.SetMode(gin.TestMode) + podRepo := &runtimeAgentHandlerPodRepo{} + events := &runtimeAgentHandlerEvents{} + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events) + + router := gin.New() + router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics) + + body := `{ + "pod_id": 9, + "cpu_millis_used": 2400, + "memory_bytes_used": 4096, + "disk_bytes_used": 8192, + "network_rx_bytes": 12, + "network_tx_bytes": 34, + "metrics": {"load": 0.42} + }` + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/metrics/report", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-ClawManager-Agent-Token", "secret") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if podRepo.updatedPodID != 9 { + t.Fatalf("updated pod id = %d, want 9", podRepo.updatedPodID) + } + if podRepo.lastMetrics.CPUMillisUsed != 2400 || podRepo.lastMetrics.MemoryBytesUsed != 4096 || podRepo.lastMetrics.DiskBytesUsed != 8192 { + t.Fatalf("metrics = %#v, want cpu/memory/disk values from request", podRepo.lastMetrics) + } + if podRepo.lastMetrics.MetricsJSON == nil || *podRepo.lastMetrics.MetricsJSON == "" { + t.Fatalf("metrics json was not persisted") + } + if events.lastType != "runtime_pod_metrics" { + t.Fatalf("event type = %q, want runtime_pod_metrics", events.lastType) + } + payload, ok := events.lastPayload.(map[string]any) + if !ok { + t.Fatalf("event payload = %#v, want map", events.lastPayload) + } + if payload["pod_id"] != int64(9) { + t.Fatalf("event pod_id = %#v, want 9", payload["pod_id"]) + } +} + +func TestRuntimeAgentHandlerGatewayReportOnlyUpdatesCurrentPodBinding(t *testing.T) { + gin.SetMode(gin.TestMode) + bindingRepo := &runtimeAgentHandlerBindingRepo{ + bindings: map[int]*models.InstanceRuntimeBinding{ + 10: {InstanceID: 10, RuntimePodID: 9, Generation: 2}, + 11: {InstanceID: 11, RuntimePodID: 99, Generation: 2}, + 12: {InstanceID: 12, RuntimePodID: 9, Generation: 3}, + }, + } + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{}) + + router := gin.New() + router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways) + + body := `{ + "pod_id": 9, + "gateways": [ + {"instance_id":10,"gateway_id":"gw-10","gateway_port":20010,"state":"running","generation":2}, + {"instance_id":11,"gateway_id":"gw-11","gateway_port":20011,"state":"running","generation":2}, + {"instance_id":12,"gateway_id":"gw-12","gateway_port":20012,"state":"running","generation":2} + ] + }` + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/gateways/report", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-ClawManager-Agent-Token", "secret") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if bindingRepo.updateRunningCalls != 1 { + t.Fatalf("UpdateRunning calls = %d, want 1", bindingRepo.updateRunningCalls) + } +} + +func TestRuntimeAgentHandlerGatewayReportSyncsInstanceRuntimeState(t *testing.T) { + gin.SetMode(gin.TestMode) + message := "gateway port 20000 is not listening" + bindingRepo := &runtimeAgentHandlerBindingRepo{ + bindings: map[int]*models.InstanceRuntimeBinding{ + 10: {InstanceID: 10, RuntimePodID: 9, Generation: 2}, + 11: {InstanceID: 11, RuntimePodID: 9, Generation: 2}, + 12: {InstanceID: 12, RuntimePodID: 9, Generation: 2}, + }, + } + instanceRepo := &runtimeAgentHandlerInstanceRepo{} + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, instanceRepo, &runtimeAgentHandlerEvents{}) + + router := gin.New() + router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways) + + body := `{ + "pod_id": 9, + "gateways": [ + {"instance_id":10,"gateway_id":"gw-10","gateway_port":20010,"state":"healthy","generation":2}, + {"instance_id":11,"gateway_id":"gw-11","gateway_port":20011,"state":"error","generation":2,"error_message":"` + message + `"}, + {"instance_id":12,"gateway_id":"gw-12","gateway_port":20012,"state":"ready","generation":2} + ] + }` + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/gateways/report", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-ClawManager-Agent-Token", "secret") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if got := instanceRepo.statusByID[10]; got != "running" { + t.Fatalf("healthy gateway synced instance status = %q, want running", got) + } + if got := instanceRepo.statusByID[11]; got != "error" { + t.Fatalf("error gateway synced instance status = %q, want error", got) + } + if got := instanceRepo.statusByID[12]; got != "running" { + t.Fatalf("ready gateway synced instance status = %q, want running", got) + } + if instanceRepo.messageByID[11] == nil || *instanceRepo.messageByID[11] != message { + t.Fatalf("error message = %#v, want %q", instanceRepo.messageByID[11], message) + } +} + +type runtimeAgentHandlerPodRepo struct { + updatedPodID int64 + lastMetrics repository.RuntimePodMetricsUpdate + lastHeartbeatCapacity int + updateMetricsCalls int + podsByID map[int64]*models.RuntimePod +} + +func (r *runtimeAgentHandlerPodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error { + if r.podsByID == nil { + r.podsByID = map[int64]*models.RuntimePod{} + } + if pod.ID == 0 { + pod.ID = int64(len(r.podsByID) + 1) + } + cp := *pod + r.podsByID[pod.ID] = &cp + return nil +} + +func (r *runtimeAgentHandlerPodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) { + if r.podsByID == nil { + return nil, nil + } + return r.podsByID[id], nil +} + +func (r *runtimeAgentHandlerPodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) { + for _, pod := range r.podsByID { + if pod.Namespace == namespace && pod.PodName == podName { + return pod, nil + } + } + return nil, nil +} + +func (r *runtimeAgentHandlerPodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerPodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerPodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) { + return false, nil +} + +func (r *runtimeAgentHandlerPodRepo) ReleaseSlot(ctx context.Context, podID int64) error { + return nil +} + +func (r *runtimeAgentHandlerPodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error { + return nil +} + +func (r *runtimeAgentHandlerPodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error { + r.updatedPodID = podID + r.lastHeartbeatCapacity = capacity + return nil +} + +func (r *runtimeAgentHandlerPodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error { + return nil +} + +func (r *runtimeAgentHandlerPodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error { + r.updatedPodID = podID + r.lastMetrics = metrics + r.updateMetricsCalls++ + return nil +} + +type runtimeAgentHandlerBindingRepo struct { + updateRunningCalls int + updateStateCalls int + bindings map[int]*models.InstanceRuntimeBinding +} + +func (r *runtimeAgentHandlerBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error { + return nil +} + +func (r *runtimeAgentHandlerBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + return r.bindings[instanceID], nil +} + +func (r *runtimeAgentHandlerBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error { + r.updateRunningCalls++ + return nil +} + +func (r *runtimeAgentHandlerBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error { + r.updateStateCalls++ + return nil +} + +func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error { + return nil +} + +func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error { + return nil +} + +type runtimeAgentHandlerInstanceRepo struct { + statusByID map[int]string + generationByID map[int]int + messageByID map[int]*string +} + +func (r *runtimeAgentHandlerInstanceRepo) Create(instance *models.Instance) error { + return nil +} + +func (r *runtimeAgentHandlerInstanceRepo) GetByID(id int) (*models.Instance, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) GetByAccessToken(accessToken string) (*models.Instance, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) GetByAgentBootstrapToken(bootstrapToken string) (*models.Instance, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) GetAll(offset, limit int) ([]models.Instance, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) CountAll() (int, error) { + return 0, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) GetByUserID(userID int, offset, limit int) ([]models.Instance, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) CountByUserID(userID int) (int, error) { + return 0, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) CountActiveByMode(ctx context.Context, mode string) (int, error) { + return 0, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) ExistsByUserIDAndName(userID int, name string) (bool, error) { + return false, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) GetAllRunning() ([]models.Instance, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerInstanceRepo) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error { + if r.statusByID == nil { + r.statusByID = map[int]string{} + } + if r.generationByID == nil { + r.generationByID = map[int]int{} + } + if r.messageByID == nil { + r.messageByID = map[int]*string{} + } + r.statusByID[id] = status + r.generationByID[id] = generation + r.messageByID[id] = message + return nil +} + +func (r *runtimeAgentHandlerInstanceRepo) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error { + return nil +} + +func (r *runtimeAgentHandlerInstanceRepo) UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error { + return nil +} + +func (r *runtimeAgentHandlerInstanceRepo) Update(instance *models.Instance) error { + return nil +} + +func (r *runtimeAgentHandlerInstanceRepo) Delete(id int) error { + return nil +} + +type runtimeAgentHandlerEvents struct { + lastType string + lastPayload any +} + +func (e *runtimeAgentHandlerEvents) Publish(ctx context.Context, eventType string, payload any) error { + e.lastType = eventType + e.lastPayload = payload + return nil +} diff --git a/backend/internal/handlers/runtime_pool_handler.go b/backend/internal/handlers/runtime_pool_handler.go new file mode 100644 index 0000000..af703ad --- /dev/null +++ b/backend/internal/handlers/runtime_pool_handler.go @@ -0,0 +1,270 @@ +package handlers + +import ( + "context" + "log" + "net/http" + "strconv" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type RuntimePoolHandler struct { + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + rolloutRepo repository.RuntimeRolloutRepository + scheduler *services.RuntimeScheduler + events runtimeEventPublisher +} + +const ( + runtimePodListFallbackHeartbeatTimeout = 10 * time.Second + runtimePodListStaleWindowMultiplier = 3 +) + +type startRuntimeRolloutRequest struct { + RuntimeType string `json:"runtime_type" binding:"required"` + TargetImageRef string `json:"target_image_ref" binding:"required"` + BatchSize int `json:"batch_size"` + MaxUnavailable int `json:"max_unavailable"` +} + +type runtimePoolPodListItem struct { + models.RuntimePod + AgentReported bool `json:"agent_reported"` +} + +func NewRuntimePoolHandler( + podRepo repository.RuntimePodRepository, + bindingRepo repository.InstanceRuntimeBindingRepository, + rolloutRepo repository.RuntimeRolloutRepository, + scheduler *services.RuntimeScheduler, + events runtimeEventPublisher, +) *RuntimePoolHandler { + return &RuntimePoolHandler{ + podRepo: podRepo, + bindingRepo: bindingRepo, + rolloutRepo: rolloutRepo, + scheduler: scheduler, + events: events, + } +} + +func (h *RuntimePoolHandler) ListPods(c *gin.Context) { + runtimeType := strings.TrimSpace(c.Query("runtime_type")) + if runtimeType != "" { + normalized, ok := services.NormalizeV2RuntimeType(runtimeType) + if !ok { + utils.Error(c, http.StatusBadRequest, "unsupported runtime type") + return + } + runtimeType = normalized + } + pods, err := h.podRepo.List(c.Request.Context(), runtimeType) + if err != nil { + utils.HandleError(c, err) + return + } + currentPods := filterCurrentRuntimePods(pods, time.Now().UTC(), h.runtimePodListHeartbeatTimeout()) + items := runtimePoolPodListItems(currentPods, true) + if h.scheduler != nil { + deploymentPods, err := h.scheduler.RuntimeDeploymentPods(c.Request.Context(), runtimeType) + if err != nil { + log.Printf("runtime pool list deployment pods failed: %v", err) + } else { + items = mergeRuntimePoolDeploymentPods(items, deploymentPods) + } + } + utils.Success(c, http.StatusOK, "Runtime pods retrieved successfully", gin.H{"pods": items}) +} + +func (h *RuntimePoolHandler) GetPodGateways(c *gin.Context) { + podID, ok := parseRuntimePodID(c) + if !ok { + return + } + bindings, err := h.bindingRepo.ListByRuntimePodID(c.Request.Context(), podID) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Runtime pod gateways retrieved successfully", gin.H{"gateways": bindings}) +} + +func (h *RuntimePoolHandler) DrainPod(c *gin.Context) { + podID, ok := parseRuntimePodID(c) + if !ok { + return + } + if h.scheduler != nil { + if err := h.scheduler.DrainPod(c.Request.Context(), podID); err != nil { + utils.HandleError(c, err) + return + } + } else if err := h.podRepo.MarkState(c.Request.Context(), podID, "draining", true); err != nil { + utils.HandleError(c, err) + return + } + h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{ + "pod_id": podID, + "state": "draining", + "draining": true, + }) + utils.Success(c, http.StatusOK, "Runtime pod drain started", nil) +} + +func (h *RuntimePoolHandler) StartRollout(c *gin.Context) { + var req startRuntimeRolloutRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + runtimeType, ok := services.NormalizeV2RuntimeType(req.RuntimeType) + if !ok { + utils.Error(c, http.StatusBadRequest, "unsupported runtime type") + return + } + targetImage := strings.TrimSpace(req.TargetImageRef) + if targetImage == "" { + utils.Error(c, http.StatusBadRequest, "target image ref is required") + return + } + batchSize := req.BatchSize + if batchSize <= 0 { + batchSize = 1 + } + maxUnavailable := req.MaxUnavailable + if maxUnavailable <= 0 { + maxUnavailable = 1 + } + startedBy := currentUserIDPtr(c) + rollout := &models.RuntimeRollout{ + RuntimeType: runtimeType, + TargetImageRef: targetImage, + Status: "pending", + BatchSize: batchSize, + MaxUnavailable: maxUnavailable, + StartedBy: startedBy, + } + if err := h.rolloutRepo.Create(c.Request.Context(), rollout); err != nil { + utils.HandleError(c, err) + return + } + if h.scheduler != nil { + if err := h.scheduler.StartRollout(c.Request.Context(), rollout.ID); err != nil { + utils.HandleError(c, err) + return + } + } + h.publish(c.Request.Context(), "runtime_rollout", map[string]any{ + "rollout_id": rollout.ID, + "runtime_type": rollout.RuntimeType, + "target_image_ref": rollout.TargetImageRef, + "status": rollout.Status, + "batch_size": rollout.BatchSize, + "max_unavailable": rollout.MaxUnavailable, + "started_by": rollout.StartedBy, + }) + utils.Success(c, http.StatusCreated, "Runtime rollout created successfully", gin.H{"rollout": rollout}) +} + +func (h *RuntimePoolHandler) publish(ctx context.Context, eventType string, payload any) { + if h.events == nil { + return + } + _ = h.events.Publish(ctx, eventType, payload) +} + +func (h *RuntimePoolHandler) runtimePodListHeartbeatTimeout() time.Duration { + if h != nil && h.scheduler != nil { + if timeout := h.scheduler.HeartbeatTimeout(); timeout > 0 { + return timeout + } + } + return runtimePodListFallbackHeartbeatTimeout +} + +func filterCurrentRuntimePods(pods []models.RuntimePod, now time.Time, heartbeatTimeout time.Duration) []models.RuntimePod { + if heartbeatTimeout <= 0 { + return pods + } + cutoff := now.UTC().Add(-runtimePodListStaleWindowMultiplier * heartbeatTimeout) + current := pods[:0] + for _, pod := range pods { + if pod.LastSeenAt != nil && pod.LastSeenAt.UTC().Before(cutoff) { + continue + } + current = append(current, pod) + } + return current +} + +func runtimePoolPodListItems(pods []models.RuntimePod, agentReported bool) []runtimePoolPodListItem { + items := make([]runtimePoolPodListItem, 0, len(pods)) + for _, pod := range pods { + items = append(items, runtimePoolPodListItem{ + RuntimePod: pod, + AgentReported: agentReported, + }) + } + return items +} + +func mergeRuntimePoolDeploymentPods(items []runtimePoolPodListItem, deploymentPods []models.RuntimePod) []runtimePoolPodListItem { + seen := map[string]struct{}{} + for _, item := range items { + seen[runtimePoolPodKey(item.Namespace, item.PodName)] = struct{}{} + } + for _, pod := range deploymentPods { + key := runtimePoolPodKey(pod.Namespace, pod.PodName) + if key == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + items = append(items, runtimePoolPodListItem{ + RuntimePod: pod, + AgentReported: false, + }) + } + return items +} + +func runtimePoolPodKey(namespace, podName string) string { + namespace = strings.TrimSpace(namespace) + podName = strings.TrimSpace(podName) + if namespace == "" || podName == "" { + return "" + } + return namespace + "/" + podName +} + +func parseRuntimePodID(c *gin.Context) (int64, bool) { + podID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || podID <= 0 { + utils.Error(c, http.StatusBadRequest, "invalid runtime pod ID") + return 0, false + } + return podID, true +} + +func currentUserIDPtr(c *gin.Context) *int { + raw, ok := c.Get("userID") + if !ok { + return nil + } + userID, ok := raw.(int) + if !ok { + return nil + } + return &userID +} diff --git a/backend/internal/handlers/runtime_pool_handler_test.go b/backend/internal/handlers/runtime_pool_handler_test.go new file mode 100644 index 0000000..51405c9 --- /dev/null +++ b/backend/internal/handlers/runtime_pool_handler_test.go @@ -0,0 +1,532 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "clawreef/internal/middleware" + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services" + "clawreef/internal/services/k8s" + + "github.com/gin-gonic/gin" +) + +func TestRuntimePoolHandlerRejectsNonAdmin(t *testing.T) { + gin.SetMode(gin.TestMode) + handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{}) + + router := runtimePoolHandlerRouter(20, "user", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, body = %s, want 403", rec.Code, rec.Body.String()) + } +} + +func TestRuntimePoolHandlerListPodsReturnsMetricsForAdmin(t *testing.T) { + gin.SetMode(gin.TestMode) + now := time.Now().UTC() + podIP := "10.42.0.31" + nodeName := "node-a" + handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{pods: []models.RuntimePod{{ + ID: 9, + RuntimeType: "openclaw", + PodName: "openclaw-runtime-abcde", + PodIP: &podIP, + NodeName: &nodeName, + State: "ready", + UsedSlots: 37, + Capacity: 100, + Draining: false, + CPUMillisUsed: 13600, + MemoryBytesUsed: 42949672960, + DiskBytesUsed: 214748364800, + NetworkRXBytes: 9223372, + NetworkTXBytes: 19223372, + LastSeenAt: &now, + }}}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{}) + + router := runtimePoolHandlerRouter(1, "admin", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + + var resp struct { + Data struct { + Pods []struct { + ID int64 `json:"id"` + RuntimeType string `json:"runtime_type"` + PodName string `json:"pod_name"` + CPUMillisUsed int64 `json:"cpu_millis_used"` + MemoryBytesUsed int64 `json:"memory_bytes_used"` + DiskBytesUsed int64 `json:"disk_bytes_used"` + NetworkRXBytes int64 `json:"network_rx_bytes"` + NetworkTXBytes int64 `json:"network_tx_bytes"` + LastSeenAt string `json:"last_seen_at"` + } `json:"pods"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Data.Pods) != 1 { + t.Fatalf("pods length = %d, want 1", len(resp.Data.Pods)) + } + pod := resp.Data.Pods[0] + if pod.ID != 9 || pod.RuntimeType != "openclaw" || pod.PodName != "openclaw-runtime-abcde" { + t.Fatalf("pod identity = %#v, want runtime pod data", pod) + } + if pod.CPUMillisUsed != 13600 || pod.MemoryBytesUsed != 42949672960 || pod.DiskBytesUsed != 214748364800 || pod.NetworkRXBytes != 9223372 || pod.NetworkTXBytes != 19223372 { + t.Fatalf("pod metrics = %#v, want persisted metrics", pod) + } +} + +func TestRuntimePoolHandlerListPodsOmitsStaleRuntimePods(t *testing.T) { + gin.SetMode(gin.TestMode) + recentSeen := time.Now().UTC() + staleSeen := recentSeen.Add(-5 * time.Minute) + handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{pods: []models.RuntimePod{ + {ID: 1, RuntimeType: "openclaw", PodName: "openclaw-runtime-live", State: "ready", LastSeenAt: &recentSeen}, + {ID: 2, RuntimeType: "openclaw", PodName: "openclaw-runtime-deleted", State: "unhealthy", LastSeenAt: &staleSeen}, + {ID: 3, RuntimeType: "hermes", PodName: "hermes-runtime-recent-unhealthy", State: "unhealthy", LastSeenAt: &recentSeen}, + }}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{}) + + router := runtimePoolHandlerRouter(1, "admin", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + + var resp struct { + Data struct { + Pods []struct { + ID int64 `json:"id"` + PodName string `json:"pod_name"` + } `json:"pods"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Data.Pods) != 2 { + t.Fatalf("pods length = %d, want 2: %#v", len(resp.Data.Pods), resp.Data.Pods) + } + for _, pod := range resp.Data.Pods { + if pod.ID == 2 || pod.PodName == "openclaw-runtime-deleted" { + t.Fatalf("stale deleted pod was returned: %#v", pod) + } + } +} + +func TestRuntimePoolHandlerListPodsIncludesUnreportedDeploymentPods(t *testing.T) { + gin.SetMode(gin.TestMode) + deployments := &runtimePoolHandlerDeploymentService{ + pods: []k8s.RuntimeDeploymentPod{{ + RuntimeType: "openclaw", + Namespace: "clawmanager-system", + DeploymentName: "openclaw-runtime", + PodName: "openclaw-runtime-current", + ImageRef: "registry/openclaw-lite:final2", + State: "ready", + }}, + } + scheduler := services.NewRuntimeScheduler( + nil, + &runtimePoolHandlerPodRepo{}, + nil, + &runtimePoolHandlerRolloutRepo{}, + &runtimePoolHandlerAgentClient{}, + services.NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + services.WithRuntimeSchedulerNamespace("clawmanager-system"), + services.WithRuntimeSchedulerMaxGatewaysPerPod(33), + ) + handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, scheduler, &runtimePoolHandlerEvents{}) + + router := runtimePoolHandlerRouter(1, "admin", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods?runtime_type=openclaw", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + + var resp struct { + Data struct { + Pods []struct { + ID int64 `json:"id"` + RuntimeType string `json:"runtime_type"` + PodName string `json:"pod_name"` + ImageRef string `json:"image_ref"` + State string `json:"state"` + Capacity int `json:"capacity"` + AgentReported bool `json:"agent_reported"` + DeploymentName string `json:"deployment_name"` + } `json:"pods"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Data.Pods) != 1 { + t.Fatalf("pods length = %d, want 1: %#v", len(resp.Data.Pods), resp.Data.Pods) + } + pod := resp.Data.Pods[0] + if pod.ID >= 0 || pod.AgentReported { + t.Fatalf("fallback pod identity = id:%d agent_reported:%t, want synthetic unreported pod", pod.ID, pod.AgentReported) + } + if pod.RuntimeType != "openclaw" || pod.PodName != "openclaw-runtime-current" || pod.DeploymentName != "openclaw-runtime" { + t.Fatalf("fallback pod metadata = %#v, want OpenClaw deployment pod", pod) + } + if pod.ImageRef != "registry/openclaw-lite:final2" { + t.Fatalf("fallback pod image = %q, want registry/openclaw-lite:final2", pod.ImageRef) + } + if pod.State != "unhealthy" { + t.Fatalf("fallback pod state = %q, want unhealthy while agent is not reporting", pod.State) + } + if pod.Capacity != 33 { + t.Fatalf("fallback pod capacity = %d, want configured capacity 33", pod.Capacity) + } +} + +func TestRuntimePoolHandlerStartRolloutStoresRequesterAndPublishesEvent(t *testing.T) { + gin.SetMode(gin.TestMode) + rolloutRepo := &runtimePoolHandlerRolloutRepo{} + events := &runtimePoolHandlerEvents{} + handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, rolloutRepo, nil, events) + + router := runtimePoolHandlerRouter(7, "admin", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/runtime-rollouts", bytes.NewBufferString(`{ + "runtime_type": "hermes", + "target_image_ref": "ghcr.io/example/hermes:v2", + "batch_size": 2, + "max_unavailable": 1 + }`)) + req.Header.Set("Content-Type", "application/json") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s, want 201", rec.Code, rec.Body.String()) + } + if rolloutRepo.created == nil { + t.Fatalf("rollout was not created") + } + if rolloutRepo.created.StartedBy == nil || *rolloutRepo.created.StartedBy != 7 { + t.Fatalf("started_by = %#v, want 7", rolloutRepo.created.StartedBy) + } + if events.lastType != "runtime_rollout" { + t.Fatalf("event type = %q, want runtime_rollout", events.lastType) + } +} + +func TestRuntimePoolHandlerStartRolloutRunsSchedulerImmediately(t *testing.T) { + gin.SetMode(gin.TestMode) + rolloutRepo := &runtimePoolHandlerRolloutRepo{} + podRepo := &runtimePoolHandlerPodRepo{pods: []models.RuntimePod{{ + ID: 21, + RuntimeType: "hermes", + Namespace: "clawmanager-system", + PodName: "hermes-runtime-old", + DeploymentName: "hermes-runtime", + ImageRef: "registry/hermes:v1", + State: "ready", + Capacity: 100, + UsedSlots: 7, + }}} + deployments := &runtimePoolHandlerDeploymentService{} + scheduler := services.NewRuntimeScheduler( + nil, + podRepo, + nil, + rolloutRepo, + &runtimePoolHandlerAgentClient{}, + services.NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + handler := NewRuntimePoolHandler(podRepo, &runtimePoolHandlerBindingRepo{}, rolloutRepo, scheduler, &runtimePoolHandlerEvents{}) + + router := runtimePoolHandlerRouter(7, "admin", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/runtime-rollouts", bytes.NewBufferString(`{ + "runtime_type": "hermes", + "target_image_ref": "registry/hermes:v2", + "batch_size": 1, + "max_unavailable": 1 + }`)) + req.Header.Set("Content-Type", "application/json") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s, want 201", rec.Code, rec.Body.String()) + } + if got := len(deployments.rollouts); got != 1 { + t.Fatalf("deployment rollouts = %d, want 1", got) + } + rollout := deployments.rollouts[0] + if rollout.namespace != "clawmanager-system" || rollout.name != "hermes-runtime" || rollout.image != "registry/hermes:v2" { + t.Fatalf("deployment rollout = %+v, want hermes-runtime registry/hermes:v2", rollout) + } + if podRepo.markedPodID != 21 || podRepo.markedState != "draining" || !podRepo.markedDraining { + t.Fatalf("pod mark = id:%d state:%q draining:%t, want drain pod 21", podRepo.markedPodID, podRepo.markedState, podRepo.markedDraining) + } +} + +func runtimePoolHandlerRouter(userID int, role string, handler *RuntimePoolHandler) *gin.Engine { + userRepo := &runtimePoolHandlerUserRepo{users: map[int]*models.User{ + userID: {ID: userID, Username: "tester", Role: role}, + }} + router := gin.New() + admin := router.Group("/api/v1/admin") + admin.Use(func(c *gin.Context) { + c.Set("userID", userID) + c.Next() + }) + admin.Use(middleware.SetUserInfo(userRepo)) + admin.Use(middleware.NewAdminAuth(userRepo)) + { + admin.GET("/runtime-pods", handler.ListPods) + admin.POST("/runtime-rollouts", handler.StartRollout) + } + return router +} + +type runtimePoolHandlerUserRepo struct { + users map[int]*models.User +} + +func (r *runtimePoolHandlerUserRepo) Create(user *models.User) error { return nil } +func (r *runtimePoolHandlerUserRepo) GetByID(id int) (*models.User, error) { + return r.users[id], nil +} +func (r *runtimePoolHandlerUserRepo) GetByUsername(username string) (*models.User, error) { + return nil, nil +} +func (r *runtimePoolHandlerUserRepo) GetByEmail(email string) (*models.User, error) { + return nil, nil +} +func (r *runtimePoolHandlerUserRepo) Update(user *models.User) error { return nil } +func (r *runtimePoolHandlerUserRepo) Delete(id int) error { return nil } +func (r *runtimePoolHandlerUserRepo) List(offset, limit int) ([]models.User, error) { + return nil, nil +} +func (r *runtimePoolHandlerUserRepo) Count() (int, error) { return 0, nil } + +type runtimePoolHandlerPodRepo struct { + pods []models.RuntimePod + markedPodID int64 + markedState string + markedDraining bool +} + +func (r *runtimePoolHandlerPodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error { + return nil +} +func (r *runtimePoolHandlerPodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) { + for _, pod := range r.pods { + if pod.ID == id { + cp := pod + return &cp, nil + } + } + return nil, nil +} +func (r *runtimePoolHandlerPodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) { + return nil, nil +} +func (r *runtimePoolHandlerPodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + if runtimeType == "" { + return append([]models.RuntimePod(nil), r.pods...), nil + } + var pods []models.RuntimePod + for _, pod := range r.pods { + if pod.RuntimeType == runtimeType { + pods = append(pods, pod) + } + } + return pods, nil +} +func (r *runtimePoolHandlerPodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + return nil, nil +} +func (r *runtimePoolHandlerPodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) { + return false, nil +} +func (r *runtimePoolHandlerPodRepo) ReleaseSlot(ctx context.Context, podID int64) error { + return nil +} +func (r *runtimePoolHandlerPodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error { + r.markedPodID = podID + r.markedState = state + r.markedDraining = draining + return nil +} +func (r *runtimePoolHandlerPodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error { + return nil +} +func (r *runtimePoolHandlerPodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error { + return nil +} +func (r *runtimePoolHandlerPodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error { + return nil +} + +type runtimePoolHandlerBindingRepo struct { + bindings []models.InstanceRuntimeBinding +} + +func (r *runtimePoolHandlerBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error { + return nil +} +func (r *runtimePoolHandlerBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + return nil, nil +} +func (r *runtimePoolHandlerBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + return nil, nil +} +func (r *runtimePoolHandlerBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) { + var bindings []models.InstanceRuntimeBinding + for _, binding := range r.bindings { + if binding.RuntimePodID == runtimePodID { + bindings = append(bindings, binding) + } + } + return bindings, nil +} +func (r *runtimePoolHandlerBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) { + return nil, nil +} +func (r *runtimePoolHandlerBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error { + return nil +} +func (r *runtimePoolHandlerBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error { + return nil +} +func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error { + return nil +} +func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error { + return nil +} + +type runtimePoolHandlerRolloutRepo struct { + created *models.RuntimeRollout + byID map[int64]*models.RuntimeRollout +} + +func (r *runtimePoolHandlerRolloutRepo) Create(ctx context.Context, rollout *models.RuntimeRollout) error { + rollout.ID = 12 + cp := *rollout + r.created = &cp + if r.byID == nil { + r.byID = map[int64]*models.RuntimeRollout{} + } + r.byID[rollout.ID] = &cp + return nil +} +func (r *runtimePoolHandlerRolloutRepo) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) { + if r.byID == nil { + return nil, nil + } + return r.byID[id], nil +} +func (r *runtimePoolHandlerRolloutRepo) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) { + return nil, nil +} +func (r *runtimePoolHandlerRolloutRepo) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error { + return nil +} + +type runtimePoolHandlerEvents struct { + lastType string + lastPayload any +} + +func (e *runtimePoolHandlerEvents) Publish(ctx context.Context, eventType string, payload any) error { + e.lastType = eventType + e.lastPayload = payload + return nil +} + +type runtimePoolHandlerDeploymentService struct { + rollouts []runtimePoolHandlerDeploymentRollout + pods []k8s.RuntimeDeploymentPod +} + +type runtimePoolHandlerDeploymentRollout struct { + namespace string + name string + image string + maxUnavailable int + maxSurge int +} + +func (s *runtimePoolHandlerDeploymentService) Ensure(ctx context.Context, spec k8s.RuntimeDeploymentSpec) error { + return nil +} +func (s *runtimePoolHandlerDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error { + return nil +} +func (s *runtimePoolHandlerDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error { + s.rollouts = append(s.rollouts, runtimePoolHandlerDeploymentRollout{ + namespace: namespace, + name: name, + image: image, + maxUnavailable: maxUnavailable, + maxSurge: maxSurge, + }) + return nil +} +func (s *runtimePoolHandlerDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]k8s.RuntimeDeploymentPod, error) { + var pods []k8s.RuntimeDeploymentPod + for _, pod := range s.pods { + if namespace != "" && pod.Namespace != namespace { + continue + } + if runtimeType != "" && pod.RuntimeType != runtimeType { + continue + } + pods = append(pods, pod) + } + return pods, nil +} + +type runtimePoolHandlerAgentClient struct{} + +func (c *runtimePoolHandlerAgentClient) Health(ctx context.Context, endpoint string) error { + return nil +} +func (c *runtimePoolHandlerAgentClient) CreateGateway(ctx context.Context, endpoint string, req services.RuntimeAgentCreateGatewayRequest) (*services.RuntimeAgentCreateGatewayResponse, error) { + return nil, nil +} +func (c *runtimePoolHandlerAgentClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error { + return nil +} +func (c *runtimePoolHandlerAgentClient) Drain(ctx context.Context, endpoint string) error { return nil } diff --git a/backend/internal/handlers/security_handler.go b/backend/internal/handlers/security_handler.go new file mode 100644 index 0000000..54e367f --- /dev/null +++ b/backend/internal/handlers/security_handler.go @@ -0,0 +1,104 @@ +package handlers + +import ( + "net/http" + "strconv" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type SecurityHandler struct { + service services.SecurityScanService +} + +func NewSecurityHandler(service services.SecurityScanService) *SecurityHandler { + return &SecurityHandler{service: service} +} + +func (h *SecurityHandler) GetConfig(c *gin.Context) { + item, err := h.service.GetConfig() + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Security scan config retrieved successfully", item) +} + +func (h *SecurityHandler) SaveConfig(c *gin.Context) { + var req services.SecurityScanConfigPayload + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + userID, _ := c.Get("userID") + item, err := h.service.SaveConfig(userID.(int), req) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Security scan config saved successfully", item) +} + +func (h *SecurityHandler) StartScan(c *gin.Context) { + var req services.StartSecurityScanRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + userID, _ := c.Get("userID") + item, err := h.service.StartScan(userID.(int), req) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Security scan started successfully", item) +} + +func (h *SecurityHandler) ListJobs(c *gin.Context) { + limit, _ := strconv.Atoi(c.DefaultQuery("limit", "20")) + items, err := h.service.ListJobs(limit) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Security scan jobs retrieved successfully", items) +} + +func (h *SecurityHandler) GetJob(c *gin.Context) { + jobID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid job ID") + return + } + item, err := h.service.GetJob(jobID) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Security scan job retrieved successfully", item) +} + +func (h *SecurityHandler) RescanSkill(c *gin.Context) { + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + var req struct { + ScanMode string `json:"scan_mode"` + } + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + userID, _ := c.Get("userID") + item, err := h.service.RescanSkill(userID.(int), skillID, req.ScanMode) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Skill rescan started successfully", item) +} diff --git a/backend/internal/handlers/skill_handler.go b/backend/internal/handlers/skill_handler.go new file mode 100644 index 0000000..debb7fa --- /dev/null +++ b/backend/internal/handlers/skill_handler.go @@ -0,0 +1,250 @@ +package handlers + +import ( + "fmt" + "net/http" + "strconv" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type SkillHandler struct { + service services.SkillService + instanceService services.InstanceService +} + +func NewSkillHandler(service services.SkillService, instanceService services.InstanceService) *SkillHandler { + return &SkillHandler{service: service, instanceService: instanceService} +} + +func (h *SkillHandler) ImportSkills(c *gin.Context) { + userID, _ := c.Get("userID") + fileHeader, err := c.FormFile("file") + if err != nil { + utils.Error(c, http.StatusBadRequest, "file is required") + return + } + items, err := h.service.ImportArchive(c.Request.Context(), userID.(int), fileHeader) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Skills imported successfully", items) +} + +func (h *SkillHandler) ListSkills(c *gin.Context) { + userID, _ := c.Get("userID") + items, err := h.service.ListSkills(userID.(int)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skills retrieved successfully", items) +} + +func (h *SkillHandler) ListAllSkills(c *gin.Context) { + items, err := h.service.ListAllSkills() + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "All skills retrieved successfully", items) +} + +func (h *SkillHandler) GetSkill(c *gin.Context) { + userID, _ := c.Get("userID") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + item, err := h.service.GetSkill(userID.(int), skillID) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill retrieved successfully", item) +} + +func (h *SkillHandler) UpdateSkill(c *gin.Context) { + userID, _ := c.Get("userID") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + var req services.UpdateSkillRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + item, err := h.service.UpdateSkill(userID.(int), skillID, req) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill updated successfully", item) +} + +func (h *SkillHandler) DeleteSkill(c *gin.Context) { + userID, _ := c.Get("userID") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + if err := h.service.DeleteSkill(userID.(int), skillID); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill deleted successfully", nil) +} + +func (h *SkillHandler) DownloadSkill(c *gin.Context) { + userID, _ := c.Get("userID") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + content, fileName, err := h.service.DownloadSkill(userID.(int), skillID) + if err != nil { + utils.HandleError(c, err) + return + } + c.Header("Content-Type", "application/zip") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName)) + c.Data(http.StatusOK, "application/zip", content) +} + +func (h *SkillHandler) DownloadSkillVersionForAgent(c *gin.Context) { + content, fileName, err := h.service.DownloadSkillVersionByExternalID(c.Param("skillVersion")) + if err != nil { + utils.HandleError(c, err) + return + } + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName)) + c.Data(http.StatusOK, "application/octet-stream", content) +} + +func (h *SkillHandler) ListVersions(c *gin.Context) { + userID, _ := c.Get("userID") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + items, err := h.service.ListVersions(userID.(int), skillID) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill versions retrieved successfully", items) +} + +func (h *SkillHandler) ListScanResults(c *gin.Context) { + userID, _ := c.Get("userID") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + items, err := h.service.ListScanResults(userID.(int), skillID) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill scan results retrieved successfully", items) +} + +func (h *SkillHandler) ListAvailableInstanceSkills(c *gin.Context) { + instanceID, ok := h.authorizeOwnedInstance(c) + if !ok { + return + } + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + items, err := h.service.ListAvailableSkillsForInstance(instanceID, userID.(int), fmt.Sprint(userRole)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Available instance skills retrieved successfully", items) +} +func (h *SkillHandler) ListInstanceSkills(c *gin.Context) { + instanceID, ok := h.authorizeOwnedInstance(c) + if !ok { + return + } + items, err := h.service.ListInstanceSkills(instanceID) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Instance skills retrieved successfully", items) +} + +func (h *SkillHandler) AttachSkillToInstance(c *gin.Context) { + instanceID, ok := h.authorizeOwnedInstance(c) + if !ok { + return + } + var req services.AttachSkillToInstanceRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + item, err := h.service.AttachSkillToInstanceForActor(instanceID, req.SkillID, userID.(int), fmt.Sprint(userRole)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Skill attached to instance successfully", item) +} + +func (h *SkillHandler) RemoveSkillFromInstance(c *gin.Context) { + instanceID, ok := h.authorizeOwnedInstance(c) + if !ok { + return + } + skillID, err := strconv.Atoi(c.Param("skillId")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + if err := h.service.RemoveSkillFromInstance(instanceID, skillID); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill removed from instance successfully", nil) +} + +func (h *SkillHandler) authorizeOwnedInstance(c *gin.Context) (int, bool) { + instanceID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid instance ID") + return 0, false + } + instance, err := h.instanceService.GetByID(instanceID) + if err != nil { + utils.HandleError(c, err) + return 0, false + } + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return 0, false + } + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if userRole != "admin" && instance.UserID != userID.(int) { + utils.Error(c, http.StatusForbidden, "Access denied") + return 0, false + } + return instanceID, true +} diff --git a/backend/internal/handlers/system_settings_handler.go b/backend/internal/handlers/system_settings_handler.go new file mode 100644 index 0000000..4008aa6 --- /dev/null +++ b/backend/internal/handlers/system_settings_handler.go @@ -0,0 +1,82 @@ +package handlers + +import ( + "net/http" + "strconv" + "strings" + + "clawreef/internal/models" + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type SystemSettingsHandler struct { + systemImageSettingService services.SystemImageSettingService +} + +type UpsertSystemImageSettingRequest struct { + ID int `json:"id,omitempty"` + InstanceType string `json:"instance_type" binding:"required"` + RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=desktop gateway shell"` + DisplayName string `json:"display_name"` + Image string `json:"image" binding:"required"` +} + +func NewSystemSettingsHandler(systemImageSettingService services.SystemImageSettingService) *SystemSettingsHandler { + return &SystemSettingsHandler{systemImageSettingService: systemImageSettingService} +} + +func (h *SystemSettingsHandler) ListSystemImageSettings(c *gin.Context) { + settings, err := h.systemImageSettingService.List() + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "System image settings retrieved successfully", gin.H{ + "items": settings, + }) +} + +func (h *SystemSettingsHandler) UpsertSystemImageSetting(c *gin.Context) { + var req UpsertSystemImageSettingRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + setting := &models.SystemImageSetting{ + ID: req.ID, + InstanceType: strings.TrimSpace(req.InstanceType), + RuntimeType: strings.TrimSpace(req.RuntimeType), + DisplayName: strings.TrimSpace(req.DisplayName), + Image: strings.TrimSpace(req.Image), + } + + saved, err := h.systemImageSettingService.Save(setting) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "System image setting saved successfully", saved) +} + +func (h *SystemSettingsHandler) DeleteSystemImageSetting(c *gin.Context) { + target := strings.TrimSpace(c.Param("instanceType")) + if id, err := strconv.Atoi(target); err == nil { + if err := h.systemImageSettingService.DeleteByID(id); err != nil { + utils.HandleError(c, err) + return + } + } else { + if err := h.systemImageSettingService.DisableType(target); err != nil { + utils.HandleError(c, err) + return + } + } + + utils.Success(c, http.StatusOK, "System image setting deleted successfully", nil) +} diff --git a/backend/internal/handlers/team_handler.go b/backend/internal/handlers/team_handler.go new file mode 100644 index 0000000..7b3ff26 --- /dev/null +++ b/backend/internal/handlers/team_handler.go @@ -0,0 +1,291 @@ +package handlers + +import ( + "io" + "net/http" + "strconv" + "strings" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type TeamHandler struct { + teamService services.TeamService +} + +func NewTeamHandler(teamService services.TeamService) *TeamHandler { + return &TeamHandler{teamService: teamService} +} + +func (h *TeamHandler) CreateTeam(c *gin.Context) { + userID, _ := c.Get("userID") + var req services.CreateTeamRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + team, err := h.teamService.CreateTeam(userID.(int), req) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Team created successfully", team) +} + +func (h *TeamHandler) ListTeams(c *gin.Context) { + userID, _ := c.Get("userID") + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + limit, _ := strconv.Atoi(c.DefaultQuery("limit", "20")) + if page <= 0 { + page = 1 + } + if limit <= 0 { + limit = 20 + } + teams, err := h.teamService.ListTeams(userID.(int), (page-1)*limit, limit) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Teams retrieved successfully", gin.H{ + "teams": teams.Teams, + "total": teams.Total, + "page": page, + "limit": limit, + }) +} + +func (h *TeamHandler) GetTeam(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + team, err := h.teamService.GetTeam(userID.(int), teamID) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Team retrieved successfully", team) +} + +func (h *TeamHandler) ListTasks(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + tasks, err := h.teamService.ListTeamTasks( + userID.(int), + teamID, + parseOptionalIntQuery(c, "before_id"), + parseOptionalIntQuery(c, "limit"), + ) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Team tasks retrieved successfully", tasks) +} + +func (h *TeamHandler) ListEvents(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + events, err := h.teamService.ListTeamEvents( + userID.(int), + teamID, + parseOptionalIntQuery(c, "before_id"), + parseOptionalIntQuery(c, "limit"), + ) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Team events retrieved successfully", events) +} + +func (h *TeamHandler) ListWorkspaceFiles(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + payload, err := h.teamService.ListWorkspaceFiles(c.Request.Context(), userID.(int), teamID, c.Query("path")) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Team workspace files retrieved successfully", payload) +} + +func (h *TeamHandler) PreviewWorkspaceFile(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + payload, err := h.teamService.PreviewWorkspaceFile(c.Request.Context(), userID.(int), teamID, c.Query("path")) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Team workspace file preview retrieved successfully", payload) +} + +func (h *TeamHandler) DownloadWorkspaceFile(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + payload, err := h.teamService.DownloadWorkspaceFile(c.Request.Context(), userID.(int), teamID, c.Query("path")) + if err != nil { + utils.HandleError(c, err) + return + } + c.Header("Content-Disposition", `attachment; filename="`+strings.ReplaceAll(payload.Name, `"`, "")+`"`) + c.Data(http.StatusOK, payload.ContentType, payload.Data) +} + +func (h *TeamHandler) CreateWorkspaceFolder(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + var req services.TeamWorkspaceFolderRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + if err := h.teamService.CreateWorkspaceFolder(c.Request.Context(), userID.(int), teamID, req); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Team workspace folder created successfully", nil) +} + +func (h *TeamHandler) RenameWorkspaceEntry(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + var req services.TeamWorkspaceRenameRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + if err := h.teamService.RenameWorkspaceEntry(c.Request.Context(), userID.(int), teamID, req); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Team workspace entry renamed successfully", nil) +} + +func (h *TeamHandler) DeleteWorkspaceEntry(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + if err := h.teamService.DeleteWorkspaceEntry(c.Request.Context(), userID.(int), teamID, c.Query("path")); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Team workspace entry deleted successfully", nil) +} + +func (h *TeamHandler) UploadWorkspaceFiles(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + form, err := c.MultipartForm() + if err != nil { + utils.ValidationError(c, err) + return + } + files := form.File["files"] + relativePaths := form.Value["relative_paths"] + if err := h.teamService.UploadWorkspaceFiles(c.Request.Context(), userID.(int), teamID, c.PostForm("path"), files, relativePaths); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Team workspace files uploaded successfully", nil) +} + +func (h *TeamHandler) DispatchTask(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + + var req services.DispatchTeamTaskRequest + if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF { + utils.ValidationError(c, err) + return + } + task, err := h.teamService.DispatchTask(userID.(int), teamID, req) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Team task dispatched successfully", task) +} + +func (h *TeamHandler) DeleteTeam(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + if err := h.teamService.DeleteTeam(userID.(int), teamID); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Team deleted successfully", gin.H{"id": teamID}) +} + +func (h *TeamHandler) DeleteMember(c *gin.Context) { + userID, _ := c.Get("userID") + teamID, ok := parseTeamID(c) + if !ok { + return + } + memberID := c.Param("memberID") + if err := h.teamService.DeleteMember(userID.(int), teamID, memberID); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Team member deleted successfully", gin.H{"member_id": memberID}) +} + +func parseTeamID(c *gin.Context) (int, bool) { + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid Team ID") + return 0, false + } + return id, true +} + +func parseOptionalIntQuery(c *gin.Context, key string) int { + raw := c.Query(key) + if raw == "" { + return 0 + } + value, err := strconv.Atoi(raw) + if err != nil || value < 0 { + return 0 + } + return value +} diff --git a/backend/internal/handlers/user_handler.go b/backend/internal/handlers/user_handler.go new file mode 100644 index 0000000..50e3373 --- /dev/null +++ b/backend/internal/handlers/user_handler.go @@ -0,0 +1,587 @@ +package handlers + +import ( + "encoding/csv" + "fmt" + "net/http" + "regexp" + "strconv" + "strings" + + "clawreef/internal/models" + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +// UserHandler handles user management requests +type UserHandler struct { + userService services.UserService + quotaService services.QuotaService +} + +// NewUserHandler creates a new user handler +func NewUserHandler(userService services.UserService, quotaService services.QuotaService) *UserHandler { + return &UserHandler{ + userService: userService, + quotaService: quotaService, + } +} + +// ListUsersRequest represents a list users request +type ListUsersRequest struct { + Page int `form:"page,default=1"` + Limit int `form:"limit,default=20"` +} + +// UpdateUserRequest represents an update user request +type UpdateUserRequest struct { + Email string `json:"email" binding:"omitempty,email"` + IsActive *bool `json:"is_active" binding:"omitempty"` +} + +// UpdateRoleRequest represents an update role request +type UpdateRoleRequest struct { + Role string `json:"role" binding:"required,oneof=admin user"` +} + +// UpdateQuotaRequest represents an update quota request +type UpdateQuotaRequest struct { + MaxInstances int `json:"max_instances" binding:"min=0"` + MaxCPUCores float64 `json:"max_cpu_cores" binding:"min=0"` + MaxMemoryGB int `json:"max_memory_gb" binding:"min=0"` + MaxStorageGB int `json:"max_storage_gb" binding:"min=0"` + MaxGPUCount int `json:"max_gpu_count" binding:"min=0"` +} + +// CreateUserRequest represents a create user request (admin only) +type CreateUserRequest struct { + Username string `json:"username" binding:"required,min=3,max=32,alphanum"` + Email string `json:"email" binding:"required,email"` + Password string `json:"password" binding:"omitempty,min=8"` + Role string `json:"role" binding:"required,oneof=admin user"` +} + +type importUserResult struct { + Line int `json:"line"` + Username string `json:"username"` + Error string `json:"error"` +} + +type importedUserCredential struct { + Username string `json:"username"` + Email string `json:"email"` + Role string `json:"role"` + MaxInstances int `json:"max_instances"` + MaxCPUCores float64 `json:"max_cpu_cores"` + MaxMemoryGB int `json:"max_memory_gb"` + MaxStorageGB int `json:"max_storage_gb"` + MaxGPUCount int `json:"max_gpu_count"` + InitialPassword string `json:"initial_password"` +} + +var importUsernamePattern = regexp.MustCompile(`^[a-zA-Z0-9]+$`) + +// ListUsers lists all users (admin only) +func (h *UserHandler) ListUsers(c *gin.Context) { + var req ListUsersRequest + if err := c.ShouldBindQuery(&req); err != nil { + utils.ValidationError(c, err) + return + } + + // Calculate offset + offset := (req.Page - 1) * req.Limit + + users, err := h.userService.ListUsers(offset, req.Limit) + if err != nil { + utils.HandleError(c, err) + return + } + + // Get total count + total, err := h.userService.CountUsers() + if err != nil { + utils.HandleError(c, err) + return + } + + response := map[string]interface{}{ + "users": users, + "total": total, + "page": req.Page, + "limit": req.Limit, + } + + utils.Success(c, http.StatusOK, "Users retrieved successfully", response) +} + +// CreateUser creates a new user (admin only) +func (h *UserHandler) CreateUser(c *gin.Context) { + var req CreateUserRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + user, err := h.userService.CreateUser(req.Username, req.Email, req.Password, req.Role) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusCreated, "User created successfully", user) +} + +// ImportUsers imports users from a CSV file (admin only) +func (h *UserHandler) ImportUsers(c *gin.Context) { + file, err := c.FormFile("file") + if err != nil { + utils.Error(c, http.StatusBadRequest, "User import file is required") + return + } + if !strings.HasSuffix(strings.ToLower(file.Filename), ".csv") { + utils.Error(c, http.StatusBadRequest, "Only CSV files are supported") + return + } + + src, err := file.Open() + if err != nil { + utils.Error(c, http.StatusBadRequest, "Failed to open import file") + return + } + defer src.Close() + + reader := csv.NewReader(src) + reader.TrimLeadingSpace = true + reader.FieldsPerRecord = -1 + + rows, err := reader.ReadAll() + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid CSV format") + return + } + + if len(rows) == 0 { + utils.Error(c, http.StatusBadRequest, "Import file is empty") + return + } + + results := make([]importUserResult, 0) + createdUsers := make([]importedUserCredential, 0) + + headerMap := buildImportHeaderMap(rows[0]) + if len(headerMap) == 0 { + utils.Error(c, http.StatusBadRequest, "Import file must include the required CSV headers") + return + } + + requiredHeaders := []string{"username", "role", "maxinstances", "maxcpucores", "maxmemorygb", "maxstoragegb"} + for _, header := range requiredHeaders { + if _, ok := headerMap[header]; !ok { + utils.Error(c, http.StatusBadRequest, fmt.Sprintf("%s column is required", headerLabel(header))) + return + } + } + + for i := 1; i < len(rows); i++ { + lineNumber := i + 1 + fields := normalizeImportFields(rows[i]) + if len(fields) == 0 { + continue + } + + username := importFieldValue(fields, headerMap, "username") + email := importFieldValue(fields, headerMap, "email") + role := importFieldValue(fields, headerMap, "role") + password := importFieldValue(fields, headerMap, "password") + maxInstances, parseErr := parseImportInt(fields, headerMap, "maxinstances", true) + if parseErr != "" { + results = append(results, importUserResult{Line: lineNumber, Username: username, Error: parseErr}) + continue + } + maxCPUCores, parseErr := parseImportFloat(fields, headerMap, "maxcpucores", true) + if parseErr != "" { + results = append(results, importUserResult{Line: lineNumber, Username: username, Error: parseErr}) + continue + } + maxMemoryGB, parseErr := parseImportInt(fields, headerMap, "maxmemorygb", true) + if parseErr != "" { + results = append(results, importUserResult{Line: lineNumber, Username: username, Error: parseErr}) + continue + } + maxStorageGB, parseErr := parseImportInt(fields, headerMap, "maxstoragegb", true) + if parseErr != "" { + results = append(results, importUserResult{Line: lineNumber, Username: username, Error: parseErr}) + continue + } + maxGPUCount, parseErr := parseImportInt(fields, headerMap, "maxgpucount", false) + if parseErr != "" { + results = append(results, importUserResult{Line: lineNumber, Username: username, Error: parseErr}) + continue + } + + if email == "" && username != "" { + email = fmt.Sprintf("%s@import.clawmanager.local", strings.ToLower(username)) + } + + if validationErr := validateImportedUser(username, email, password, role); validationErr != "" { + results = append(results, importUserResult{ + Line: lineNumber, + Username: username, + Error: validationErr, + }) + continue + } + + if password == "" { + password = servicesDefaultPasswordForRole(role) + } + + user, createErr := h.userService.CreateUser(username, email, password, role) + if createErr != nil { + results = append(results, importUserResult{ + Line: lineNumber, + Username: username, + Error: createErr.Error(), + }) + continue + } + + if quotaErr := h.quotaService.UpdateUserQuota(user.ID, &models.UserQuota{ + MaxInstances: maxInstances, + MaxCPUCores: maxCPUCores, + MaxMemoryGB: maxMemoryGB, + MaxStorageGB: maxStorageGB, + MaxGPUCount: maxGPUCount, + }); quotaErr != nil { + results = append(results, importUserResult{ + Line: lineNumber, + Username: username, + Error: quotaErr.Error(), + }) + continue + } + + createdUsers = append(createdUsers, importedUserCredential{ + Username: user.Username, + Email: user.Email, + Role: user.Role, + MaxInstances: maxInstances, + MaxCPUCores: maxCPUCores, + MaxMemoryGB: maxMemoryGB, + MaxStorageGB: maxStorageGB, + MaxGPUCount: maxGPUCount, + InitialPassword: password, + }) + } + + utils.Success(c, http.StatusCreated, "Users imported successfully", gin.H{ + "created_count": len(createdUsers), + "failed_count": len(results), + "created_users": createdUsers, + "errors": results, + }) +} + +func normalizeImportFields(record []string) []string { + fields := make([]string, 0, len(record)) + for _, field := range record { + trimmed := strings.TrimSpace(field) + if trimmed == "" { + fields = append(fields, "") + continue + } + fields = append(fields, trimmed) + } + return fields +} + +func buildImportHeaderMap(record []string) map[string]int { + headers := map[string]int{} + for index, raw := range record { + key := normalizeImportHeader(raw) + switch key { + case "username", "email", "role", "password", "maxinstances", "maxcpucores", "maxmemorygb", "maxstoragegb", "maxgpucount": + headers[key] = index + } + } + return headers +} + +func normalizeImportHeader(raw string) string { + key := strings.ToLower(strings.TrimSpace(raw)) + replacer := strings.NewReplacer(" ", "", "_", "", "(", "", ")", "", "-", "") + return replacer.Replace(key) +} + +func importFieldValue(fields []string, headerMap map[string]int, key string) string { + index, ok := headerMap[key] + if !ok || index >= len(fields) { + return "" + } + return strings.TrimSpace(fields[index]) +} + +func validateImportedUser(username, email, password, role string) string { + if len(username) < 3 || len(username) > 32 { + return "Username must be between 3 and 32 characters" + } + if !importUsernamePattern.MatchString(username) { + return "Username must be alphanumeric" + } + if email == "" || !strings.Contains(email, "@") { + return "Email must be a valid email" + } + if password != "" && len(password) < 8 { + return "Password must be at least 8 characters" + } + if role != "admin" && role != "user" { + return "Role must be admin or user" + } + return "" +} + +func parseImportInt(fields []string, headerMap map[string]int, key string, required bool) (int, string) { + value := importFieldValue(fields, headerMap, key) + if value == "" { + if required { + return 0, fmt.Sprintf("%s is required", headerLabel(key)) + } + return 0, "" + } + parsed, err := strconv.Atoi(value) + if err != nil || parsed < 0 { + return 0, fmt.Sprintf("%s must be a non-negative integer", headerLabel(key)) + } + return parsed, "" +} + +func parseImportFloat(fields []string, headerMap map[string]int, key string, required bool) (float64, string) { + value := importFieldValue(fields, headerMap, key) + if value == "" { + if required { + return 0, fmt.Sprintf("%s is required", headerLabel(key)) + } + return 0, "" + } + parsed, err := strconv.ParseFloat(value, 64) + if err != nil || parsed < 0 { + return 0, fmt.Sprintf("%s must be a non-negative number", headerLabel(key)) + } + return parsed, "" +} + +func headerLabel(key string) string { + switch key { + case "username": + return "Username" + case "role": + return "Role" + case "maxinstances": + return "Max Instances" + case "maxcpucores": + return "Max CPU Cores" + case "maxmemorygb": + return "Max Memory (GB)" + case "maxstoragegb": + return "Max Storage (GB)" + case "maxgpucount": + return "Max GPU Count" + default: + return key + } +} + +func servicesDefaultPasswordForRole(role string) string { + return services.DefaultPasswordForRole(role) +} + +// GetUser gets a user by ID +func (h *UserHandler) GetUser(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid user ID") + return + } + + // Get current user ID from context + currentUserID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + + // Only admin or the user themselves can view user details + if userRole != "admin" && currentUserID.(int) != id { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + user, err := h.userService.GetUserByID(id) + if err != nil { + utils.HandleError(c, err) + return + } + + if user == nil { + utils.Error(c, http.StatusNotFound, "User not found") + return + } + + utils.Success(c, http.StatusOK, "User retrieved successfully", user) +} + +// UpdateUser updates a user +func (h *UserHandler) UpdateUser(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid user ID") + return + } + + var req UpdateUserRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + // Get current user ID from context + currentUserID, _ := c.Get("userID") + + // Users can only update their own profile + if currentUserID.(int) != id { + utils.Error(c, http.StatusForbidden, "Can only update your own profile") + return + } + + user := &models.User{ + ID: id, + Email: req.Email, + IsActive: true, + } + + if req.IsActive != nil { + user.IsActive = *req.IsActive + } + + if err := h.userService.UpdateUser(user); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "User updated successfully", user) +} + +// DeleteUser deletes a user (admin only) +func (h *UserHandler) DeleteUser(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid user ID") + return + } + + // Prevent admin from deleting themselves + currentUserID, _ := c.Get("userID") + if currentUserID.(int) == id { + utils.Error(c, http.StatusBadRequest, "Cannot delete yourself") + return + } + + if err := h.userService.DeleteUser(id); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "User deleted successfully", nil) +} + +// UpdateRole updates a user's role (admin only) +func (h *UserHandler) UpdateRole(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid user ID") + return + } + + var req UpdateRoleRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + // Prevent admin from changing their own role + currentUserID, _ := c.Get("userID") + if currentUserID.(int) == id { + utils.Error(c, http.StatusBadRequest, "Cannot change your own role") + return + } + + if err := h.userService.UpdateUserRole(id, req.Role); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "User role updated successfully", nil) +} + +// GetUserQuota gets a user's quota +func (h *UserHandler) GetUserQuota(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid user ID") + return + } + + // Get current user ID from context + currentUserID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + + // Only admin or the user themselves can view quota + if userRole != "admin" && currentUserID.(int) != id { + utils.Error(c, http.StatusForbidden, "Access denied") + return + } + + quota, err := h.quotaService.GetUserQuota(id) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Quota retrieved successfully", quota) +} + +// UpdateUserQuota updates a user's quota (admin only) +func (h *UserHandler) UpdateUserQuota(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid user ID") + return + } + + var req UpdateQuotaRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + + quota := &models.UserQuota{ + MaxInstances: req.MaxInstances, + MaxCPUCores: req.MaxCPUCores, + MaxMemoryGB: req.MaxMemoryGB, + MaxStorageGB: req.MaxStorageGB, + MaxGPUCount: req.MaxGPUCount, + } + + if err := h.quotaService.UpdateUserQuota(id, quota); err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Quota updated successfully", quota) +} diff --git a/backend/internal/handlers/user_handler_test.go b/backend/internal/handlers/user_handler_test.go new file mode 100644 index 0000000..9e357bd --- /dev/null +++ b/backend/internal/handlers/user_handler_test.go @@ -0,0 +1,15 @@ +package handlers + +import "testing" + +func TestValidateImportedUserAllowsEmptyPassword(t *testing.T) { + if err := validateImportedUser("alice", "alice@example.com", "", "user"); err != "" { + t.Fatalf("expected empty import password to be allowed, got %q", err) + } +} + +func TestValidateImportedUserRejectsShortExplicitPassword(t *testing.T) { + if err := validateImportedUser("alice", "alice@example.com", "user123", "user"); err != "Password must be at least 8 characters" { + t.Fatalf("expected short explicit password error, got %q", err) + } +} diff --git a/backend/internal/handlers/websocket_handler.go b/backend/internal/handlers/websocket_handler.go new file mode 100644 index 0000000..98c29b4 --- /dev/null +++ b/backend/internal/handlers/websocket_handler.go @@ -0,0 +1,51 @@ +package handlers + +import ( + "net/http" + + "clawreef/internal/services" + + "github.com/gin-gonic/gin" +) + +// WebSocketHandler handles WebSocket connections +type WebSocketHandler struct { + hub *services.Hub +} + +// NewWebSocketHandler creates a new WebSocket handler +func NewWebSocketHandler(hub *services.Hub) *WebSocketHandler { + return &WebSocketHandler{ + hub: hub, + } +} + +// HandleWebSocket handles WebSocket upgrade requests +func (h *WebSocketHandler) HandleWebSocket(c *gin.Context) { + userID, exists := c.Get("userID") + if !exists { + c.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"}) + return + } + roleRaw, _ := c.Get("userRole") + role, _ := roleRaw.(string) + topic := services.WebSocketTopicUser + if c.Query("topic") == string(services.WebSocketTopicRuntimeAdmin) { + if role != "admin" { + c.JSON(http.StatusForbidden, gin.H{"error": "Admin access required"}) + return + } + topic = services.WebSocketTopicRuntimeAdmin + } + + // Upgrade HTTP connection to WebSocket + services.ServeWSWithOptions(h.hub, c.Writer, c.Request, userID.(int), role, topic) +} + +// GetConnectionCount returns the number of active WebSocket connections +func (h *WebSocketHandler) GetConnectionCount(c *gin.Context) { + count := h.hub.GetClientCount() + c.JSON(http.StatusOK, gin.H{ + "count": count, + }) +} diff --git a/backend/internal/handlers/workspace_file_handler.go b/backend/internal/handlers/workspace_file_handler.go new file mode 100644 index 0000000..ea1d8ab --- /dev/null +++ b/backend/internal/handlers/workspace_file_handler.go @@ -0,0 +1,336 @@ +package handlers + +import ( + "errors" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "path/filepath" + "strconv" + "strings" + "time" + "unicode" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type WorkspaceFileHandler struct { + instanceService services.InstanceService + fileService services.WorkspaceFileService + runtimeWorkspaceFileService services.WorkspaceFileService + skillRepo repository.SkillRepository +} + +type createWorkspaceFolderRequest struct { + Path string `json:"path" binding:"required"` +} + +type renameWorkspaceEntryRequest struct { + OldPath string `json:"old_path" binding:"required"` + NewPath string `json:"new_path" binding:"required"` +} + +func NewWorkspaceFileHandler(instanceService services.InstanceService, fileService services.WorkspaceFileService, runtimeWorkspaceFileService ...services.WorkspaceFileService) *WorkspaceFileHandler { + runtimeFileService := fileService + if len(runtimeWorkspaceFileService) > 0 && runtimeWorkspaceFileService[0] != nil { + runtimeFileService = runtimeWorkspaceFileService[0] + } + return &WorkspaceFileHandler{ + instanceService: instanceService, + fileService: fileService, + runtimeWorkspaceFileService: runtimeFileService, + } +} + +func (h *WorkspaceFileHandler) SetSkillRepository(skillRepo repository.SkillRepository) { + h.skillRepo = skillRepo +} + +func (h *WorkspaceFileHandler) List(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + entries, err := service.List(c.Request.Context(), scope, c.Query("path")) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusOK, "Workspace files retrieved successfully", gin.H{"entries": entries}) +} + +func (h *WorkspaceFileHandler) Preview(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + + if strings.EqualFold(strings.TrimSpace(c.Query("raw")), "1") { + file, contentType, size, err := service.OpenPreview(c.Request.Context(), scope, c.Query("path")) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + defer file.Close() + filename := safeWorkspaceDownloadName(filepath.Base(c.Query("path"))) + streamWorkspaceFile(c, file, filename, contentType, "inline", size) + return + } + + preview, err := service.Preview(c.Request.Context(), scope, c.Query("path")) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusOK, "Workspace preview retrieved successfully", gin.H{"preview": preview}) +} + +func (h *WorkspaceFileHandler) Download(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + file, filename, size, err := service.OpenDownload(c.Request.Context(), scope, c.Query("path")) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + defer file.Close() + + contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(filename))) + if contentType == "" { + contentType = "application/octet-stream" + } + streamWorkspaceFile(c, file, filename, contentType, "attachment", size) +} + +func (h *WorkspaceFileHandler) Upload(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + + maxBytes := services.WorkspaceUploadMaxBytes() + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes+(1<<20)) + fileHeader, err := c.FormFile("file") + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + utils.Error(c, http.StatusRequestEntityTooLarge, "workspace upload exceeds maximum size") + return + } + utils.Error(c, http.StatusBadRequest, "file is required") + return + } + if fileHeader.Size > maxBytes { + utils.Error(c, http.StatusRequestEntityTooLarge, "workspace upload exceeds maximum size") + return + } + file, err := fileHeader.Open() + if err != nil { + handleWorkspaceFileError(c, err) + return + } + defer file.Close() + + entry, err := service.Upload(c.Request.Context(), scope, c.Query("path"), fileHeader.Filename, file, fileHeader.Size) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Workspace file uploaded successfully", entry) +} + +func (h *WorkspaceFileHandler) Mkdir(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + var req createWorkspaceFolderRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + entry, err := service.Mkdir(c.Request.Context(), scope, req.Path) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Workspace folder created successfully", entry) +} + +func (h *WorkspaceFileHandler) Rename(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + var req renameWorkspaceEntryRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + entry, err := service.Rename(c.Request.Context(), scope, req.OldPath, req.NewPath) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusOK, "Workspace entry renamed successfully", entry) +} + +func (h *WorkspaceFileHandler) Delete(c *gin.Context) { + instance, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + path := c.Query("path") + if err := service.Delete(c.Request.Context(), scope, path); err != nil { + handleWorkspaceFileError(c, err) + return + } + if err := h.syncSkillDeletionFromWorkspacePath(instance.ID, path); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Workspace entry deleted successfully", nil) +} + +func (h *WorkspaceFileHandler) syncSkillDeletionFromWorkspacePath(instanceID int, path string) error { + if h.skillRepo == nil { + return nil + } + return h.skillRepo.MarkInstanceSkillsRemovedByWorkspacePath(instanceID, path, time.Now().UTC()) +} + +func (h *WorkspaceFileHandler) workspaceScope(c *gin.Context) (*models.Instance, services.WorkspaceFileService, services.WorkspaceFileScope, bool) { + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return nil, nil, services.WorkspaceFileScope{}, false + } + + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return nil, nil, services.WorkspaceFileScope{}, false + } + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return nil, nil, services.WorkspaceFileScope{}, false + } + + userIDRaw, ok := c.Get("userID") + if !ok { + utils.Error(c, http.StatusUnauthorized, "Unauthorized") + return nil, nil, services.WorkspaceFileScope{}, false + } + userID, ok := userIDRaw.(int) + if !ok { + utils.Error(c, http.StatusUnauthorized, "Unauthorized") + return nil, nil, services.WorkspaceFileScope{}, false + } + roleRaw, _ := c.Get("userRole") + userRole, _ := roleRaw.(string) + if userRole != "admin" && instance.UserID != userID { + utils.Error(c, http.StatusForbidden, "Access denied") + return nil, nil, services.WorkspaceFileScope{}, false + } + if isDesktopWorkspaceInstance(instance) { + return instance, h.runtimeWorkspaceFileService, services.WorkspaceFileScope{ + InstanceID: instance.ID, + UserID: instance.UserID, + WorkspacePath: "/config", + }, true + } + if instance.WorkspacePath == nil || strings.TrimSpace(*instance.WorkspacePath) == "" { + utils.Error(c, http.StatusNotFound, "Workspace not found") + return nil, nil, services.WorkspaceFileScope{}, false + } + + return instance, h.fileService, services.WorkspaceFileScope{ + InstanceID: instance.ID, + UserID: instance.UserID, + WorkspacePath: *instance.WorkspacePath, + }, true +} + +func isDesktopWorkspaceInstance(instance *models.Instance) bool { + if instance == nil { + return false + } + return strings.EqualFold(strings.TrimSpace(instance.InstanceMode), services.InstanceModePro) || + strings.EqualFold(strings.TrimSpace(instance.RuntimeType), services.RuntimeBackendDesktop) +} + +func streamWorkspaceFile(c *gin.Context, file io.ReadSeeker, filename, contentType, disposition string, size int64) { + safeName := safeWorkspaceDownloadName(filename) + c.Header("Content-Type", contentType) + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Content-Disposition", workspaceContentDisposition(disposition, safeName)) + if size >= 0 { + c.Header("Content-Length", strconv.FormatInt(size, 10)) + } + if _, err := io.Copy(c.Writer, file); err != nil && !c.Writer.Written() { + utils.HandleError(c, err) + } +} + +func workspaceContentDisposition(disposition, filename string) string { + disposition = strings.TrimSpace(strings.ToLower(disposition)) + if disposition != "inline" { + disposition = "attachment" + } + escaped := strings.ReplaceAll(filename, `\`, `\\`) + escaped = strings.ReplaceAll(escaped, `"`, `\"`) + return fmt.Sprintf("%s; filename=\"%s\"; filename*=UTF-8''%s", disposition, escaped, url.PathEscape(filename)) +} + +func safeWorkspaceDownloadName(name string) string { + name = strings.TrimSpace(strings.ReplaceAll(name, "\\", "/")) + name = filepath.Base(name) + if name == "." || name == "/" || name == "" { + name = "download" + } + name = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + switch r { + case '/', '\\', ':', '*', '?', '"', '<', '>', '|': + return '-' + default: + return r + } + }, name) + name = strings.TrimSpace(name) + if name == "" || name == "." || name == ".." { + return "download" + } + return name +} + +func handleWorkspaceFileError(c *gin.Context, err error) { + switch { + case errors.Is(err, services.ErrWorkspacePathNotFound): + utils.Error(c, http.StatusNotFound, err.Error()) + case errors.Is(err, services.ErrWorkspacePreviewTooLarge), errors.Is(err, services.ErrWorkspaceUploadTooLarge): + utils.Error(c, http.StatusRequestEntityTooLarge, err.Error()) + case errors.Is(err, services.ErrWorkspacePathEscape): + utils.Error(c, http.StatusForbidden, "Access denied") + case errors.Is(err, services.ErrWorkspacePathInvalid), + errors.Is(err, services.ErrWorkspaceDirectoryExpected), + errors.Is(err, services.ErrWorkspaceFileExpected), + errors.Is(err, services.ErrWorkspaceRootOperation), + errors.Is(err, services.ErrWorkspaceFileNameInvalid), + errors.Is(err, services.ErrWorkspaceEntryExists): + utils.Error(c, http.StatusBadRequest, err.Error()) + default: + utils.HandleError(c, err) + } +} diff --git a/backend/internal/handlers/workspace_file_handler_test.go b/backend/internal/handlers/workspace_file_handler_test.go new file mode 100644 index 0000000..fa95a8d --- /dev/null +++ b/backend/internal/handlers/workspace_file_handler_test.go @@ -0,0 +1,242 @@ +package handlers + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/services" + + "github.com/gin-gonic/gin" +) + +type fakeWorkspaceHandlerInstanceService struct { + instances map[int]*models.Instance +} + +func (s *fakeWorkspaceHandlerInstanceService) Create(userID int, req services.CreateInstanceRequest) (*models.Instance, error) { + return nil, nil +} +func (s *fakeWorkspaceHandlerInstanceService) ValidateCreateRequests(userID int, requests []services.CreateInstanceRequest) error { + return nil +} +func (s *fakeWorkspaceHandlerInstanceService) GetByID(id int) (*models.Instance, error) { + return s.instances[id], nil +} +func (s *fakeWorkspaceHandlerInstanceService) GetByUserID(userID int, offset, limit int) ([]models.Instance, int, error) { + return nil, 0, nil +} +func (s *fakeWorkspaceHandlerInstanceService) GetAllInstances(offset, limit int) ([]models.Instance, int, error) { + return nil, 0, nil +} +func (s *fakeWorkspaceHandlerInstanceService) Start(instanceID int) error { return nil } +func (s *fakeWorkspaceHandlerInstanceService) Stop(instanceID int) error { return nil } +func (s *fakeWorkspaceHandlerInstanceService) Restart(instanceID int) error { return nil } +func (s *fakeWorkspaceHandlerInstanceService) Delete(instanceID int) error { return nil } +func (s *fakeWorkspaceHandlerInstanceService) Update(instanceID int, req services.UpdateInstanceRequest) error { + return nil +} +func (s *fakeWorkspaceHandlerInstanceService) GetInstanceStatus(instanceID int) (*services.InstanceStatus, error) { + return nil, nil +} +func (s *fakeWorkspaceHandlerInstanceService) ForceSyncInstance(instanceID int) error { return nil } + +type fakeWorkspaceFileService struct { + lastScope services.WorkspaceFileScope + listCalls int + file *os.File + filename string + size int64 +} + +func (s *fakeWorkspaceFileService) List(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) ([]services.WorkspaceEntry, error) { + s.lastScope = scope + s.listCalls++ + return []services.WorkspaceEntry{{Name: "readme.md", Path: "readme.md", ModifiedAt: time.Unix(1, 0), Previewable: true, Downloadable: true}}, nil +} +func (s *fakeWorkspaceFileService) Preview(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*services.WorkspacePreview, error) { + s.lastScope = scope + return &services.WorkspacePreview{Kind: "text", Text: "ok"}, nil +} +func (s *fakeWorkspaceFileService) OpenPreview(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + s.lastScope = scope + return s.file, "text/plain; charset=utf-8", s.size, nil +} +func (s *fakeWorkspaceFileService) OpenDownload(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + s.lastScope = scope + return s.file, s.filename, s.size, nil +} +func (s *fakeWorkspaceFileService) Upload(ctx context.Context, scope services.WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*services.WorkspaceEntry, error) { + s.lastScope = scope + return &services.WorkspaceEntry{Name: filename, Path: filename, Downloadable: true}, nil +} +func (s *fakeWorkspaceFileService) Mkdir(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*services.WorkspaceEntry, error) { + s.lastScope = scope + return &services.WorkspaceEntry{Name: "docs", Path: relativePath, IsDir: true}, nil +} +func (s *fakeWorkspaceFileService) Rename(ctx context.Context, scope services.WorkspaceFileScope, oldPath, newPath string) (*services.WorkspaceEntry, error) { + s.lastScope = scope + return &services.WorkspaceEntry{Name: "renamed", Path: newPath}, nil +} +func (s *fakeWorkspaceFileService) Delete(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) error { + s.lastScope = scope + return nil +} + +func TestWorkspaceFileHandlerRejectsNonOwnerUser(t *testing.T) { + gin.SetMode(gin.TestMode) + workspacePath := t.TempDir() + instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 77: {ID: 77, UserID: 20, WorkspacePath: &workspacePath}, + }} + fileService := &fakeWorkspaceFileService{} + handler := NewWorkspaceFileHandler(instanceService, fileService) + + router := workspaceFileTestRouter(10, "user") + router.GET("/api/v1/instances/:id/workspace/files", handler.List) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, body = %s, want 403", rec.Code, rec.Body.String()) + } + if fileService.listCalls != 0 { + t.Fatalf("workspace file service called %d times, want 0", fileService.listCalls) + } +} + +func TestWorkspaceFileHandlerAdminUsesInstanceOwnerScope(t *testing.T) { + gin.SetMode(gin.TestMode) + workspacePath := t.TempDir() + instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 77: {ID: 77, UserID: 20, WorkspacePath: &workspacePath}, + }} + fileService := &fakeWorkspaceFileService{} + handler := NewWorkspaceFileHandler(instanceService, fileService) + + router := workspaceFileTestRouter(10, "admin") + router.GET("/api/v1/instances/:id/workspace/files", handler.List) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files?path=", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if fileService.lastScope.InstanceID != 77 || fileService.lastScope.UserID != 20 || fileService.lastScope.WorkspacePath != workspacePath { + t.Fatalf("scope = %#v, want instance owner scope", fileService.lastScope) + } +} + +func TestWorkspaceFileHandlerRequiresWorkspacePath(t *testing.T) { + gin.SetMode(gin.TestMode) + instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 77: {ID: 77, UserID: 10}, + }} + fileService := &fakeWorkspaceFileService{} + handler := NewWorkspaceFileHandler(instanceService, fileService) + + router := workspaceFileTestRouter(10, "user") + router.GET("/api/v1/instances/:id/workspace/files", handler.List) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, body = %s, want 404", rec.Code, rec.Body.String()) + } + if fileService.listCalls != 0 { + t.Fatalf("workspace file service called %d times, want 0", fileService.listCalls) + } +} + +func TestWorkspaceFileHandlerProDesktopUsesRuntimeConfigWorkspace(t *testing.T) { + gin.SetMode(gin.TestMode) + instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 77: { + ID: 77, + UserID: 10, + RuntimeType: services.RuntimeBackendDesktop, + InstanceMode: services.InstanceModePro, + }, + }} + localFileService := &fakeWorkspaceFileService{} + runtimeFileService := &fakeWorkspaceFileService{} + handler := NewWorkspaceFileHandler(instanceService, localFileService, runtimeFileService) + + router := workspaceFileTestRouter(10, "user") + router.GET("/api/v1/instances/:id/workspace/files", handler.List) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files?path=", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if localFileService.listCalls != 0 { + t.Fatalf("local workspace service called %d times, want 0", localFileService.listCalls) + } + if runtimeFileService.listCalls != 1 { + t.Fatalf("runtime workspace service called %d times, want 1", runtimeFileService.listCalls) + } + if runtimeFileService.lastScope.WorkspacePath != "/config" { + t.Fatalf("runtime workspace path = %q, want /config", runtimeFileService.lastScope.WorkspacePath) + } +} + +func TestWorkspaceFileHandlerDownloadUsesSafeContentDisposition(t *testing.T) { + gin.SetMode(gin.TestMode) + workspacePath := t.TempDir() + downloadFile, err := os.CreateTemp(t.TempDir(), "download-*") + if err != nil { + t.Fatal(err) + } + if _, err := downloadFile.WriteString("body"); err != nil { + t.Fatal(err) + } + if _, err := downloadFile.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + + instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 77: {ID: 77, UserID: 10, WorkspacePath: &workspacePath}, + }} + fileService := &fakeWorkspaceFileService{file: downloadFile, filename: `bad"name/ignored.txt`, size: int64(len("body"))} + handler := NewWorkspaceFileHandler(instanceService, fileService) + + router := workspaceFileTestRouter(10, "user") + router.GET("/api/v1/instances/:id/workspace/download", handler.Download) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/download?path=report.txt", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + disposition := rec.Header().Get("Content-Disposition") + if !strings.Contains(disposition, "attachment;") || strings.Contains(disposition, `"name/`) || strings.Contains(disposition, "\r") || strings.Contains(disposition, "\n") { + t.Fatalf("Content-Disposition = %q, want safe attachment filename", disposition) + } +} + +func workspaceFileTestRouter(userID int, role string) *gin.Engine { + router := gin.New() + router.Use(func(c *gin.Context) { + c.Set("userID", userID) + c.Set("userRole", role) + c.Next() + }) + return router +} diff --git a/backend/internal/middleware/auth_middleware.go b/backend/internal/middleware/auth_middleware.go new file mode 100644 index 0000000..521dfe2 --- /dev/null +++ b/backend/internal/middleware/auth_middleware.go @@ -0,0 +1,166 @@ +package middleware + +import ( + "errors" + "net/http" + "os" + "strings" + + "clawreef/internal/repository" + "clawreef/internal/utils" + "github.com/gin-gonic/gin" +) + +// Auth middleware validates JWT token +func Auth() gin.HandlerFunc { + return func(c *gin.Context) { + tokenString, ok := extractToken(c) + if !ok { + c.JSON(http.StatusUnauthorized, gin.H{ + "success": false, + "error": "Authorization header required", + }) + c.Abort() + return + } + + claims, err := validateUserAccessToken(tokenString) + if err != nil { + c.JSON(http.StatusUnauthorized, gin.H{ + "success": false, + "error": "Invalid or expired token", + }) + c.Abort() + return + } + + // Check token type + if claims.TokenType != "access" { + c.JSON(http.StatusUnauthorized, gin.H{ + "success": false, + "error": "Invalid token type", + }) + c.Abort() + return + } + + // Set user ID in context + c.Set("userID", claims.UserID) + c.Next() + } +} + +// GatewayAuth accepts either a normal user access JWT or an instance lifecycle gateway token. +func GatewayAuth(instanceRepo repository.InstanceRepository, bindingRepos ...repository.InstanceRuntimeBindingRepository) gin.HandlerFunc { + var bindingRepo repository.InstanceRuntimeBindingRepository + if len(bindingRepos) > 0 { + bindingRepo = bindingRepos[0] + } + return func(c *gin.Context) { + tokenString, ok := extractToken(c) + if !ok { + c.JSON(http.StatusUnauthorized, gin.H{ + "success": false, + "error": "Authorization header required", + }) + c.Abort() + return + } + + if claims, err := validateUserAccessToken(tokenString); err == nil { + c.Set("userID", claims.UserID) + c.Set("gatewayAuthType", "user") + c.Next() + return + } + + instance, err := instanceRepo.GetByAccessToken(tokenString) + if err != nil { + c.JSON(http.StatusUnauthorized, gin.H{ + "success": false, + "error": "Invalid gateway token", + }) + c.Abort() + return + } + if instance == nil || instance.AccessToken == nil || strings.TrimSpace(*instance.AccessToken) == "" { + c.JSON(http.StatusUnauthorized, gin.H{ + "success": false, + "error": "Invalid gateway token", + }) + c.Abort() + return + } + + c.Set("userID", instance.UserID) + c.Set("instanceID", instance.ID) + c.Set("instanceMode", gatewayInstanceMode(instance.InstanceMode, instance.RuntimeType)) + c.Set("runtimeType", strings.TrimSpace(instance.RuntimeType)) + if bindingRepo != nil { + if binding, err := bindingRepo.GetRunningByInstanceID(c.Request.Context(), instance.ID); err == nil && binding != nil { + c.Set("gatewayID", strings.TrimSpace(binding.GatewayID)) + c.Set("runtimePodID", binding.RuntimePodID) + } + } + c.Set("gatewayAuthType", "instance") + c.Next() + } +} + +func gatewayInstanceMode(instanceMode, runtimeType string) string { + mode := strings.ToLower(strings.TrimSpace(instanceMode)) + if mode == "lite" || mode == "pro" { + return mode + } + switch strings.ToLower(strings.TrimSpace(runtimeType)) { + case "gateway": + return "lite" + case "desktop", "shell": + return "pro" + default: + return mode + } +} + +func extractToken(c *gin.Context) (string, bool) { + authHeader := c.GetHeader("Authorization") + if authHeader != "" { + parts := strings.SplitN(authHeader, " ", 2) + if len(parts) != 2 || strings.ToLower(parts[0]) != "bearer" { + return "", false + } + return parts[1], true + } + + // Browsers cannot set custom Authorization headers for native WebSocket + // handshakes, so allow `?token=` specifically for upgrade requests. + if strings.EqualFold(c.GetHeader("Upgrade"), "websocket") { + token := strings.TrimSpace(c.Query("token")) + if token != "" { + return token, true + } + } + + return "", false +} + +func getJWTSecret() string { + // Get from environment variable, fallback to default + // Must match the secret used in config.go + if secret := os.Getenv("JWT_SECRET"); secret != "" { + return secret + } + return "clawreef-dev-secret-key-change-in-production" +} + +func validateUserAccessToken(tokenString string) (*utils.TokenClaims, error) { + jwtSecret := getJWTSecret() + claims, err := utils.ValidateToken(tokenString, jwtSecret) + if err != nil { + return nil, err + } + if claims.TokenType != "access" { + return nil, errors.New("invalid token type") + } + return claims, nil +} diff --git a/backend/internal/middleware/cors_middleware.go b/backend/internal/middleware/cors_middleware.go new file mode 100644 index 0000000..fe9e899 --- /dev/null +++ b/backend/internal/middleware/cors_middleware.go @@ -0,0 +1,22 @@ +package middleware + +import ( + "github.com/gin-gonic/gin" +) + +// CORS middleware handles Cross-Origin Resource Sharing +func CORS() gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer.Header().Set("Access-Control-Allow-Origin", "*") + c.Writer.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS, PATCH") + c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With") + c.Writer.Header().Set("Access-Control-Max-Age", "86400") + + if c.Request.Method == "OPTIONS" { + c.AbortWithStatus(204) + return + } + + c.Next() + } +} diff --git a/backend/internal/middleware/error_handler.go b/backend/internal/middleware/error_handler.go new file mode 100644 index 0000000..0a4c3fd --- /dev/null +++ b/backend/internal/middleware/error_handler.go @@ -0,0 +1,27 @@ +package middleware + +import ( + "fmt" + "log" + "net/http" + "runtime/debug" + + "github.com/gin-gonic/gin" +) + +// ErrorHandler middleware handles panics and errors +func ErrorHandler() gin.HandlerFunc { + return func(c *gin.Context) { + defer func() { + if err := recover(); err != nil { + log.Printf("[PANIC] %v\n%s", err, debug.Stack()) + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "error": fmt.Sprintf("Internal server error: %v", err), + }) + c.Abort() + } + }() + c.Next() + } +} diff --git a/backend/internal/middleware/rbac_middleware.go b/backend/internal/middleware/rbac_middleware.go new file mode 100644 index 0000000..53a8722 --- /dev/null +++ b/backend/internal/middleware/rbac_middleware.go @@ -0,0 +1,122 @@ +package middleware + +import ( + "net/http" + + "clawreef/internal/models" + "clawreef/internal/repository" + "github.com/gin-gonic/gin" +) + +// AdminAuth middleware checks if user is admin +type AdminAuth struct { + userRepo repository.UserRepository +} + +// NewAdminAuth creates a new admin auth middleware +func NewAdminAuth(userRepo repository.UserRepository) gin.HandlerFunc { + return func(c *gin.Context) { + userID, exists := c.Get("userID") + if !exists { + c.JSON(http.StatusUnauthorized, gin.H{ + "success": false, + "error": "Unauthorized", + }) + c.Abort() + return + } + + // Get user from database + user, err := userRepo.GetByID(userID.(int)) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "error": "Failed to get user", + }) + c.Abort() + return + } + + if user == nil { + c.JSON(http.StatusUnauthorized, gin.H{ + "success": false, + "error": "User not found", + }) + c.Abort() + return + } + + // Check if user is admin + if user.Role != "admin" { + c.JSON(http.StatusForbidden, gin.H{ + "success": false, + "error": "Admin access required", + }) + c.Abort() + return + } + + // Set user role in context + c.Set("userRole", user.Role) + c.Next() + } +} + +// RoleAuth middleware checks if user has required role +func RoleAuth(allowedRoles ...string) gin.HandlerFunc { + return func(c *gin.Context) { + userRole, exists := c.Get("userRole") + if !exists { + c.JSON(http.StatusForbidden, gin.H{ + "success": false, + "error": "Role not found", + }) + c.Abort() + return + } + + role := userRole.(string) + for _, allowedRole := range allowedRoles { + if role == allowedRole { + c.Next() + return + } + } + + c.JSON(http.StatusForbidden, gin.H{ + "success": false, + "error": "Insufficient permissions", + }) + c.Abort() + } +} + +// SetUserInfo middleware sets user info in context +func SetUserInfo(userRepo repository.UserRepository) gin.HandlerFunc { + return func(c *gin.Context) { + userID, exists := c.Get("userID") + if !exists { + c.Next() + return + } + + user, err := userRepo.GetByID(userID.(int)) + if err != nil || user == nil { + c.Next() + return + } + + c.Set("userRole", user.Role) + c.Set("user", user) + c.Next() + } +} + +// GetCurrentUser gets current user from context +func GetCurrentUser(c *gin.Context) *models.User { + user, exists := c.Get("user") + if !exists { + return nil + } + return user.(*models.User) +} diff --git a/backend/internal/models/ai_governance_constants.go b/backend/internal/models/ai_governance_constants.go new file mode 100644 index 0000000..1382478 --- /dev/null +++ b/backend/internal/models/ai_governance_constants.go @@ -0,0 +1,38 @@ +package models + +const ( + // Traffic classes + TrafficClassDesktopStream = "desktop_stream" + TrafficClassLLM = "llm" + TrafficClassToolMCP = "tool_mcp" + TrafficClassGenericEgress = "generic_egress" +) + +const ( + // Model invocation statuses + ModelInvocationStatusPending = "pending" + ModelInvocationStatusCompleted = "completed" + ModelInvocationStatusFailed = "failed" + ModelInvocationStatusBlocked = "blocked" +) + +const ( + // Audit severities + AuditSeverityInfo = "info" + AuditSeverityWarn = "warn" + AuditSeverityError = "error" +) + +const ( + // Risk severities + RiskSeverityLow = "low" + RiskSeverityMedium = "medium" + RiskSeverityHigh = "high" +) + +const ( + // Risk actions + RiskActionAllow = "allow" + RiskActionRouteSecureModel = "route_secure_model" + RiskActionBlock = "block" +) diff --git a/backend/internal/models/audit_event.go b/backend/internal/models/audit_event.go new file mode 100644 index 0000000..a367d86 --- /dev/null +++ b/backend/internal/models/audit_event.go @@ -0,0 +1,29 @@ +package models + +import "time" + +// AuditEvent stores normalized AI governance events. +type AuditEvent struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TraceID string `db:"trace_id" json:"trace_id"` + SessionID *string `db:"session_id" json:"session_id,omitempty"` + RequestID *string `db:"request_id" json:"request_id,omitempty"` + UserID *int `db:"user_id" json:"user_id,omitempty"` + InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"` + RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"` + GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"` + RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"` + InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"` + EventType string `db:"event_type" json:"event_type"` + TrafficClass string `db:"traffic_class" json:"traffic_class"` + Severity string `db:"severity" json:"severity"` + Message string `db:"message" json:"message"` + Details *string `db:"details" json:"details,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +// TableName returns the table name for audit events. +func (a AuditEvent) TableName() string { + return "audit_events" +} diff --git a/backend/internal/models/audit_log.go b/backend/internal/models/audit_log.go new file mode 100644 index 0000000..dfbb8ff --- /dev/null +++ b/backend/internal/models/audit_log.go @@ -0,0 +1,22 @@ +package models + +import ( + "time" +) + +// AuditLog represents an audit log entry +type AuditLog struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + UserID *int `db:"user_id" json:"user_id,omitempty"` + Action string `db:"action" json:"action"` + ResourceType string `db:"resource_type" json:"resource_type"` + ResourceID *int `db:"resource_id" json:"resource_id,omitempty"` + Details *string `db:"details" json:"details,omitempty"` + IPAddress *string `db:"ip_address" json:"ip_address,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +// TableName returns the table name for the AuditLog model +func (a AuditLog) TableName() string { + return "audit_logs" +} diff --git a/backend/internal/models/backup.go b/backend/internal/models/backup.go new file mode 100644 index 0000000..5bd2753 --- /dev/null +++ b/backend/internal/models/backup.go @@ -0,0 +1,24 @@ +package models + +import ( + "time" +) + +// Backup represents a backup of an instance +type Backup struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + BackupName string `db:"backup_name" json:"backup_name"` + BackupSizeGB *int `db:"backup_size_gb" json:"backup_size_gb,omitempty"` + BackupPath *string `db:"backup_path" json:"backup_path,omitempty"` + Status string `db:"status" json:"status"` + BackupType string `db:"backup_type" json:"backup_type"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + CompletedAt *time.Time `db:"completed_at" json:"completed_at,omitempty"` + ExpiresAt *time.Time `db:"expires_at" json:"expires_at,omitempty"` +} + +// TableName returns the table name for the Backup model +func (b Backup) TableName() string { + return "backups" +} diff --git a/backend/internal/models/backup_schedule.go b/backend/internal/models/backup_schedule.go new file mode 100644 index 0000000..d5664e7 --- /dev/null +++ b/backend/internal/models/backup_schedule.go @@ -0,0 +1,22 @@ +package models + +import ( + "time" +) + +// BackupSchedule represents a scheduled backup task +type BackupSchedule struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + ScheduleName *string `db:"schedule_name" json:"schedule_name,omitempty"` + CronExpression string `db:"cron_expression" json:"cron_expression"` + RetentionDays int `db:"retention_days" json:"retention_days"` + IsActive bool `db:"is_active" json:"is_active"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +// TableName returns the table name for the BackupSchedule model +func (b BackupSchedule) TableName() string { + return "backup_schedules" +} diff --git a/backend/internal/models/chat_message.go b/backend/internal/models/chat_message.go new file mode 100644 index 0000000..6467152 --- /dev/null +++ b/backend/internal/models/chat_message.go @@ -0,0 +1,23 @@ +package models + +import "time" + +// ChatMessageRecord stores a persisted message within a chat session. +type ChatMessageRecord struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TraceID string `db:"trace_id" json:"trace_id"` + SessionID string `db:"session_id" json:"session_id"` + RequestID *string `db:"request_id" json:"request_id,omitempty"` + UserID *int `db:"user_id" json:"user_id,omitempty"` + InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"` + Role string `db:"role" json:"role"` + Content string `db:"content" json:"content"` + SequenceNo int `db:"sequence_no" json:"sequence_no"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +// TableName returns the table name for chat messages. +func (c ChatMessageRecord) TableName() string { + return "chat_messages" +} diff --git a/backend/internal/models/chat_session.go b/backend/internal/models/chat_session.go new file mode 100644 index 0000000..62c582e --- /dev/null +++ b/backend/internal/models/chat_session.go @@ -0,0 +1,20 @@ +package models + +import "time" + +// ChatSession stores a persisted AI chat session. +type ChatSession struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + SessionID string `db:"session_id" json:"session_id"` + UserID *int `db:"user_id" json:"user_id,omitempty"` + InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + Title *string `db:"title" json:"title,omitempty"` + LastTraceID *string `db:"last_trace_id" json:"last_trace_id,omitempty"` + StartedAt time.Time `db:"started_at" json:"started_at"` + LastActivityAt time.Time `db:"last_activity_at" json:"last_activity_at"` +} + +// TableName returns the table name for chat sessions. +func (c ChatSession) TableName() string { + return "chat_sessions" +} diff --git a/backend/internal/models/cost_record.go b/backend/internal/models/cost_record.go new file mode 100644 index 0000000..7bdcf69 --- /dev/null +++ b/backend/internal/models/cost_record.go @@ -0,0 +1,36 @@ +package models + +import "time" + +// CostRecord stores token and money accounting snapshots for a model call. +type CostRecord struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TraceID string `db:"trace_id" json:"trace_id"` + SessionID *string `db:"session_id" json:"session_id,omitempty"` + RequestID *string `db:"request_id" json:"request_id,omitempty"` + UserID *int `db:"user_id" json:"user_id,omitempty"` + InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"` + RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"` + GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"` + RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"` + InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"` + ModelID *int `db:"model_id" json:"model_id,omitempty"` + ProviderType string `db:"provider_type" json:"provider_type"` + ModelName string `db:"model_name" json:"model_name"` + Currency string `db:"currency" json:"currency"` + PromptTokens int `db:"prompt_tokens" json:"prompt_tokens"` + CompletionTokens int `db:"completion_tokens" json:"completion_tokens"` + TotalTokens int `db:"total_tokens" json:"total_tokens"` + InputUnitPrice float64 `db:"input_unit_price" json:"input_unit_price"` + OutputUnitPrice float64 `db:"output_unit_price" json:"output_unit_price"` + EstimatedCost float64 `db:"estimated_cost" json:"estimated_cost"` + ActualCost *float64 `db:"actual_cost" json:"actual_cost,omitempty"` + InternalCost float64 `db:"internal_cost" json:"internal_cost"` + RecordedAt time.Time `db:"recorded_at" json:"recorded_at"` +} + +// TableName returns the table name for cost records. +func (c CostRecord) TableName() string { + return "cost_records" +} diff --git a/backend/internal/models/instance.go b/backend/internal/models/instance.go new file mode 100644 index 0000000..5b9f62b --- /dev/null +++ b/backend/internal/models/instance.go @@ -0,0 +1,51 @@ +package models + +import ( + "time" +) + +// Instance represents a virtual desktop instance +type Instance struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + UserID int `db:"user_id" json:"user_id"` + Name string `db:"name" json:"name"` + Description *string `db:"description" json:"description,omitempty"` + Type string `db:"type" json:"type"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + InstanceMode string `db:"instance_mode" json:"instance_mode"` + Status string `db:"status" json:"status"` + CPUCores float64 `db:"cpu_cores" json:"cpu_cores"` + MemoryGB int `db:"memory_gb" json:"memory_gb"` + DiskGB int `db:"disk_gb" json:"disk_gb"` + GPUEnabled bool `db:"gpu_enabled" json:"gpu_enabled"` + GPUType *string `db:"gpu_type" json:"gpu_type,omitempty"` + GPUCount int `db:"gpu_count" json:"gpu_count"` + OSType string `db:"os_type" json:"os_type"` + OSVersion string `db:"os_version" json:"os_version"` + ImageRegistry *string `db:"image_registry" json:"image_registry,omitempty"` + ImageTag *string `db:"image_tag" json:"image_tag,omitempty"` + EnvironmentOverridesJSON *string `db:"environment_overrides_json" json:"-"` + DesktopStreamProfile string `db:"-" json:"desktop_stream_profile,omitempty"` + StorageClass string `db:"storage_class" json:"storage_class"` + MountPath string `db:"mount_path" json:"mount_path"` + WorkspacePath *string `db:"workspace_path" json:"workspace_path,omitempty"` + WorkspaceUsageBytes int64 `db:"workspace_usage_bytes" json:"workspace_usage_bytes"` + RuntimeGeneration int `db:"runtime_generation" json:"runtime_generation"` + RuntimeErrorMessage *string `db:"runtime_error_message" json:"runtime_error_message,omitempty"` + PodName *string `db:"pod_name" json:"pod_name,omitempty"` + PodNamespace *string `db:"pod_namespace" json:"pod_namespace,omitempty"` + PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"` + AccessURL *string `db:"access_url" json:"access_url,omitempty"` + AccessToken *string `db:"access_token" json:"-"` + AgentBootstrapToken *string `db:"agent_bootstrap_token" json:"-"` + OpenClawConfigSnapshotID *int `db:"openclaw_config_snapshot_id" json:"openclaw_config_snapshot_id,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + StoppedAt *time.Time `db:"stopped_at" json:"stopped_at,omitempty"` +} + +// TableName returns the table name for the Instance model +func (i Instance) TableName() string { + return "instances" +} diff --git a/backend/internal/models/instance_agent.go b/backend/internal/models/instance_agent.go new file mode 100644 index 0000000..556f980 --- /dev/null +++ b/backend/internal/models/instance_agent.go @@ -0,0 +1,26 @@ +package models + +import "time" + +type InstanceAgent struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + AgentID string `db:"agent_id" json:"agent_id"` + AgentVersion string `db:"agent_version" json:"agent_version"` + ProtocolVersion string `db:"protocol_version" json:"protocol_version"` + Status string `db:"status" json:"status"` + CapabilitiesJSON string `db:"capabilities_json" json:"-"` + HostInfoJSON *string `db:"host_info_json" json:"-"` + SessionToken *string `db:"session_token" json:"-"` + SessionExpiresAt *time.Time `db:"session_expires_at" json:"session_expires_at,omitempty"` + LastHeartbeatAt *time.Time `db:"last_heartbeat_at" json:"last_heartbeat_at,omitempty"` + LastReportedAt *time.Time `db:"last_reported_at" json:"last_reported_at,omitempty"` + LastSeenIP *string `db:"last_seen_ip" json:"last_seen_ip,omitempty"` + RegisteredAt *time.Time `db:"registered_at" json:"registered_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (a InstanceAgent) TableName() string { + return "instance_agents" +} diff --git a/backend/internal/models/instance_command.go b/backend/internal/models/instance_command.go new file mode 100644 index 0000000..c310cb2 --- /dev/null +++ b/backend/internal/models/instance_command.go @@ -0,0 +1,27 @@ +package models + +import "time" + +type InstanceCommand struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + AgentID *string `db:"agent_id" json:"agent_id,omitempty"` + CommandType string `db:"command_type" json:"command_type"` + PayloadJSON *string `db:"payload_json" json:"-"` + Status string `db:"status" json:"status"` + IdempotencyKey string `db:"idempotency_key" json:"idempotency_key"` + IssuedBy *int `db:"issued_by" json:"issued_by,omitempty"` + IssuedAt time.Time `db:"issued_at" json:"issued_at"` + DispatchedAt *time.Time `db:"dispatched_at" json:"dispatched_at,omitempty"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"` + TimeoutSeconds int `db:"timeout_seconds" json:"timeout_seconds"` + ResultJSON *string `db:"result_json" json:"-"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (c InstanceCommand) TableName() string { + return "instance_commands" +} diff --git a/backend/internal/models/instance_config_revision.go b/backend/internal/models/instance_config_revision.go new file mode 100644 index 0000000..01678f7 --- /dev/null +++ b/backend/internal/models/instance_config_revision.go @@ -0,0 +1,23 @@ +package models + +import "time" + +type InstanceConfigRevision struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + SourceSnapshotID *int `db:"source_snapshot_id" json:"source_snapshot_id,omitempty"` + SourceBundleID *int `db:"source_bundle_id" json:"source_bundle_id,omitempty"` + RevisionNo int `db:"revision_no" json:"revision_no"` + ContentJSON string `db:"content_json" json:"-"` + Checksum string `db:"checksum" json:"checksum"` + Status string `db:"status" json:"status"` + PublishedBy *int `db:"published_by" json:"published_by,omitempty"` + PublishedAt *time.Time `db:"published_at" json:"published_at,omitempty"` + ActivatedAt *time.Time `db:"activated_at" json:"activated_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (r InstanceConfigRevision) TableName() string { + return "instance_config_revisions" +} diff --git a/backend/internal/models/instance_desired_state.go b/backend/internal/models/instance_desired_state.go new file mode 100644 index 0000000..2df54e1 --- /dev/null +++ b/backend/internal/models/instance_desired_state.go @@ -0,0 +1,18 @@ +package models + +import "time" + +type InstanceDesiredState struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + DesiredPowerState string `db:"desired_power_state" json:"desired_power_state"` + DesiredConfigRevisionID *int `db:"desired_config_revision_id" json:"desired_config_revision_id,omitempty"` + DesiredRuntimeAction *string `db:"desired_runtime_action" json:"desired_runtime_action,omitempty"` + UpdatedBy *int `db:"updated_by" json:"updated_by,omitempty"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +func (s InstanceDesiredState) TableName() string { + return "instance_desired_state" +} diff --git a/backend/internal/models/instance_external_access.go b/backend/internal/models/instance_external_access.go new file mode 100644 index 0000000..1acc5f3 --- /dev/null +++ b/backend/internal/models/instance_external_access.go @@ -0,0 +1,25 @@ +package models + +import "time" + +type InstanceExternalAccess struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + Enabled bool `db:"enabled" json:"enabled"` + AuthMode string `db:"auth_mode" json:"auth_mode"` + PublicSlug *string `db:"public_slug" json:"-"` + PublicTokenHash *string `db:"public_token_hash" json:"-"` + ShortCodeHash *string `db:"short_code_hash" json:"-"` + PasswordHash *string `db:"api_key_hash" json:"-"` + PasswordValue *string `db:"password_value" json:"-"` + PasswordHint *string `db:"api_key_prefix" json:"password_hint,omitempty"` + ExpiresAt *time.Time `db:"expires_at" json:"expires_at,omitempty"` + CreatedBy *int `db:"created_by" json:"created_by,omitempty"` + LastUsedAt *time.Time `db:"last_used_at" json:"last_used_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (InstanceExternalAccess) TableName() string { + return "instance_external_access" +} diff --git a/backend/internal/models/instance_runtime_binding.go b/backend/internal/models/instance_runtime_binding.go new file mode 100644 index 0000000..1dff9e5 --- /dev/null +++ b/backend/internal/models/instance_runtime_binding.go @@ -0,0 +1,24 @@ +package models + +import "time" + +type InstanceRuntimeBinding struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + RuntimePodID int64 `db:"runtime_pod_id" json:"runtime_pod_id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + GatewayID string `db:"gateway_id" json:"gateway_id"` + GatewayPort int `db:"gateway_port" json:"gateway_port"` + GatewayPID *int `db:"gateway_pid" json:"gateway_pid,omitempty"` + WorkspacePath string `db:"workspace_path" json:"workspace_path"` + State string `db:"state" json:"state"` + Generation int `db:"generation" json:"generation"` + LastHealthAt *time.Time `db:"last_health_at" json:"last_health_at,omitempty"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (InstanceRuntimeBinding) TableName() string { + return "instance_runtime_bindings" +} diff --git a/backend/internal/models/instance_runtime_status.go b/backend/internal/models/instance_runtime_status.go new file mode 100644 index 0000000..b38f2c4 --- /dev/null +++ b/backend/internal/models/instance_runtime_status.go @@ -0,0 +1,25 @@ +package models + +import "time" + +type InstanceRuntimeStatus struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + InfraStatus string `db:"infra_status" json:"infra_status"` + AgentStatus string `db:"agent_status" json:"agent_status"` + OpenClawStatus string `db:"openclaw_status" json:"openclaw_status"` + OpenClawPID *int `db:"openclaw_pid" json:"openclaw_pid,omitempty"` + OpenClawVersion *string `db:"openclaw_version" json:"openclaw_version,omitempty"` + CurrentConfigRevisionID *int `db:"current_config_revision_id" json:"current_config_revision_id,omitempty"` + DesiredConfigRevisionID *int `db:"desired_config_revision_id" json:"desired_config_revision_id,omitempty"` + SummaryJSON *string `db:"summary_json" json:"-"` + SystemInfoJSON *string `db:"system_info_json" json:"-"` + HealthJSON *string `db:"health_json" json:"-"` + LastReportedAt *time.Time `db:"last_reported_at" json:"last_reported_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (s InstanceRuntimeStatus) TableName() string { + return "instance_runtime_status" +} diff --git a/backend/internal/models/instance_usage.go b/backend/internal/models/instance_usage.go new file mode 100644 index 0000000..4066732 --- /dev/null +++ b/backend/internal/models/instance_usage.go @@ -0,0 +1,22 @@ +package models + +import ( + "time" +) + +// InstanceUsage represents resource usage statistics for an instance +type InstanceUsage struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + CPUUsagePercent *float64 `db:"cpu_usage_percent" json:"cpu_usage_percent,omitempty"` + MemoryUsageGB *float64 `db:"memory_usage_gb" json:"memory_usage_gb,omitempty"` + DiskUsageGB *float64 `db:"disk_usage_gb" json:"disk_usage_gb,omitempty"` + GPUUsagePercent *float64 `db:"gpu_usage_percent" json:"gpu_usage_percent,omitempty"` + UptimeSeconds *int `db:"uptime_seconds" json:"uptime_seconds,omitempty"` + RecordedAt time.Time `db:"recorded_at" json:"recorded_at"` +} + +// TableName returns the table name for the InstanceUsage model +func (i InstanceUsage) TableName() string { + return "instance_usage" +} diff --git a/backend/internal/models/llm_model.go b/backend/internal/models/llm_model.go new file mode 100644 index 0000000..c0e986d --- /dev/null +++ b/backend/internal/models/llm_model.go @@ -0,0 +1,28 @@ +package models + +import "time" + +// LLMModel stores an admin-managed AI model configuration. +type LLMModel struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + DisplayName string `db:"display_name" json:"display_name"` + Description *string `db:"description" json:"description,omitempty"` + ProviderType string `db:"provider_type" json:"provider_type"` + ProtocolType string `db:"protocol_type" json:"protocol_type,omitempty"` + BaseURL string `db:"base_url" json:"base_url"` + ProviderModelName string `db:"provider_model_name" json:"provider_model_name"` + APIKey *string `db:"api_key" json:"api_key,omitempty"` + APIKeySecretRef *string `db:"api_key_secret_ref" json:"api_key_secret_ref,omitempty"` + IsSecure bool `db:"is_secure" json:"is_secure"` + IsActive bool `db:"is_active" json:"is_active"` + InputPrice float64 `db:"input_price" json:"input_price"` + OutputPrice float64 `db:"output_price" json:"output_price"` + Currency string `db:"currency" json:"currency"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +// TableName returns the table name for the LLM model. +func (m LLMModel) TableName() string { + return "llm_models" +} diff --git a/backend/internal/models/llm_protocol.go b/backend/internal/models/llm_protocol.go new file mode 100644 index 0000000..6b4e46f --- /dev/null +++ b/backend/internal/models/llm_protocol.go @@ -0,0 +1,85 @@ +package models + +import ( + "fmt" + "strings" +) + +const ( + ProviderTypeOpenAI = "openai" + ProviderTypeOpenAICompatible = "openai-compatible" + ProviderTypeAnthropic = "anthropic" + ProviderTypeGoogle = "google" + ProviderTypeAzureOpenAI = "azure-openai" + ProviderTypeLocal = "local" + + ProtocolTypeOpenAI = "openai" + ProtocolTypeOpenAICompatible = "openai-compatible" + ProtocolTypeAnthropic = "anthropic" + ProtocolTypeGoogle = "google" + ProtocolTypeAzureOpenAI = "azure-openai" +) + +func normalizeLLMType(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +// ResolveLLMProtocolType validates and normalizes the effective transport protocol for a model. +func ResolveLLMProtocolType(providerType, protocolType string) (string, error) { + normalizedProviderType := normalizeLLMType(providerType) + normalizedProtocolType := normalizeLLMType(protocolType) + + switch normalizedProviderType { + case "": + return "", fmt.Errorf("provider type is required") + case ProviderTypeLocal: + switch normalizedProtocolType { + case "", ProtocolTypeOpenAI, ProtocolTypeOpenAICompatible: + return ProtocolTypeOpenAICompatible, nil + case ProtocolTypeAnthropic: + return ProtocolTypeAnthropic, nil + default: + return "", fmt.Errorf("protocol type %s is not supported for local provider", normalizedProtocolType) + } + case ProviderTypeOpenAI: + return ProtocolTypeOpenAI, nil + case ProviderTypeOpenAICompatible: + return ProtocolTypeOpenAICompatible, nil + case ProviderTypeAnthropic: + return ProtocolTypeAnthropic, nil + case ProviderTypeGoogle: + return ProtocolTypeGoogle, nil + case ProviderTypeAzureOpenAI: + return ProtocolTypeAzureOpenAI, nil + default: + if normalizedProtocolType != "" { + return normalizedProtocolType, nil + } + return normalizedProviderType, nil + } +} + +// ResolveLLMProtocolTypeOrDefault returns a normalized protocol type and falls back to a sensible default. +func ResolveLLMProtocolTypeOrDefault(providerType, protocolType string) string { + resolved, err := ResolveLLMProtocolType(providerType, protocolType) + if err == nil { + return resolved + } + + switch normalizeLLMType(providerType) { + case ProviderTypeLocal: + return ProtocolTypeOpenAICompatible + case ProviderTypeOpenAI: + return ProtocolTypeOpenAI + case ProviderTypeAnthropic: + return ProtocolTypeAnthropic + case ProviderTypeGoogle: + return ProtocolTypeGoogle + case ProviderTypeAzureOpenAI: + return ProtocolTypeAzureOpenAI + case ProviderTypeOpenAICompatible: + return ProtocolTypeOpenAICompatible + default: + return normalizeLLMType(providerType) + } +} diff --git a/backend/internal/models/llm_protocol_test.go b/backend/internal/models/llm_protocol_test.go new file mode 100644 index 0000000..66c770f --- /dev/null +++ b/backend/internal/models/llm_protocol_test.go @@ -0,0 +1,67 @@ +package models + +import "testing" + +func TestResolveLLMProtocolType(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + providerType string + protocolType string + want string + wantErr bool + }{ + { + name: "local defaults to openai compatible", + providerType: ProviderTypeLocal, + want: ProtocolTypeOpenAICompatible, + }, + { + name: "local accepts anthropic", + providerType: ProviderTypeLocal, + protocolType: ProtocolTypeAnthropic, + want: ProtocolTypeAnthropic, + }, + { + name: "local normalizes openai to openai compatible", + providerType: ProviderTypeLocal, + protocolType: ProtocolTypeOpenAI, + want: ProtocolTypeOpenAICompatible, + }, + { + name: "local rejects unsupported protocol", + providerType: ProviderTypeLocal, + protocolType: ProtocolTypeGoogle, + wantErr: true, + }, + { + name: "anthropic keeps its own protocol", + providerType: ProviderTypeAnthropic, + protocolType: ProtocolTypeOpenAICompatible, + want: ProtocolTypeAnthropic, + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := ResolveLLMProtocolType(tc.providerType, tc.protocolType) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error, got protocol %q", got) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if got != tc.want { + t.Fatalf("expected %q, got %q", tc.want, got) + } + }) + } +} diff --git a/backend/internal/models/model_invocation.go b/backend/internal/models/model_invocation.go new file mode 100644 index 0000000..9f9f87b --- /dev/null +++ b/backend/internal/models/model_invocation.go @@ -0,0 +1,40 @@ +package models + +import "time" + +// ModelInvocation stores a governed LLM request/response record. +type ModelInvocation struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TraceID string `db:"trace_id" json:"trace_id"` + SessionID *string `db:"session_id" json:"session_id,omitempty"` + RequestID string `db:"request_id" json:"request_id"` + UserID *int `db:"user_id" json:"user_id,omitempty"` + InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"` + RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"` + GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"` + RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"` + ModelID *int `db:"model_id" json:"model_id,omitempty"` + ProviderType string `db:"provider_type" json:"provider_type"` + RequestedModel string `db:"requested_model" json:"requested_model"` + ActualProviderModel string `db:"actual_provider_model" json:"actual_provider_model"` + TrafficClass string `db:"traffic_class" json:"traffic_class"` + RequestPayload *string `db:"request_payload" json:"request_payload,omitempty"` + ResponsePayload *string `db:"response_payload" json:"response_payload,omitempty"` + PromptTokens int `db:"prompt_tokens" json:"prompt_tokens"` + CompletionTokens int `db:"completion_tokens" json:"completion_tokens"` + TotalTokens int `db:"total_tokens" json:"total_tokens"` + CachedTokens *int `db:"cached_tokens" json:"cached_tokens,omitempty"` + ReasoningTokens *int `db:"reasoning_tokens" json:"reasoning_tokens,omitempty"` + LatencyMs *int `db:"latency_ms" json:"latency_ms,omitempty"` + IsStreaming bool `db:"is_streaming" json:"is_streaming"` + Status string `db:"status" json:"status"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + CompletedAt *time.Time `db:"completed_at" json:"completed_at,omitempty"` +} + +// TableName returns the table name for model invocations. +func (m ModelInvocation) TableName() string { + return "model_invocations" +} diff --git a/backend/internal/models/openclaw_config.go b/backend/internal/models/openclaw_config.go new file mode 100644 index 0000000..6c56642 --- /dev/null +++ b/backend/internal/models/openclaw_config.go @@ -0,0 +1,89 @@ +package models + +import "time" + +// OpenClawConfigResource stores a reusable OpenClaw bootstrap resource. +type OpenClawConfigResource struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + UserID int `db:"user_id" json:"user_id"` + ResourceType string `db:"resource_type" json:"resource_type"` + ResourceKey string `db:"resource_key" json:"resource_key"` + Name string `db:"name" json:"name"` + Description *string `db:"description" json:"description,omitempty"` + Enabled bool `db:"enabled" json:"enabled"` + Version int `db:"version" json:"version"` + TagsJSON string `db:"tags_json" json:"-"` + ContentJSON string `db:"content_json" json:"-"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (r OpenClawConfigResource) TableName() string { + return "openclaw_config_resources" +} + +// OpenClawConfigBundle stores a reusable bundle of OpenClaw resources. +type OpenClawConfigBundle struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + UserID int `db:"user_id" json:"user_id"` + Name string `db:"name" json:"name"` + Description *string `db:"description" json:"description,omitempty"` + Enabled bool `db:"enabled" json:"enabled"` + Version int `db:"version" json:"version"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (b OpenClawConfigBundle) TableName() string { + return "openclaw_config_bundles" +} + +// OpenClawConfigBundleItem stores a resource membership inside a bundle. +type OpenClawConfigBundleItem struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + BundleID int `db:"bundle_id" json:"bundle_id"` + ResourceID int `db:"resource_id" json:"resource_id"` + SortOrder int `db:"sort_order" json:"sort_order"` + Required bool `db:"required" json:"required"` +} + +func (i OpenClawConfigBundleItem) TableName() string { + return "openclaw_config_bundle_items" +} + +// OpenClawConfigBundleSkill stores an uploaded skill membership inside a bundle. +type OpenClawConfigBundleSkill struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + BundleID int `db:"bundle_id" json:"bundle_id"` + SkillID int `db:"skill_id" json:"skill_id"` + SortOrder int `db:"sort_order" json:"sort_order"` + Required bool `db:"required" json:"required"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +func (i OpenClawConfigBundleSkill) TableName() string { + return "openclaw_config_bundle_skills" +} + +// OpenClawInjectionSnapshot stores the rendered bootstrap payload used by an instance. +type OpenClawInjectionSnapshot struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + UserID int `db:"user_id" json:"user_id"` + Mode string `db:"mode" json:"mode"` + BundleID *int `db:"bundle_id" json:"bundle_id,omitempty"` + SelectedResourceIDsJSON string `db:"selected_resource_ids_json" json:"-"` + ResolvedResourcesJSON string `db:"resolved_resources_json" json:"-"` + RenderedManifestJSON string `db:"rendered_manifest_json" json:"-"` + RenderedEnvJSON string `db:"rendered_env_json" json:"-"` + SecretName *string `db:"secret_name" json:"secret_name,omitempty"` + Status string `db:"status" json:"status"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + ActivatedAt *time.Time `db:"activated_at" json:"activated_at,omitempty"` +} + +func (s OpenClawInjectionSnapshot) TableName() string { + return "openclaw_injection_snapshots" +} diff --git a/backend/internal/models/persistent_volume.go b/backend/internal/models/persistent_volume.go new file mode 100644 index 0000000..d611d01 --- /dev/null +++ b/backend/internal/models/persistent_volume.go @@ -0,0 +1,24 @@ +package models + +import ( + "time" +) + +// PersistentVolume represents a persistent storage volume +type PersistentVolume struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + PVCName string `db:"pvc_name" json:"pvc_name"` + PVCNamespace string `db:"pvc_namespace" json:"pvc_namespace"` + StorageSizeGB int `db:"storage_size_gb" json:"storage_size_gb"` + StorageClass *string `db:"storage_class" json:"storage_class,omitempty"` + MountPath *string `db:"mount_path" json:"mount_path,omitempty"` + Status string `db:"status" json:"status"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +// TableName returns the table name for the PersistentVolume model +func (p PersistentVolume) TableName() string { + return "persistent_volumes" +} diff --git a/backend/internal/models/risk_hit.go b/backend/internal/models/risk_hit.go new file mode 100644 index 0000000..3bdc613 --- /dev/null +++ b/backend/internal/models/risk_hit.go @@ -0,0 +1,29 @@ +package models + +import "time" + +// RiskHit stores a normalized sensitive-data detection result. +type RiskHit struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TraceID string `db:"trace_id" json:"trace_id"` + SessionID *string `db:"session_id" json:"session_id,omitempty"` + RequestID *string `db:"request_id" json:"request_id,omitempty"` + UserID *int `db:"user_id" json:"user_id,omitempty"` + InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"` + RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"` + GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"` + RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"` + InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"` + RuleID string `db:"rule_id" json:"rule_id"` + RuleName string `db:"rule_name" json:"rule_name"` + Severity string `db:"severity" json:"severity"` + Action string `db:"action" json:"action"` + MatchSummary string `db:"match_summary" json:"match_summary"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +// TableName returns the table name for risk hits. +func (r RiskHit) TableName() string { + return "risk_hits" +} diff --git a/backend/internal/models/risk_rule.go b/backend/internal/models/risk_rule.go new file mode 100644 index 0000000..c4f2aa6 --- /dev/null +++ b/backend/internal/models/risk_rule.go @@ -0,0 +1,23 @@ +package models + +import "time" + +// RiskRule stores an admin-managed sensitive detection rule. +type RiskRule struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + RuleID string `db:"rule_id" json:"rule_id"` + DisplayName string `db:"display_name" json:"display_name"` + Description *string `db:"description" json:"description,omitempty"` + Pattern string `db:"pattern" json:"pattern"` + Severity string `db:"severity" json:"severity"` + Action string `db:"action" json:"action"` + IsEnabled bool `db:"is_enabled" json:"is_enabled"` + SortOrder int `db:"sort_order" json:"sort_order"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +// TableName returns the table name for risk rules. +func (r RiskRule) TableName() string { + return "risk_rules" +} diff --git a/backend/internal/models/runtime_pod.go b/backend/internal/models/runtime_pod.go new file mode 100644 index 0000000..69c90ef --- /dev/null +++ b/backend/internal/models/runtime_pod.go @@ -0,0 +1,33 @@ +package models + +import "time" + +type RuntimePod struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + Namespace string `db:"namespace" json:"namespace"` + PodName string `db:"pod_name" json:"pod_name"` + PodUID *string `db:"pod_uid" json:"pod_uid,omitempty"` + PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"` + NodeName *string `db:"node_name" json:"node_name,omitempty"` + DeploymentName string `db:"deployment_name" json:"deployment_name"` + ImageRef string `db:"image_ref" json:"image_ref"` + AgentEndpoint *string `db:"agent_endpoint" json:"agent_endpoint,omitempty"` + State string `db:"state" json:"state"` + Capacity int `db:"capacity" json:"capacity"` + UsedSlots int `db:"used_slots" json:"used_slots"` + Draining bool `db:"draining" json:"draining"` + CPUMillisUsed int64 `db:"cpu_millis_used" json:"cpu_millis_used"` + MemoryBytesUsed int64 `db:"memory_bytes_used" json:"memory_bytes_used"` + DiskBytesUsed int64 `db:"disk_bytes_used" json:"disk_bytes_used"` + NetworkRXBytes int64 `db:"network_rx_bytes" json:"network_rx_bytes"` + NetworkTXBytes int64 `db:"network_tx_bytes" json:"network_tx_bytes"` + MetricsJSON *string `db:"metrics_json" json:"-"` + LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (RuntimePod) TableName() string { + return "runtime_pods" +} diff --git a/backend/internal/models/runtime_rollout.go b/backend/internal/models/runtime_rollout.go new file mode 100644 index 0000000..5b7502e --- /dev/null +++ b/backend/internal/models/runtime_rollout.go @@ -0,0 +1,22 @@ +package models + +import "time" + +type RuntimeRollout struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + TargetImageRef string `db:"target_image_ref" json:"target_image_ref"` + Status string `db:"status" json:"status"` + BatchSize int `db:"batch_size" json:"batch_size"` + MaxUnavailable int `db:"max_unavailable" json:"max_unavailable"` + StartedBy *int `db:"started_by" json:"started_by,omitempty"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (RuntimeRollout) TableName() string { + return "runtime_rollouts" +} diff --git a/backend/internal/models/security_scan.go b/backend/internal/models/security_scan.go new file mode 100644 index 0000000..9118f0d --- /dev/null +++ b/backend/internal/models/security_scan.go @@ -0,0 +1,68 @@ +package models + +import "time" + +type SecurityScanConfig struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + DefaultMode string `db:"default_mode" json:"default_mode"` + QuickAnalyzersJSON string `db:"quick_analyzers_json" json:"-"` + DeepAnalyzersJSON string `db:"deep_analyzers_json" json:"-"` + QuickTimeoutSeconds int `db:"quick_timeout_seconds" json:"quick_timeout_seconds"` + DeepTimeoutSeconds int `db:"deep_timeout_seconds" json:"deep_timeout_seconds"` + AllowFallback bool `db:"allow_fallback" json:"allow_fallback"` + UpdatedBy *int `db:"updated_by" json:"updated_by,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (s SecurityScanConfig) TableName() string { return "security_scan_configs" } + +type SecurityScanJob struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + AssetType string `db:"asset_type" json:"asset_type"` + ScanMode string `db:"scan_mode" json:"scan_mode"` + Status string `db:"status" json:"status"` + RequestedBy *int `db:"requested_by" json:"requested_by,omitempty"` + ScopeJSON *string `db:"scope_json" json:"-"` + TotalItems int `db:"total_items" json:"total_items"` + CompletedItems int `db:"completed_items" json:"completed_items"` + FailedItems int `db:"failed_items" json:"failed_items"` + CurrentItemName *string `db:"current_item_name" json:"current_item_name,omitempty"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (s SecurityScanJob) TableName() string { return "security_scan_jobs" } + +type SecurityScanJobItem struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + JobID int `db:"job_id" json:"job_id"` + AssetType string `db:"asset_type" json:"asset_type"` + AssetID int `db:"asset_id" json:"asset_id"` + AssetName string `db:"asset_name" json:"asset_name"` + Status string `db:"status" json:"status"` + ProgressPct int `db:"progress_pct" json:"progress_pct"` + RiskLevel *string `db:"risk_level" json:"risk_level,omitempty"` + Summary *string `db:"summary" json:"summary,omitempty"` + ScanResultID *int `db:"scan_result_id" json:"scan_result_id,omitempty"` + CachedResult bool `db:"cached_result" json:"cached_result"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (s SecurityScanJobItem) TableName() string { return "security_scan_job_items" } + +type SecurityScanReport struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + JobID int `db:"job_id" json:"job_id"` + SummaryJSON string `db:"summary_json" json:"-"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (s SecurityScanReport) TableName() string { return "security_scan_reports" } diff --git a/backend/internal/models/skill.go b/backend/internal/models/skill.go new file mode 100644 index 0000000..a17fbdf --- /dev/null +++ b/backend/internal/models/skill.go @@ -0,0 +1,84 @@ +package models + +import "time" + +type Skill struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + UserID int `db:"user_id" json:"user_id"` + SkillKey string `db:"skill_key" json:"skill_key"` + Name string `db:"name" json:"name"` + Description *string `db:"description" json:"description,omitempty"` + CurrentVersionID *int `db:"current_version_id" json:"current_version_id,omitempty"` + SourceType string `db:"source_type" json:"source_type"` + Status string `db:"status" json:"status"` + RiskLevel string `db:"risk_level" json:"risk_level"` + LastScannedAt *time.Time `db:"last_scanned_at" json:"last_scanned_at,omitempty"` + LastScanResultID *int `db:"last_scan_result_id" json:"last_scan_result_id,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (s Skill) TableName() string { return "skills" } + +type SkillBlob struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + ContentHash string `db:"content_hash" json:"content_hash"` + ArchiveHash string `db:"archive_hash" json:"archive_hash"` + ObjectKey string `db:"object_key" json:"object_key"` + FileName string `db:"file_name" json:"file_name"` + MediaType string `db:"media_type" json:"media_type"` + SizeBytes int64 `db:"size_bytes" json:"size_bytes"` + ScanStatus string `db:"scan_status" json:"scan_status"` + RiskLevel string `db:"risk_level" json:"risk_level"` + LastScannedAt *time.Time `db:"last_scanned_at" json:"last_scanned_at,omitempty"` + LastScanResultID *int `db:"last_scan_result_id" json:"last_scan_result_id,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (s SkillBlob) TableName() string { return "skill_blobs" } + +type SkillVersion struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + SkillID int `db:"skill_id" json:"skill_id"` + BlobID int `db:"blob_id" json:"blob_id"` + VersionNo int `db:"version_no" json:"version_no"` + ManifestJSON *string `db:"manifest_json" json:"-"` + SourceType string `db:"source_type" json:"source_type"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (s SkillVersion) TableName() string { return "skill_versions" } + +type InstanceSkill struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + SkillID int `db:"skill_id" json:"skill_id"` + SkillVersionID *int `db:"skill_version_id" json:"skill_version_id,omitempty"` + SourceType string `db:"source_type" json:"source_type"` + InstallPath *string `db:"install_path" json:"install_path,omitempty"` + ObservedHash *string `db:"observed_hash" json:"observed_hash,omitempty"` + Status string `db:"status" json:"status"` + LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"` + RemovedAt *time.Time `db:"removed_at" json:"removed_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (s InstanceSkill) TableName() string { return "instance_skills" } + +type SkillScanResult struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + BlobID int `db:"blob_id" json:"blob_id"` + Engine string `db:"engine" json:"engine"` + RiskLevel string `db:"risk_level" json:"risk_level"` + Status string `db:"status" json:"status"` + Summary *string `db:"summary" json:"summary,omitempty"` + FindingsJSON *string `db:"findings_json" json:"-"` + ScannedAt *time.Time `db:"scanned_at" json:"scanned_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (s SkillScanResult) TableName() string { return "skill_scan_results" } diff --git a/backend/internal/models/system_image_setting.go b/backend/internal/models/system_image_setting.go new file mode 100644 index 0000000..4242776 --- /dev/null +++ b/backend/internal/models/system_image_setting.go @@ -0,0 +1,20 @@ +package models + +import "time" + +// SystemImageSetting stores admin-managed runtime image overrides for instance types. +type SystemImageSetting struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceType string `db:"instance_type" json:"instance_type"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + DisplayName string `db:"display_name" json:"display_name"` + Image string `db:"image" json:"image"` + IsEnabled bool `db:"is_enabled" json:"is_enabled"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +// TableName returns the table name for the SystemImageSetting model. +func (s SystemImageSetting) TableName() string { + return "system_image_settings" +} diff --git a/backend/internal/models/team.go b/backend/internal/models/team.go new file mode 100644 index 0000000..2051525 --- /dev/null +++ b/backend/internal/models/team.go @@ -0,0 +1,204 @@ +package models + +import "time" + +const ( + TeamStatusCreating = "creating" + TeamStatusRunning = "running" + TeamStatusFailed = "failed" + + TeamMemberStatusCreating = "creating" + TeamMemberStatusIdle = "idle" + TeamMemberStatusBusy = "busy" + TeamMemberStatusFailed = "failed" + TeamMemberStatusOffline = "offline" + TeamMemberStatusDeleting = "deleting" + TeamMemberStatusDeleted = "deleted" + + TeamMemberAvailabilityUnknown = "unknown" + TeamMemberAvailabilityIdle = "idle" + TeamMemberAvailabilityBusy = "busy" + TeamMemberAvailabilityBlocked = "blocked" + TeamMemberAvailabilityOffline = "offline" + + TeamStatusDeleting = "deleting" + TeamStatusDeleted = "deleted" + + TeamTaskStatusPending = "pending" + TeamTaskStatusDispatched = "dispatched" + TeamTaskStatusRunning = "running" + TeamTaskStatusSucceeded = "succeeded" + TeamTaskStatusFailed = "failed" + TeamTaskStatusStale = "stale" +) + +type Team struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + UserID int `db:"user_id" json:"user_id"` + Name string `db:"name" json:"name"` + Description *string `db:"description" json:"description,omitempty"` + Status string `db:"status" json:"status"` + CommunicationMode string `db:"communication_mode" json:"communication_mode"` + RedisURLSecretName *string `db:"redis_url_secret_name" json:"-"` + RedisURLSecretKey *string `db:"redis_url_secret_key" json:"-"` + TeamTokenSecretName *string `db:"team_token_secret_name" json:"-"` + TeamTokenSecretKey *string `db:"team_token_secret_key" json:"-"` + RedisEventsLastID string `db:"redis_events_last_id" json:"redis_events_last_id"` + SharedPVCName *string `db:"shared_pvc_name" json:"shared_pvc_name,omitempty"` + SharedPVCNamespace *string `db:"shared_pvc_namespace" json:"shared_pvc_namespace,omitempty"` + SharedMountPath string `db:"shared_mount_path" json:"shared_mount_path"` + StorageClass *string `db:"storage_class" json:"storage_class,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (Team) TableName() string { + return "teams" +} + +type TeamMember struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TeamID int `db:"team_id" json:"team_id"` + InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + UserID int `db:"user_id" json:"user_id"` + MemberKey string `db:"member_key" json:"member_key"` + DisplayName string `db:"display_name" json:"display_name"` + Role string `db:"role" json:"role"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + InstanceMode string `db:"instance_mode" json:"instance_mode"` + Description *string `db:"description" json:"description,omitempty"` + Status string `db:"status" json:"status"` + CurrentTaskID *int `db:"current_task_id" json:"current_task_id,omitempty"` + Progress int `db:"progress" json:"progress"` + LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"` + Availability string `db:"availability" json:"availability"` + RuntimeStatus *string `db:"runtime_status" json:"runtime_status,omitempty"` + RuntimeTaskID *string `db:"runtime_task_id" json:"runtime_task_id,omitempty"` + RuntimeIntent *string `db:"runtime_intent" json:"runtime_intent,omitempty"` + BlockedReason *string `db:"blocked_reason" json:"blocked_reason,omitempty"` + LastSummary *string `db:"last_summary" json:"last_summary,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (TeamMember) TableName() string { + return "team_members" +} + +type TeamTask struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TeamID int `db:"team_id" json:"team_id"` + TargetMemberID int `db:"target_member_id" json:"target_member_id"` + CreatedBy *int `db:"created_by" json:"created_by,omitempty"` + MessageID string `db:"message_id" json:"message_id"` + Status string `db:"status" json:"status"` + WorkflowState string `db:"workflow_state" json:"workflow_state"` + PlanVersion int64 `db:"plan_version" json:"plan_version"` + LedgerVersion int64 `db:"ledger_version" json:"ledger_version"` + CurrentPhaseID *string `db:"current_phase_id" json:"current_phase_id,omitempty"` + AcceptedCompletionID *string `db:"accepted_completion_id" json:"accepted_completion_id,omitempty"` + PayloadJSON string `db:"payload_json" json:"-"` + ResultJSON *string `db:"result_json" json:"-"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + RedisStreamID *string `db:"redis_stream_id" json:"redis_stream_id,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + DispatchedAt *time.Time `db:"dispatched_at" json:"dispatched_at,omitempty"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (TeamTask) TableName() string { + return "team_tasks" +} + +type TeamEvent struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + EventID *string `db:"event_id" json:"event_id,omitempty"` + CompletionID *string `db:"completion_id" json:"completion_id,omitempty"` + SequenceNo int64 `db:"sequence_no" json:"sequence_no"` + TeamID int `db:"team_id" json:"team_id"` + MemberID *int `db:"member_id" json:"member_id,omitempty"` + TaskID *int `db:"task_id" json:"task_id,omitempty"` + MessageID *string `db:"message_id" json:"message_id,omitempty"` + EventType string `db:"event_type" json:"event_type"` + PayloadJSON *string `db:"payload_json" json:"-"` + RedisStreamID *string `db:"redis_stream_id" json:"redis_stream_id,omitempty"` + OccurredAt *time.Time `db:"occurred_at" json:"occurred_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +func (TeamEvent) TableName() string { + return "team_events" +} + +type TeamWorkItem struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TeamID int `db:"team_id" json:"team_id"` + RootTaskID int `db:"root_task_id" json:"root_task_id"` + WorkID string `db:"work_id" json:"work_id"` + AssignmentID *string `db:"assignment_id" json:"assignment_id,omitempty"` + CanonicalWorkID *string `db:"canonical_work_id" json:"canonical_work_id,omitempty"` + PhaseID *string `db:"phase_id" json:"phase_id,omitempty"` + Revision int `db:"revision" json:"revision"` + RequiredForRoot bool `db:"required_for_root" json:"required_for_root"` + SupersededBy *string `db:"superseded_by" json:"superseded_by,omitempty"` + ReviewRequired bool `db:"review_required" json:"review_required"` + ValidatedRevision *int `db:"validated_revision" json:"validated_revision,omitempty"` + OwnerMemberID *int `db:"owner_member_id" json:"owner_member_id,omitempty"` + Title string `db:"title" json:"title"` + Status string `db:"status" json:"status"` + DependsOnJSON *string `db:"depends_on_json" json:"-"` + ResultJSON *string `db:"result_json" json:"-"` + ArtifactRefsJSON *string `db:"artifact_refs_json" json:"-"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +type TeamWorkflowPhase struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TeamID int `db:"team_id" json:"team_id"` + RootTaskID int `db:"root_task_id" json:"root_task_id"` + PhaseID string `db:"phase_id" json:"phase_id"` + PlanVersion int64 `db:"plan_version" json:"plan_version"` + SequenceNo int `db:"sequence_no" json:"sequence_no"` + Status string `db:"status" json:"status"` + RequiredForRoot bool `db:"required_for_root" json:"required_for_root"` + DecisionRequired bool `db:"decision_required" json:"decision_required"` + DependsOnJSON *string `db:"depends_on_json" json:"-"` + NextPhaseID *string `db:"next_phase_id" json:"next_phase_id,omitempty"` + CompletionPolicy *string `db:"completion_policy" json:"completion_policy,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + CompletedAt *time.Time `db:"completed_at" json:"completed_at,omitempty"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (TeamWorkflowPhase) TableName() string { + return "team_workflow_phases" +} + +type TeamEventOutbox struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TeamID int `db:"team_id" json:"team_id"` + SourceEventID string `db:"source_event_id" json:"source_event_id"` + Destination string `db:"destination" json:"destination"` + MessageID string `db:"message_id" json:"message_id"` + PayloadJSON string `db:"payload_json" json:"-"` + Status string `db:"status" json:"status"` + Attempts int `db:"attempts" json:"attempts"` + AvailableAt time.Time `db:"available_at" json:"available_at"` + LastError *string `db:"last_error" json:"last_error,omitempty"` + DeliveredAt *time.Time `db:"delivered_at" json:"delivered_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (TeamEventOutbox) TableName() string { + return "team_event_outbox" +} + +func (TeamWorkItem) TableName() string { + return "team_work_items" +} diff --git a/backend/internal/models/user.go b/backend/internal/models/user.go new file mode 100644 index 0000000..c0d9c0d --- /dev/null +++ b/backend/internal/models/user.go @@ -0,0 +1,23 @@ +package models + +import ( + "time" +) + +// User represents a user in the system +type User struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + Username string `db:"username" json:"username"` + Email string `db:"email" json:"email"` + PasswordHash string `db:"password_hash" json:"-"` + Role string `db:"role" json:"role"` + IsActive bool `db:"is_active" json:"is_active"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + LastLogin *time.Time `db:"last_login" json:"last_login,omitempty"` +} + +// TableName returns the table name for the User model +func (u User) TableName() string { + return "users" +} diff --git a/backend/internal/models/user_quota.go b/backend/internal/models/user_quota.go new file mode 100644 index 0000000..1e64594 --- /dev/null +++ b/backend/internal/models/user_quota.go @@ -0,0 +1,23 @@ +package models + +import ( + "time" +) + +// UserQuota represents resource quota for a user +type UserQuota struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + UserID int `db:"user_id" json:"user_id"` + MaxInstances int `db:"max_instances" json:"max_instances"` + MaxCPUCores float64 `db:"max_cpu_cores" json:"max_cpu_cores"` + MaxMemoryGB int `db:"max_memory_gb" json:"max_memory_gb"` + MaxStorageGB int `db:"max_storage_gb" json:"max_storage_gb"` + MaxGPUCount int `db:"max_gpu_count" json:"max_gpu_count"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +// TableName returns the table name for the UserQuota model +func (u UserQuota) TableName() string { + return "user_quotas" +} diff --git a/backend/internal/models/workspace_file_audit.go b/backend/internal/models/workspace_file_audit.go new file mode 100644 index 0000000..eb28539 --- /dev/null +++ b/backend/internal/models/workspace_file_audit.go @@ -0,0 +1,17 @@ +package models + +import "time" + +type WorkspaceFileAudit struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + UserID int `db:"user_id" json:"user_id"` + Action string `db:"action" json:"action"` + RelativePath string `db:"relative_path" json:"relative_path"` + Bytes int64 `db:"bytes" json:"bytes"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +func (WorkspaceFileAudit) TableName() string { + return "workspace_file_audits" +} diff --git a/backend/internal/repository/audit_event_repository.go b/backend/internal/repository/audit_event_repository.go new file mode 100644 index 0000000..377a364 --- /dev/null +++ b/backend/internal/repository/audit_event_repository.go @@ -0,0 +1,94 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +// AuditEventRepository defines repository operations for governance audit events. +type AuditEventRepository interface { + Create(event *models.AuditEvent) error + ListByTraceID(traceID string) ([]models.AuditEvent, error) + ListRecent(limit int) ([]models.AuditEvent, error) +} + +type auditEventRepository struct { + sess db.Session +} + +// NewAuditEventRepository creates a new audit event repository and ensures its table exists. +func NewAuditEventRepository(sess db.Session) AuditEventRepository { + repo := &auditEventRepository{sess: sess} + repo.ensureTable() + return repo +} + +func (r *auditEventRepository) ensureTable() { + const query = ` +CREATE TABLE IF NOT EXISTS audit_events ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + invocation_id INT NULL, + event_type VARCHAR(100) NOT NULL, + traffic_class VARCHAR(50) NOT NULL, + severity VARCHAR(20) NOT NULL, + message VARCHAR(500) NOT NULL, + details LONGTEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_audit_events_trace_id (trace_id), + INDEX idx_audit_events_request_id (request_id), + INDEX idx_audit_events_user_id (user_id), + INDEX idx_audit_events_gateway_id (gateway_id), + INDEX idx_audit_events_invocation_id (invocation_id), + INDEX idx_audit_events_event_type (event_type), + INDEX idx_audit_events_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +` + + if _, err := r.sess.SQL().Exec(query); err != nil { + panic(fmt.Errorf("failed to ensure audit_events table: %w", err)) + } +} + +func (r *auditEventRepository) Create(event *models.AuditEvent) error { + if event.CreatedAt.IsZero() { + event.CreatedAt = time.Now() + } + res, err := r.sess.Collection("audit_events").Insert(event) + if err != nil { + return fmt.Errorf("failed to create audit event: %w", err) + } + event.ID = int(res.ID().(int64)) + return nil +} + +func (r *auditEventRepository) ListByTraceID(traceID string) ([]models.AuditEvent, error) { + var items []models.AuditEvent + if err := r.sess.Collection("audit_events").Find(db.Cond{"trace_id": traceID}).OrderBy("id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list audit events by trace: %w", err) + } + return items, nil +} + +func (r *auditEventRepository) ListRecent(limit int) ([]models.AuditEvent, error) { + var items []models.AuditEvent + if limit <= 0 { + limit = 100 + } + if err := r.sess.Collection("audit_events").Find().OrderBy("-created_at").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list recent audit events: %w", err) + } + return items, nil +} diff --git a/backend/internal/repository/chat_message_repository.go b/backend/internal/repository/chat_message_repository.go new file mode 100644 index 0000000..d814ae8 --- /dev/null +++ b/backend/internal/repository/chat_message_repository.go @@ -0,0 +1,74 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +// ChatMessageRepository defines repository operations for persisted chat messages. +type ChatMessageRepository interface { + Create(message *models.ChatMessageRecord) error + ListByTraceID(traceID string) ([]models.ChatMessageRecord, error) +} + +type chatMessageRepository struct { + sess db.Session +} + +// NewChatMessageRepository creates a new chat message repository and ensures its table exists. +func NewChatMessageRepository(sess db.Session) ChatMessageRepository { + repo := &chatMessageRepository{sess: sess} + repo.ensureTable() + return repo +} + +func (r *chatMessageRepository) ensureTable() { + const query = ` +CREATE TABLE IF NOT EXISTS chat_messages ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NOT NULL, + request_id VARCHAR(100) NULL, + user_id INT NULL, + instance_id INT NULL, + invocation_id INT NULL, + role VARCHAR(50) NOT NULL, + content LONGTEXT NOT NULL, + sequence_no INT NOT NULL DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_chat_messages_trace_id (trace_id), + INDEX idx_chat_messages_session_id (session_id), + INDEX idx_chat_messages_request_id (request_id), + INDEX idx_chat_messages_user_id (user_id), + INDEX idx_chat_messages_sequence_no (sequence_no) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +` + + if _, err := r.sess.SQL().Exec(query); err != nil { + panic(fmt.Errorf("failed to ensure chat_messages table: %w", err)) + } +} + +func (r *chatMessageRepository) Create(message *models.ChatMessageRecord) error { + if message.CreatedAt.IsZero() { + message.CreatedAt = time.Now() + } + res, err := r.sess.Collection("chat_messages").Insert(message) + if err != nil { + return fmt.Errorf("failed to create chat message: %w", err) + } + message.ID = int(res.ID().(int64)) + return nil +} + +func (r *chatMessageRepository) ListByTraceID(traceID string) ([]models.ChatMessageRecord, error) { + var items []models.ChatMessageRecord + if err := r.sess.Collection("chat_messages").Find(db.Cond{"trace_id": traceID}).OrderBy("sequence_no", "id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list chat messages by trace: %w", err) + } + return items, nil +} diff --git a/backend/internal/repository/chat_session_repository.go b/backend/internal/repository/chat_session_repository.go new file mode 100644 index 0000000..d48b7fc --- /dev/null +++ b/backend/internal/repository/chat_session_repository.go @@ -0,0 +1,99 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +// ChatSessionRepository defines repository operations for chat sessions. +type ChatSessionRepository interface { + GetBySessionID(sessionID string) (*models.ChatSession, error) + Save(session *models.ChatSession) error +} + +type chatSessionRepository struct { + sess db.Session +} + +// NewChatSessionRepository creates a new chat session repository and ensures its table exists. +func NewChatSessionRepository(sess db.Session) ChatSessionRepository { + repo := &chatSessionRepository{sess: sess} + repo.ensureTable() + return repo +} + +func (r *chatSessionRepository) ensureTable() { + const query = ` +CREATE TABLE IF NOT EXISTS chat_sessions ( + id INT AUTO_INCREMENT PRIMARY KEY, + session_id VARCHAR(100) NOT NULL UNIQUE, + user_id INT NULL, + instance_id INT NULL, + title VARCHAR(255) NULL, + last_trace_id VARCHAR(100) NULL, + started_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + last_activity_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_chat_sessions_user_id (user_id), + INDEX idx_chat_sessions_instance_id (instance_id), + INDEX idx_chat_sessions_last_activity_at (last_activity_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +` + + if _, err := r.sess.SQL().Exec(query); err != nil { + panic(fmt.Errorf("failed to ensure chat_sessions table: %w", err)) + } +} + +func (r *chatSessionRepository) GetBySessionID(sessionID string) (*models.ChatSession, error) { + var item models.ChatSession + err := r.sess.Collection("chat_sessions").Find(db.Cond{"session_id": sessionID}).One(&item) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get chat session by session id: %w", err) + } + return &item, nil +} + +func (r *chatSessionRepository) Save(session *models.ChatSession) error { + now := time.Now() + existing, err := r.GetBySessionID(session.SessionID) + if err != nil { + return err + } + + if existing == nil { + if session.StartedAt.IsZero() { + session.StartedAt = now + } + if session.LastActivityAt.IsZero() { + session.LastActivityAt = now + } + res, err := r.sess.Collection("chat_sessions").Insert(session) + if err != nil { + return fmt.Errorf("failed to create chat session: %w", err) + } + session.ID = int(res.ID().(int64)) + return nil + } + + existing.UserID = session.UserID + existing.InstanceID = session.InstanceID + if session.Title != nil { + existing.Title = session.Title + } + existing.LastTraceID = session.LastTraceID + existing.LastActivityAt = now + if err := r.sess.Collection("chat_sessions").Find(db.Cond{"id": existing.ID}).Update(existing); err != nil { + return fmt.Errorf("failed to update chat session: %w", err) + } + session.ID = existing.ID + session.StartedAt = existing.StartedAt + session.LastActivityAt = existing.LastActivityAt + return nil +} diff --git a/backend/internal/repository/cost_record_repository.go b/backend/internal/repository/cost_record_repository.go new file mode 100644 index 0000000..51bb713 --- /dev/null +++ b/backend/internal/repository/cost_record_repository.go @@ -0,0 +1,112 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +// CostRecordRepository defines repository operations for token and money accounting. +type CostRecordRepository interface { + Create(record *models.CostRecord) error + ListByTraceID(traceID string) ([]models.CostRecord, error) + ListByUserID(userID, limit int) ([]models.CostRecord, error) + ListRecent(limit int) ([]models.CostRecord, error) +} + +type costRecordRepository struct { + sess db.Session +} + +// NewCostRecordRepository creates a new cost record repository and ensures its table exists. +func NewCostRecordRepository(sess db.Session) CostRecordRepository { + repo := &costRecordRepository{sess: sess} + repo.ensureTable() + return repo +} + +func (r *costRecordRepository) ensureTable() { + const query = ` +CREATE TABLE IF NOT EXISTS cost_records ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + invocation_id INT NULL, + model_id INT NULL, + provider_type VARCHAR(100) NOT NULL, + model_name VARCHAR(255) NOT NULL, + currency VARCHAR(16) NOT NULL DEFAULT 'USD', + prompt_tokens INT NOT NULL DEFAULT 0, + completion_tokens INT NOT NULL DEFAULT 0, + total_tokens INT NOT NULL DEFAULT 0, + input_unit_price DECIMAL(18,8) NOT NULL DEFAULT 0, + output_unit_price DECIMAL(18,8) NOT NULL DEFAULT 0, + estimated_cost DECIMAL(18,8) NOT NULL DEFAULT 0, + actual_cost DECIMAL(18,8) NULL, + internal_cost DECIMAL(18,8) NOT NULL DEFAULT 0, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_cost_records_trace_id (trace_id), + INDEX idx_cost_records_user_id (user_id), + INDEX idx_cost_records_gateway_id (gateway_id), + INDEX idx_cost_records_model_id (model_id), + INDEX idx_cost_records_provider_type (provider_type), + INDEX idx_cost_records_recorded_at (recorded_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +` + + if _, err := r.sess.SQL().Exec(query); err != nil { + panic(fmt.Errorf("failed to ensure cost_records table: %w", err)) + } +} + +func (r *costRecordRepository) Create(record *models.CostRecord) error { + if record.RecordedAt.IsZero() { + record.RecordedAt = time.Now() + } + res, err := r.sess.Collection("cost_records").Insert(record) + if err != nil { + return fmt.Errorf("failed to create cost record: %w", err) + } + record.ID = int(res.ID().(int64)) + return nil +} + +func (r *costRecordRepository) ListByTraceID(traceID string) ([]models.CostRecord, error) { + var items []models.CostRecord + if err := r.sess.Collection("cost_records").Find(db.Cond{"trace_id": traceID}).OrderBy("id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list cost records by trace: %w", err) + } + return items, nil +} + +func (r *costRecordRepository) ListByUserID(userID, limit int) ([]models.CostRecord, error) { + var items []models.CostRecord + if limit <= 0 { + limit = 50 + } + if err := r.sess.Collection("cost_records").Find(db.Cond{"user_id": userID}).OrderBy("-recorded_at").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list cost records by user: %w", err) + } + return items, nil +} + +func (r *costRecordRepository) ListRecent(limit int) ([]models.CostRecord, error) { + var items []models.CostRecord + if limit <= 0 { + limit = 100 + } + if err := r.sess.Collection("cost_records").Find().OrderBy("-recorded_at").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list recent cost records: %w", err) + } + return items, nil +} diff --git a/backend/internal/repository/instance_agent_repository.go b/backend/internal/repository/instance_agent_repository.go new file mode 100644 index 0000000..b1fffbe --- /dev/null +++ b/backend/internal/repository/instance_agent_repository.go @@ -0,0 +1,69 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type InstanceAgentRepository interface { + GetByInstanceID(instanceID int) (*models.InstanceAgent, error) + GetBySessionToken(sessionToken string) (*models.InstanceAgent, error) + Create(agent *models.InstanceAgent) error + Update(agent *models.InstanceAgent) error +} + +type instanceAgentRepository struct { + sess db.Session +} + +func NewInstanceAgentRepository(sess db.Session) InstanceAgentRepository { + return &instanceAgentRepository{sess: sess} +} + +func (r *instanceAgentRepository) GetByInstanceID(instanceID int) (*models.InstanceAgent, error) { + var item models.InstanceAgent + if err := r.sess.Collection("instance_agents").Find(db.Cond{"instance_id": instanceID}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance agent: %w", err) + } + return &item, nil +} + +func (r *instanceAgentRepository) GetBySessionToken(sessionToken string) (*models.InstanceAgent, error) { + var item models.InstanceAgent + if err := r.sess.Collection("instance_agents").Find(db.Cond{"session_token": sessionToken}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance agent by session token: %w", err) + } + return &item, nil +} + +func (r *instanceAgentRepository) Create(agent *models.InstanceAgent) error { + ensureTimestamps(&agent.CreatedAt, &agent.UpdatedAt) + res, err := r.sess.Collection("instance_agents").Insert(agent) + if err != nil { + return fmt.Errorf("failed to create instance agent: %w", err) + } + if id, ok := res.ID().(int64); ok { + agent.ID = int(id) + } + return nil +} + +func (r *instanceAgentRepository) Update(agent *models.InstanceAgent) error { + if agent.UpdatedAt.IsZero() { + agent.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("instance_agents").Find(db.Cond{"id": agent.ID}).Update(agent); err != nil { + return fmt.Errorf("failed to update instance agent: %w", err) + } + return nil +} diff --git a/backend/internal/repository/instance_command_repository.go b/backend/internal/repository/instance_command_repository.go new file mode 100644 index 0000000..29032f7 --- /dev/null +++ b/backend/internal/repository/instance_command_repository.go @@ -0,0 +1,97 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type InstanceCommandRepository interface { + Create(command *models.InstanceCommand) error + Update(command *models.InstanceCommand) error + GetByID(id int) (*models.InstanceCommand, error) + GetByInstanceIdempotencyKey(instanceID int, idempotencyKey string) (*models.InstanceCommand, error) + GetNextPendingByInstance(instanceID int) (*models.InstanceCommand, error) + ListByInstanceID(instanceID int, limit int) ([]models.InstanceCommand, error) +} + +type instanceCommandRepository struct { + sess db.Session +} + +func NewInstanceCommandRepository(sess db.Session) InstanceCommandRepository { + return &instanceCommandRepository{sess: sess} +} + +func (r *instanceCommandRepository) Create(command *models.InstanceCommand) error { + ensureTimestamps(&command.CreatedAt, &command.UpdatedAt) + if command.IssuedAt.IsZero() { + command.IssuedAt = time.Now().UTC() + } + res, err := r.sess.Collection("instance_commands").Insert(command) + if err != nil { + return fmt.Errorf("failed to create instance command: %w", err) + } + if id, ok := res.ID().(int64); ok { + command.ID = int(id) + } + return nil +} + +func (r *instanceCommandRepository) Update(command *models.InstanceCommand) error { + if command.UpdatedAt.IsZero() { + command.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("instance_commands").Find(db.Cond{"id": command.ID}).Update(command); err != nil { + return fmt.Errorf("failed to update instance command: %w", err) + } + return nil +} + +func (r *instanceCommandRepository) GetByID(id int) (*models.InstanceCommand, error) { + var item models.InstanceCommand + if err := r.sess.Collection("instance_commands").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance command: %w", err) + } + return &item, nil +} + +func (r *instanceCommandRepository) GetByInstanceIdempotencyKey(instanceID int, idempotencyKey string) (*models.InstanceCommand, error) { + var item models.InstanceCommand + if err := r.sess.Collection("instance_commands").Find(db.Cond{"instance_id": instanceID, "idempotency_key": idempotencyKey}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance command by idempotency key: %w", err) + } + return &item, nil +} + +func (r *instanceCommandRepository) GetNextPendingByInstance(instanceID int) (*models.InstanceCommand, error) { + var item models.InstanceCommand + if err := r.sess.Collection("instance_commands").Find(db.Cond{"instance_id": instanceID, "status": "pending"}).OrderBy("issued_at", "id").One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get next pending instance command: %w", err) + } + return &item, nil +} + +func (r *instanceCommandRepository) ListByInstanceID(instanceID int, limit int) ([]models.InstanceCommand, error) { + if limit <= 0 { + limit = 20 + } + + var items []models.InstanceCommand + if err := r.sess.Collection("instance_commands").Find(db.Cond{"instance_id": instanceID}).OrderBy("-issued_at", "-id").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list instance commands: %w", err) + } + return items, nil +} diff --git a/backend/internal/repository/instance_config_revision_repository.go b/backend/internal/repository/instance_config_revision_repository.go new file mode 100644 index 0000000..0d2b60f --- /dev/null +++ b/backend/internal/repository/instance_config_revision_repository.go @@ -0,0 +1,82 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type InstanceConfigRevisionRepository interface { + Create(revision *models.InstanceConfigRevision) error + Update(revision *models.InstanceConfigRevision) error + GetByID(id int) (*models.InstanceConfigRevision, error) + GetLatestByInstanceID(instanceID int) (*models.InstanceConfigRevision, error) + ListByInstanceID(instanceID int, limit int) ([]models.InstanceConfigRevision, error) +} + +type instanceConfigRevisionRepository struct { + sess db.Session +} + +func NewInstanceConfigRevisionRepository(sess db.Session) InstanceConfigRevisionRepository { + return &instanceConfigRevisionRepository{sess: sess} +} + +func (r *instanceConfigRevisionRepository) Create(revision *models.InstanceConfigRevision) error { + ensureTimestamps(&revision.CreatedAt, &revision.UpdatedAt) + res, err := r.sess.Collection("instance_config_revisions").Insert(revision) + if err != nil { + return fmt.Errorf("failed to create instance config revision: %w", err) + } + if id, ok := res.ID().(int64); ok { + revision.ID = int(id) + } + return nil +} + +func (r *instanceConfigRevisionRepository) Update(revision *models.InstanceConfigRevision) error { + if revision.UpdatedAt.IsZero() { + revision.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("instance_config_revisions").Find(db.Cond{"id": revision.ID}).Update(revision); err != nil { + return fmt.Errorf("failed to update instance config revision: %w", err) + } + return nil +} + +func (r *instanceConfigRevisionRepository) GetByID(id int) (*models.InstanceConfigRevision, error) { + var item models.InstanceConfigRevision + if err := r.sess.Collection("instance_config_revisions").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance config revision: %w", err) + } + return &item, nil +} + +func (r *instanceConfigRevisionRepository) GetLatestByInstanceID(instanceID int) (*models.InstanceConfigRevision, error) { + var item models.InstanceConfigRevision + if err := r.sess.Collection("instance_config_revisions").Find(db.Cond{"instance_id": instanceID}).OrderBy("-revision_no", "-id").One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get latest instance config revision: %w", err) + } + return &item, nil +} + +func (r *instanceConfigRevisionRepository) ListByInstanceID(instanceID int, limit int) ([]models.InstanceConfigRevision, error) { + if limit <= 0 { + limit = 20 + } + + var items []models.InstanceConfigRevision + if err := r.sess.Collection("instance_config_revisions").Find(db.Cond{"instance_id": instanceID}).OrderBy("-revision_no", "-id").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list instance config revisions: %w", err) + } + return items, nil +} diff --git a/backend/internal/repository/instance_desired_state_repository.go b/backend/internal/repository/instance_desired_state_repository.go new file mode 100644 index 0000000..e90928d --- /dev/null +++ b/backend/internal/repository/instance_desired_state_repository.go @@ -0,0 +1,57 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type InstanceDesiredStateRepository interface { + GetByInstanceID(instanceID int) (*models.InstanceDesiredState, error) + Create(state *models.InstanceDesiredState) error + Update(state *models.InstanceDesiredState) error +} + +type instanceDesiredStateRepository struct { + sess db.Session +} + +func NewInstanceDesiredStateRepository(sess db.Session) InstanceDesiredStateRepository { + return &instanceDesiredStateRepository{sess: sess} +} + +func (r *instanceDesiredStateRepository) GetByInstanceID(instanceID int) (*models.InstanceDesiredState, error) { + var item models.InstanceDesiredState + if err := r.sess.Collection("instance_desired_state").Find(db.Cond{"instance_id": instanceID}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance desired state: %w", err) + } + return &item, nil +} + +func (r *instanceDesiredStateRepository) Create(state *models.InstanceDesiredState) error { + ensureTimestamps(&state.CreatedAt, &state.UpdatedAt) + res, err := r.sess.Collection("instance_desired_state").Insert(state) + if err != nil { + return fmt.Errorf("failed to create instance desired state: %w", err) + } + if id, ok := res.ID().(int64); ok { + state.ID = int(id) + } + return nil +} + +func (r *instanceDesiredStateRepository) Update(state *models.InstanceDesiredState) error { + if state.UpdatedAt.IsZero() { + state.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("instance_desired_state").Find(db.Cond{"id": state.ID}).Update(state); err != nil { + return fmt.Errorf("failed to update instance desired state: %w", err) + } + return nil +} diff --git a/backend/internal/repository/instance_external_access_repository.go b/backend/internal/repository/instance_external_access_repository.go new file mode 100644 index 0000000..cd99504 --- /dev/null +++ b/backend/internal/repository/instance_external_access_repository.go @@ -0,0 +1,107 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "clawreef/internal/models" + "github.com/upper/db/v4" +) + +type InstanceExternalAccessRepository interface { + GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) + GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error) + Upsert(ctx context.Context, access *models.InstanceExternalAccess) error + Disable(ctx context.Context, instanceID int) error + MarkUsed(ctx context.Context, id int64) error +} + +type instanceExternalAccessRepository struct { + sess db.Session +} + +func NewInstanceExternalAccessRepository(sess db.Session) InstanceExternalAccessRepository { + return &instanceExternalAccessRepository{sess: sess} +} + +func (r *instanceExternalAccessRepository) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) { + _ = ctx + var access models.InstanceExternalAccess + if err := r.sess.Collection(access.TableName()).Find(db.Cond{"instance_id": instanceID}).One(&access); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance external access: %w", err) + } + return &access, nil +} + +func (r *instanceExternalAccessRepository) GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error) { + _ = ctx + var access models.InstanceExternalAccess + if err := r.sess.Collection(access.TableName()).Find(db.Cond{"short_code_hash": codeHash}).One(&access); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance external access by short code hash: %w", err) + } + return &access, nil +} + +func (r *instanceExternalAccessRepository) Upsert(ctx context.Context, access *models.InstanceExternalAccess) error { + now := time.Now().UTC() + if access.CreatedAt.IsZero() { + access.CreatedAt = now + } + access.UpdatedAt = now + _, err := r.sess.SQL().ExecContext(ctx, ` + INSERT INTO instance_external_access ( + instance_id, enabled, auth_mode, public_slug, short_code_hash, public_token_hash, + api_key_hash, password_value, api_key_prefix, expires_at, created_by, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + enabled = VALUES(enabled), + auth_mode = VALUES(auth_mode), + public_slug = VALUES(public_slug), + short_code_hash = VALUES(short_code_hash), + public_token_hash = VALUES(public_token_hash), + api_key_hash = VALUES(api_key_hash), + password_value = VALUES(password_value), + api_key_prefix = VALUES(api_key_prefix), + expires_at = VALUES(expires_at), + created_by = VALUES(created_by), + updated_at = VALUES(updated_at) + `, access.InstanceID, access.Enabled, access.AuthMode, access.PublicSlug, access.ShortCodeHash, access.PublicTokenHash, + access.PasswordHash, access.PasswordValue, access.PasswordHint, access.ExpiresAt, access.CreatedBy, + access.CreatedAt, access.UpdatedAt) + if err != nil { + return fmt.Errorf("failed to upsert instance external access: %w", err) + } + return nil +} + +func (r *instanceExternalAccessRepository) Disable(ctx context.Context, instanceID int) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instance_external_access + SET enabled = 0, updated_at = ? + WHERE instance_id = ? + `, time.Now().UTC(), instanceID) + if err != nil { + return fmt.Errorf("failed to disable instance external access: %w", err) + } + return nil +} + +func (r *instanceExternalAccessRepository) MarkUsed(ctx context.Context, id int64) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instance_external_access + SET last_used_at = ?, updated_at = ? + WHERE id = ? + `, time.Now().UTC(), time.Now().UTC(), id) + if err != nil { + return fmt.Errorf("failed to mark instance external access used: %w", err) + } + return nil +} diff --git a/backend/internal/repository/instance_repository.go b/backend/internal/repository/instance_repository.go new file mode 100644 index 0000000..1b593e3 --- /dev/null +++ b/backend/internal/repository/instance_repository.go @@ -0,0 +1,340 @@ +package repository + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "clawreef/internal/models" + "github.com/upper/db/v4" +) + +var ErrStaleRuntimeGeneration = errors.New("stale runtime generation") + +// InstanceRepository defines the interface for instance data operations +type InstanceRepository interface { + Create(instance *models.Instance) error + GetByID(id int) (*models.Instance, error) + GetByAccessToken(accessToken string) (*models.Instance, error) + GetByAgentBootstrapToken(bootstrapToken string) (*models.Instance, error) + GetAll(offset, limit int) ([]models.Instance, error) + CountAll() (int, error) + GetByUserID(userID int, offset, limit int) ([]models.Instance, error) + CountByUserID(userID int) (int, error) + CountActiveByMode(ctx context.Context, mode string) (int, error) + ExistsByUserIDAndName(userID int, name string) (bool, error) + GetAllRunning() ([]models.Instance, error) + GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) + GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) + UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error + SetWorkspacePath(ctx context.Context, id int, workspacePath string) error + UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error + Update(instance *models.Instance) error + Delete(id int) error +} + +// instanceRepository implements InstanceRepository +type instanceRepository struct { + sess db.Session +} + +// NewInstanceRepository creates a new instance repository +func NewInstanceRepository(sess db.Session) InstanceRepository { + return &instanceRepository{sess: sess} +} + +// Create creates a new instance +func (r *instanceRepository) Create(instance *models.Instance) error { + res, err := r.sess.Collection("instances").Insert(instance) + if err != nil { + return fmt.Errorf("failed to create instance: %w", err) + } + // Get the generated ID + if id, ok := res.ID().(int64); ok { + instance.ID = int(id) + } + return nil +} + +// GetByID gets an instance by ID +func (r *instanceRepository) GetByID(id int) (*models.Instance, error) { + var instance models.Instance + err := r.sess.Collection("instances").Find(db.Cond{"id": id}).One(&instance) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance: %w", err) + } + return &instance, nil +} + +// GetByAccessToken gets an instance by its lifecycle gateway token. +func (r *instanceRepository) GetByAccessToken(accessToken string) (*models.Instance, error) { + var instance models.Instance + err := r.sess.Collection("instances").Find(db.Cond{"access_token": accessToken}).One(&instance) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance by access token: %w", err) + } + return &instance, nil +} + +func (r *instanceRepository) GetByAgentBootstrapToken(bootstrapToken string) (*models.Instance, error) { + var instance models.Instance + err := r.sess.Collection("instances").Find(db.Cond{"agent_bootstrap_token": bootstrapToken}).One(&instance) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance by agent bootstrap token: %w", err) + } + return &instance, nil +} + +func (r *instanceRepository) GetAll(offset, limit int) ([]models.Instance, error) { + var instances []models.Instance + err := r.sess.Collection("instances").Find().Offset(offset).Limit(limit).All(&instances) + if err != nil { + return nil, fmt.Errorf("failed to get all instances: %w", err) + } + return instances, nil +} + +func (r *instanceRepository) CountAll() (int, error) { + count, err := r.sess.Collection("instances").Find().Count() + if err != nil { + return 0, fmt.Errorf("failed to count all instances: %w", err) + } + return int(count), nil +} + +// GetByUserID gets instances by user ID with pagination +func (r *instanceRepository) GetByUserID(userID int, offset, limit int) ([]models.Instance, error) { + var instances []models.Instance + err := r.sess.Collection("instances").Find(db.Cond{"user_id": userID}).OrderBy("-created_at", "-id").Offset(offset).Limit(limit).All(&instances) + if err != nil { + return nil, fmt.Errorf("failed to get instances: %w", err) + } + return instances, nil +} + +// CountByUserID counts instances by user ID +func (r *instanceRepository) CountByUserID(userID int) (int, error) { + count, err := r.sess.Collection("instances").Find(db.Cond{"user_id": userID}).Count() + if err != nil { + return 0, fmt.Errorf("failed to count instances: %w", err) + } + return int(count), nil +} + +func (r *instanceRepository) CountActiveByMode(ctx context.Context, mode string) (int, error) { + normalized := strings.TrimSpace(strings.ToLower(mode)) + if normalized == "" { + return 0, nil + } + row, err := r.sess.SQL().QueryRowContext(ctx, ` + SELECT COUNT(*) + FROM instances + WHERE instance_mode = ? + AND status IN ('creating', 'running') + `, normalized) + if err != nil { + return 0, fmt.Errorf("failed to count active instances by mode: %w", err) + } + var count int + if err := row.Scan(&count); err != nil { + return 0, fmt.Errorf("failed to scan active instances by mode count: %w", err) + } + return count, nil +} + +// ExistsByUserIDAndName checks whether the user already has an instance with the same display name. +func (r *instanceRepository) ExistsByUserIDAndName(userID int, name string) (bool, error) { + instances, err := r.GetByUserID(userID, 0, 1000) + if err != nil { + return false, err + } + + normalized := strings.TrimSpace(strings.ToLower(name)) + for _, instance := range instances { + if strings.TrimSpace(strings.ToLower(instance.Name)) == normalized { + return true, nil + } + } + + return false, nil +} + +// GetAllRunning gets all instances that are not in stopped or error state (for sync) +func (r *instanceRepository) GetAllRunning() ([]models.Instance, error) { + var instances []models.Instance + err := r.sess.Collection("instances").Find( + db.Or( + db.Cond{"status": "running"}, + db.Cond{"status": "creating"}, + db.Cond{"status": "stopped"}, + db.Cond{"status": "error"}, + ), + ).All(&instances) + if err != nil { + return nil, fmt.Errorf("failed to get running instances: %w", err) + } + return instances, nil +} + +func (r *instanceRepository) GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if limit <= 0 { + limit = 100 + } + var instances []models.Instance + query, args := buildV2SchedulerInstanceQuery(v2DesiredRunningStatuses(), limit) + iter := r.sess.SQL().IteratorContext(ctx, query, args...) + defer iter.Close() + if err := iter.All(&instances); err != nil { + return nil, fmt.Errorf("failed to get v2 desired running instances: %w", err) + } + return instances, nil +} + +func v2DesiredRunningStatuses() []string { + return []string{"creating", "running", "error"} +} + +func (r *instanceRepository) GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if limit <= 0 { + limit = 100 + } + var instances []models.Instance + query, args := buildV2SchedulerInstanceQuery([]string{"creating"}, limit) + iter := r.sess.SQL().IteratorContext(ctx, query, args...) + defer iter.Close() + if err := iter.All(&instances); err != nil { + return nil, fmt.Errorf("failed to get v2 creating instances: %w", err) + } + return instances, nil +} + +func buildV2SchedulerInstanceQuery(statuses []string, limit int) (string, []any) { + if limit <= 0 { + limit = 100 + } + if len(statuses) == 0 { + statuses = []string{"creating", "running"} + } + statusPlaceholders := strings.TrimRight(strings.Repeat("?, ", len(statuses)), ", ") + args := make([]any, 0, len(statuses)+5) + for _, status := range statuses { + args = append(args, status) + } + args = append(args, "gateway", "lite", "openclaw", "hermes", limit) + return fmt.Sprintf(` + SELECT * + FROM instances + WHERE status IN (%s) + AND runtime_type = ? + AND instance_mode = ? + AND workspace_path IS NOT NULL + AND TRIM(workspace_path) <> '' + AND type IN (?, ?) + ORDER BY id + LIMIT ? + `, statusPlaceholders), args +} + +func (r *instanceRepository) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error { + res, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instances + SET status = ?, runtime_generation = ?, runtime_error_message = ?, updated_at = ? + WHERE id = ? AND runtime_generation <= ? + `, status, generation, message, time.Now().UTC(), id, generation) + if err != nil { + return fmt.Errorf("failed to update instance runtime state: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("failed to inspect instance runtime state update: %w", err) + } + if affected == 0 { + currentGeneration, err := r.getRuntimeGeneration(ctx, id) + if err != nil { + return err + } + if currentGeneration > generation { + return ErrStaleRuntimeGeneration + } + } + return nil +} + +func (r *instanceRepository) getRuntimeGeneration(ctx context.Context, id int) (int, error) { + var currentGeneration int + row, err := r.sess.SQL().QueryRowContext(ctx, ` + SELECT runtime_generation + FROM instances + WHERE id = ? + `, id) + if err != nil { + return 0, fmt.Errorf("failed to query instance runtime generation: %w", err) + } + if err := row.Scan(¤tGeneration); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return 0, ErrStaleRuntimeGeneration + } + return 0, fmt.Errorf("failed to scan instance runtime generation: %w", err) + } + return currentGeneration, nil +} + +func (r *instanceRepository) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instances + SET workspace_path = ?, updated_at = ? + WHERE id = ? + `, workspacePath, time.Now().UTC(), id) + if err != nil { + return fmt.Errorf("failed to set instance workspace path: %w", err) + } + return nil +} + +func (r *instanceRepository) UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instances + SET workspace_usage_bytes = ?, updated_at = ? + WHERE id = ? + `, usageBytes, time.Now().UTC(), id) + if err != nil { + return fmt.Errorf("failed to update instance workspace usage: %w", err) + } + return nil +} + +// Update updates an instance +func (r *instanceRepository) Update(instance *models.Instance) error { + err := r.sess.Collection("instances").Find(db.Cond{"id": instance.ID}).Update(instance) + if err != nil { + return fmt.Errorf("failed to update instance: %w", err) + } + return nil +} + +// Delete deletes an instance +func (r *instanceRepository) Delete(id int) error { + err := r.sess.Collection("instances").Find(db.Cond{"id": id}).Delete() + if err != nil { + return fmt.Errorf("failed to delete instance: %w", err) + } + return nil +} diff --git a/backend/internal/repository/instance_repository_test.go b/backend/internal/repository/instance_repository_test.go new file mode 100644 index 0000000..1a45955 --- /dev/null +++ b/backend/internal/repository/instance_repository_test.go @@ -0,0 +1,58 @@ +package repository + +import ( + "reflect" + "strings" + "testing" +) + +func TestBuildV2SchedulerInstanceQueryRequiresWorkspaceTypeAndStatuses(t *testing.T) { + query, args := buildV2SchedulerInstanceQuery([]string{"creating", "running"}, 25) + normalized := normalizeSQLForTest(query) + + requiredFragments := []string{ + "FROM instances", + "status IN (?, ?)", + "runtime_type = ?", + "instance_mode = ?", + "workspace_path IS NOT NULL", + "TRIM(workspace_path) <> ''", + "type IN (?, ?)", + "ORDER BY id", + "LIMIT ?", + } + for _, fragment := range requiredFragments { + if !strings.Contains(normalized, fragment) { + t.Fatalf("query %q does not contain %q", normalized, fragment) + } + } + + wantArgs := []any{"creating", "running", "gateway", "lite", "openclaw", "hermes", 25} + if !reflect.DeepEqual(args, wantArgs) { + t.Fatalf("args = %#v, want %#v", args, wantArgs) + } +} + +func TestV2DesiredRunningStatusesIncludeRecoverableErrorCandidates(t *testing.T) { + want := []string{"creating", "running", "error"} + if got := v2DesiredRunningStatuses(); !reflect.DeepEqual(got, want) { + t.Fatalf("desired running statuses = %#v, want %#v", got, want) + } +} + +func TestBuildV2SchedulerInstanceQueryDefaultsLimit(t *testing.T) { + query, args := buildV2SchedulerInstanceQuery([]string{"creating"}, 0) + normalized := normalizeSQLForTest(query) + + if !strings.Contains(normalized, "status IN (?)") { + t.Fatalf("query %q does not contain single status predicate", normalized) + } + wantArgs := []any{"creating", "gateway", "lite", "openclaw", "hermes", 100} + if !reflect.DeepEqual(args, wantArgs) { + t.Fatalf("args = %#v, want %#v", args, wantArgs) + } +} + +func normalizeSQLForTest(query string) string { + return strings.Join(strings.Fields(query), " ") +} diff --git a/backend/internal/repository/instance_runtime_binding_repository.go b/backend/internal/repository/instance_runtime_binding_repository.go new file mode 100644 index 0000000..f5d2969 --- /dev/null +++ b/backend/internal/repository/instance_runtime_binding_repository.go @@ -0,0 +1,213 @@ +package repository + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type InstanceRuntimeBindingRepository interface { + Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error + GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) + GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) + ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) + ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) + UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error + UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error + DeleteByInstanceID(ctx context.Context, instanceID int) error + DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error +} + +type instanceRuntimeBindingRepository struct { + sess db.Session +} + +func NewInstanceRuntimeBindingRepository(sess db.Session) InstanceRuntimeBindingRepository { + return &instanceRuntimeBindingRepository{sess: sess} +} + +func (r *instanceRuntimeBindingRepository) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error { + if err := ctx.Err(); err != nil { + return err + } + ensureTimestamps(&binding.CreatedAt, &binding.UpdatedAt) + res, err := r.sess.Collection("instance_runtime_bindings").Insert(binding) + if err != nil { + return fmt.Errorf("failed to create instance runtime binding: %w", err) + } + if id, ok := res.ID().(int64); ok { + binding.ID = id + } + return nil +} + +func (r *instanceRuntimeBindingRepository) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var binding models.InstanceRuntimeBinding + if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"instance_id": instanceID}).One(&binding); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance runtime binding: %w", err) + } + return &binding, nil +} + +func (r *instanceRuntimeBindingRepository) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var binding models.InstanceRuntimeBinding + if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"instance_id": instanceID, "state": "running"}).One(&binding); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get running instance runtime binding: %w", err) + } + return &binding, nil +} + +func (r *instanceRuntimeBindingRepository) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var bindings []models.InstanceRuntimeBinding + if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"runtime_pod_id": runtimePodID}).OrderBy("id").All(&bindings); err != nil { + return nil, fmt.Errorf("failed to list instance runtime bindings: %w", err) + } + return bindings, nil +} + +func (r *instanceRuntimeBindingRepository) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if len(runtimePodIDs) == 0 { + return []models.InstanceRuntimeBinding{}, nil + } + var bindings []models.InstanceRuntimeBinding + if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"runtime_pod_id IN": runtimePodIDs}).OrderBy("runtime_pod_id", "id").All(&bindings); err != nil { + return nil, fmt.Errorf("failed to list instance runtime bindings by pods: %w", err) + } + return bindings, nil +} + +func (r *instanceRuntimeBindingRepository) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error { + now := time.Now().UTC() + res, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instance_runtime_bindings + SET state = 'running', generation = ?, gateway_id = ?, gateway_port = ?, gateway_pid = ?, + last_health_at = ?, error_message = NULL, updated_at = ? + WHERE instance_id = ? AND generation <= ? + `, generation, gatewayID, port, pid, now, now, instanceID, generation) + if err != nil { + return fmt.Errorf("failed to update running instance runtime binding: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("failed to inspect running instance runtime binding update: %w", err) + } + if affected == 0 { + currentGeneration, err := r.getGeneration(ctx, instanceID) + if err != nil { + return err + } + if currentGeneration > generation { + return ErrStaleRuntimeGeneration + } + } + return nil +} + +func (r *instanceRuntimeBindingRepository) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error { + res, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instance_runtime_bindings + SET state = ?, generation = ?, error_message = ?, updated_at = ? + WHERE instance_id = ? AND generation <= ? + `, state, generation, message, time.Now().UTC(), instanceID, generation) + if err != nil { + return fmt.Errorf("failed to update instance runtime binding state: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("failed to inspect instance runtime binding state update: %w", err) + } + if affected == 0 { + currentGeneration, err := r.getGeneration(ctx, instanceID) + if err != nil { + return err + } + if currentGeneration > generation { + return ErrStaleRuntimeGeneration + } + } + return nil +} + +func (r *instanceRuntimeBindingRepository) getGeneration(ctx context.Context, instanceID int) (int, error) { + var currentGeneration int + row, err := r.sess.SQL().QueryRowContext(ctx, ` + SELECT generation + FROM instance_runtime_bindings + WHERE instance_id = ? + `, instanceID) + if err != nil { + return 0, fmt.Errorf("failed to query instance runtime binding generation: %w", err) + } + if err := row.Scan(¤tGeneration); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return 0, ErrStaleRuntimeGeneration + } + return 0, fmt.Errorf("failed to scan instance runtime binding generation: %w", err) + } + return currentGeneration, nil +} + +func (r *instanceRuntimeBindingRepository) DeleteByInstanceID(ctx context.Context, instanceID int) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + DELETE FROM instance_runtime_bindings + WHERE instance_id = ? + `, instanceID) + if err != nil { + return fmt.Errorf("failed to delete instance runtime binding: %w", err) + } + return nil +} + +func (r *instanceRuntimeBindingRepository) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error { + if err := ctx.Err(); err != nil { + return err + } + return r.sess.TxContext(ctx, func(tx db.Session) error { + res, err := tx.SQL().ExecContext(ctx, ` + DELETE FROM instance_runtime_bindings + WHERE instance_id = ? AND runtime_pod_id = ? + `, instanceID, runtimePodID) + if err != nil { + return fmt.Errorf("failed to delete instance runtime binding: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("failed to inspect instance runtime binding delete: %w", err) + } + if affected == 0 { + return nil + } + if _, err := tx.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET used_slots = CASE WHEN used_slots > 0 THEN used_slots - 1 ELSE 0 END, updated_at = ? + WHERE id = ? + `, time.Now().UTC(), runtimePodID); err != nil { + return fmt.Errorf("failed to release runtime pod slot: %w", err) + } + return nil + }, nil) +} diff --git a/backend/internal/repository/instance_runtime_status_repository.go b/backend/internal/repository/instance_runtime_status_repository.go new file mode 100644 index 0000000..56d339a --- /dev/null +++ b/backend/internal/repository/instance_runtime_status_repository.go @@ -0,0 +1,57 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type InstanceRuntimeStatusRepository interface { + GetByInstanceID(instanceID int) (*models.InstanceRuntimeStatus, error) + Create(status *models.InstanceRuntimeStatus) error + Update(status *models.InstanceRuntimeStatus) error +} + +type instanceRuntimeStatusRepository struct { + sess db.Session +} + +func NewInstanceRuntimeStatusRepository(sess db.Session) InstanceRuntimeStatusRepository { + return &instanceRuntimeStatusRepository{sess: sess} +} + +func (r *instanceRuntimeStatusRepository) GetByInstanceID(instanceID int) (*models.InstanceRuntimeStatus, error) { + var item models.InstanceRuntimeStatus + if err := r.sess.Collection("instance_runtime_status").Find(db.Cond{"instance_id": instanceID}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance runtime status: %w", err) + } + return &item, nil +} + +func (r *instanceRuntimeStatusRepository) Create(status *models.InstanceRuntimeStatus) error { + ensureTimestamps(&status.CreatedAt, &status.UpdatedAt) + res, err := r.sess.Collection("instance_runtime_status").Insert(status) + if err != nil { + return fmt.Errorf("failed to create instance runtime status: %w", err) + } + if id, ok := res.ID().(int64); ok { + status.ID = int(id) + } + return nil +} + +func (r *instanceRuntimeStatusRepository) Update(status *models.InstanceRuntimeStatus) error { + if status.UpdatedAt.IsZero() { + status.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("instance_runtime_status").Find(db.Cond{"id": status.ID}).Update(status); err != nil { + return fmt.Errorf("failed to update instance runtime status: %w", err) + } + return nil +} diff --git a/backend/internal/repository/llm_model_repository.go b/backend/internal/repository/llm_model_repository.go new file mode 100644 index 0000000..9a2e2f6 --- /dev/null +++ b/backend/internal/repository/llm_model_repository.go @@ -0,0 +1,157 @@ +package repository + +import ( + "fmt" + "strings" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +// LLMModelRepository defines repository operations for admin-managed models. +type LLMModelRepository interface { + List() ([]models.LLMModel, error) + ListActive() ([]models.LLMModel, error) + GetByID(id int) (*models.LLMModel, error) + GetByDisplayName(displayName string) (*models.LLMModel, error) + Save(model *models.LLMModel) error + Delete(id int) error +} + +type llmModelRepository struct { + sess db.Session +} + +// NewLLMModelRepository creates a new model repository and ensures its table exists. +func NewLLMModelRepository(sess db.Session) LLMModelRepository { + repo := &llmModelRepository{sess: sess} + repo.ensureTable() + return repo +} + +func (r *llmModelRepository) ensureTable() { + const query = ` +CREATE TABLE IF NOT EXISTS llm_models ( + id INT AUTO_INCREMENT PRIMARY KEY, + display_name VARCHAR(255) NOT NULL UNIQUE, + description TEXT NULL, + provider_type VARCHAR(100) NOT NULL, + protocol_type VARCHAR(100) NULL, + base_url VARCHAR(500) NOT NULL, + provider_model_name VARCHAR(255) NOT NULL, + api_key TEXT NULL, + api_key_secret_ref VARCHAR(255) NULL, + is_secure BOOLEAN NOT NULL DEFAULT FALSE, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + input_price DECIMAL(18,8) NOT NULL DEFAULT 0, + output_price DECIMAL(18,8) NOT NULL DEFAULT 0, + currency VARCHAR(16) NOT NULL DEFAULT 'USD', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_llm_models_provider_type (provider_type), + INDEX idx_llm_models_is_active (is_active), + INDEX idx_llm_models_is_secure (is_secure) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +` + + if _, err := r.sess.SQL().Exec(query); err != nil { + panic(fmt.Errorf("failed to ensure llm_models table: %w", err)) + } + + const alterProtocolTypeQuery = ` +ALTER TABLE llm_models + ADD COLUMN protocol_type VARCHAR(100) NULL AFTER provider_type; +` + + if _, err := r.sess.SQL().Exec(alterProtocolTypeQuery); err != nil { + if !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") { + panic(fmt.Errorf("failed to ensure llm_models protocol_type column: %w", err)) + } + } + + const backfillProtocolTypeQuery = ` +UPDATE llm_models +SET protocol_type = CASE + WHEN LOWER(TRIM(provider_type)) = 'local' THEN 'openai-compatible' + WHEN LOWER(TRIM(provider_type)) = 'openai' THEN 'openai' + WHEN LOWER(TRIM(provider_type)) = 'anthropic' THEN 'anthropic' + WHEN LOWER(TRIM(provider_type)) = 'google' THEN 'google' + WHEN LOWER(TRIM(provider_type)) = 'azure-openai' THEN 'azure-openai' + ELSE 'openai-compatible' +END +WHERE protocol_type IS NULL OR TRIM(protocol_type) = ''; +` + + if _, err := r.sess.SQL().Exec(backfillProtocolTypeQuery); err != nil { + panic(fmt.Errorf("failed to ensure llm_models protocol_type column: %w", err)) + } +} + +func (r *llmModelRepository) List() ([]models.LLMModel, error) { + var items []models.LLMModel + if err := r.sess.Collection("llm_models").Find().OrderBy("-is_secure", "display_name").All(&items); err != nil { + return nil, fmt.Errorf("failed to list llm models: %w", err) + } + return items, nil +} + +func (r *llmModelRepository) ListActive() ([]models.LLMModel, error) { + var items []models.LLMModel + if err := r.sess.Collection("llm_models").Find(db.Cond{"is_active": true}).OrderBy("-is_secure", "display_name").All(&items); err != nil { + return nil, fmt.Errorf("failed to list active llm models: %w", err) + } + return items, nil +} + +func (r *llmModelRepository) GetByID(id int) (*models.LLMModel, error) { + var item models.LLMModel + err := r.sess.Collection("llm_models").Find(db.Cond{"id": id}).One(&item) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get llm model by id: %w", err) + } + return &item, nil +} + +func (r *llmModelRepository) GetByDisplayName(displayName string) (*models.LLMModel, error) { + var item models.LLMModel + err := r.sess.Collection("llm_models").Find(db.Cond{"display_name": displayName}).One(&item) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get llm model by display name: %w", err) + } + return &item, nil +} + +func (r *llmModelRepository) Save(model *models.LLMModel) error { + now := time.Now() + if model.ID == 0 { + model.CreatedAt = now + model.UpdatedAt = now + res, err := r.sess.Collection("llm_models").Insert(model) + if err != nil { + return fmt.Errorf("failed to create llm model: %w", err) + } + model.ID = int(res.ID().(int64)) + return nil + } + + model.UpdatedAt = now + if err := r.sess.Collection("llm_models").Find(db.Cond{"id": model.ID}).Update(model); err != nil { + return fmt.Errorf("failed to update llm model: %w", err) + } + return nil +} + +func (r *llmModelRepository) Delete(id int) error { + if err := r.sess.Collection("llm_models").Find(db.Cond{"id": id}).Delete(); err != nil { + return fmt.Errorf("failed to delete llm model: %w", err) + } + return nil +} diff --git a/backend/internal/repository/model_invocation_repository.go b/backend/internal/repository/model_invocation_repository.go new file mode 100644 index 0000000..a581cd7 --- /dev/null +++ b/backend/internal/repository/model_invocation_repository.go @@ -0,0 +1,156 @@ +package repository + +import ( + "fmt" + "strings" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +// ModelInvocationRepository defines repository operations for governed model calls. +type ModelInvocationRepository interface { + Create(invocation *models.ModelInvocation) error + GetByID(id int) (*models.ModelInvocation, error) + ListByTraceID(traceID string) ([]models.ModelInvocation, error) + ListBySessionID(sessionID string, limit int) ([]models.ModelInvocation, error) + ListByUserID(userID, limit int) ([]models.ModelInvocation, error) + ListRecent(limit int) ([]models.ModelInvocation, error) +} + +type modelInvocationRepository struct { + sess db.Session +} + +// NewModelInvocationRepository creates a new invocation repository and ensures its table exists. +func NewModelInvocationRepository(sess db.Session) ModelInvocationRepository { + repo := &modelInvocationRepository{sess: sess} + repo.ensureTable() + return repo +} + +func (r *modelInvocationRepository) ensureTable() { + const query = ` +CREATE TABLE IF NOT EXISTS model_invocations ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NOT NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + model_id INT NULL, + provider_type VARCHAR(100) NOT NULL, + requested_model VARCHAR(255) NOT NULL, + actual_provider_model VARCHAR(255) NOT NULL, + traffic_class VARCHAR(50) NOT NULL, + request_payload LONGTEXT NULL, + response_payload LONGTEXT NULL, + prompt_tokens INT NOT NULL DEFAULT 0, + completion_tokens INT NOT NULL DEFAULT 0, + total_tokens INT NOT NULL DEFAULT 0, + cached_tokens INT NULL, + reasoning_tokens INT NULL, + latency_ms INT NULL, + is_streaming BOOLEAN NOT NULL DEFAULT FALSE, + status VARCHAR(50) NOT NULL, + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP NULL, + INDEX idx_model_invocations_trace_id (trace_id), + INDEX idx_model_invocations_request_id (request_id), + INDEX idx_model_invocations_user_id (user_id), + INDEX idx_model_invocations_instance_id (instance_id), + INDEX idx_model_invocations_gateway_id (gateway_id), + INDEX idx_model_invocations_model_id (model_id), + INDEX idx_model_invocations_status (status), + INDEX idx_model_invocations_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +` + + if _, err := r.sess.SQL().Exec(query); err != nil { + panic(fmt.Errorf("failed to ensure model_invocations table: %w", err)) + } + if _, err := r.sess.SQL().Exec("ALTER TABLE model_invocations ADD INDEX idx_model_invocations_session_id (session_id)"); err != nil && !isDuplicateIndexError(err) { + panic(fmt.Errorf("failed to ensure session index on model_invocations: %w", err)) + } +} + +func (r *modelInvocationRepository) Create(invocation *models.ModelInvocation) error { + now := time.Now() + if invocation.CreatedAt.IsZero() { + invocation.CreatedAt = now + } + res, err := r.sess.Collection("model_invocations").Insert(invocation) + if err != nil { + return fmt.Errorf("failed to create model invocation: %w", err) + } + invocation.ID = int(res.ID().(int64)) + return nil +} + +func (r *modelInvocationRepository) GetByID(id int) (*models.ModelInvocation, error) { + var invocation models.ModelInvocation + err := r.sess.Collection("model_invocations").Find(db.Cond{"id": id}).One(&invocation) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get model invocation by id: %w", err) + } + return &invocation, nil +} + +func (r *modelInvocationRepository) ListByTraceID(traceID string) ([]models.ModelInvocation, error) { + var items []models.ModelInvocation + if err := r.sess.Collection("model_invocations").Find(db.Cond{"trace_id": traceID}).OrderBy("id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list model invocations by trace: %w", err) + } + return items, nil +} + +func (r *modelInvocationRepository) ListBySessionID(sessionID string, limit int) ([]models.ModelInvocation, error) { + var items []models.ModelInvocation + if limit <= 0 { + limit = 50 + } + if err := r.sess.Collection("model_invocations").Find(db.Cond{"session_id": sessionID}).OrderBy("-created_at").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list model invocations by session: %w", err) + } + return items, nil +} + +func (r *modelInvocationRepository) ListByUserID(userID, limit int) ([]models.ModelInvocation, error) { + var items []models.ModelInvocation + if limit <= 0 { + limit = 50 + } + if err := r.sess.Collection("model_invocations").Find(db.Cond{"user_id": userID}).OrderBy("-created_at").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list model invocations by user: %w", err) + } + return items, nil +} + +func (r *modelInvocationRepository) ListRecent(limit int) ([]models.ModelInvocation, error) { + var items []models.ModelInvocation + if limit <= 0 { + limit = 100 + } + if err := r.sess.Collection("model_invocations").Find().OrderBy("-created_at").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list recent model invocations: %w", err) + } + return items, nil +} + +func isDuplicateIndexError(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + return strings.Contains(message, "duplicate key name") || strings.Contains(message, "already exists") +} diff --git a/backend/internal/repository/openclaw_config_repository.go b/backend/internal/repository/openclaw_config_repository.go new file mode 100644 index 0000000..eb4fc0e --- /dev/null +++ b/backend/internal/repository/openclaw_config_repository.go @@ -0,0 +1,277 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +// OpenClawConfigRepository defines storage operations for the OpenClaw config center. +type OpenClawConfigRepository interface { + ListResources(userID int, resourceType string) ([]models.OpenClawConfigResource, error) + GetResourceByID(id int) (*models.OpenClawConfigResource, error) + GetResourceByUserTypeKey(userID int, resourceType, resourceKey string) (*models.OpenClawConfigResource, error) + CreateResource(resource *models.OpenClawConfigResource) error + UpdateResource(resource *models.OpenClawConfigResource) error + DeleteResource(id int) error + + ListBundles(userID int) ([]models.OpenClawConfigBundle, error) + GetBundleByID(id int) (*models.OpenClawConfigBundle, error) + CreateBundle(bundle *models.OpenClawConfigBundle) error + UpdateBundle(bundle *models.OpenClawConfigBundle) error + DeleteBundle(id int) error + ListBundleItems(bundleID int) ([]models.OpenClawConfigBundleItem, error) + ReplaceBundleItems(bundleID int, items []models.OpenClawConfigBundleItem) error + ListBundleSkills(bundleID int) ([]models.OpenClawConfigBundleSkill, error) + ReplaceBundleSkills(bundleID int, items []models.OpenClawConfigBundleSkill) error + + CreateSnapshot(snapshot *models.OpenClawInjectionSnapshot) error + UpdateSnapshot(snapshot *models.OpenClawInjectionSnapshot) error + GetSnapshotByID(id int) (*models.OpenClawInjectionSnapshot, error) + ListSnapshotsByUser(userID int, limit int) ([]models.OpenClawInjectionSnapshot, error) + ListActiveSnapshots(userID int) ([]models.OpenClawInjectionSnapshot, error) + UpdateSnapshotIfUnchanged(snapshot *models.OpenClawInjectionSnapshot, expectedUpdatedAt time.Time) (bool, error) +} + +type openClawConfigRepository struct { + sess db.Session +} + +// NewOpenClawConfigRepository creates a new OpenClaw config repository. +func NewOpenClawConfigRepository(sess db.Session) OpenClawConfigRepository { + return &openClawConfigRepository{sess: sess} +} + +func (r *openClawConfigRepository) ListResources(userID int, resourceType string) ([]models.OpenClawConfigResource, error) { + find := r.sess.Collection("openclaw_config_resources").Find(db.Cond{"user_id": userID}) + if resourceType != "" { + find = find.And("resource_type", resourceType) + } + + var items []models.OpenClawConfigResource + if err := find.OrderBy("-updated_at", "-id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list openclaw config resources: %w", err) + } + return items, nil +} + +func (r *openClawConfigRepository) GetResourceByID(id int) (*models.OpenClawConfigResource, error) { + var item models.OpenClawConfigResource + if err := r.sess.Collection("openclaw_config_resources").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get openclaw config resource: %w", err) + } + return &item, nil +} + +func (r *openClawConfigRepository) GetResourceByUserTypeKey(userID int, resourceType, resourceKey string) (*models.OpenClawConfigResource, error) { + var item models.OpenClawConfigResource + if err := r.sess.Collection("openclaw_config_resources").Find(db.Cond{ + "user_id": userID, + "resource_type": resourceType, + "resource_key": resourceKey, + }).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get openclaw config resource by key: %w", err) + } + return &item, nil +} + +func (r *openClawConfigRepository) CreateResource(resource *models.OpenClawConfigResource) error { + res, err := r.sess.Collection("openclaw_config_resources").Insert(resource) + if err != nil { + return fmt.Errorf("failed to create openclaw config resource: %w", err) + } + if id, ok := res.ID().(int64); ok { + resource.ID = int(id) + } + return nil +} + +func (r *openClawConfigRepository) UpdateResource(resource *models.OpenClawConfigResource) error { + if err := r.sess.Collection("openclaw_config_resources").Find(db.Cond{"id": resource.ID}).Update(resource); err != nil { + return fmt.Errorf("failed to update openclaw config resource: %w", err) + } + return nil +} + +func (r *openClawConfigRepository) DeleteResource(id int) error { + if err := r.sess.Collection("openclaw_config_resources").Find(db.Cond{"id": id}).Delete(); err != nil { + return fmt.Errorf("failed to delete openclaw config resource: %w", err) + } + return nil +} + +func (r *openClawConfigRepository) ListBundles(userID int) ([]models.OpenClawConfigBundle, error) { + var items []models.OpenClawConfigBundle + if err := r.sess.Collection("openclaw_config_bundles").Find(db.Cond{"user_id": userID}).OrderBy("-updated_at", "-id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list openclaw config bundles: %w", err) + } + return items, nil +} + +func (r *openClawConfigRepository) GetBundleByID(id int) (*models.OpenClawConfigBundle, error) { + var item models.OpenClawConfigBundle + if err := r.sess.Collection("openclaw_config_bundles").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get openclaw config bundle: %w", err) + } + return &item, nil +} + +func (r *openClawConfigRepository) CreateBundle(bundle *models.OpenClawConfigBundle) error { + res, err := r.sess.Collection("openclaw_config_bundles").Insert(bundle) + if err != nil { + return fmt.Errorf("failed to create openclaw config bundle: %w", err) + } + if id, ok := res.ID().(int64); ok { + bundle.ID = int(id) + } + return nil +} + +func (r *openClawConfigRepository) UpdateBundle(bundle *models.OpenClawConfigBundle) error { + if err := r.sess.Collection("openclaw_config_bundles").Find(db.Cond{"id": bundle.ID}).Update(bundle); err != nil { + return fmt.Errorf("failed to update openclaw config bundle: %w", err) + } + return nil +} + +func (r *openClawConfigRepository) DeleteBundle(id int) error { + if err := r.sess.Collection("openclaw_config_bundles").Find(db.Cond{"id": id}).Delete(); err != nil { + return fmt.Errorf("failed to delete openclaw config bundle: %w", err) + } + return nil +} + +func (r *openClawConfigRepository) ListBundleItems(bundleID int) ([]models.OpenClawConfigBundleItem, error) { + var items []models.OpenClawConfigBundleItem + if err := r.sess.Collection("openclaw_config_bundle_items").Find(db.Cond{"bundle_id": bundleID}).OrderBy("sort_order", "id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list openclaw config bundle items: %w", err) + } + return items, nil +} + +func (r *openClawConfigRepository) ReplaceBundleItems(bundleID int, items []models.OpenClawConfigBundleItem) error { + if err := r.sess.Collection("openclaw_config_bundle_items").Find(db.Cond{"bundle_id": bundleID}).Delete(); err != nil && err != db.ErrNoMoreRows { + return fmt.Errorf("failed to delete existing openclaw config bundle items: %w", err) + } + + for _, item := range items { + item.BundleID = bundleID + if _, err := r.sess.Collection("openclaw_config_bundle_items").Insert(item); err != nil { + return fmt.Errorf("failed to add openclaw config bundle item: %w", err) + } + } + + return nil +} + +func (r *openClawConfigRepository) ListBundleSkills(bundleID int) ([]models.OpenClawConfigBundleSkill, error) { + var items []models.OpenClawConfigBundleSkill + if err := r.sess.Collection("openclaw_config_bundle_skills").Find(db.Cond{"bundle_id": bundleID}).OrderBy("sort_order", "id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list openclaw config bundle skills: %w", err) + } + return items, nil +} + +func (r *openClawConfigRepository) ReplaceBundleSkills(bundleID int, items []models.OpenClawConfigBundleSkill) error { + if err := r.sess.Collection("openclaw_config_bundle_skills").Find(db.Cond{"bundle_id": bundleID}).Delete(); err != nil && err != db.ErrNoMoreRows { + return fmt.Errorf("failed to delete existing openclaw config bundle skills: %w", err) + } + + for _, item := range items { + item.BundleID = bundleID + if _, err := r.sess.Collection("openclaw_config_bundle_skills").Insert(item); err != nil { + return fmt.Errorf("failed to add openclaw config bundle skill: %w", err) + } + } + + return nil +} + +func (r *openClawConfigRepository) CreateSnapshot(snapshot *models.OpenClawInjectionSnapshot) error { + res, err := r.sess.Collection("openclaw_injection_snapshots").Insert(snapshot) + if err != nil { + return fmt.Errorf("failed to create openclaw injection snapshot: %w", err) + } + if id, ok := res.ID().(int64); ok { + snapshot.ID = int(id) + } + return nil +} + +func (r *openClawConfigRepository) UpdateSnapshot(snapshot *models.OpenClawInjectionSnapshot) error { + if err := r.sess.Collection("openclaw_injection_snapshots").Find(db.Cond{"id": snapshot.ID}).Update(snapshot); err != nil { + return fmt.Errorf("failed to update openclaw injection snapshot: %w", err) + } + return nil +} + +func (r *openClawConfigRepository) GetSnapshotByID(id int) (*models.OpenClawInjectionSnapshot, error) { + var item models.OpenClawInjectionSnapshot + if err := r.sess.Collection("openclaw_injection_snapshots").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get openclaw injection snapshot: %w", err) + } + return &item, nil +} + +func (r *openClawConfigRepository) ListSnapshotsByUser(userID int, limit int) ([]models.OpenClawInjectionSnapshot, error) { + if limit <= 0 { + limit = 50 + } + + var items []models.OpenClawInjectionSnapshot + if err := r.sess.Collection("openclaw_injection_snapshots").Find(db.Cond{"user_id": userID}).OrderBy("-created_at", "-id").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list openclaw injection snapshots: %w", err) + } + return items, nil +} + +func (r *openClawConfigRepository) ListActiveSnapshots(userID int) ([]models.OpenClawInjectionSnapshot, error) { + var items []models.OpenClawInjectionSnapshot + if err := r.sess.Collection("openclaw_injection_snapshots").Find(db.Cond{ + "user_id": userID, + "status IN": []string{"compiled", "active"}, + }).All(&items); err != nil { + return nil, fmt.Errorf("failed to list active openclaw injection snapshots: %w", err) + } + return items, nil +} + +func (r *openClawConfigRepository) UpdateSnapshotIfUnchanged( + snapshot *models.OpenClawInjectionSnapshot, + expectedUpdatedAt time.Time, +) (bool, error) { + result, err := r.sess.SQL().Exec( + `UPDATE openclaw_injection_snapshots + SET resolved_resources_json = ?, + rendered_env_json = ?, + rendered_manifest_json = ?, + updated_at = ? + WHERE id = ? AND updated_at = ?`, + snapshot.ResolvedResourcesJSON, + snapshot.RenderedEnvJSON, + snapshot.RenderedManifestJSON, + snapshot.UpdatedAt, + snapshot.ID, + expectedUpdatedAt, + ) + if err != nil { + return false, fmt.Errorf("failed to cas-update openclaw injection snapshot: %w", err) + } + rows, _ := result.RowsAffected() + return rows > 0, nil +} diff --git a/backend/internal/repository/quota_repository.go b/backend/internal/repository/quota_repository.go new file mode 100644 index 0000000..dbe89c0 --- /dev/null +++ b/backend/internal/repository/quota_repository.go @@ -0,0 +1,90 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + "github.com/upper/db/v4" +) + +// QuotaRepository defines the interface for quota data operations +type QuotaRepository interface { + Create(quota *models.UserQuota) error + GetByUserID(userID int) (*models.UserQuota, error) + Update(quota *models.UserQuota) error + DeleteByUserID(userID int) error + CreateDefaultQuota(userID int) (*models.UserQuota, error) +} + +// quotaRepository implements QuotaRepository +type quotaRepository struct { + sess db.Session +} + +// NewQuotaRepository creates a new quota repository +func NewQuotaRepository(sess db.Session) QuotaRepository { + return "aRepository{sess: sess} +} + +// Create creates a new quota +func (r *quotaRepository) Create(quota *models.UserQuota) error { + _, err := r.sess.Collection("user_quotas").Insert(quota) + if err != nil { + return fmt.Errorf("failed to create quota: %w", err) + } + return nil +} + +// GetByUserID gets quota by user ID +func (r *quotaRepository) GetByUserID(userID int) (*models.UserQuota, error) { + var quota models.UserQuota + err := r.sess.Collection("user_quotas").Find(db.Cond{"user_id": userID}).One("a) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get quota: %w", err) + } + return "a, nil +} + +// Update updates a quota +func (r *quotaRepository) Update(quota *models.UserQuota) error { + err := r.sess.Collection("user_quotas").Find(db.Cond{"id": quota.ID}).Update(quota) + if err != nil { + return fmt.Errorf("failed to update quota: %w", err) + } + return nil +} + +// DeleteByUserID deletes quota by user ID +func (r *quotaRepository) DeleteByUserID(userID int) error { + err := r.sess.Collection("user_quotas").Find(db.Cond{"user_id": userID}).Delete() + if err != nil { + return fmt.Errorf("failed to delete quota: %w", err) + } + return nil +} + +// CreateDefaultQuota creates default quota for a user +func (r *quotaRepository) CreateDefaultQuota(userID int) (*models.UserQuota, error) { + now := time.Now() + quota := &models.UserQuota{ + UserID: userID, + MaxInstances: 10, + MaxCPUCores: 40, + MaxMemoryGB: 100, + MaxStorageGB: 500, + MaxGPUCount: 2, + CreatedAt: now, + UpdatedAt: now, + } + + _, err := r.sess.Collection("user_quotas").Insert(quota) + if err != nil { + return nil, fmt.Errorf("failed to create default quota: %w", err) + } + + return quota, nil +} diff --git a/backend/internal/repository/risk_hit_repository.go b/backend/internal/repository/risk_hit_repository.go new file mode 100644 index 0000000..864187c --- /dev/null +++ b/backend/internal/repository/risk_hit_repository.go @@ -0,0 +1,82 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +// RiskHitRepository defines repository operations for sensitive-data detection records. +type RiskHitRepository interface { + Create(hit *models.RiskHit) error + ListByTraceID(traceID string) ([]models.RiskHit, error) +} + +type riskHitRepository struct { + sess db.Session +} + +// NewRiskHitRepository creates a new risk hit repository and ensures its table exists. +func NewRiskHitRepository(sess db.Session) RiskHitRepository { + repo := &riskHitRepository{sess: sess} + repo.ensureTable() + return repo +} + +func (r *riskHitRepository) ensureTable() { + const query = ` +CREATE TABLE IF NOT EXISTS risk_hits ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + invocation_id INT NULL, + rule_id VARCHAR(100) NOT NULL, + rule_name VARCHAR(255) NOT NULL, + severity VARCHAR(20) NOT NULL, + action VARCHAR(50) NOT NULL, + match_summary VARCHAR(500) NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_risk_hits_trace_id (trace_id), + INDEX idx_risk_hits_request_id (request_id), + INDEX idx_risk_hits_user_id (user_id), + INDEX idx_risk_hits_gateway_id (gateway_id), + INDEX idx_risk_hits_invocation_id (invocation_id), + INDEX idx_risk_hits_severity (severity), + INDEX idx_risk_hits_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +` + + if _, err := r.sess.SQL().Exec(query); err != nil { + panic(fmt.Errorf("failed to ensure risk_hits table: %w", err)) + } +} + +func (r *riskHitRepository) Create(hit *models.RiskHit) error { + if hit.CreatedAt.IsZero() { + hit.CreatedAt = time.Now() + } + res, err := r.sess.Collection("risk_hits").Insert(hit) + if err != nil { + return fmt.Errorf("failed to create risk hit: %w", err) + } + hit.ID = int(res.ID().(int64)) + return nil +} + +func (r *riskHitRepository) ListByTraceID(traceID string) ([]models.RiskHit, error) { + var items []models.RiskHit + if err := r.sess.Collection("risk_hits").Find(db.Cond{"trace_id": traceID}).OrderBy("id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list risk hits by trace: %w", err) + } + return items, nil +} diff --git a/backend/internal/repository/risk_rule_repository.go b/backend/internal/repository/risk_rule_repository.go new file mode 100644 index 0000000..99f7be5 --- /dev/null +++ b/backend/internal/repository/risk_rule_repository.go @@ -0,0 +1,479 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +// RiskRuleRepository defines repository operations for configurable risk rules. +type RiskRuleRepository interface { + List() ([]models.RiskRule, error) + ListEnabled() ([]models.RiskRule, error) + GetByRuleID(ruleID string) (*models.RiskRule, error) + Upsert(rule *models.RiskRule) error +} + +type riskRuleRepository struct { + sess db.Session +} + +// NewRiskRuleRepository creates a new risk rule repository and ensures its table and defaults exist. +func NewRiskRuleRepository(sess db.Session) RiskRuleRepository { + repo := &riskRuleRepository{sess: sess} + repo.ensureTable() + repo.normalizeDeprecatedActions() + repo.normalizeDefaultRuleActions() + repo.seedDefaults() + return repo +} + +func (r *riskRuleRepository) ensureTable() { + const query = ` +CREATE TABLE IF NOT EXISTS risk_rules ( + id INT AUTO_INCREMENT PRIMARY KEY, + rule_id VARCHAR(100) NOT NULL UNIQUE, + display_name VARCHAR(255) NOT NULL, + description TEXT NULL, + pattern TEXT NOT NULL, + severity VARCHAR(20) NOT NULL, + action VARCHAR(50) NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + sort_order INT NOT NULL DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_risk_rules_enabled (is_enabled), + INDEX idx_risk_rules_sort (sort_order) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +` + + if _, err := r.sess.SQL().Exec(query); err != nil { + panic(fmt.Errorf("failed to ensure risk_rules table: %w", err)) + } +} + +func (r *riskRuleRepository) seedDefaults() { + defaults := builtInRiskRules() + + for _, item := range defaults { + existing, err := r.GetByRuleID(item.RuleID) + if err != nil { + panic(fmt.Errorf("failed to inspect risk rule %s: %w", item.RuleID, err)) + } + if existing != nil { + continue + } + rule := item + now := time.Now() + rule.CreatedAt = now + rule.UpdatedAt = now + if _, err := r.sess.Collection("risk_rules").Insert(&rule); err != nil { + panic(fmt.Errorf("failed to seed risk rule %s: %w", item.RuleID, err)) + } + } +} + +func (r *riskRuleRepository) normalizeDeprecatedActions() { + if _, err := r.sess.SQL().Exec("UPDATE risk_rules SET action = ? WHERE action = ?", models.RiskActionBlock, "require_approval"); err != nil { + panic(fmt.Errorf("failed to normalize deprecated risk rule actions: %w", err)) + } +} + +func (r *riskRuleRepository) normalizeDefaultRuleActions() { + for _, rule := range builtInRiskRules() { + if _, err := r.sess.SQL().Exec( + "UPDATE risk_rules SET action = ?, updated_at = ? WHERE rule_id = ?", + models.RiskActionRouteSecureModel, + time.Now(), + rule.RuleID, + ); err != nil { + panic(fmt.Errorf("failed to normalize default risk rule action for %s: %w", rule.RuleID, err)) + } + } +} + +func builtInRiskRules() []models.RiskRule { + return []models.RiskRule{ + { + RuleID: "private_key_marker", + DisplayName: "Private key material", + Description: stringPtr("Detect PEM private key markers."), + Pattern: `-----BEGIN [A-Z ]*PRIVATE KEY-----`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 10, + }, + { + RuleID: "api_key_like", + DisplayName: "API key or cloud credential pattern", + Description: stringPtr("Detect common API key and cloud credential patterns."), + Pattern: `(?i)\b(sk-[a-z0-9]{12,}|AKIA[0-9A-Z]{16}|ASIA[0-9A-Z]{16}|ghp_[0-9A-Za-z]{20,255}|AIza[0-9A-Za-z\-_]{35})\b`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 20, + }, + { + RuleID: "credential_assignment", + DisplayName: "Credential assignment pattern", + Description: stringPtr("Detect explicit password, secret, token, or API key assignment."), + Pattern: `(?i)\b(password|passwd|pwd|secret|api[_-]?key|token|client[_-]?secret|access[_-]?key)\b\s*[:=]\s*\S+`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 30, + }, + { + RuleID: "private_ip", + DisplayName: "Private network address", + Description: stringPtr("Detect RFC1918 private IP addresses."), + Pattern: `\b(?:10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})\b`, + Severity: models.RiskSeverityMedium, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 40, + }, + { + RuleID: "email_address", + DisplayName: "Email address", + Description: stringPtr("Detect email addresses."), + Pattern: `(?i)\b[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}\b`, + Severity: models.RiskSeverityMedium, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 50, + }, + { + RuleID: "cn_mobile_number", + DisplayName: "Chinese mainland mobile number", + Description: stringPtr("Detect Chinese mainland mobile phone numbers."), + Pattern: `(?:\+?86[- ]?)?1[3-9]\d{9}`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 60, + }, + { + RuleID: "cn_id_card", + DisplayName: "Chinese ID card number", + Description: stringPtr("Detect 18-digit Chinese ID card numbers."), + Pattern: `\b\d{17}[\dXx]\b`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 70, + }, + { + RuleID: "passport_number", + DisplayName: "Passport number", + Description: stringPtr("Detect common passport number formats."), + Pattern: `(?i)\b(?:passport|护照)[^A-Za-z0-9]{0,12}[A-Z0-9]{7,10}\b`, + Severity: models.RiskSeverityMedium, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 80, + }, + { + RuleID: "bank_card_context", + DisplayName: "Bank card context", + Description: stringPtr("Detect card-number-like data near bank-card keywords."), + Pattern: `(?i)(银行卡|bank card|card number|account number).{0,24}\b\d(?:[ -]?\d){11,18}\b`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 90, + }, + { + RuleID: "personal_address_keywords", + DisplayName: "Personal address keywords", + Description: stringPtr("Detect common personal address expressions."), + Pattern: `(?i)(住址|开户地址|通信地址|家庭住址|收件地址|mailing address|home address|residential address)`, + Severity: models.RiskSeverityMedium, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 100, + }, + { + RuleID: "resume_cv_keywords", + DisplayName: "Resume or CV content", + Description: stringPtr("Detect resume and curriculum vitae related content."), + Pattern: `(?i)\b(简历|履历|resume|curriculum vitae|work experience|employment history|education history|expected salary)\b`, + Severity: models.RiskSeverityMedium, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 110, + }, + { + RuleID: "internal_domain_name", + DisplayName: "Internal domain or system hostname", + Description: stringPtr("Detect common internal domain and system naming patterns."), + Pattern: `(?i)\b(?:corp|internal|intra|intranet|gitlab|jenkins|confluence|jira|wiki|oa|vpn)\.[a-z0-9.-]+\b`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 120, + }, + { + RuleID: "internal_hostname_pattern", + DisplayName: "Internal host or environment naming", + Description: stringPtr("Detect internal hostnames and environment-specific resource names."), + Pattern: `(?i)\b(?:prod|stg|stage|dev|test|uat|db|redis|mq|k8s|kube|es|kafka|mongo|mysql|postgres)[-_][a-z0-9.-]+\b`, + Severity: models.RiskSeverityMedium, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 130, + }, + { + RuleID: "kubernetes_service_dns", + DisplayName: "Kubernetes service DNS", + Description: stringPtr("Detect Kubernetes internal service DNS references."), + Pattern: `(?i)\b[a-z0-9-]+\.[a-z0-9-]+\.svc(?:\.cluster\.local)?\b`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 140, + }, + { + RuleID: "project_codename_keywords", + DisplayName: "Project codename or roadmap wording", + Description: stringPtr("Detect project codename or unreleased roadmap wording."), + Pattern: `(?i)(项目代号|产品代号|代号|codename|roadmap|未发布|未公开|internal launch)`, + Severity: models.RiskSeverityMedium, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 150, + }, + { + RuleID: "org_roster_keywords", + DisplayName: "Organization roster or directory", + Description: stringPtr("Detect organization charts, rosters, and internal directories."), + Pattern: `(?i)(组织架构|员工名单|花名册|通讯录|organization chart|employee roster|staff roster|directory)`, + Severity: models.RiskSeverityMedium, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 160, + }, + { + RuleID: "salary_hr_keywords", + DisplayName: "Salary or HR data", + Description: stringPtr("Detect compensation, payroll, and HR evaluation wording."), + Pattern: `(?i)(薪资|薪酬|绩效|KPI|晋升|调薪|裁员|salary|compensation|payroll|performance review)`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 170, + }, + { + RuleID: "customer_list_keywords", + DisplayName: "Customer list or contact list", + Description: stringPtr("Detect customer roster and contact-list style wording."), + Pattern: `(?i)(客户名单|客户列表|联系人清单|CRM导出|customer list|client list|contact list|lead list)`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 180, + }, + { + RuleID: "contract_quote_keywords", + DisplayName: "Contract or quote document", + Description: stringPtr("Detect contract, quotation, or tender document wording."), + Pattern: `(?i)(合同|报价单|投标|采购单|订单明细|contract|quote|quotation|proposal|tender|purchase order|order details)`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 190, + }, + { + RuleID: "invoice_tax_keywords", + DisplayName: "Invoice or tax information", + Description: stringPtr("Detect invoice and tax identifier wording."), + Pattern: `(?i)(发票|税号|纳税识别号|开票信息|invoice|tax id|vat|tin)`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 200, + }, + { + RuleID: "crm_ticket_keywords", + DisplayName: "CRM or ticket data", + Description: stringPtr("Detect CRM notes, ticket content, or customer profile wording."), + Pattern: `(?i)(工单|客服记录|客户画像|客户需求|ticket|case record|crm note|customer profile)`, + Severity: models.RiskSeverityMedium, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 210, + }, + { + RuleID: "jwt_token_like", + DisplayName: "JWT token", + Description: stringPtr("Detect JWT-like bearer tokens."), + Pattern: `\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 220, + }, + { + RuleID: "cookie_session_like", + DisplayName: "Cookie or session secret wording", + Description: stringPtr("Detect session, cookie, and authorization wording."), + Pattern: `(?i)\b(sessionid|set-cookie|refresh_token|access_token|authorization|cookie)\b`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 230, + }, + { + RuleID: "db_connection_string", + DisplayName: "Database or queue connection string", + Description: stringPtr("Detect connection strings for databases, caches, and queues."), + Pattern: `(?i)\b(?:mysql|postgres(?:ql)?|mongodb(?:\+srv)?|redis|amqp|kafka):\/\/[^\s]+`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 240, + }, + { + RuleID: "kubeconfig_content", + DisplayName: "Kubeconfig content", + Description: stringPtr("Detect kubeconfig-style configuration content."), + Pattern: `(?s)apiVersion:\s*v1.{0,400}(clusters:|contexts:|users:)`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 250, + }, + { + RuleID: "env_file_secret", + DisplayName: "Environment secret variable", + Description: stringPtr("Detect common secret variable names in env-style content."), + Pattern: `(?i)\b(DATABASE_URL|SECRET_KEY|PRIVATE_KEY|ACCESS_KEY|API_SECRET|CLIENT_SECRET|WEBHOOK_SECRET)\b`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 260, + }, + { + RuleID: "financial_metrics_keywords", + DisplayName: "Financial metrics or budget data", + Description: stringPtr("Detect financial metrics, budgets, and profitability wording."), + Pattern: `(?i)(营收|收入|利润|毛利|成本|预算|现金流|revenue|gross margin|profit|budget|cash flow)`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 270, + }, + { + RuleID: "legal_document_keywords", + DisplayName: "Legal or compliance document", + Description: stringPtr("Detect legal opinions, disputes, and NDA style wording."), + Pattern: `(?i)(法务意见|诉讼|仲裁|保密协议|NDA|legal opinion|litigation|arbitration|non-disclosure agreement)`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 280, + }, + { + RuleID: "political_person_org_keywords", + DisplayName: "Political or government organization wording", + Description: stringPtr("Detect political institutions, officials, and related wording."), + Pattern: `(?i)(国务院|外交部|国安|公安部|人大|政协|党中央|中央军委|政府工作报告|政治局|state council|ministry of foreign affairs|national security)`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 290, + }, + { + RuleID: "military_security_keywords", + DisplayName: "Military or national security wording", + Description: stringPtr("Detect military deployment, weapons, and national security wording."), + Pattern: `(?i)(军事部署|武器系统|导弹|军工|部队番号|国家安全|涉密单位|military deployment|weapon system|missile|classified unit)`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 300, + }, + { + RuleID: "extremism_keywords", + DisplayName: "Extremism or violent attack wording", + Description: stringPtr("Detect extremist, violent attack, and bomb-making wording."), + Pattern: `(?i)(恐怖袭击|炸弹制造|极端组织|暴恐|terrorist attack|bomb making|extremist organization)`, + Severity: models.RiskSeverityHigh, + Action: models.RiskActionRouteSecureModel, + IsEnabled: true, + SortOrder: 310, + }, + } +} + +func (r *riskRuleRepository) List() ([]models.RiskRule, error) { + var items []models.RiskRule + if err := r.sess.Collection("risk_rules").Find().OrderBy("sort_order", "rule_id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list risk rules: %w", err) + } + return items, nil +} + +func (r *riskRuleRepository) ListEnabled() ([]models.RiskRule, error) { + var items []models.RiskRule + if err := r.sess.Collection("risk_rules").Find(db.Cond{"is_enabled": true}).OrderBy("sort_order", "rule_id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list enabled risk rules: %w", err) + } + return items, nil +} + +func (r *riskRuleRepository) GetByRuleID(ruleID string) (*models.RiskRule, error) { + var item models.RiskRule + err := r.sess.Collection("risk_rules").Find(db.Cond{"rule_id": ruleID}).One(&item) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get risk rule by rule id: %w", err) + } + return &item, nil +} + +func (r *riskRuleRepository) Upsert(rule *models.RiskRule) error { + existing, err := r.GetByRuleID(rule.RuleID) + if err != nil { + return err + } + + now := time.Now() + if existing == nil { + rule.CreatedAt = now + rule.UpdatedAt = now + res, err := r.sess.Collection("risk_rules").Insert(rule) + if err != nil { + return fmt.Errorf("failed to create risk rule: %w", err) + } + rule.ID = int(res.ID().(int64)) + return nil + } + + existing.DisplayName = rule.DisplayName + existing.Description = rule.Description + existing.Pattern = rule.Pattern + existing.Severity = rule.Severity + existing.Action = rule.Action + existing.IsEnabled = rule.IsEnabled + existing.SortOrder = rule.SortOrder + existing.UpdatedAt = now + + if err := r.sess.Collection("risk_rules").Find(db.Cond{"id": existing.ID}).Update(existing); err != nil { + return fmt.Errorf("failed to update risk rule: %w", err) + } + rule.ID = existing.ID + rule.CreatedAt = existing.CreatedAt + rule.UpdatedAt = existing.UpdatedAt + return nil +} + +func stringPtr(value string) *string { + return &value +} diff --git a/backend/internal/repository/runtime_pod_repository.go b/backend/internal/repository/runtime_pod_repository.go new file mode 100644 index 0000000..e168a12 --- /dev/null +++ b/backend/internal/repository/runtime_pod_repository.go @@ -0,0 +1,224 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type RuntimePodMetricsUpdate struct { + CPUMillisUsed int64 + MemoryBytesUsed int64 + DiskBytesUsed int64 + NetworkRXBytes int64 + NetworkTXBytes int64 + MetricsJSON *string + LastSeenAt *time.Time +} + +type RuntimePodRepository interface { + UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error + GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) + GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) + List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) + ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) + TryClaimSlot(ctx context.Context, podID int64) (bool, error) + ReleaseSlot(ctx context.Context, podID int64) error + MarkState(ctx context.Context, podID int64, state string, draining bool) error + UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error + MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error + UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error +} + +type runtimePodRepository struct { + sess db.Session +} + +func NewRuntimePodRepository(sess db.Session) RuntimePodRepository { + return &runtimePodRepository{sess: sess} +} + +func (r *runtimePodRepository) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error { + ensureTimestamps(&pod.CreatedAt, &pod.UpdatedAt) + res, err := r.sess.SQL().ExecContext(ctx, ` + INSERT INTO runtime_pods ( + runtime_type, namespace, pod_name, pod_uid, pod_ip, node_name, deployment_name, + image_ref, agent_endpoint, state, capacity, used_slots, draining, cpu_millis_used, + memory_bytes_used, disk_bytes_used, network_rx_bytes, network_tx_bytes, metrics_json, + last_seen_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + id = LAST_INSERT_ID(id), + runtime_type = VALUES(runtime_type), + pod_uid = VALUES(pod_uid), + pod_ip = VALUES(pod_ip), + node_name = VALUES(node_name), + deployment_name = VALUES(deployment_name), + image_ref = VALUES(image_ref), + agent_endpoint = VALUES(agent_endpoint), + capacity = VALUES(capacity), + cpu_millis_used = VALUES(cpu_millis_used), + memory_bytes_used = VALUES(memory_bytes_used), + disk_bytes_used = VALUES(disk_bytes_used), + network_rx_bytes = VALUES(network_rx_bytes), + network_tx_bytes = VALUES(network_tx_bytes), + metrics_json = VALUES(metrics_json), + last_seen_at = VALUES(last_seen_at), + updated_at = VALUES(updated_at) + `, pod.RuntimeType, pod.Namespace, pod.PodName, pod.PodUID, pod.PodIP, pod.NodeName, pod.DeploymentName, + pod.ImageRef, pod.AgentEndpoint, pod.State, pod.Capacity, pod.UsedSlots, pod.Draining, pod.CPUMillisUsed, + pod.MemoryBytesUsed, pod.DiskBytesUsed, pod.NetworkRXBytes, pod.NetworkTXBytes, pod.MetricsJSON, + pod.LastSeenAt, pod.CreatedAt, pod.UpdatedAt) + if err != nil { + return fmt.Errorf("failed to upsert runtime pod: %w", err) + } + if id, err := res.LastInsertId(); err == nil { + pod.ID = id + } + return nil +} + +func (r *runtimePodRepository) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var pod models.RuntimePod + if err := r.sess.Collection("runtime_pods").Find(db.Cond{"id": id}).One(&pod); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get runtime pod: %w", err) + } + return &pod, nil +} + +func (r *runtimePodRepository) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var pod models.RuntimePod + if err := r.sess.Collection("runtime_pods").Find(db.Cond{"namespace": namespace, "pod_name": podName}).One(&pod); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get runtime pod by namespace/name: %w", err) + } + return &pod, nil +} + +func (r *runtimePodRepository) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + cond := db.Cond{} + if runtimeType != "" { + cond["runtime_type"] = runtimeType + } + var pods []models.RuntimePod + if err := r.sess.Collection("runtime_pods").Find(cond).OrderBy("namespace", "pod_name").All(&pods); err != nil { + return nil, fmt.Errorf("failed to list runtime pods: %w", err) + } + return pods, nil +} + +func (r *runtimePodRepository) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var pods []models.RuntimePod + iter := r.sess.SQL().IteratorContext(ctx, ` + SELECT * + FROM runtime_pods + WHERE runtime_type = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity + ORDER BY used_slots, id + `, runtimeType) + defer iter.Close() + if err := iter.All(&pods); err != nil { + return nil, fmt.Errorf("failed to list schedulable runtime pods: %w", err) + } + return pods, nil +} + +func (r *runtimePodRepository) TryClaimSlot(ctx context.Context, podID int64) (bool, error) { + res, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET used_slots = used_slots + 1, updated_at = ? + WHERE id = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity + `, time.Now().UTC(), podID) + if err != nil { + return false, fmt.Errorf("failed to claim runtime pod slot: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("failed to inspect runtime pod slot claim: %w", err) + } + return affected == 1, nil +} + +func (r *runtimePodRepository) ReleaseSlot(ctx context.Context, podID int64) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET used_slots = CASE WHEN used_slots > 0 THEN used_slots - 1 ELSE 0 END, updated_at = ? + WHERE id = ? + `, time.Now().UTC(), podID) + if err != nil { + return fmt.Errorf("failed to release runtime pod slot: %w", err) + } + return nil +} + +func (r *runtimePodRepository) MarkState(ctx context.Context, podID int64, state string, draining bool) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET state = ?, draining = ?, updated_at = ? + WHERE id = ? + `, state, draining, time.Now().UTC(), podID) + if err != nil { + return fmt.Errorf("failed to mark runtime pod state: %w", err) + } + return nil +} + +func (r *runtimePodRepository) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET state = ?, used_slots = ?, capacity = ?, draining = ?, last_seen_at = ?, updated_at = ? + WHERE id = ? + `, state, usedSlots, capacity, draining, lastSeenAt, time.Now().UTC(), podID) + if err != nil { + return fmt.Errorf("failed to update runtime pod heartbeat: %w", err) + } + return nil +} + +func (r *runtimePodRepository) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET state = 'unhealthy', updated_at = ? + WHERE (last_seen_at IS NULL OR last_seen_at < ?) AND state <> 'unhealthy' + `, time.Now().UTC(), cutoff) + if err != nil { + return fmt.Errorf("failed to mark unseen runtime pods unhealthy: %w", err) + } + return nil +} + +func (r *runtimePodRepository) UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET cpu_millis_used = ?, memory_bytes_used = ?, disk_bytes_used = ?, + network_rx_bytes = ?, network_tx_bytes = ?, metrics_json = ?, + last_seen_at = ?, updated_at = ? + WHERE id = ? + `, metrics.CPUMillisUsed, metrics.MemoryBytesUsed, metrics.DiskBytesUsed, + metrics.NetworkRXBytes, metrics.NetworkTXBytes, metrics.MetricsJSON, + metrics.LastSeenAt, time.Now().UTC(), podID) + if err != nil { + return fmt.Errorf("failed to update runtime pod metrics: %w", err) + } + return nil +} diff --git a/backend/internal/repository/runtime_rollout_repository.go b/backend/internal/repository/runtime_rollout_repository.go new file mode 100644 index 0000000..43c5e0d --- /dev/null +++ b/backend/internal/repository/runtime_rollout_repository.go @@ -0,0 +1,82 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type RuntimeRolloutRepository interface { + Create(ctx context.Context, rollout *models.RuntimeRollout) error + GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) + ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) + UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error +} + +type runtimeRolloutRepository struct { + sess db.Session +} + +func NewRuntimeRolloutRepository(sess db.Session) RuntimeRolloutRepository { + return &runtimeRolloutRepository{sess: sess} +} + +func (r *runtimeRolloutRepository) Create(ctx context.Context, rollout *models.RuntimeRollout) error { + if err := ctx.Err(); err != nil { + return err + } + ensureTimestamps(&rollout.CreatedAt, &rollout.UpdatedAt) + res, err := r.sess.Collection("runtime_rollouts").Insert(rollout) + if err != nil { + return fmt.Errorf("failed to create runtime rollout: %w", err) + } + if id, ok := res.ID().(int64); ok { + rollout.ID = id + } + return nil +} + +func (r *runtimeRolloutRepository) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var rollout models.RuntimeRollout + if err := r.sess.Collection("runtime_rollouts").Find(db.Cond{"id": id}).One(&rollout); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get runtime rollout: %w", err) + } + return &rollout, nil +} + +func (r *runtimeRolloutRepository) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + cond := db.Cond{"status IN": []string{"pending", "running"}} + if runtimeType != "" { + cond["runtime_type"] = runtimeType + } + var rollouts []models.RuntimeRollout + if err := r.sess.Collection("runtime_rollouts").Find(cond).OrderBy("created_at", "id").All(&rollouts); err != nil { + return nil, fmt.Errorf("failed to list active runtime rollouts: %w", err) + } + return rollouts, nil +} + +func (r *runtimeRolloutRepository) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_rollouts + SET status = ?, started_at = ?, finished_at = ?, error_message = ?, updated_at = ? + WHERE id = ? + `, status, startedAt, finishedAt, message, time.Now().UTC(), id) + if err != nil { + return fmt.Errorf("failed to update runtime rollout status: %w", err) + } + return nil +} diff --git a/backend/internal/repository/security_scan_repository.go b/backend/internal/repository/security_scan_repository.go new file mode 100644 index 0000000..5471e9b --- /dev/null +++ b/backend/internal/repository/security_scan_repository.go @@ -0,0 +1,254 @@ +package repository + +import ( + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type SecurityScanRepository interface { + GetConfig() (*models.SecurityScanConfig, error) + UpsertConfig(config *models.SecurityScanConfig) error + CreateJob(job *models.SecurityScanJob) error + UpdateJob(job *models.SecurityScanJob) error + GetJobByID(id int) (*models.SecurityScanJob, error) + ListJobs(limit int) ([]models.SecurityScanJob, error) + CreateJobItem(item *models.SecurityScanJobItem) error + UpdateJobItem(item *models.SecurityScanJobItem) error + ListJobItems(jobID int) ([]models.SecurityScanJobItem, error) + UpsertReport(report *models.SecurityScanReport) error + GetReportByJobID(jobID int) (*models.SecurityScanReport, error) +} + +type securityScanRepository struct { + sess db.Session +} + +func NewSecurityScanRepository(sess db.Session) SecurityScanRepository { + repo := &securityScanRepository{sess: sess} + repo.ensureTables() + return repo +} + +func (r *securityScanRepository) ensureTables() { + statements := []string{ + `CREATE TABLE IF NOT EXISTS security_scan_configs ( + id INT AUTO_INCREMENT PRIMARY KEY, + default_mode VARCHAR(20) NOT NULL DEFAULT 'quick', + quick_analyzers_json LONGTEXT NOT NULL, + deep_analyzers_json LONGTEXT NOT NULL, + quick_timeout_seconds INT NOT NULL DEFAULT 30, + deep_timeout_seconds INT NOT NULL DEFAULT 120, + allow_fallback BOOLEAN NOT NULL DEFAULT TRUE, + updated_by INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_security_scan_configs_singleton (id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, + `CREATE TABLE IF NOT EXISTS security_scan_jobs ( + id INT AUTO_INCREMENT PRIMARY KEY, + asset_type VARCHAR(30) NOT NULL DEFAULT 'skill', + scan_mode VARCHAR(20) NOT NULL DEFAULT 'quick', + status VARCHAR(20) NOT NULL DEFAULT 'pending', + requested_by INT NULL, + scope_json LONGTEXT NULL, + total_items INT NOT NULL DEFAULT 0, + completed_items INT NOT NULL DEFAULT 0, + failed_items INT NOT NULL DEFAULT 0, + current_item_name VARCHAR(255) NULL, + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_security_scan_jobs_status (status, created_at), + INDEX idx_security_scan_jobs_asset_type (asset_type, created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, + `CREATE TABLE IF NOT EXISTS security_scan_job_items ( + id INT AUTO_INCREMENT PRIMARY KEY, + job_id INT NOT NULL, + asset_type VARCHAR(30) NOT NULL DEFAULT 'skill', + asset_id INT NOT NULL, + asset_name VARCHAR(255) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + progress_pct INT NOT NULL DEFAULT 0, + risk_level VARCHAR(30) NULL, + summary TEXT NULL, + scan_result_id INT NULL, + cached_result BOOLEAN NOT NULL DEFAULT FALSE, + error_message TEXT NULL, + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (job_id) REFERENCES security_scan_jobs(id) ON DELETE CASCADE, + UNIQUE KEY uk_security_scan_job_items_job_asset (job_id, asset_type, asset_id), + INDEX idx_security_scan_job_items_job (job_id, status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, + `CREATE TABLE IF NOT EXISTS security_scan_reports ( + id INT AUTO_INCREMENT PRIMARY KEY, + job_id INT NOT NULL, + summary_json LONGTEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (job_id) REFERENCES security_scan_jobs(id) ON DELETE CASCADE, + UNIQUE KEY uk_security_scan_reports_job (job_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, + } + for _, statement := range statements { + if _, err := r.sess.SQL().Exec(statement); err != nil { + panic(fmt.Errorf("failed to ensure security scan tables: %w", err)) + } + } +} + +func (r *securityScanRepository) GetConfig() (*models.SecurityScanConfig, error) { + var item models.SecurityScanConfig + err := r.sess.Collection("security_scan_configs").Find(db.Cond{"id": 1}).One(&item) + if err == db.ErrNoMoreRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to get security scan config: %w", err) + } + return &item, nil +} + +func (r *securityScanRepository) UpsertConfig(config *models.SecurityScanConfig) error { + existing, err := r.GetConfig() + if err != nil { + return err + } + now := time.Now().UTC() + if existing == nil { + config.ID = 1 + config.CreatedAt = now + config.UpdatedAt = now + if _, err := r.sess.Collection("security_scan_configs").Insert(config); err != nil { + return fmt.Errorf("failed to create security scan config: %w", err) + } + return nil + } + config.ID = existing.ID + config.CreatedAt = existing.CreatedAt + config.UpdatedAt = now + if err := r.sess.Collection("security_scan_configs").Find(db.Cond{"id": existing.ID}).Update(config); err != nil { + return fmt.Errorf("failed to update security scan config: %w", err) + } + return nil +} + +func (r *securityScanRepository) CreateJob(job *models.SecurityScanJob) error { + ensureTimestamps(&job.CreatedAt, &job.UpdatedAt) + res, err := r.sess.Collection("security_scan_jobs").Insert(job) + if err != nil { + return fmt.Errorf("failed to create security scan job: %w", err) + } + if id, ok := res.ID().(int64); ok { + job.ID = int(id) + } + return nil +} + +func (r *securityScanRepository) UpdateJob(job *models.SecurityScanJob) error { + if job.UpdatedAt.IsZero() { + job.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("security_scan_jobs").Find(db.Cond{"id": job.ID}).Update(job); err != nil { + return fmt.Errorf("failed to update security scan job: %w", err) + } + return nil +} + +func (r *securityScanRepository) GetJobByID(id int) (*models.SecurityScanJob, error) { + var item models.SecurityScanJob + err := r.sess.Collection("security_scan_jobs").Find(db.Cond{"id": id}).One(&item) + if err == db.ErrNoMoreRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to get security scan job: %w", err) + } + return &item, nil +} + +func (r *securityScanRepository) ListJobs(limit int) ([]models.SecurityScanJob, error) { + if limit <= 0 { + limit = 20 + } + var items []models.SecurityScanJob + if err := r.sess.Collection("security_scan_jobs").Find().OrderBy("-created_at", "-id").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list security scan jobs: %w", err) + } + return items, nil +} + +func (r *securityScanRepository) CreateJobItem(item *models.SecurityScanJobItem) error { + ensureTimestamps(&item.CreatedAt, &item.UpdatedAt) + res, err := r.sess.Collection("security_scan_job_items").Insert(item) + if err != nil { + return fmt.Errorf("failed to create security scan job item: %w", err) + } + if id, ok := res.ID().(int64); ok { + item.ID = int(id) + } + return nil +} + +func (r *securityScanRepository) UpdateJobItem(item *models.SecurityScanJobItem) error { + if item.UpdatedAt.IsZero() { + item.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("security_scan_job_items").Find(db.Cond{"id": item.ID}).Update(item); err != nil { + return fmt.Errorf("failed to update security scan job item: %w", err) + } + return nil +} + +func (r *securityScanRepository) ListJobItems(jobID int) ([]models.SecurityScanJobItem, error) { + var items []models.SecurityScanJobItem + if err := r.sess.Collection("security_scan_job_items").Find(db.Cond{"job_id": jobID}).OrderBy("id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list security scan job items: %w", err) + } + return items, nil +} + +func (r *securityScanRepository) UpsertReport(report *models.SecurityScanReport) error { + var existing models.SecurityScanReport + err := r.sess.Collection("security_scan_reports").Find(db.Cond{"job_id": report.JobID}).One(&existing) + if err == db.ErrNoMoreRows { + ensureTimestamps(&report.CreatedAt, &report.UpdatedAt) + res, err := r.sess.Collection("security_scan_reports").Insert(report) + if err != nil { + return fmt.Errorf("failed to create security scan report: %w", err) + } + if id, ok := res.ID().(int64); ok { + report.ID = int(id) + } + return nil + } + if err != nil { + return fmt.Errorf("failed to get security scan report: %w", err) + } + report.ID = existing.ID + report.CreatedAt = existing.CreatedAt + report.UpdatedAt = time.Now().UTC() + if err := r.sess.Collection("security_scan_reports").Find(db.Cond{"id": existing.ID}).Update(report); err != nil { + return fmt.Errorf("failed to update security scan report: %w", err) + } + return nil +} + +func (r *securityScanRepository) GetReportByJobID(jobID int) (*models.SecurityScanReport, error) { + var item models.SecurityScanReport + err := r.sess.Collection("security_scan_reports").Find(db.Cond{"job_id": jobID}).One(&item) + if err == db.ErrNoMoreRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to get security scan report: %w", err) + } + return &item, nil +} diff --git a/backend/internal/repository/skill_repository.go b/backend/internal/repository/skill_repository.go new file mode 100644 index 0000000..40e36f9 --- /dev/null +++ b/backend/internal/repository/skill_repository.go @@ -0,0 +1,493 @@ +package repository + +import ( + "fmt" + "strings" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type SkillRepository interface { + ListSkillsByUser(userID int) ([]models.Skill, error) + ListAllSkills() ([]models.Skill, error) + GetSkillByID(id int) (*models.Skill, error) + GetSkillByUserKey(userID int, skillKey string) (*models.Skill, error) + CreateSkill(skill *models.Skill) error + UpdateSkill(skill *models.Skill) error + DeleteSkill(id int) error + GetBlobByContentHash(hash string) (*models.SkillBlob, error) + GetBlobByID(id int) (*models.SkillBlob, error) + CreateBlob(blob *models.SkillBlob) error + UpdateBlob(blob *models.SkillBlob) error + ListVersionsBySkillID(skillID int) ([]models.SkillVersion, error) + GetVersionByID(id int) (*models.SkillVersion, error) + GetVersionBySkillAndBlob(skillID, blobID int) (*models.SkillVersion, error) + GetLatestVersionBySkillID(skillID int) (*models.SkillVersion, error) + CreateVersion(version *models.SkillVersion) error + ListInstanceSkills(instanceID int) ([]models.InstanceSkill, error) + GetInstanceSkill(instanceID, skillID int) (*models.InstanceSkill, error) + UpsertInstanceSkill(item *models.InstanceSkill) error + MarkInstanceSkillRemoved(instanceID int, skillID int, observedAt time.Time) error + MarkInstanceSkillRemovedBySkillKey(instanceID int, skillKey string, observedAt time.Time) error + MarkInstanceSkillsRemovedByWorkspacePath(instanceID int, workspacePath string, observedAt time.Time) error + MarkMissingInstanceSkills(instanceID int, activeSkillIDs []int, observedAt time.Time) error + CreateScanResult(result *models.SkillScanResult) error + GetScanResultByID(id int) (*models.SkillScanResult, error) + ListScanResultsByBlobID(blobID int) ([]models.SkillScanResult, error) + GetLatestScanResultByBlobID(blobID int) (*models.SkillScanResult, error) + GetLatestScanResultBySkillID(skillID int) (*models.SkillScanResult, error) +} + +type skillRepository struct{ sess db.Session } + +func NewSkillRepository(sess db.Session) SkillRepository { return &skillRepository{sess: sess} } + +func (r *skillRepository) ListSkillsByUser(userID int) ([]models.Skill, error) { + var items []models.Skill + if err := r.sess.Collection("skills").Find(db.Cond{"user_id": userID}).OrderBy("-updated_at", "-id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list skills by user: %w", err) + } + return items, nil +} + +func (r *skillRepository) ListAllSkills() ([]models.Skill, error) { + var items []models.Skill + if err := r.sess.Collection("skills").Find().OrderBy("-updated_at", "-id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list all skills: %w", err) + } + return items, nil +} + +func (r *skillRepository) GetSkillByID(id int) (*models.Skill, error) { + var item models.Skill + if err := r.sess.Collection("skills").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get skill: %w", err) + } + return &item, nil +} + +func (r *skillRepository) GetSkillByUserKey(userID int, skillKey string) (*models.Skill, error) { + var item models.Skill + if err := r.sess.Collection("skills").Find(db.Cond{"user_id": userID, "skill_key": skillKey}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get skill by key: %w", err) + } + return &item, nil +} + +func (r *skillRepository) CreateSkill(skill *models.Skill) error { + ensureTimestamps(&skill.CreatedAt, &skill.UpdatedAt) + res, err := r.sess.Collection("skills").Insert(skill) + if err != nil { + if isDuplicateEntryError(err) { + existing, findErr := r.GetSkillByUserKey(skill.UserID, skill.SkillKey) + if findErr != nil { + return findErr + } + if existing != nil { + *skill = *existing + return nil + } + } + return fmt.Errorf("failed to create skill: %w", err) + } + if id, ok := res.ID().(int64); ok { + skill.ID = int(id) + } + return nil +} + +func (r *skillRepository) UpdateSkill(skill *models.Skill) error { + if skill.UpdatedAt.IsZero() { + skill.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("skills").Find(db.Cond{"id": skill.ID}).Update(skill); err != nil { + return fmt.Errorf("failed to update skill: %w", err) + } + return nil +} + +func (r *skillRepository) DeleteSkill(id int) error { + if err := r.sess.Collection("skills").Find(db.Cond{"id": id}).Delete(); err != nil { + return fmt.Errorf("failed to delete skill: %w", err) + } + return nil +} + +func (r *skillRepository) GetBlobByContentHash(hash string) (*models.SkillBlob, error) { + var item models.SkillBlob + if err := r.sess.Collection("skill_blobs").Find(db.Cond{"content_hash": hash}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get skill blob by content hash: %w", err) + } + return &item, nil +} + +func (r *skillRepository) GetBlobByID(id int) (*models.SkillBlob, error) { + var item models.SkillBlob + if err := r.sess.Collection("skill_blobs").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get skill blob: %w", err) + } + return &item, nil +} + +func (r *skillRepository) CreateBlob(blob *models.SkillBlob) error { + ensureTimestamps(&blob.CreatedAt, &blob.UpdatedAt) + res, err := r.sess.Collection("skill_blobs").Insert(blob) + if err != nil { + return fmt.Errorf("failed to create skill blob: %w", err) + } + if id, ok := res.ID().(int64); ok { + blob.ID = int(id) + } + return nil +} + +func (r *skillRepository) UpdateBlob(blob *models.SkillBlob) error { + if blob.UpdatedAt.IsZero() { + blob.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("skill_blobs").Find(db.Cond{"id": blob.ID}).Update(blob); err != nil { + return fmt.Errorf("failed to update skill blob: %w", err) + } + return nil +} + +func (r *skillRepository) ListVersionsBySkillID(skillID int) ([]models.SkillVersion, error) { + var items []models.SkillVersion + if err := r.sess.Collection("skill_versions").Find(db.Cond{"skill_id": skillID}).OrderBy("-version_no", "-id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list skill versions: %w", err) + } + return items, nil +} + +func (r *skillRepository) GetVersionByID(id int) (*models.SkillVersion, error) { + var item models.SkillVersion + if err := r.sess.Collection("skill_versions").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get skill version: %w", err) + } + return &item, nil +} + +func (r *skillRepository) GetVersionBySkillAndBlob(skillID, blobID int) (*models.SkillVersion, error) { + var item models.SkillVersion + if err := r.sess.Collection("skill_versions").Find(db.Cond{"skill_id": skillID, "blob_id": blobID}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get skill version by blob: %w", err) + } + return &item, nil +} + +func (r *skillRepository) GetLatestVersionBySkillID(skillID int) (*models.SkillVersion, error) { + var item models.SkillVersion + if err := r.sess.Collection("skill_versions").Find(db.Cond{"skill_id": skillID}).OrderBy("-version_no", "-id").One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get latest skill version: %w", err) + } + return &item, nil +} + +func (r *skillRepository) CreateVersion(version *models.SkillVersion) error { + ensureTimestamps(&version.CreatedAt, &version.UpdatedAt) + res, err := r.sess.Collection("skill_versions").Insert(version) + if err != nil { + return fmt.Errorf("failed to create skill version: %w", err) + } + if id, ok := res.ID().(int64); ok { + version.ID = int(id) + } + return nil +} + +func (r *skillRepository) ListInstanceSkills(instanceID int) ([]models.InstanceSkill, error) { + var items []models.InstanceSkill + if err := r.sess.Collection("instance_skills").Find(db.Cond{"instance_id": instanceID}).OrderBy("-updated_at", "-id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list instance skills: %w", err) + } + return items, nil +} + +func (r *skillRepository) GetInstanceSkill(instanceID, skillID int) (*models.InstanceSkill, error) { + var item models.InstanceSkill + if err := r.sess.Collection("instance_skills").Find(db.Cond{"instance_id": instanceID, "skill_id": skillID}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance skill: %w", err) + } + return &item, nil +} + +func (r *skillRepository) UpsertInstanceSkill(item *models.InstanceSkill) error { + existing, err := r.GetInstanceSkill(item.InstanceID, item.SkillID) + if err != nil { + return err + } + if existing == nil { + ensureTimestamps(&item.CreatedAt, &item.UpdatedAt) + res, err := r.sess.Collection("instance_skills").Insert(item) + if err != nil { + if !isDuplicateEntryError(err) { + return fmt.Errorf("failed to create instance skill: %w", err) + } + existing, err = r.GetInstanceSkill(item.InstanceID, item.SkillID) + if err != nil { + return err + } + if existing == nil { + return fmt.Errorf("failed to create instance skill: %w", err) + } + item.ID = existing.ID + item.CreatedAt = existing.CreatedAt + if item.UpdatedAt.IsZero() { + item.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": existing.ID}).Update(item); err != nil { + return fmt.Errorf("failed to update instance skill after duplicate insert: %w", err) + } + return nil + } + if id, ok := res.ID().(int64); ok { + item.ID = int(id) + } + return nil + } + item.ID = existing.ID + item.CreatedAt = existing.CreatedAt + if item.UpdatedAt.IsZero() { + item.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": existing.ID}).Update(item); err != nil { + return fmt.Errorf("failed to update instance skill: %w", err) + } + return nil +} + +func isDuplicateEntryError(err error) bool { + if err == nil { + return false + } + return strings.Contains(strings.ToLower(err.Error()), "duplicate entry") +} + +func (r *skillRepository) MarkInstanceSkillRemoved(instanceID int, skillID int, observedAt time.Time) error { + item, err := r.GetInstanceSkill(instanceID, skillID) + if err != nil { + return err + } + if item == nil { + return nil + } + if item.UpdatedAt.After(observedAt) { + return nil + } + item.Status = "removed" + item.RemovedAt = &observedAt + item.UpdatedAt = observedAt + if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": item.ID}).Update(item); err != nil { + return fmt.Errorf("failed to mark instance skill removed: %w", err) + } + return nil +} + +func (r *skillRepository) MarkInstanceSkillRemovedBySkillKey(instanceID int, skillKey string, observedAt time.Time) error { + skillKey = strings.TrimSpace(skillKey) + if skillKey == "" { + return nil + } + var skills []models.Skill + if err := r.sess.Collection("skills").Find(db.Cond{"skill_key": skillKey}).All(&skills); err != nil && err != db.ErrNoMoreRows { + return fmt.Errorf("failed to find skills by key for removal: %w", err) + } + for _, skill := range skills { + if err := r.MarkInstanceSkillRemoved(instanceID, skill.ID, observedAt); err != nil { + return err + } + } + return nil +} + +func (r *skillRepository) MarkInstanceSkillsRemovedByWorkspacePath(instanceID int, workspacePath string, observedAt time.Time) error { + deletedPath := normalizeInstanceSkillWorkspacePath(workspacePath) + if deletedPath == "" { + return nil + } + var items []models.InstanceSkill + if err := r.sess.Collection("instance_skills").Find(db.Cond{"instance_id": instanceID}).All(&items); err != nil && err != db.ErrNoMoreRows { + return fmt.Errorf("failed to list instance skills for workspace deletion: %w", err) + } + for _, item := range items { + if isRemovedInstanceSkillRecord(item) { + continue + } + matches := instanceSkillInstallPathMatchesDelete(item, deletedPath) + if !matches { + skill, err := r.GetSkillByID(item.SkillID) + if err != nil { + return err + } + matches = skill != nil && workspaceDeleteTargetsSkillKey(deletedPath, skill.SkillKey) + } + if !matches { + continue + } + item.Status = "removed" + item.RemovedAt = &observedAt + item.UpdatedAt = observedAt + if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": item.ID}).Update(&item); err != nil { + return fmt.Errorf("failed to mark instance skill removed after workspace deletion: %w", err) + } + } + return nil +} + +func normalizeInstanceSkillWorkspacePath(value string) string { + value = strings.TrimSpace(strings.ReplaceAll(value, "\\", "/")) + value = strings.TrimPrefix(value, "/") + parts := strings.Split(value, "/") + clean := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" || part == "." { + continue + } + clean = append(clean, part) + } + if len(clean) > 0 && strings.EqualFold(clean[0], "config") { + clean = clean[1:] + } + return strings.Join(clean, "/") +} + +func instanceSkillInstallPathMatchesDelete(item models.InstanceSkill, deletedPath string) bool { + if item.InstallPath == nil { + return false + } + installPath := normalizeInstanceSkillWorkspacePath(*item.InstallPath) + if installPath == "" || deletedPath == "" { + return false + } + return installPath == deletedPath || + strings.HasPrefix(installPath, deletedPath+"/") || + strings.HasPrefix(deletedPath, installPath+"/") || + strings.HasSuffix(installPath, "/"+deletedPath) || + strings.HasSuffix(deletedPath, "/"+installPath) +} + +func workspaceDeleteTargetsSkillKey(deletedPath string, skillKey string) bool { + key := strings.ToLower(strings.TrimSpace(skillKey)) + if key == "" || deletedPath == "" { + return false + } + segments := strings.Split(strings.ToLower(deletedPath), "/") + last := segments[len(segments)-1] + if last != key { + return false + } + if len(segments) <= 2 { + return true + } + for i, segment := range segments { + if segment == "skills" && i+1 < len(segments) && segments[i+1] == key { + return true + } + if segment == ".openclaw" || segment == "openclaw" { + return true + } + } + return false +} + +func isRemovedInstanceSkillRecord(item models.InstanceSkill) bool { + return strings.EqualFold(strings.TrimSpace(item.Status), "removed") || item.RemovedAt != nil +} + +func (r *skillRepository) MarkMissingInstanceSkills(instanceID int, activeSkillIDs []int, observedAt time.Time) error { + find := r.sess.Collection("instance_skills").Find(db.Cond{"instance_id": instanceID}) + if len(activeSkillIDs) > 0 { + find = find.And(db.Cond{"skill_id NOT IN": activeSkillIDs}) + } + var items []models.InstanceSkill + if err := find.All(&items); err != nil && err != db.ErrNoMoreRows { + return fmt.Errorf("failed to list stale instance skills: %w", err) + } + for _, item := range items { + item.Status = "removed" + item.RemovedAt = &observedAt + item.UpdatedAt = observedAt + if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": item.ID}).Update(item); err != nil { + return fmt.Errorf("failed to mark instance skill removed: %w", err) + } + } + return nil +} + +func (r *skillRepository) CreateScanResult(result *models.SkillScanResult) error { + ensureTimestamps(&result.CreatedAt, &result.UpdatedAt) + res, err := r.sess.Collection("skill_scan_results").Insert(result) + if err != nil { + return fmt.Errorf("failed to create skill scan result: %w", err) + } + if id, ok := res.ID().(int64); ok { + result.ID = int(id) + } + return nil +} + +func (r *skillRepository) GetScanResultByID(id int) (*models.SkillScanResult, error) { + var item models.SkillScanResult + if err := r.sess.Collection("skill_scan_results").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get skill scan result: %w", err) + } + return &item, nil +} + +func (r *skillRepository) ListScanResultsByBlobID(blobID int) ([]models.SkillScanResult, error) { + var items []models.SkillScanResult + if err := r.sess.Collection("skill_scan_results").Find(db.Cond{"blob_id": blobID}).OrderBy("-scanned_at", "-id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list skill scan results: %w", err) + } + return items, nil +} + +func (r *skillRepository) GetLatestScanResultByBlobID(blobID int) (*models.SkillScanResult, error) { + var item models.SkillScanResult + if err := r.sess.Collection("skill_scan_results").Find(db.Cond{"blob_id": blobID}).OrderBy("-scanned_at", "-id").One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get latest skill scan result: %w", err) + } + return &item, nil +} + +func (r *skillRepository) GetLatestScanResultBySkillID(skillID int) (*models.SkillScanResult, error) { + skill, err := r.GetSkillByID(skillID) + if err != nil || skill == nil || skill.LastScanResultID == nil { + return nil, err + } + return r.GetScanResultByID(*skill.LastScanResultID) +} diff --git a/backend/internal/repository/system_image_setting_repository.go b/backend/internal/repository/system_image_setting_repository.go new file mode 100644 index 0000000..f9f9e08 --- /dev/null +++ b/backend/internal/repository/system_image_setting_repository.go @@ -0,0 +1,268 @@ +package repository + +import ( + "fmt" + "strings" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +// SystemImageSettingRepository defines repository operations for runtime image settings. +type SystemImageSettingRepository interface { + List() ([]models.SystemImageSetting, error) + GetByID(id int) (*models.SystemImageSetting, error) + ListByInstanceType(instanceType string) ([]models.SystemImageSetting, error) + Save(setting *models.SystemImageSetting) error + DeleteByID(id int) error + DeleteByInstanceType(instanceType string) error +} + +type systemImageSettingRepository struct { + sess db.Session +} + +// NewSystemImageSettingRepository creates a new repository and ensures the table exists. +func NewSystemImageSettingRepository(sess db.Session) SystemImageSettingRepository { + repo := &systemImageSettingRepository{sess: sess} + repo.ensureTable() + return repo +} + +func (r *systemImageSettingRepository) ensureTable() { + const query = ` +CREATE TABLE IF NOT EXISTS system_image_settings ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_type VARCHAR(50) NOT NULL, + runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop', + display_name VARCHAR(255) NOT NULL, + image VARCHAR(500) NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_instance_type (instance_type) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +` + + if _, err := r.sess.SQL().Exec(query); err != nil { + panic(fmt.Errorf("failed to ensure system_image_settings table: %w", err)) + } + + r.ensureIsEnabledColumn() + r.ensureRuntimeTypeColumn() + r.ensureRuntimeTypeAllowsGateway() + r.ensureInstanceTypeIsNotUnique() + r.ensureInstanceTypeIndex() +} + +func (r *systemImageSettingRepository) ensureIsEnabledColumn() { + var count int + row, err := r.sess.SQL().QueryRow(` +SELECT COUNT(*) +FROM information_schema.columns +WHERE table_schema = DATABASE() + AND table_name = 'system_image_settings' + AND column_name = 'is_enabled' +`) + if err != nil { + panic(fmt.Errorf("failed to inspect system_image_settings columns: %w", err)) + } + if err := row.Scan(&count); err != nil { + panic(fmt.Errorf("failed to scan system_image_settings column count: %w", err)) + } + + if count == 0 { + if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings ADD COLUMN is_enabled BOOLEAN NOT NULL DEFAULT TRUE"); err != nil { + panic(fmt.Errorf("failed to ensure system_image_settings.is_enabled column: %w", err)) + } + } +} + +func (r *systemImageSettingRepository) ensureRuntimeTypeColumn() { + var count int + row, err := r.sess.SQL().QueryRow(` +SELECT COUNT(*) +FROM information_schema.columns +WHERE table_schema = DATABASE() + AND table_name = 'system_image_settings' + AND column_name = 'runtime_type' +`) + if err != nil { + panic(fmt.Errorf("failed to inspect system_image_settings runtime_type column: %w", err)) + } + if err := row.Scan(&count); err != nil { + panic(fmt.Errorf("failed to scan system_image_settings runtime_type column count: %w", err)) + } + + if count == 0 { + if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop' AFTER instance_type"); err != nil { + panic(fmt.Errorf("failed to ensure system_image_settings.runtime_type column: %w", err)) + } + } +} + +func (r *systemImageSettingRepository) ensureRuntimeTypeAllowsGateway() { + var columnType string + row, err := r.sess.SQL().QueryRow(` +SELECT COLUMN_TYPE +FROM information_schema.columns +WHERE table_schema = DATABASE() + AND table_name = 'system_image_settings' + AND column_name = 'runtime_type' +`) + if err != nil { + panic(fmt.Errorf("failed to inspect system_image_settings runtime_type enum: %w", err)) + } + if err := row.Scan(&columnType); err != nil { + panic(fmt.Errorf("failed to scan system_image_settings runtime_type enum: %w", err)) + } + + if !strings.Contains(columnType, "'gateway'") { + if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop'"); err != nil { + panic(fmt.Errorf("failed to allow system_image_settings.runtime_type gateway: %w", err)) + } + } +} + +func (r *systemImageSettingRepository) ensureInstanceTypeIsNotUnique() { + rows, err := r.sess.SQL().Query(` +SELECT DISTINCT INDEX_NAME +FROM information_schema.statistics +WHERE table_schema = DATABASE() + AND table_name = 'system_image_settings' + AND column_name = 'instance_type' + AND NON_UNIQUE = 0 + AND INDEX_NAME <> 'PRIMARY' +`) + if err != nil { + panic(fmt.Errorf("failed to inspect system_image_settings indexes: %w", err)) + } + defer rows.Close() + + var indexNames []string + for rows.Next() { + var indexName string + if err := rows.Scan(&indexName); err != nil { + panic(fmt.Errorf("failed to scan system_image_settings unique index: %w", err)) + } + indexNames = append(indexNames, indexName) + } + + for _, indexName := range indexNames { + escapedName := strings.ReplaceAll(indexName, "`", "``") + statement := fmt.Sprintf("ALTER TABLE system_image_settings DROP INDEX `%s`", escapedName) + if _, err := r.sess.SQL().Exec(statement); err != nil { + panic(fmt.Errorf("failed to drop unique instance_type index %s: %w", indexName, err)) + } + } +} + +func (r *systemImageSettingRepository) ensureInstanceTypeIndex() { + var count int + row, err := r.sess.SQL().QueryRow(` +SELECT COUNT(*) +FROM information_schema.statistics +WHERE table_schema = DATABASE() + AND table_name = 'system_image_settings' + AND index_name = 'idx_instance_type' +`) + if err != nil { + panic(fmt.Errorf("failed to inspect system_image_settings idx_instance_type index: %w", err)) + } + if err := row.Scan(&count); err != nil { + panic(fmt.Errorf("failed to scan system_image_settings idx_instance_type index count: %w", err)) + } + + if count == 0 { + if _, err := r.sess.SQL().Exec("CREATE INDEX idx_instance_type ON system_image_settings (instance_type)"); err != nil { + panic(fmt.Errorf("failed to create idx_instance_type index: %w", err)) + } + } +} + +func (r *systemImageSettingRepository) List() ([]models.SystemImageSetting, error) { + var settings []models.SystemImageSetting + if err := r.sess.Collection("system_image_settings").Find().OrderBy("instance_type", "runtime_type", "-updated_at", "-id").All(&settings); err != nil { + return nil, fmt.Errorf("failed to list system image settings: %w", err) + } + return settings, nil +} + +func (r *systemImageSettingRepository) GetByID(id int) (*models.SystemImageSetting, error) { + var setting models.SystemImageSetting + err := r.sess.Collection("system_image_settings").Find(db.Cond{"id": id}).One(&setting) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get system image setting by id: %w", err) + } + return &setting, nil +} + +func (r *systemImageSettingRepository) ListByInstanceType(instanceType string) ([]models.SystemImageSetting, error) { + var settings []models.SystemImageSetting + if err := r.sess.Collection("system_image_settings").Find(db.Cond{"instance_type": instanceType}).OrderBy("-updated_at", "-id").All(&settings); err != nil { + return nil, fmt.Errorf("failed to list system image settings by instance type: %w", err) + } + return settings, nil +} + +func (r *systemImageSettingRepository) Save(setting *models.SystemImageSetting) error { + now := time.Now() + if setting.ID > 0 { + existing, err := r.GetByID(setting.ID) + if err != nil { + return err + } + if existing == nil { + return fmt.Errorf("system image setting not found") + } + + existing.InstanceType = setting.InstanceType + existing.RuntimeType = setting.RuntimeType + existing.DisplayName = setting.DisplayName + existing.Image = setting.Image + existing.IsEnabled = setting.IsEnabled + existing.UpdatedAt = now + if err := r.sess.Collection("system_image_settings").Find(db.Cond{"id": existing.ID}).Update(existing); err != nil { + return fmt.Errorf("failed to update system image setting: %w", err) + } + + *setting = *existing + return nil + } + + setting.CreatedAt = now + setting.UpdatedAt = now + res, err := r.sess.Collection("system_image_settings").Insert(setting) + if err != nil { + return fmt.Errorf("failed to create system image setting: %w", err) + } + if id, ok := res.ID().(int64); ok { + setting.ID = int(id) + } + return nil +} + +func (r *systemImageSettingRepository) DeleteByID(id int) error { + if err := r.sess.Collection("system_image_settings").Find(db.Cond{"id": id}).Delete(); err != nil { + if err == db.ErrNoMoreRows { + return nil + } + return fmt.Errorf("failed to delete system image setting by id: %w", err) + } + return nil +} + +func (r *systemImageSettingRepository) DeleteByInstanceType(instanceType string) error { + if err := r.sess.Collection("system_image_settings").Find(db.Cond{"instance_type": instanceType}).Delete(); err != nil { + if err == db.ErrNoMoreRows { + return nil + } + return fmt.Errorf("failed to delete system image settings by instance type: %w", err) + } + return nil +} diff --git a/backend/internal/repository/team_repository.go b/backend/internal/repository/team_repository.go new file mode 100644 index 0000000..8cdf2b8 --- /dev/null +++ b/backend/internal/repository/team_repository.go @@ -0,0 +1,736 @@ +package repository + +import ( + "errors" + "fmt" + "strings" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +var ErrDuplicateTeamEvent = errors.New("duplicate team event") + +type TeamRepository interface { + CreateTeam(team *models.Team) error + UpdateTeam(team *models.Team) error + GetTeamByID(id int) (*models.Team, error) + GetTeamByUserIDAndName(userID int, name string) (*models.Team, error) + ExistsByUserIDAndName(userID int, name string) (bool, error) + ListTeamsByUserID(userID int, offset, limit int) ([]models.Team, error) + ListActiveTeams() ([]models.Team, error) + CountTeamsByUserID(userID int) (int, error) + + CreateMember(member *models.TeamMember) error + UpdateMember(member *models.TeamMember) error + GetMemberByID(id int) (*models.TeamMember, error) + GetMemberByTeamKey(teamID int, memberKey string) (*models.TeamMember, error) + ListMembersByTeamID(teamID int) ([]models.TeamMember, error) + + CreateTask(task *models.TeamTask) error + UpdateTask(task *models.TeamTask) error + GetTaskByID(id int) (*models.TeamTask, error) + GetTaskByMessageID(teamID int, messageID string) (*models.TeamTask, error) + ListTasksByTeamID(teamID int, limit int) ([]models.TeamTask, error) + ListTasksBeforeID(teamID, beforeID, limit int) ([]models.TeamTask, error) + ListStaleCandidateTasks(cutoff time.Time, limit int) ([]models.TeamTask, error) + + CreateEvent(event *models.TeamEvent) error + EventExistsByStreamID(teamID int, streamID string) (bool, error) + EventExistsByEventID(teamID int, eventID string) (bool, error) + EventExistsByCompletionID(teamID int, completionID string) (bool, error) + ListEventsByTeamID(teamID int, limit int) ([]models.TeamEvent, error) + ListEventsBeforeID(teamID, beforeID, limit int) ([]models.TeamEvent, error) + + UpsertWorkItem(item *models.TeamWorkItem) error + InvalidateWorkItemReview(workItemID int, updatedAt time.Time) error + ListWorkItemsByRootTaskID(rootTaskID int) ([]models.TeamWorkItem, error) + ListWorkItemsByTeamID(teamID int, limit int) ([]models.TeamWorkItem, error) + UpsertWorkflowPhase(phase *models.TeamWorkflowPhase) error + ListWorkflowPhasesByRootTaskID(rootTaskID int) ([]models.TeamWorkflowPhase, error) + AcceptRootCompletion(task *models.TeamTask, expectedLedgerVersion int64, event *models.TeamEvent, outbox *models.TeamEventOutbox) (bool, error) + ConfirmWorkItemResult(item *models.TeamWorkItem, event *models.TeamEvent, outbox *models.TeamEventOutbox) error + CreateEventOutbox(outbox *models.TeamEventOutbox) error + ListPendingEventOutbox(now time.Time, limit int) ([]models.TeamEventOutbox, error) + MarkEventOutboxDelivered(id int, deliveredAt time.Time) error + MarkEventOutboxFailed(id int, availableAt time.Time, cause string) error +} + +type teamRepository struct { + sess db.Session +} + +func NewTeamRepository(sess db.Session) TeamRepository { + return &teamRepository{sess: sess} +} + +func (r *teamRepository) CreateTeam(team *models.Team) error { + ensureTimestamps(&team.CreatedAt, &team.UpdatedAt) + res, err := r.sess.Collection("teams").Insert(team) + if err != nil { + if strings.Contains(err.Error(), "Duplicate entry") && strings.Contains(err.Error(), "uk_teams_user_name") { + return fmt.Errorf("team name already exists") + } + return fmt.Errorf("failed to create team: %w", err) + } + if id, ok := res.ID().(int64); ok { + team.ID = int(id) + } + return nil +} + +func (r *teamRepository) UpdateTeam(team *models.Team) error { + if team.UpdatedAt.IsZero() { + team.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("teams").Find(db.Cond{"id": team.ID}).Update(team); err != nil { + return fmt.Errorf("failed to update team: %w", err) + } + return nil +} + +func (r *teamRepository) GetTeamByID(id int) (*models.Team, error) { + var team models.Team + if err := r.sess.Collection("teams").Find(db.Cond{"id": id}).One(&team); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get team: %w", err) + } + return &team, nil +} + +func (r *teamRepository) GetTeamByUserIDAndName(userID int, name string) (*models.Team, error) { + var team models.Team + if err := r.sess.Collection("teams").Find(db.Cond{"user_id": userID, "name": name}).One(&team); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get team by name: %w", err) + } + return &team, nil +} + +func (r *teamRepository) ExistsByUserIDAndName(userID int, name string) (bool, error) { + count, err := r.sess.Collection("teams").Find(db.Cond{"user_id": userID, "name": name}).Count() + if err != nil { + return false, fmt.Errorf("failed to check team name: %w", err) + } + return count > 0, nil +} + +func (r *teamRepository) ListTeamsByUserID(userID int, offset, limit int) ([]models.Team, error) { + if limit <= 0 { + limit = 20 + } + var teams []models.Team + if err := r.sess.Collection("teams").Find(db.Cond{"user_id": userID}).OrderBy("-created_at", "-id").Offset(offset).Limit(limit).All(&teams); err != nil { + return nil, fmt.Errorf("failed to list teams: %w", err) + } + return teams, nil +} + +func (r *teamRepository) ListActiveTeams() ([]models.Team, error) { + var teams []models.Team + if err := r.sess.Collection("teams").Find(db.Cond{"status IN": []string{models.TeamStatusCreating, models.TeamStatusRunning}}).All(&teams); err != nil { + return nil, fmt.Errorf("failed to list active teams: %w", err) + } + return teams, nil +} + +func (r *teamRepository) CountTeamsByUserID(userID int) (int, error) { + count, err := r.sess.Collection("teams").Find(db.Cond{"user_id": userID}).Count() + if err != nil { + return 0, fmt.Errorf("failed to count teams: %w", err) + } + return int(count), nil +} + +func (r *teamRepository) CreateMember(member *models.TeamMember) error { + ensureTimestamps(&member.CreatedAt, &member.UpdatedAt) + res, err := r.sess.Collection("team_members").Insert(member) + if err != nil { + return fmt.Errorf("failed to create team member: %w", err) + } + if id, ok := res.ID().(int64); ok { + member.ID = int(id) + } + return nil +} + +func (r *teamRepository) UpdateMember(member *models.TeamMember) error { + if member.UpdatedAt.IsZero() { + member.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("team_members").Find(db.Cond{"id": member.ID}).Update(member); err != nil { + return fmt.Errorf("failed to update team member: %w", err) + } + return nil +} + +func (r *teamRepository) GetMemberByID(id int) (*models.TeamMember, error) { + var member models.TeamMember + if err := r.sess.Collection("team_members").Find(db.Cond{"id": id}).One(&member); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get team member: %w", err) + } + return &member, nil +} + +func (r *teamRepository) GetMemberByTeamKey(teamID int, memberKey string) (*models.TeamMember, error) { + var member models.TeamMember + if err := r.sess.Collection("team_members").Find(db.Cond{"team_id": teamID, "member_key": memberKey}).One(&member); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get team member by key: %w", err) + } + return &member, nil +} + +func (r *teamRepository) ListMembersByTeamID(teamID int) ([]models.TeamMember, error) { + var members []models.TeamMember + if err := r.sess.Collection("team_members").Find(db.Cond{"team_id": teamID}).OrderBy("id").All(&members); err != nil { + return nil, fmt.Errorf("failed to list team members: %w", err) + } + return members, nil +} + +func (r *teamRepository) CreateTask(task *models.TeamTask) error { + ensureTimestamps(&task.CreatedAt, &task.UpdatedAt) + res, err := r.sess.Collection("team_tasks").Insert(task) + if err != nil { + return fmt.Errorf("failed to create team task: %w", err) + } + if id, ok := res.ID().(int64); ok { + task.ID = int(id) + } + return nil +} + +func (r *teamRepository) UpdateTask(task *models.TeamTask) error { + if task.UpdatedAt.IsZero() { + task.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("team_tasks").Find(db.Cond{"id": task.ID}).Update(task); err != nil { + return fmt.Errorf("failed to update team task: %w", err) + } + return nil +} + +func (r *teamRepository) GetTaskByID(id int) (*models.TeamTask, error) { + var task models.TeamTask + if err := r.sess.Collection("team_tasks").Find(db.Cond{"id": id}).One(&task); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get team task: %w", err) + } + return &task, nil +} + +func (r *teamRepository) GetTaskByMessageID(teamID int, messageID string) (*models.TeamTask, error) { + var task models.TeamTask + if err := r.sess.Collection("team_tasks").Find(db.Cond{"team_id": teamID, "message_id": messageID}).One(&task); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get team task by message id: %w", err) + } + return &task, nil +} + +func (r *teamRepository) ListTasksByTeamID(teamID int, limit int) ([]models.TeamTask, error) { + if limit <= 0 { + limit = 20 + } + var tasks []models.TeamTask + if err := r.sess.Collection("team_tasks").Find(db.Cond{"team_id": teamID}).OrderBy("-created_at", "-id").Limit(limit).All(&tasks); err != nil { + return nil, fmt.Errorf("failed to list team tasks: %w", err) + } + return tasks, nil +} + +func (r *teamRepository) ListTasksBeforeID(teamID, beforeID, limit int) ([]models.TeamTask, error) { + if limit <= 0 { + limit = 20 + } + cond := db.Cond{"team_id": teamID} + if beforeID > 0 { + cond["id <"] = beforeID + } + var tasks []models.TeamTask + if err := r.sess.Collection("team_tasks").Find(cond).OrderBy("-created_at", "-id").Limit(limit).All(&tasks); err != nil { + return nil, fmt.Errorf("failed to list team task history: %w", err) + } + return tasks, nil +} + +func (r *teamRepository) ListStaleCandidateTasks(cutoff time.Time, limit int) ([]models.TeamTask, error) { + if limit <= 0 { + limit = 100 + } + var tasks []models.TeamTask + statuses := []string{ + models.TeamTaskStatusDispatched, + models.TeamTaskStatusRunning, + } + if err := r.sess.Collection("team_tasks").Find(db.Cond{ + "status IN": statuses, + "updated_at <": cutoff, + }).OrderBy("updated_at", "id").Limit(limit).All(&tasks); err != nil { + return nil, fmt.Errorf("failed to list stale candidate team tasks: %w", err) + } + return tasks, nil +} + +func (r *teamRepository) CreateEvent(event *models.TeamEvent) error { + if event.CreatedAt.IsZero() { + event.CreatedAt = time.Now().UTC() + } + res, err := r.sess.Collection("team_events").Insert(event) + if err != nil { + if strings.Contains(err.Error(), "uk_team_events_stream_id") || + strings.Contains(err.Error(), "uk_team_events_event_id") || + strings.Contains(err.Error(), "uk_team_events_completion_id") { + return ErrDuplicateTeamEvent + } + return fmt.Errorf("failed to create team event: %w", err) + } + if id, ok := res.ID().(int64); ok { + event.ID = int(id) + } + if event.SequenceNo <= 0 && event.ID > 0 { + event.SequenceNo = int64(event.ID) + if err := r.sess.Collection("team_events").Find(db.Cond{"id": event.ID}).Update(map[string]interface{}{"sequence_no": event.SequenceNo}); err != nil { + return fmt.Errorf("failed to assign team event sequence: %w", err) + } + } + return nil +} + +func (r *teamRepository) ConfirmWorkItemResult(item *models.TeamWorkItem, event *models.TeamEvent, outbox *models.TeamEventOutbox) error { + if item == nil || event == nil || outbox == nil { + return fmt.Errorf("work item, confirmation event and outbox are required") + } + return r.sess.Tx(func(sess db.Session) error { + txRepo := &teamRepository{sess: sess} + if err := txRepo.UpsertWorkItem(item); err != nil { + return err + } + if err := txRepo.CreateEvent(event); err != nil && !errors.Is(err, ErrDuplicateTeamEvent) { + return err + } + if outbox.Status == "" { + outbox.Status = "pending" + } + now := time.Now().UTC() + if outbox.AvailableAt.IsZero() { + outbox.AvailableAt = now + } + if outbox.CreatedAt.IsZero() { + outbox.CreatedAt = now + } + if outbox.UpdatedAt.IsZero() { + outbox.UpdatedAt = now + } + res, err := sess.Collection("team_event_outbox").Insert(outbox) + if err != nil { + if strings.Contains(err.Error(), "uk_team_event_outbox_message") { + return nil + } + return fmt.Errorf("failed to create team event outbox: %w", err) + } + if id, ok := res.ID().(int64); ok { + outbox.ID = int(id) + } + return nil + }) +} + +func (r *teamRepository) UpsertWorkflowPhase(phase *models.TeamWorkflowPhase) error { + if phase == nil || phase.RootTaskID <= 0 || strings.TrimSpace(phase.PhaseID) == "" { + return fmt.Errorf("team workflow phase is required") + } + if phase.PlanVersion <= 0 { + phase.PlanVersion = 1 + } + var existing models.TeamWorkflowPhase + err := r.sess.Collection("team_workflow_phases").Find(db.Cond{ + "root_task_id": phase.RootTaskID, + "phase_id": phase.PhaseID, + "plan_version": phase.PlanVersion, + }).One(&existing) + if err != nil && err != db.ErrNoMoreRows { + return fmt.Errorf("failed to get team workflow phase: %w", err) + } + if err == nil { + phase.ID = existing.ID + phase.CreatedAt = existing.CreatedAt + if phase.DependsOnJSON == nil { + phase.DependsOnJSON = existing.DependsOnJSON + } + if phase.NextPhaseID == nil { + phase.NextPhaseID = existing.NextPhaseID + } + if phase.CompletionPolicy == nil { + phase.CompletionPolicy = existing.CompletionPolicy + } + if phase.CompletedAt == nil { + phase.CompletedAt = existing.CompletedAt + } + if phase.UpdatedAt.IsZero() { + phase.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("team_workflow_phases").Find(db.Cond{"id": existing.ID}).Update(phase); err != nil { + return fmt.Errorf("failed to update team workflow phase: %w", err) + } + return nil + } + ensureTimestamps(&phase.CreatedAt, &phase.UpdatedAt) + res, err := r.sess.Collection("team_workflow_phases").Insert(phase) + if err != nil { + return fmt.Errorf("failed to create team workflow phase: %w", err) + } + if id, ok := res.ID().(int64); ok { + phase.ID = int(id) + } + return nil +} + +func (r *teamRepository) ListWorkflowPhasesByRootTaskID(rootTaskID int) ([]models.TeamWorkflowPhase, error) { + var phases []models.TeamWorkflowPhase + if err := r.sess.Collection("team_workflow_phases").Find(db.Cond{"root_task_id": rootTaskID}).OrderBy("plan_version", "sequence_no", "id").All(&phases); err != nil { + return nil, fmt.Errorf("failed to list team workflow phases: %w", err) + } + return phases, nil +} + +func (r *teamRepository) AcceptRootCompletion(task *models.TeamTask, expectedLedgerVersion int64, event *models.TeamEvent, outbox *models.TeamEventOutbox) (bool, error) { + if task == nil || event == nil || outbox == nil || task.ID <= 0 { + return false, fmt.Errorf("task, accepted completion event and outbox are required") + } + accepted := false + err := r.sess.Tx(func(sess db.Session) error { + result, err := sess.SQL().Exec(` +UPDATE team_tasks +SET status = ?, workflow_state = ?, plan_version = ?, ledger_version = ?, current_phase_id = ?, + accepted_completion_id = ?, result_json = ?, error_message = ?, finished_at = ?, updated_at = ? +WHERE id = ? AND ledger_version = ? AND accepted_completion_id IS NULL + AND status NOT IN (?, ?, ?) +`, + task.Status, + task.WorkflowState, + task.PlanVersion, + task.LedgerVersion, + task.CurrentPhaseID, + task.AcceptedCompletionID, + task.ResultJSON, + task.ErrorMessage, + task.FinishedAt, + task.UpdatedAt, + task.ID, + expectedLedgerVersion, + models.TeamTaskStatusSucceeded, + models.TeamTaskStatusFailed, + models.TeamTaskStatusStale, + ) + if err != nil { + return fmt.Errorf("failed to atomically accept team completion: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("failed to inspect accepted team completion: %w", err) + } + if affected != 1 { + return nil + } + accepted = true + if _, err := sess.SQL().Exec(` +UPDATE team_workflow_phases +SET status = 'completed', completed_at = ?, updated_at = ? +WHERE root_task_id = ? AND (? = 0 OR plan_version = ?) + AND status NOT IN ('cancelled', 'superseded') +`, task.UpdatedAt, task.UpdatedAt, task.ID, task.PlanVersion, task.PlanVersion); err != nil { + return fmt.Errorf("failed to complete team workflow phases: %w", err) + } + txRepo := &teamRepository{sess: sess} + if err := txRepo.CreateEvent(event); err != nil && !errors.Is(err, ErrDuplicateTeamEvent) { + return err + } + if outbox.Status == "" { + outbox.Status = "pending" + } + now := time.Now().UTC() + if outbox.AvailableAt.IsZero() { + outbox.AvailableAt = now + } + if outbox.CreatedAt.IsZero() { + outbox.CreatedAt = now + } + if outbox.UpdatedAt.IsZero() { + outbox.UpdatedAt = now + } + insertResult, err := sess.Collection("team_event_outbox").Insert(outbox) + if err != nil { + if strings.Contains(err.Error(), "uk_team_event_outbox_message") { + return nil + } + return fmt.Errorf("failed to create completion acknowledgement outbox: %w", err) + } + if id, ok := insertResult.ID().(int64); ok { + outbox.ID = int(id) + } + return nil + }) + return accepted, err +} + +func (r *teamRepository) ListPendingEventOutbox(now time.Time, limit int) ([]models.TeamEventOutbox, error) { + if limit <= 0 { + limit = 100 + } + var rows []models.TeamEventOutbox + if err := r.sess.Collection("team_event_outbox").Find(db.Cond{ + "status": "pending", + "available_at <=": now, + }).OrderBy("available_at", "id").Limit(limit).All(&rows); err != nil { + return nil, fmt.Errorf("failed to list pending team event outbox: %w", err) + } + return rows, nil +} + +func (r *teamRepository) CreateEventOutbox(outbox *models.TeamEventOutbox) error { + if outbox == nil { + return fmt.Errorf("team event outbox is required") + } + if outbox.Status == "" { + outbox.Status = "pending" + } + now := time.Now().UTC() + if outbox.AvailableAt.IsZero() { + outbox.AvailableAt = now + } + if outbox.CreatedAt.IsZero() { + outbox.CreatedAt = now + } + if outbox.UpdatedAt.IsZero() { + outbox.UpdatedAt = now + } + res, err := r.sess.Collection("team_event_outbox").Insert(outbox) + if err != nil { + if strings.Contains(err.Error(), "uk_team_event_outbox_message") { + return nil + } + return fmt.Errorf("failed to create team event outbox: %w", err) + } + if id, ok := res.ID().(int64); ok { + outbox.ID = int(id) + } + return nil +} + +func (r *teamRepository) MarkEventOutboxDelivered(id int, deliveredAt time.Time) error { + if id <= 0 { + return nil + } + return r.sess.Collection("team_event_outbox").Find(db.Cond{"id": id}).Update(map[string]interface{}{ + "status": "delivered", + "delivered_at": deliveredAt, + "last_error": nil, + "updated_at": deliveredAt, + }) +} + +func (r *teamRepository) MarkEventOutboxFailed(id int, availableAt time.Time, cause string) error { + if id <= 0 { + return nil + } + return r.sess.Collection("team_event_outbox").Find(db.Cond{"id": id}).Update(map[string]interface{}{ + "status": "pending", + "attempts": db.Raw("attempts + 1"), + "available_at": availableAt, + "last_error": cause, + "updated_at": time.Now().UTC(), + }) +} + +func (r *teamRepository) EventExistsByStreamID(teamID int, streamID string) (bool, error) { + if streamID == "" { + return false, nil + } + count, err := r.sess.Collection("team_events").Find(db.Cond{"team_id": teamID, "redis_stream_id": streamID}).Count() + if err != nil { + return false, fmt.Errorf("failed to check team event stream id: %w", err) + } + return count > 0, nil +} + +func (r *teamRepository) EventExistsByEventID(teamID int, eventID string) (bool, error) { + return r.teamEventFieldExists(teamID, "event_id", eventID) +} + +func (r *teamRepository) EventExistsByCompletionID(teamID int, completionID string) (bool, error) { + return r.teamEventFieldExists(teamID, "completion_id", completionID) +} + +func (r *teamRepository) teamEventFieldExists(teamID int, field, value string) (bool, error) { + value = strings.TrimSpace(value) + if value == "" { + return false, nil + } + count, err := r.sess.Collection("team_events").Find(db.Cond{"team_id": teamID, field: value}).Count() + if err != nil { + return false, fmt.Errorf("failed to check team event %s: %w", field, err) + } + return count > 0, nil +} + +func (r *teamRepository) ListEventsByTeamID(teamID int, limit int) ([]models.TeamEvent, error) { + if limit <= 0 { + limit = 50 + } + var events []models.TeamEvent + if err := r.sess.Collection("team_events").Find(db.Cond{"team_id": teamID}).OrderBy("-sequence_no", "-id").Limit(limit).All(&events); err != nil { + return nil, fmt.Errorf("failed to list team events: %w", err) + } + return events, nil +} + +func (r *teamRepository) ListEventsBeforeID(teamID, beforeID, limit int) ([]models.TeamEvent, error) { + if limit <= 0 { + limit = 50 + } + cond := db.Cond{"team_id": teamID} + if beforeID > 0 { + cond["id <"] = beforeID + } + var events []models.TeamEvent + if err := r.sess.Collection("team_events").Find(cond).OrderBy("-sequence_no", "-id").Limit(limit).All(&events); err != nil { + return nil, fmt.Errorf("failed to list team event history: %w", err) + } + return events, nil +} + +func (r *teamRepository) UpsertWorkItem(item *models.TeamWorkItem) error { + if item == nil { + return fmt.Errorf("team work item is required") + } + var existing models.TeamWorkItem + err := r.sess.Collection("team_work_items").Find(db.Cond{ + "team_id": item.TeamID, "root_task_id": item.RootTaskID, "work_id": item.WorkID, + }).One(&existing) + if err != nil && err != db.ErrNoMoreRows { + return fmt.Errorf("failed to get team work item: %w", err) + } + if err == nil { + item.ID = existing.ID + item.CreatedAt = existing.CreatedAt + newRevision := item.Revision > existing.Revision + existingTerminal := existing.Status == models.TeamTaskStatusSucceeded || existing.Status == models.TeamTaskStatusFailed || existing.Status == models.TeamTaskStatusStale + itemTerminal := item.Status == models.TeamTaskStatusSucceeded || item.Status == models.TeamTaskStatusFailed || item.Status == models.TeamTaskStatusStale + reopeningCurrent := !newRevision && existingTerminal && !itemTerminal && + !item.UpdatedAt.IsZero() && + (existing.UpdatedAt.IsZero() || item.UpdatedAt.After(existing.UpdatedAt)) + if !newRevision && existingTerminal && !reopeningCurrent { + if !itemTerminal { + item.Status = existing.Status + } + } + if item.OwnerMemberID == nil { + item.OwnerMemberID = existing.OwnerMemberID + } + if item.StartedAt == nil { + item.StartedAt = existing.StartedAt + } + if item.FinishedAt == nil && !newRevision && !reopeningCurrent { + item.FinishedAt = existing.FinishedAt + } + if item.DependsOnJSON == nil { + item.DependsOnJSON = existing.DependsOnJSON + } + if item.AssignmentID == nil { + item.AssignmentID = existing.AssignmentID + } + if item.CanonicalWorkID == nil { + item.CanonicalWorkID = existing.CanonicalWorkID + } + if item.PhaseID == nil { + item.PhaseID = existing.PhaseID + } + if item.Revision <= 0 { + item.Revision = existing.Revision + } + if item.SupersededBy == nil && !reopeningCurrent { + item.SupersededBy = existing.SupersededBy + } + if item.ValidatedRevision == nil && !newRevision && !reopeningCurrent { + item.ValidatedRevision = existing.ValidatedRevision + } + if existing.ReviewRequired && !newRevision && !reopeningCurrent { + item.ReviewRequired = true + } + if item.ResultJSON == nil && !newRevision && !reopeningCurrent { + item.ResultJSON = existing.ResultJSON + } + if item.ArtifactRefsJSON == nil && !newRevision && !reopeningCurrent { + item.ArtifactRefsJSON = existing.ArtifactRefsJSON + } + if item.UpdatedAt.IsZero() { + item.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("team_work_items").Find(db.Cond{"id": existing.ID}).Update(item); err != nil { + return fmt.Errorf("failed to update team work item: %w", err) + } + return nil + } + ensureTimestamps(&item.CreatedAt, &item.UpdatedAt) + res, err := r.sess.Collection("team_work_items").Insert(item) + if err != nil { + return fmt.Errorf("failed to create team work item: %w", err) + } + if id, ok := res.ID().(int64); ok { + item.ID = int(id) + } + return nil +} + +func (r *teamRepository) InvalidateWorkItemReview(workItemID int, updatedAt time.Time) error { + if workItemID <= 0 { + return fmt.Errorf("team work item id is required") + } + if updatedAt.IsZero() { + updatedAt = time.Now().UTC() + } + if _, err := r.sess.SQL().Exec(` +UPDATE team_work_items +SET validated_revision = NULL, updated_at = ? +WHERE id = ? AND review_required = TRUE AND validated_revision IS NOT NULL +`, updatedAt, workItemID); err != nil { + return fmt.Errorf("failed to invalidate team work item review: %w", err) + } + return nil +} + +func (r *teamRepository) ListWorkItemsByRootTaskID(rootTaskID int) ([]models.TeamWorkItem, error) { + var items []models.TeamWorkItem + if err := r.sess.Collection("team_work_items").Find(db.Cond{"root_task_id": rootTaskID}).OrderBy("id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list team work items: %w", err) + } + return items, nil +} + +func (r *teamRepository) ListWorkItemsByTeamID(teamID int, limit int) ([]models.TeamWorkItem, error) { + if limit <= 0 { + limit = 200 + } + var items []models.TeamWorkItem + if err := r.sess.Collection("team_work_items").Find(db.Cond{"team_id": teamID}).OrderBy("-root_task_id", "id").Limit(limit).All(&items); err != nil { + return nil, fmt.Errorf("failed to list team work items by team: %w", err) + } + return items, nil +} diff --git a/backend/internal/repository/timestamps.go b/backend/internal/repository/timestamps.go new file mode 100644 index 0000000..5d9da59 --- /dev/null +++ b/backend/internal/repository/timestamps.go @@ -0,0 +1,13 @@ +package repository + +import "time" + +func ensureTimestamps(createdAt *time.Time, updatedAt *time.Time) { + now := time.Now().UTC() + if createdAt != nil && createdAt.IsZero() { + *createdAt = now + } + if updatedAt != nil && updatedAt.IsZero() { + *updatedAt = now + } +} diff --git a/backend/internal/repository/user_repository.go b/backend/internal/repository/user_repository.go new file mode 100644 index 0000000..0c87c6e --- /dev/null +++ b/backend/internal/repository/user_repository.go @@ -0,0 +1,117 @@ +package repository + +import ( + "fmt" + + "clawreef/internal/models" + "github.com/upper/db/v4" +) + +// UserRepository defines the interface for user data operations +type UserRepository interface { + Create(user *models.User) error + GetByID(id int) (*models.User, error) + GetByUsername(username string) (*models.User, error) + GetByEmail(email string) (*models.User, error) + Update(user *models.User) error + Delete(id int) error + List(offset, limit int) ([]models.User, error) + Count() (int, error) +} + +// userRepository implements UserRepository +type userRepository struct { + sess db.Session +} + +// NewUserRepository creates a new user repository +func NewUserRepository(sess db.Session) UserRepository { + return &userRepository{sess: sess} +} + +// Create creates a new user +func (r *userRepository) Create(user *models.User) error { + res, err := r.sess.Collection("users").Insert(user) + if err != nil { + return fmt.Errorf("failed to create user: %w", err) + } + // Get the last insert ID + user.ID = int(res.ID().(int64)) + return nil +} + +// GetByID gets a user by ID +func (r *userRepository) GetByID(id int) (*models.User, error) { + var user models.User + err := r.sess.Collection("users").Find(db.Cond{"id": id}).One(&user) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get user by ID: %w", err) + } + return &user, nil +} + +// GetByUsername gets a user by username +func (r *userRepository) GetByUsername(username string) (*models.User, error) { + var user models.User + err := r.sess.Collection("users").Find(db.Cond{"username": username}).One(&user) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get user by username: %w", err) + } + return &user, nil +} + +// GetByEmail gets a user by email +func (r *userRepository) GetByEmail(email string) (*models.User, error) { + var user models.User + err := r.sess.Collection("users").Find(db.Cond{"email": email}).One(&user) + if err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get user by email: %w", err) + } + return &user, nil +} + +// Update updates a user +func (r *userRepository) Update(user *models.User) error { + err := r.sess.Collection("users").Find(db.Cond{"id": user.ID}).Update(user) + if err != nil { + return fmt.Errorf("failed to update user: %w", err) + } + return nil +} + +// Delete deletes a user by ID +func (r *userRepository) Delete(id int) error { + err := r.sess.Collection("users").Find(db.Cond{"id": id}).Delete() + if err != nil { + return fmt.Errorf("failed to delete user: %w", err) + } + return nil +} + +// List returns a list of users with pagination +func (r *userRepository) List(offset, limit int) ([]models.User, error) { + var users []models.User + err := r.sess.Collection("users").Find().OrderBy("id").Offset(offset).Limit(limit).All(&users) + if err != nil { + return nil, fmt.Errorf("failed to list users: %w", err) + } + return users, nil +} + +// Count returns the total number of users +func (r *userRepository) Count() (int, error) { + count, err := r.sess.Collection("users").Find().Count() + if err != nil { + return 0, fmt.Errorf("failed to count users: %w", err) + } + return int(count), nil +} diff --git a/backend/internal/repository/workspace_file_audit_repository.go b/backend/internal/repository/workspace_file_audit_repository.go new file mode 100644 index 0000000..5cac1c1 --- /dev/null +++ b/backend/internal/repository/workspace_file_audit_repository.go @@ -0,0 +1,40 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type WorkspaceFileAuditRepository interface { + Create(ctx context.Context, audit *models.WorkspaceFileAudit) error +} + +type workspaceFileAuditRepository struct { + sess db.Session +} + +func NewWorkspaceFileAuditRepository(sess db.Session) WorkspaceFileAuditRepository { + return &workspaceFileAuditRepository{sess: sess} +} + +func (r *workspaceFileAuditRepository) Create(ctx context.Context, audit *models.WorkspaceFileAudit) error { + if err := ctx.Err(); err != nil { + return err + } + if audit.CreatedAt.IsZero() { + audit.CreatedAt = time.Now().UTC() + } + res, err := r.sess.Collection("workspace_file_audits").Insert(audit) + if err != nil { + return fmt.Errorf("failed to create workspace file audit: %w", err) + } + if id, ok := res.ID().(int64); ok { + audit.ID = id + } + return nil +} diff --git a/backend/internal/services/ai_observability_service.go b/backend/internal/services/ai_observability_service.go new file mode 100644 index 0000000..36ce777 --- /dev/null +++ b/backend/internal/services/ai_observability_service.go @@ -0,0 +1,1849 @@ +package services + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// AuditQuery contains query options for AI audit list views. +type AuditQuery struct { + Page int + Limit int + Search string + Status string + Model string +} + +// AuditListResult is the paginated audit list response. +type AuditListResult struct { + Items []AuditListItem `json:"items"` + Total int `json:"total"` + Page int `json:"page"` + Limit int `json:"limit"` +} + +// AuditListItem is a flattened list row for AI audit views. +type AuditListItem struct { + TraceID string `json:"trace_id"` + RequestID string `json:"request_id"` + SessionID *string `json:"session_id,omitempty"` + UserID *int `json:"user_id,omitempty"` + Username string `json:"username,omitempty"` + InstanceID *int `json:"instance_id,omitempty"` + RequestedModel string `json:"requested_model"` + ActualProviderModel string `json:"actual_provider_model"` + ProviderType string `json:"provider_type"` + Status string `json:"status"` + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + LatencyMs *int `json:"latency_ms,omitempty"` + ErrorMessage *string `json:"error_message,omitempty"` + CreatedAt time.Time `json:"created_at"` + CompletedAt *time.Time `json:"completed_at,omitempty"` +} + +// AuditTraceDetail bundles invocation, audit event, and cost records for a trace. +type AuditTraceDetail struct { + TraceID string `json:"trace_id"` + Username string `json:"username,omitempty"` + Invocations []models.ModelInvocation `json:"invocations"` + AuditEvents []models.AuditEvent `json:"audit_events"` + CostRecords []models.CostRecord `json:"cost_records"` + RiskHits []models.RiskHit `json:"risk_hits"` + Messages []models.ChatMessageRecord `json:"messages"` + FlowNodes []AuditFlowNode `json:"flow_nodes"` +} + +// AuditFlowNode is a synthesized node in a trace flow visualization. +type AuditFlowNode struct { + ID string `json:"id"` + Kind string `json:"kind"` + Title string `json:"title"` + RequestID string `json:"request_id,omitempty"` + InvocationID *int `json:"invocation_id,omitempty"` + Model string `json:"model,omitempty"` + Status string `json:"status,omitempty"` + Summary string `json:"summary,omitempty"` + InputPayload string `json:"input_payload,omitempty"` + OutputPayload string `json:"output_payload,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// CostOverview contains a compact cost summary for admin reporting. +type CostOverview struct { + TotalPromptTokens int `json:"total_prompt_tokens"` + TotalCompletionTokens int `json:"total_completion_tokens"` + TotalTokens int `json:"total_tokens"` + TotalEstimatedCost float64 `json:"total_estimated_cost"` + TotalInternalCost float64 `json:"total_internal_cost"` + Currency string `json:"currency"` + UserSummary []CostAggregateItem `json:"user_summary"` + InstanceSummary []CostAggregateItem `json:"instance_summary"` + TopModels []CostBreakdownItem `json:"top_models"` + TopUsers []CostBreakdownItem `json:"top_users"` + DailyTrend []CostTrendPoint `json:"daily_trend"` + ModelTrends []CostTrendSeries `json:"model_trends"` + UserTrends []CostTrendSeries `json:"user_trends"` + RecentRecords []CostRecordView `json:"recent_records"` + TotalRecentRecords int `json:"total_recent_records"` + Page int `json:"page"` + Limit int `json:"limit"` +} + +// CostQuery contains query options for cost list views. +type CostQuery struct { + Page int + Limit int + Search string +} + +// CostBreakdownItem is a grouped summary row. +type CostBreakdownItem struct { + Label string `json:"label"` + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + EstimatedCost float64 `json:"estimated_cost"` + InternalCost float64 `json:"internal_cost"` +} + +// CostAggregateItem is a grouped summary row for user or instance aggregates. +type CostAggregateItem struct { + Label string `json:"label"` + Meta string `json:"meta,omitempty"` + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + EstimatedCost float64 `json:"estimated_cost"` + InternalCost float64 `json:"internal_cost"` +} + +// CostTrendPoint is a day bucket for charting token and cost trends. +type CostTrendPoint struct { + Day string `json:"day"` + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + EstimatedCost float64 `json:"estimated_cost"` + InternalCost float64 `json:"internal_cost"` +} + +// CostTrendSeries is a labeled chart series used for model/user trends. +type CostTrendSeries struct { + Label string `json:"label"` + Points []CostTrendPoint `json:"points"` +} + +// CostRecordView is a recent cost record decorated with usernames. +type CostRecordView struct { + ID int `json:"id"` + TraceID string `json:"trace_id"` + UserID *int `json:"user_id,omitempty"` + Username string `json:"username,omitempty"` + InstanceID *int `json:"instance_id,omitempty"` + InstanceName string `json:"instance_name,omitempty"` + ModelName string `json:"model_name"` + ProviderType string `json:"provider_type"` + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + EstimatedCost float64 `json:"estimated_cost"` + InternalCost float64 `json:"internal_cost"` + Currency string `json:"currency"` + RecordedAt time.Time `json:"recorded_at"` +} + +// AIObservabilityService provides read APIs for audit and cost reporting. +type AIObservabilityService interface { + ListAuditItems(query AuditQuery) (*AuditListResult, error) + GetTraceDetail(traceID string) (*AuditTraceDetail, error) + GetCostOverview(query CostQuery) (*CostOverview, error) +} + +type aiObservabilityService struct { + invocationRepo repository.ModelInvocationRepository + auditRepo repository.AuditEventRepository + costRepo repository.CostRecordRepository + riskHitRepo repository.RiskHitRepository + chatMessageRepo repository.ChatMessageRepository + llmModelRepo repository.LLMModelRepository + instanceRepo repository.InstanceRepository + userRepo repository.UserRepository +} + +// NewAIObservabilityService creates a new observability reporting service. +func NewAIObservabilityService( + invocationRepo repository.ModelInvocationRepository, + auditRepo repository.AuditEventRepository, + costRepo repository.CostRecordRepository, + riskHitRepo repository.RiskHitRepository, + chatMessageRepo repository.ChatMessageRepository, + llmModelRepo repository.LLMModelRepository, + instanceRepo repository.InstanceRepository, + userRepo repository.UserRepository, +) AIObservabilityService { + return &aiObservabilityService{ + invocationRepo: invocationRepo, + auditRepo: auditRepo, + costRepo: costRepo, + riskHitRepo: riskHitRepo, + chatMessageRepo: chatMessageRepo, + llmModelRepo: llmModelRepo, + instanceRepo: instanceRepo, + userRepo: userRepo, + } +} + +func (s *aiObservabilityService) ListAuditItems(query AuditQuery) (*AuditListResult, error) { + page, limit := normalizePageLimit(query.Page, query.Limit, 20, 100) + + items, err := s.invocationRepo.ListRecent(expandFetchWindow(page, limit)) + if err != nil { + return nil, fmt.Errorf("failed to list audit items: %w", err) + } + events, err := s.auditRepo.ListRecent(expandFetchWindow(page, limit)) + if err != nil { + return nil, fmt.Errorf("failed to list recent audit events: %w", err) + } + costs, err := s.costRepo.ListRecent(expandFetchWindow(page, limit)) + if err != nil { + return nil, fmt.Errorf("failed to list recent cost records: %w", err) + } + + usernames, _ := s.loadUsernamesForAudit(items, events, costs) + costByTrace := aggregateRecentCostByTrace(costs) + traceItems := aggregateInvocationsByTrace(items, usernames) + + search := strings.ToLower(strings.TrimSpace(query.Search)) + status := strings.ToLower(strings.TrimSpace(query.Status)) + model := strings.ToLower(strings.TrimSpace(query.Model)) + + filtered := make([]AuditListItem, 0, len(traceItems)) + seenTraceIDs := make(map[string]struct{}, len(traceItems)) + for _, item := range traceItems { + if status != "" && strings.ToLower(item.Status) != status { + continue + } + if model != "" && !strings.Contains(strings.ToLower(item.RequestedModel), model) && !strings.Contains(strings.ToLower(item.ActualProviderModel), model) { + continue + } + if search != "" { + haystacks := []string{ + strings.ToLower(item.TraceID), + strings.ToLower(item.RequestID), + strings.ToLower(item.RequestedModel), + strings.ToLower(item.ActualProviderModel), + strings.ToLower(item.ProviderType), + strings.ToLower(item.Username), + } + matched := false + for _, candidate := range haystacks { + if strings.Contains(candidate, search) { + matched = true + break + } + } + if !matched { + continue + } + } + + filtered = append(filtered, item) + seenTraceIDs[item.TraceID] = struct{}{} + } + + for _, event := range events { + if event.TraceID == "" { + continue + } + if _, exists := seenTraceIDs[event.TraceID]; exists { + continue + } + + username := usernames[valueOrZero(event.UserID)] + if username == "" { + if cost := costByTrace[event.TraceID]; cost != nil && cost.UserID != nil { + username = usernames[*cost.UserID] + } + } + fallbackItem := AuditListItem{ + TraceID: event.TraceID, + RequestID: valueOrString(event.RequestID), + SessionID: event.SessionID, + UserID: firstAvailableUserID(event.UserID, costByTrace[event.TraceID]), + Username: username, + InstanceID: firstAvailableInstanceID(event.InstanceID, costByTrace[event.TraceID]), + RequestedModel: "Auto", + ActualProviderModel: inferActualProviderLabel(event, costByTrace[event.TraceID]), + ProviderType: "gateway", + Status: inferAuditStatusFromEvent(event), + PromptTokens: valueOrCostPromptTokens(costByTrace[event.TraceID]), + CompletionTokens: valueOrCostCompletionTokens(costByTrace[event.TraceID]), + TotalTokens: valueOrCostTotalTokens(costByTrace[event.TraceID]), + CreatedAt: event.CreatedAt, + CompletedAt: nil, + } + + if search != "" { + haystacks := []string{ + strings.ToLower(fallbackItem.TraceID), + strings.ToLower(fallbackItem.RequestID), + strings.ToLower(fallbackItem.RequestedModel), + strings.ToLower(fallbackItem.ActualProviderModel), + strings.ToLower(fallbackItem.ProviderType), + strings.ToLower(username), + strings.ToLower(event.EventType), + strings.ToLower(event.Message), + } + matched := false + for _, candidate := range haystacks { + if strings.Contains(candidate, search) { + matched = true + break + } + } + if !matched { + continue + } + } + if status != "" && strings.ToLower(fallbackItem.Status) != status { + continue + } + if model != "" && !strings.Contains(strings.ToLower(fallbackItem.RequestedModel), model) && !strings.Contains(strings.ToLower(fallbackItem.ActualProviderModel), model) { + continue + } + + filtered = append(filtered, fallbackItem) + seenTraceIDs[event.TraceID] = struct{}{} + } + + sort.Slice(filtered, func(i, j int) bool { + return filtered[i].CreatedAt.After(filtered[j].CreatedAt) + }) + + total := len(filtered) + start, end := paginateBounds(total, page, limit) + if start >= total { + return &AuditListResult{ + Items: []AuditListItem{}, + Total: total, + Page: page, + Limit: limit, + }, nil + } + + return &AuditListResult{ + Items: filtered[start:end], + Total: total, + Page: page, + Limit: limit, + }, nil +} + +func (s *aiObservabilityService) GetTraceDetail(traceID string) (*AuditTraceDetail, error) { + traceID = strings.TrimSpace(traceID) + if traceID == "" { + return nil, fmt.Errorf("trace id is required") + } + + invocations, err := s.invocationRepo.ListByTraceID(traceID) + if err != nil { + return nil, fmt.Errorf("failed to load trace invocations: %w", err) + } + events, err := s.auditRepo.ListByTraceID(traceID) + if err != nil { + return nil, fmt.Errorf("failed to load trace audit events: %w", err) + } + costs, err := s.costRepo.ListByTraceID(traceID) + if err != nil { + return nil, fmt.Errorf("failed to load trace cost records: %w", err) + } + riskHits, err := s.riskHitRepo.ListByTraceID(traceID) + if err != nil { + return nil, fmt.Errorf("failed to load trace risk hits: %w", err) + } + messages, err := s.chatMessageRepo.ListByTraceID(traceID) + if err != nil { + return nil, fmt.Errorf("failed to load trace messages: %w", err) + } + + if len(invocations) == 0 { + invocations = synthesizeInvocationsFromTrace(traceID, events, costs, messages) + } + + username := s.resolveTraceUsername(invocations, events, costs, messages) + + return &AuditTraceDetail{ + TraceID: traceID, + Username: username, + Invocations: invocations, + AuditEvents: events, + CostRecords: costs, + RiskHits: riskHits, + Messages: messages, + FlowNodes: buildAuditFlowNodes(invocations), + }, nil +} + +func (s *aiObservabilityService) GetCostOverview(query CostQuery) (*CostOverview, error) { + page, limit := normalizePageLimit(query.Page, query.Limit, 20, 100) + search := strings.ToLower(strings.TrimSpace(query.Search)) + + records, err := s.costRepo.ListRecent(expandFetchWindow(page, limit)) + if err != nil { + return nil, fmt.Errorf("failed to list cost records: %w", err) + } + + userIDs := make(map[int]struct{}) + instanceIDs := make(map[int]struct{}) + for _, record := range records { + if record.UserID != nil { + userIDs[*record.UserID] = struct{}{} + } + if record.InstanceID != nil { + instanceIDs[*record.InstanceID] = struct{}{} + } + } + + usernames := make(map[int]string, len(userIDs)) + for userID := range userIDs { + user, err := s.userRepo.GetByID(userID) + if err == nil && user != nil { + usernames[userID] = user.Username + } + } + + instanceNames := make(map[int]string, len(instanceIDs)) + for instanceID := range instanceIDs { + instance, err := s.instanceRepo.GetByID(instanceID) + if err == nil && instance != nil { + instanceNames[instanceID] = instance.Name + } + } + + filteredRecords := make([]models.CostRecord, 0, len(records)) + for _, record := range records { + if search != "" { + haystacks := []string{ + strings.ToLower(record.TraceID), + strings.ToLower(record.ModelName), + strings.ToLower(record.ProviderType), + strings.ToLower(usernames[valueOrZero(record.UserID)]), + } + matched := false + for _, candidate := range haystacks { + if strings.Contains(candidate, search) { + matched = true + break + } + } + if !matched { + continue + } + } + filteredRecords = append(filteredRecords, record) + } + + overview := &CostOverview{ + Currency: "USD", + UserSummary: []CostAggregateItem{}, + InstanceSummary: []CostAggregateItem{}, + TopModels: []CostBreakdownItem{}, + TopUsers: []CostBreakdownItem{}, + DailyTrend: []CostTrendPoint{}, + ModelTrends: []CostTrendSeries{}, + UserTrends: []CostTrendSeries{}, + RecentRecords: []CostRecordView{}, + TotalRecentRecords: len(filteredRecords), + Page: page, + Limit: limit, + } + + modelTotals := map[string]*CostBreakdownItem{} + userTotals := map[string]*CostBreakdownItem{} + userAggregate := map[string]*CostAggregateItem{} + instanceAggregate := map[string]*CostAggregateItem{} + + for _, record := range filteredRecords { + overview.TotalPromptTokens += record.PromptTokens + overview.TotalCompletionTokens += record.CompletionTokens + overview.TotalTokens += record.TotalTokens + overview.TotalEstimatedCost += record.EstimatedCost + overview.TotalInternalCost += record.InternalCost + if strings.TrimSpace(record.Currency) != "" { + overview.Currency = record.Currency + } + + modelRow := modelTotals[record.ModelName] + if modelRow == nil { + modelRow = &CostBreakdownItem{Label: record.ModelName} + modelTotals[record.ModelName] = modelRow + } + aggregateBreakdown(modelRow, record) + + username := usernames[valueOrZero(record.UserID)] + userLabel := username + if userLabel == "" { + if record.UserID != nil { + userLabel = fmt.Sprintf("User #%d", *record.UserID) + } else { + userLabel = "Unknown" + } + } + userRow := userTotals[userLabel] + if userRow == nil { + userRow = &CostBreakdownItem{Label: userLabel} + userTotals[userLabel] = userRow + } + aggregateBreakdown(userRow, record) + + userSummary := userAggregate[userLabel] + if userSummary == nil { + userSummary = &CostAggregateItem{Label: userLabel} + userAggregate[userLabel] = userSummary + } + aggregateSummary(userSummary, record) + + instanceLabel := "Unassigned" + instanceMeta := "" + if record.InstanceID != nil { + instanceLabel = instanceNames[*record.InstanceID] + if instanceLabel == "" { + instanceLabel = fmt.Sprintf("Instance #%d", *record.InstanceID) + } + instanceMeta = fmt.Sprintf("ID %d", *record.InstanceID) + } + instanceSummary := instanceAggregate[instanceLabel] + if instanceSummary == nil { + instanceSummary = &CostAggregateItem{Label: instanceLabel, Meta: instanceMeta} + instanceAggregate[instanceLabel] = instanceSummary + } + aggregateSummary(instanceSummary, record) + + overview.RecentRecords = append(overview.RecentRecords, CostRecordView{ + ID: record.ID, + TraceID: record.TraceID, + UserID: record.UserID, + Username: username, + InstanceID: record.InstanceID, + InstanceName: instanceNames[valueOrZero(record.InstanceID)], + ModelName: record.ModelName, + ProviderType: record.ProviderType, + PromptTokens: record.PromptTokens, + CompletionTokens: record.CompletionTokens, + TotalTokens: record.TotalTokens, + EstimatedCost: record.EstimatedCost, + InternalCost: record.InternalCost, + Currency: record.Currency, + RecordedAt: record.RecordedAt, + }) + } + + if len(overview.RecentRecords) > 0 { + start, end := paginateBounds(len(overview.RecentRecords), page, limit) + if start < len(overview.RecentRecords) { + overview.RecentRecords = overview.RecentRecords[start:end] + } else { + overview.RecentRecords = []CostRecordView{} + } + } + + overview.TopModels = s.completeModelBreakdowns(modelTotals) + overview.TopUsers = s.completeUserBreakdowns(userTotals) + overview.UserSummary = sortAggregateItems(userAggregate, 6) + overview.InstanceSummary = sortAggregateItems(instanceAggregate, 6) + overview.DailyTrend = aggregateDailyTrend(filteredRecords, 7) + overview.ModelTrends = aggregateNamedTrends(filteredRecords, usernames, overview.TopModels, "model", 7) + overview.UserTrends = aggregateNamedTrends(filteredRecords, usernames, overview.TopUsers, "user", 7) + return overview, nil +} + +func (s *aiObservabilityService) completeModelBreakdowns(modelTotals map[string]*CostBreakdownItem) []CostBreakdownItem { + items := cloneBreakdowns(modelTotals) + + activeModels, err := s.llmModelRepo.ListActive() + if err == nil { + existing := make(map[string]struct{}, len(items)) + for _, item := range items { + existing[item.Label] = struct{}{} + } + for _, model := range activeModels { + if _, ok := existing[model.DisplayName]; ok { + continue + } + items = append(items, CostBreakdownItem{Label: model.DisplayName}) + } + } + + sort.Slice(items, func(i, j int) bool { + if items[i].EstimatedCost == items[j].EstimatedCost { + if items[i].TotalTokens == items[j].TotalTokens { + return items[i].Label < items[j].Label + } + return items[i].TotalTokens > items[j].TotalTokens + } + return items[i].EstimatedCost > items[j].EstimatedCost + }) + + if len(items) > 5 { + items = items[:5] + } + return items +} + +func (s *aiObservabilityService) completeUserBreakdowns(userTotals map[string]*CostBreakdownItem) []CostBreakdownItem { + items := cloneBreakdowns(userTotals) + + users, err := s.userRepo.List(0, 1000) + if err == nil { + existing := make(map[string]struct{}, len(items)) + for _, item := range items { + existing[item.Label] = struct{}{} + } + for _, user := range users { + if !user.IsActive { + continue + } + if _, ok := existing[user.Username]; ok { + continue + } + items = append(items, CostBreakdownItem{Label: user.Username}) + } + } + + sort.Slice(items, func(i, j int) bool { + if items[i].EstimatedCost == items[j].EstimatedCost { + if items[i].TotalTokens == items[j].TotalTokens { + return items[i].Label < items[j].Label + } + return items[i].TotalTokens > items[j].TotalTokens + } + return items[i].EstimatedCost > items[j].EstimatedCost + }) + + if len(items) > 5 { + items = items[:5] + } + return items +} + +func cloneBreakdowns(items map[string]*CostBreakdownItem) []CostBreakdownItem { + result := make([]CostBreakdownItem, 0, len(items)) + for _, item := range items { + result = append(result, *item) + } + return result +} + +func sortAggregateItems(items map[string]*CostAggregateItem, maxItems int) []CostAggregateItem { + result := make([]CostAggregateItem, 0, len(items)) + for _, item := range items { + result = append(result, *item) + } + sort.Slice(result, func(i, j int) bool { + if result[i].EstimatedCost == result[j].EstimatedCost { + if result[i].TotalTokens == result[j].TotalTokens { + return result[i].Label < result[j].Label + } + return result[i].TotalTokens > result[j].TotalTokens + } + return result[i].EstimatedCost > result[j].EstimatedCost + }) + if maxItems > 0 && len(result) > maxItems { + result = result[:maxItems] + } + return result +} + +func normalizePageLimit(page, limit, defaultLimit, maxLimit int) (int, int) { + if page <= 0 { + page = 1 + } + if limit <= 0 { + limit = defaultLimit + } + if limit > maxLimit { + limit = maxLimit + } + return page, limit +} + +func expandFetchWindow(page, limit int) int { + window := page * limit * 5 + if window < 200 { + return 200 + } + if window > 1000 { + return 1000 + } + return window +} + +func paginateBounds(total, page, limit int) (int, int) { + start := (page - 1) * limit + if start < 0 { + start = 0 + } + end := start + limit + if end > total { + end = total + } + return start, end +} + +func aggregateDailyTrend(records []models.CostRecord, days int) []CostTrendPoint { + labels := recentDayLabels(days) + byDay := make(map[string]*CostTrendPoint, len(labels)) + for _, day := range labels { + byDay[day] = &CostTrendPoint{Day: day} + } + + for _, record := range records { + day := record.RecordedAt.Format("2006-01-02") + point, ok := byDay[day] + if !ok { + continue + } + accumulateTrendPoint(point, record) + } + + points := make([]CostTrendPoint, 0, len(labels)) + for _, day := range labels { + points = append(points, *byDay[day]) + } + return points +} + +func aggregateNamedTrends(records []models.CostRecord, usernames map[int]string, topItems []CostBreakdownItem, dimension string, days int) []CostTrendSeries { + if len(topItems) == 0 { + return []CostTrendSeries{} + } + + labels := recentDayLabels(days) + allowed := make(map[string]struct{}, len(topItems)) + for _, item := range topItems { + allowed[item.Label] = struct{}{} + } + + seriesMap := make(map[string]map[string]*CostTrendPoint, len(topItems)) + for label := range allowed { + seriesMap[label] = make(map[string]*CostTrendPoint, len(labels)) + for _, day := range labels { + seriesMap[label][day] = &CostTrendPoint{Day: day} + } + } + + for _, record := range records { + day := record.RecordedAt.Format("2006-01-02") + var label string + switch dimension { + case "model": + label = record.ModelName + case "user": + username := usernames[valueOrZero(record.UserID)] + if username != "" { + label = username + } else if record.UserID != nil { + label = fmt.Sprintf("User #%d", *record.UserID) + } else { + label = "Unknown" + } + default: + continue + } + + dayPoints, ok := seriesMap[label] + if !ok { + continue + } + point, ok := dayPoints[day] + if !ok { + continue + } + accumulateTrendPoint(point, record) + } + + series := make([]CostTrendSeries, 0, min(len(topItems), 5)) + for _, item := range topItems { + pointsByDay, ok := seriesMap[item.Label] + if !ok { + continue + } + points := make([]CostTrendPoint, 0, len(labels)) + for _, day := range labels { + points = append(points, *pointsByDay[day]) + } + series = append(series, CostTrendSeries{ + Label: item.Label, + Points: points, + }) + if len(series) >= 5 { + break + } + } + return series +} + +func accumulateTrendPoint(point *CostTrendPoint, record models.CostRecord) { + point.PromptTokens += record.PromptTokens + point.CompletionTokens += record.CompletionTokens + point.TotalTokens += record.TotalTokens + point.EstimatedCost += record.EstimatedCost + point.InternalCost += record.InternalCost +} + +func recentDayLabels(days int) []string { + if days <= 0 { + days = 7 + } + labels := make([]string, 0, days) + now := time.Now() + for offset := days - 1; offset >= 0; offset-- { + labels = append(labels, now.AddDate(0, 0, -offset).Format("2006-01-02")) + } + return labels +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func (s *aiObservabilityService) loadUsernames(items []models.ModelInvocation) (map[int]string, error) { + userIDs := make(map[int]struct{}) + for _, item := range items { + if item.UserID != nil { + userIDs[*item.UserID] = struct{}{} + } + } + + usernames := make(map[int]string, len(userIDs)) + for userID := range userIDs { + user, err := s.userRepo.GetByID(userID) + if err != nil { + return nil, err + } + if user != nil { + usernames[userID] = user.Username + } + } + return usernames, nil +} + +func (s *aiObservabilityService) loadUsernamesForAudit(items []models.ModelInvocation, events []models.AuditEvent, costs []models.CostRecord) (map[int]string, error) { + userIDs := make(map[int]struct{}) + for _, item := range items { + if item.UserID != nil { + userIDs[*item.UserID] = struct{}{} + } + } + for _, event := range events { + if event.UserID != nil { + userIDs[*event.UserID] = struct{}{} + } + } + for _, cost := range costs { + if cost.UserID != nil { + userIDs[*cost.UserID] = struct{}{} + } + } + + usernames := make(map[int]string, len(userIDs)) + for userID := range userIDs { + user, err := s.userRepo.GetByID(userID) + if err != nil { + return nil, err + } + if user != nil { + usernames[userID] = user.Username + } + } + return usernames, nil +} + +func (s *aiObservabilityService) resolveTraceUsername( + invocations []models.ModelInvocation, + events []models.AuditEvent, + costs []models.CostRecord, + messages []models.ChatMessageRecord, +) string { + userID := firstTraceUserID(invocations, events, costs, messages) + if userID == nil { + return "" + } + + user, err := s.userRepo.GetByID(*userID) + if err != nil || user == nil { + return "" + } + return user.Username +} + +func firstTraceUserID( + invocations []models.ModelInvocation, + events []models.AuditEvent, + costs []models.CostRecord, + messages []models.ChatMessageRecord, +) *int { + for _, item := range invocations { + if item.UserID != nil { + return item.UserID + } + } + for _, event := range events { + if event.UserID != nil { + return event.UserID + } + } + for _, cost := range costs { + if cost.UserID != nil { + return cost.UserID + } + } + for _, message := range messages { + if message.UserID != nil { + return message.UserID + } + } + return nil +} + +func synthesizeInvocationsFromTrace( + traceID string, + events []models.AuditEvent, + costs []models.CostRecord, + messages []models.ChatMessageRecord, +) []models.ModelInvocation { + if len(events) == 0 && len(costs) == 0 && len(messages) == 0 { + return []models.ModelInvocation{} + } + + requestID := firstTraceRequestID(events, costs, messages) + sessionID := firstTraceSessionID(events, costs, messages) + userID := firstTraceUserID(nil, events, costs, messages) + instanceID := firstTraceInstanceID(events, costs, messages) + requestedModel := inferRequestedModel(events) + actualModel := inferActualModel(events, costs) + status := inferTraceStatus(events, costs) + createdAt := firstTraceTimestamp(events, costs, messages) + + var errorMessage *string + if status == models.ModelInvocationStatusBlocked || status == models.ModelInvocationStatusFailed { + if message := firstRelevantEventMessage(events); message != "" { + errorMessage = &message + } + } + + return []models.ModelInvocation{ + { + ID: 0, + TraceID: traceID, + SessionID: sessionID, + RequestID: requestID, + UserID: userID, + InstanceID: instanceID, + ProviderType: "gateway", + RequestedModel: requestedModel, + ActualProviderModel: actualModel, + TrafficClass: models.TrafficClassLLM, + Status: status, + ErrorMessage: errorMessage, + CreatedAt: createdAt, + }, + } +} + +func firstTraceRequestID(events []models.AuditEvent, costs []models.CostRecord, messages []models.ChatMessageRecord) string { + for _, event := range events { + if event.RequestID != nil && strings.TrimSpace(*event.RequestID) != "" { + return strings.TrimSpace(*event.RequestID) + } + } + for _, cost := range costs { + if cost.RequestID != nil && strings.TrimSpace(*cost.RequestID) != "" { + return strings.TrimSpace(*cost.RequestID) + } + } + for _, message := range messages { + if message.RequestID != nil && strings.TrimSpace(*message.RequestID) != "" { + return strings.TrimSpace(*message.RequestID) + } + } + return "synthetic" +} + +func firstTraceSessionID(events []models.AuditEvent, costs []models.CostRecord, messages []models.ChatMessageRecord) *string { + for _, event := range events { + if event.SessionID != nil && strings.TrimSpace(*event.SessionID) != "" { + value := strings.TrimSpace(*event.SessionID) + return &value + } + } + for _, cost := range costs { + if cost.SessionID != nil && strings.TrimSpace(*cost.SessionID) != "" { + value := strings.TrimSpace(*cost.SessionID) + return &value + } + } + for _, message := range messages { + if strings.TrimSpace(message.SessionID) != "" { + value := strings.TrimSpace(message.SessionID) + return &value + } + } + return nil +} + +func firstTraceInstanceID(events []models.AuditEvent, costs []models.CostRecord, messages []models.ChatMessageRecord) *int { + for _, event := range events { + if event.InstanceID != nil { + return event.InstanceID + } + } + for _, cost := range costs { + if cost.InstanceID != nil { + return cost.InstanceID + } + } + for _, message := range messages { + if message.InstanceID != nil { + return message.InstanceID + } + } + return nil +} + +func firstTraceTimestamp(events []models.AuditEvent, costs []models.CostRecord, messages []models.ChatMessageRecord) time.Time { + for _, event := range events { + if !event.CreatedAt.IsZero() { + return event.CreatedAt + } + } + for _, cost := range costs { + if !cost.RecordedAt.IsZero() { + return cost.RecordedAt + } + } + for _, message := range messages { + if !message.CreatedAt.IsZero() { + return message.CreatedAt + } + } + return time.Now() +} + +func inferRequestedModel(events []models.AuditEvent) string { + for _, event := range events { + if strings.Contains(event.Message, "model ") { + parts := strings.Split(event.Message, "model ") + candidate := strings.TrimSpace(parts[len(parts)-1]) + if candidate != "" { + return candidate + } + } + } + return "Auto" +} + +func inferActualModel(events []models.AuditEvent, costs []models.CostRecord) string { + for _, cost := range costs { + if strings.TrimSpace(cost.ModelName) != "" { + return cost.ModelName + } + } + for _, event := range events { + if strings.Contains(event.EventType, "rerouted") { + return "secure-route" + } + } + return "not-routed" +} + +func inferTraceStatus(events []models.AuditEvent, costs []models.CostRecord) string { + if len(costs) > 0 { + return models.ModelInvocationStatusCompleted + } + for _, event := range events { + switch { + case strings.Contains(event.EventType, "completed"): + return models.ModelInvocationStatusCompleted + case strings.Contains(event.EventType, "blocked"): + return models.ModelInvocationStatusBlocked + case strings.Contains(event.EventType, "failed"): + return models.ModelInvocationStatusFailed + } + } + return models.ModelInvocationStatusPending +} + +func firstRelevantEventMessage(events []models.AuditEvent) string { + for _, event := range events { + if strings.Contains(event.EventType, "blocked") || strings.Contains(event.EventType, "failed") { + return strings.TrimSpace(event.Message) + } + } + return "" +} + +func aggregateInvocationsByTrace(items []models.ModelInvocation, usernames map[int]string) []AuditListItem { + type traceAggregate struct { + item AuditListItem + hasCompleted bool + hasBlocked bool + hasFailed bool + hasPending bool + } + + byTrace := make(map[string]*traceAggregate, len(items)) + for _, invocation := range items { + traceID := strings.TrimSpace(invocation.TraceID) + if traceID == "" { + continue + } + + aggregate := byTrace[traceID] + if aggregate == nil { + requestID := strings.TrimSpace(invocation.RequestID) + aggregate = &traceAggregate{ + item: AuditListItem{ + TraceID: traceID, + RequestID: requestID, + SessionID: invocation.SessionID, + UserID: invocation.UserID, + Username: usernames[valueOrZero(invocation.UserID)], + InstanceID: invocation.InstanceID, + RequestedModel: invocation.RequestedModel, + ActualProviderModel: invocation.ActualProviderModel, + ProviderType: invocation.ProviderType, + CreatedAt: invocation.CreatedAt, + CompletedAt: invocation.CompletedAt, + }, + } + byTrace[traceID] = aggregate + } + + if invocation.CreatedAt.Before(aggregate.item.CreatedAt) { + aggregate.item.CreatedAt = invocation.CreatedAt + if trimmed := strings.TrimSpace(invocation.RequestID); trimmed != "" { + aggregate.item.RequestID = trimmed + } + } + if aggregate.item.CompletedAt == nil || (invocation.CompletedAt != nil && invocation.CompletedAt.After(*aggregate.item.CompletedAt)) { + aggregate.item.CompletedAt = invocation.CompletedAt + } + + if aggregate.item.SessionID == nil && invocation.SessionID != nil { + aggregate.item.SessionID = invocation.SessionID + } + if aggregate.item.UserID == nil && invocation.UserID != nil { + aggregate.item.UserID = invocation.UserID + aggregate.item.Username = usernames[valueOrZero(invocation.UserID)] + } + if aggregate.item.Username == "" { + aggregate.item.Username = usernames[valueOrZero(invocation.UserID)] + } + if aggregate.item.InstanceID == nil && invocation.InstanceID != nil { + aggregate.item.InstanceID = invocation.InstanceID + } + if strings.TrimSpace(aggregate.item.RequestedModel) == "" && strings.TrimSpace(invocation.RequestedModel) != "" { + aggregate.item.RequestedModel = invocation.RequestedModel + } + if strings.TrimSpace(invocation.ActualProviderModel) != "" { + aggregate.item.ActualProviderModel = invocation.ActualProviderModel + } + if strings.TrimSpace(invocation.ProviderType) != "" { + aggregate.item.ProviderType = invocation.ProviderType + } + if aggregate.item.ErrorMessage == nil && invocation.ErrorMessage != nil && strings.TrimSpace(*invocation.ErrorMessage) != "" { + aggregate.item.ErrorMessage = invocation.ErrorMessage + } + + aggregate.item.PromptTokens += invocation.PromptTokens + aggregate.item.CompletionTokens += invocation.CompletionTokens + aggregate.item.TotalTokens += invocation.TotalTokens + if invocation.LatencyMs != nil { + if aggregate.item.LatencyMs == nil { + aggregate.item.LatencyMs = pointerToInt(0) + } + totalLatency := *aggregate.item.LatencyMs + *invocation.LatencyMs + aggregate.item.LatencyMs = pointerToInt(totalLatency) + } + + switch strings.ToLower(strings.TrimSpace(invocation.Status)) { + case models.ModelInvocationStatusCompleted: + aggregate.hasCompleted = true + case models.ModelInvocationStatusBlocked: + aggregate.hasBlocked = true + case models.ModelInvocationStatusFailed: + aggregate.hasFailed = true + default: + aggregate.hasPending = true + } + } + + result := make([]AuditListItem, 0, len(byTrace)) + for _, aggregate := range byTrace { + switch { + case aggregate.hasFailed: + aggregate.item.Status = models.ModelInvocationStatusFailed + case aggregate.hasBlocked: + aggregate.item.Status = models.ModelInvocationStatusBlocked + case aggregate.hasPending && !aggregate.hasCompleted: + aggregate.item.Status = models.ModelInvocationStatusPending + default: + aggregate.item.Status = models.ModelInvocationStatusCompleted + } + result = append(result, aggregate.item) + } + + sort.Slice(result, func(i, j int) bool { + return result[i].CreatedAt.After(result[j].CreatedAt) + }) + + return result +} + +type auditPayloadRequest struct { + Messages []auditPayloadMessage `json:"messages"` +} + +type auditPayloadResponse struct { + Choices []struct { + Index int `json:"index"` + Message auditPayloadMessage `json:"message"` + } `json:"choices"` +} + +type auditStreamChunk struct { + Choices []struct { + Index int `json:"index"` + Delta auditPayloadMessage `json:"delta"` + } `json:"choices"` +} + +type auditPayloadMessage struct { + Role string `json:"role"` + Content interface{} `json:"content"` + Name string `json:"name,omitempty"` + ToolCalls []auditToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` +} + +type auditToolCall struct { + ID string `json:"id,omitempty"` + Type string `json:"type,omitempty"` + Index *int `json:"index,omitempty"` + Function *auditToolCallFunction `json:"function,omitempty"` +} + +type auditToolCallFunction struct { + Name string `json:"name,omitempty"` + Arguments string `json:"arguments,omitempty"` +} + +func buildAuditFlowNodes(invocations []models.ModelInvocation) []AuditFlowNode { + if len(invocations) == 0 { + return []AuditFlowNode{} + } + + ordered := append([]models.ModelInvocation(nil), invocations...) + sort.Slice(ordered, func(i, j int) bool { + if ordered[i].CreatedAt.Equal(ordered[j].CreatedAt) { + return ordered[i].ID < ordered[j].ID + } + return ordered[i].CreatedAt.Before(ordered[j].CreatedAt) + }) + + nodes := make([]AuditFlowNode, 0, len(ordered)*3) + transcript := make([]auditPayloadMessage, 0) + for _, invocation := range ordered { + requestMessages := parseAuditRequestMessages(invocation.RequestPayload) + newMessages := diffAuditMessages(transcript, requestMessages) + nodes = append(nodes, buildRequestMessageNodes(newMessages, invocation)...) + nodes = append(nodes, buildInvocationNode(invocation, sanitizeAuditRequestPayload(invocation.RequestPayload))) + + responseMessages := parseAuditResponseMessages(invocation.ResponsePayload) + nodes = append(nodes, buildResponseMessageNodes(responseMessages, invocation)...) + transcript = append(cloneAuditMessages(requestMessages), cloneAuditMessages(responseMessages)...) + } + + return nodes +} + +func parseAuditRequestMessages(payload *string) []auditPayloadMessage { + if payload == nil || strings.TrimSpace(*payload) == "" { + return []auditPayloadMessage{} + } + + var request auditPayloadRequest + if err := json.Unmarshal([]byte(*payload), &request); err != nil { + return []auditPayloadMessage{} + } + return currentAuditTurnMessages(request.Messages) +} + +func parseAuditResponseMessages(payload *string) []auditPayloadMessage { + if payload == nil { + return []auditPayloadMessage{} + } + + trimmed := strings.TrimSpace(*payload) + if trimmed == "" { + return []auditPayloadMessage{} + } + if strings.HasPrefix(trimmed, "data:") { + return parseAuditStreamMessages(trimmed) + } + + var response auditPayloadResponse + if err := json.Unmarshal([]byte(trimmed), &response); err != nil { + return []auditPayloadMessage{} + } + + items := make([]auditPayloadMessage, 0, len(response.Choices)) + for _, choice := range response.Choices { + items = append(items, choice.Message) + } + return items +} + +func parseAuditStreamMessages(payload string) []auditPayloadMessage { + lines := strings.Split(payload, "\n") + byChoice := make(map[int]*auditPayloadMessage) + order := make([]int, 0) + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "data:") { + continue + } + chunkPayload := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:")) + if chunkPayload == "" || chunkPayload == "[DONE]" { + continue + } + + var chunk auditStreamChunk + if err := json.Unmarshal([]byte(chunkPayload), &chunk); err != nil { + continue + } + + for _, choice := range chunk.Choices { + message := byChoice[choice.Index] + if message == nil { + message = &auditPayloadMessage{Role: "assistant"} + byChoice[choice.Index] = message + order = append(order, choice.Index) + } + if role := strings.TrimSpace(choice.Delta.Role); role != "" { + message.Role = role + } + message.Content = mergeAuditContent(message.Content, choice.Delta.Content) + mergeAuditToolCalls(&message.ToolCalls, choice.Delta.ToolCalls) + } + } + + sort.Ints(order) + items := make([]auditPayloadMessage, 0, len(order)) + for _, index := range order { + message := byChoice[index] + if message == nil || !hasAuditMessageBody(*message) { + continue + } + items = append(items, *message) + } + return items +} + +func diffAuditMessages(existing, current []auditPayloadMessage) []auditPayloadMessage { + limit := len(existing) + if len(current) < limit { + limit = len(current) + } + + matched := 0 + for matched < limit && auditMessagesEquivalent(existing[matched], current[matched]) { + matched++ + } + + return cloneAuditMessages(current[matched:]) +} + +func cloneAuditMessages(messages []auditPayloadMessage) []auditPayloadMessage { + if len(messages) == 0 { + return []auditPayloadMessage{} + } + + cloned := make([]auditPayloadMessage, 0, len(messages)) + for _, message := range messages { + copyMessage := message + if len(message.ToolCalls) > 0 { + copyMessage.ToolCalls = append([]auditToolCall(nil), message.ToolCalls...) + } + cloned = append(cloned, copyMessage) + } + return cloned +} + +func buildRequestMessageNodes(messages []auditPayloadMessage, invocation models.ModelInvocation) []AuditFlowNode { + nodes := make([]AuditFlowNode, 0, len(messages)) + for index, message := range messages { + content := strings.TrimSpace(flattenAuditMessageContent(message.Content)) + switch strings.ToLower(strings.TrimSpace(message.Role)) { + case "user": + nodes = append(nodes, AuditFlowNode{ + ID: fmt.Sprintf("request-user-%d-%d", invocation.ID, index), + Kind: "user_message", + Title: "User Prompt", + RequestID: invocation.RequestID, + InvocationID: pointerToInt(invocation.ID), + Model: invocation.RequestedModel, + OutputPayload: content, + CreatedAt: invocation.CreatedAt, + }) + case "tool": + title := strings.TrimSpace(message.Name) + if title == "" { + title = "Tool Output" + } + summary := strings.TrimSpace(message.ToolCallID) + nodes = append(nodes, AuditFlowNode{ + ID: fmt.Sprintf("request-tool-%d-%d", invocation.ID, index), + Kind: "tool_output", + Title: title, + RequestID: invocation.RequestID, + InvocationID: pointerToInt(invocation.ID), + Model: invocation.RequestedModel, + Summary: summary, + OutputPayload: content, + CreatedAt: invocation.CreatedAt, + }) + } + } + return nodes +} + +func buildInvocationNode(invocation models.ModelInvocation, inputPayload string) AuditFlowNode { + summary := strings.TrimSpace(invocation.RequestedModel) + if actual := strings.TrimSpace(invocation.ActualProviderModel); actual != "" { + if summary != "" { + summary = fmt.Sprintf("%s -> %s", summary, actual) + } else { + summary = actual + } + } + + return AuditFlowNode{ + ID: fmt.Sprintf("invocation-%d", invocation.ID), + Kind: "llm_call", + Title: "LLM Call", + RequestID: invocation.RequestID, + InvocationID: pointerToInt(invocation.ID), + Model: invocation.ActualProviderModel, + Status: invocation.Status, + Summary: summary, + InputPayload: inputPayload, + OutputPayload: valueOrPayload(invocation.ResponsePayload), + CreatedAt: invocation.CreatedAt, + } +} + +func buildResponseMessageNodes(messages []auditPayloadMessage, invocation models.ModelInvocation) []AuditFlowNode { + nodes := make([]AuditFlowNode, 0, len(messages)) + for index, message := range messages { + if len(message.ToolCalls) > 0 { + for toolIndex, toolCall := range message.ToolCalls { + title := "Tool Call" + if toolCall.Function != nil && strings.TrimSpace(toolCall.Function.Name) != "" { + title = strings.TrimSpace(toolCall.Function.Name) + } + summary := strings.TrimSpace(toolCall.ID) + inputPayload := "" + if toolCall.Function != nil { + inputPayload = strings.TrimSpace(toolCall.Function.Arguments) + } + nodes = append(nodes, AuditFlowNode{ + ID: fmt.Sprintf("response-tool-call-%d-%d-%d", invocation.ID, index, toolIndex), + Kind: "tool_call", + Title: title, + RequestID: invocation.RequestID, + InvocationID: pointerToInt(invocation.ID), + Model: invocation.ActualProviderModel, + Summary: summary, + InputPayload: inputPayload, + CreatedAt: coalesceTime(invocation.CompletedAt, invocation.CreatedAt), + }) + } + continue + } + + content := strings.TrimSpace(flattenAuditMessageContent(message.Content)) + if content == "" { + continue + } + nodes = append(nodes, AuditFlowNode{ + ID: fmt.Sprintf("response-assistant-%d-%d", invocation.ID, index), + Kind: "assistant_response", + Title: "Assistant Response", + RequestID: invocation.RequestID, + InvocationID: pointerToInt(invocation.ID), + Model: invocation.ActualProviderModel, + OutputPayload: content, + CreatedAt: coalesceTime(invocation.CompletedAt, invocation.CreatedAt), + }) + } + return nodes +} + +func mergeAuditContent(existing, incoming interface{}) interface{} { + existingText := strings.TrimSpace(flattenAuditMessageContent(existing)) + incomingText := strings.TrimSpace(flattenAuditMessageContent(incoming)) + switch { + case existingText == "": + return incoming + case incomingText == "": + return existing + default: + return existingText + incomingText + } +} + +func currentAuditTurnMessages(messages []auditPayloadMessage) []auditPayloadMessage { + startIndex := 0 + for index := len(messages) - 1; index >= 0; index-- { + if strings.EqualFold(strings.TrimSpace(messages[index].Role), "user") { + startIndex = index + break + } + } + return cloneAuditMessages(messages[startIndex:]) +} + +func sanitizeAuditRequestPayload(payload *string) string { + if payload == nil { + return "" + } + + trimmed := strings.TrimSpace(*payload) + if trimmed == "" { + return "" + } + + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(trimmed), &raw); err != nil { + return trimmed + } + + currentTurn := parseAuditRequestMessages(payload) + if len(currentTurn) == 0 { + return trimmed + } + + encodedMessages, err := json.Marshal(currentTurn) + if err != nil { + return trimmed + } + raw["messages"] = encodedMessages + + sanitized, err := json.Marshal(raw) + if err != nil { + return trimmed + } + return string(sanitized) +} + +func mergeAuditToolCalls(target *[]auditToolCall, incoming []auditToolCall) { + if len(incoming) == 0 { + return + } + + for _, item := range incoming { + index := len(*target) + if item.Index != nil && *item.Index >= 0 { + index = *item.Index + } + for len(*target) <= index { + *target = append(*target, auditToolCall{}) + } + + current := &(*target)[index] + if trimmed := strings.TrimSpace(item.ID); trimmed != "" { + current.ID = trimmed + } + if trimmed := strings.TrimSpace(item.Type); trimmed != "" { + current.Type = trimmed + } + if item.Function != nil { + if current.Function == nil { + current.Function = &auditToolCallFunction{} + } + if trimmed := strings.TrimSpace(item.Function.Name); trimmed != "" && current.Function.Name == "" { + current.Function.Name = trimmed + } + if strings.TrimSpace(item.Function.Arguments) != "" { + current.Function.Arguments += item.Function.Arguments + } + } + } +} + +func auditMessagesEquivalent(left, right auditPayloadMessage) bool { + if !strings.EqualFold(strings.TrimSpace(left.Role), strings.TrimSpace(right.Role)) { + return false + } + if strings.TrimSpace(left.Name) != strings.TrimSpace(right.Name) { + return false + } + if strings.TrimSpace(left.ToolCallID) != strings.TrimSpace(right.ToolCallID) { + return false + } + if strings.TrimSpace(flattenAuditMessageContent(left.Content)) != strings.TrimSpace(flattenAuditMessageContent(right.Content)) { + return false + } + if len(left.ToolCalls) != len(right.ToolCalls) { + return false + } + for index := range left.ToolCalls { + if !auditToolCallsEquivalent(left.ToolCalls[index], right.ToolCalls[index]) { + return false + } + } + return true +} + +func auditToolCallsEquivalent(left, right auditToolCall) bool { + if strings.TrimSpace(left.ID) != strings.TrimSpace(right.ID) || strings.TrimSpace(left.Type) != strings.TrimSpace(right.Type) { + return false + } + leftName, leftArgs := "", "" + if left.Function != nil { + leftName = strings.TrimSpace(left.Function.Name) + leftArgs = strings.TrimSpace(left.Function.Arguments) + } + rightName, rightArgs := "", "" + if right.Function != nil { + rightName = strings.TrimSpace(right.Function.Name) + rightArgs = strings.TrimSpace(right.Function.Arguments) + } + return leftName == rightName && leftArgs == rightArgs +} + +func hasAuditMessageBody(message auditPayloadMessage) bool { + return strings.TrimSpace(flattenAuditMessageContent(message.Content)) != "" || len(message.ToolCalls) > 0 +} + +func flattenAuditMessageContent(content interface{}) string { + switch value := content.(type) { + case nil: + return "" + case string: + return value + case []interface{}: + parts := make([]string, 0, len(value)) + for _, item := range value { + if text := flattenAuditStructuredContentPart(item); text != "" { + parts = append(parts, text) + } + } + if len(parts) > 0 { + return strings.Join(parts, "\n") + } + case map[string]interface{}: + if text := flattenAuditStructuredContentPart(value); text != "" { + return text + } + } + + data, err := json.Marshal(content) + if err != nil { + return "" + } + return string(data) +} + +func flattenAuditStructuredContentPart(content interface{}) string { + part, ok := content.(map[string]interface{}) + if !ok { + return "" + } + + if text, ok := part["text"].(string); ok { + return strings.TrimSpace(text) + } + if text, ok := part["input_text"].(string); ok { + return strings.TrimSpace(text) + } + if text, ok := part["output_text"].(string); ok { + return strings.TrimSpace(text) + } + return "" +} + +func coalesceTime(value *time.Time, fallback time.Time) time.Time { + if value != nil && !value.IsZero() { + return *value + } + return fallback +} + +func valueOrPayload(value *string) string { + if value == nil { + return "" + } + return strings.TrimSpace(*value) +} + +func aggregateBreakdown(target *CostBreakdownItem, record models.CostRecord) { + target.PromptTokens += record.PromptTokens + target.CompletionTokens += record.CompletionTokens + target.TotalTokens += record.TotalTokens + target.EstimatedCost += record.EstimatedCost + target.InternalCost += record.InternalCost +} + +func aggregateSummary(target *CostAggregateItem, record models.CostRecord) { + target.PromptTokens += record.PromptTokens + target.CompletionTokens += record.CompletionTokens + target.TotalTokens += record.TotalTokens + target.EstimatedCost += record.EstimatedCost + target.InternalCost += record.InternalCost +} + +func sortBreakdowns(items map[string]*CostBreakdownItem) []CostBreakdownItem { + result := make([]CostBreakdownItem, 0, len(items)) + for _, item := range items { + result = append(result, *item) + } + sort.Slice(result, func(i, j int) bool { + if result[i].EstimatedCost == result[j].EstimatedCost { + return result[i].TotalTokens > result[j].TotalTokens + } + return result[i].EstimatedCost > result[j].EstimatedCost + }) + return result +} + +func valueOrZero(value *int) int { + if value == nil { + return 0 + } + return *value +} + +func valueOrString(value *string) string { + if value == nil { + return "" + } + return *value +} + +func valueOrEventLabel(eventType string) string { + trimmed := strings.TrimSpace(eventType) + if trimmed == "" { + return "gateway.event" + } + return trimmed +} + +func inferAuditStatusFromEvent(event models.AuditEvent) string { + switch { + case strings.Contains(event.EventType, "completed"): + return models.ModelInvocationStatusCompleted + case strings.Contains(event.EventType, "blocked"): + return models.ModelInvocationStatusBlocked + case strings.Contains(event.EventType, "failed"): + return models.ModelInvocationStatusFailed + default: + return models.ModelInvocationStatusPending + } +} + +func aggregateRecentCostByTrace(costs []models.CostRecord) map[string]*models.CostRecord { + byTrace := make(map[string]*models.CostRecord, len(costs)) + for _, cost := range costs { + existing := byTrace[cost.TraceID] + if existing == nil { + copy := cost + byTrace[cost.TraceID] = © + continue + } + existing.PromptTokens += cost.PromptTokens + existing.CompletionTokens += cost.CompletionTokens + existing.TotalTokens += cost.TotalTokens + existing.EstimatedCost += cost.EstimatedCost + existing.InternalCost += cost.InternalCost + if existing.UserID == nil && cost.UserID != nil { + existing.UserID = cost.UserID + } + if existing.InstanceID == nil && cost.InstanceID != nil { + existing.InstanceID = cost.InstanceID + } + if strings.TrimSpace(existing.ModelName) == "" && strings.TrimSpace(cost.ModelName) != "" { + existing.ModelName = cost.ModelName + } + if strings.TrimSpace(existing.ProviderType) == "" && strings.TrimSpace(cost.ProviderType) != "" { + existing.ProviderType = cost.ProviderType + } + } + return byTrace +} + +func firstAvailableUserID(primary *int, cost *models.CostRecord) *int { + if primary != nil { + return primary + } + if cost != nil { + return cost.UserID + } + return nil +} + +func firstAvailableInstanceID(primary *int, cost *models.CostRecord) *int { + if primary != nil { + return primary + } + if cost != nil { + return cost.InstanceID + } + return nil +} + +func inferActualProviderLabel(event models.AuditEvent, cost *models.CostRecord) string { + if cost != nil && strings.TrimSpace(cost.ModelName) != "" { + return cost.ModelName + } + return valueOrEventLabel(event.EventType) +} + +func valueOrCostPromptTokens(cost *models.CostRecord) int { + if cost == nil { + return 0 + } + return cost.PromptTokens +} + +func valueOrCostCompletionTokens(cost *models.CostRecord) int { + if cost == nil { + return 0 + } + return cost.CompletionTokens +} + +func valueOrCostTotalTokens(cost *models.CostRecord) int { + if cost == nil { + return 0 + } + return cost.TotalTokens +} + +func pointerToInt(value int) *int { + return &value +} diff --git a/backend/internal/services/ai_observability_service_test.go b/backend/internal/services/ai_observability_service_test.go new file mode 100644 index 0000000..9ac2682 --- /dev/null +++ b/backend/internal/services/ai_observability_service_test.go @@ -0,0 +1,131 @@ +package services + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "clawreef/internal/models" +) + +func TestBuildAuditFlowNodesBuildsSingleToolLoopTimeline(t *testing.T) { + startedAt := time.Date(2026, 3, 26, 7, 53, 0, 0, time.UTC) + completedAt := startedAt.Add(2 * time.Second) + secondStartedAt := startedAt.Add(10 * time.Second) + secondCompletedAt := secondStartedAt.Add(2 * time.Second) + + invocations := []models.ModelInvocation{ + { + ID: 11, + TraceID: "trc_weather", + RequestID: "req_1", + RequestedModel: "auto", + ActualProviderModel: "gpt-weather", + Status: models.ModelInvocationStatusCompleted, + RequestPayload: stringPointer(`{"messages":[{"role":"user","content":"北京今天天气怎么样"}]}`), + ResponsePayload: stringPointer(`{"choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_weather","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Beijing\"}"}}]}}]}`), + CreatedAt: startedAt, + CompletedAt: &completedAt, + }, + { + ID: 12, + TraceID: "trc_weather", + RequestID: "req_2", + RequestedModel: "auto", + ActualProviderModel: "gpt-weather", + Status: models.ModelInvocationStatusCompleted, + RequestPayload: stringPointer(`{"messages":[{"role":"user","content":"北京今天天气怎么样"},{"role":"assistant","tool_calls":[{"id":"call_weather","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Beijing\"}"}}]},{"role":"tool","tool_call_id":"call_weather","name":"get_weather","content":"{\"temp_c\":25}"}]}`), + ResponsePayload: stringPointer(`{"choices":[{"index":0,"message":{"role":"assistant","content":"北京今天多云,25C。"}}]}`), + CreatedAt: secondStartedAt, + CompletedAt: &secondCompletedAt, + }, + } + + nodes := buildAuditFlowNodes(invocations) + kinds := make([]string, 0, len(nodes)) + for _, node := range nodes { + kinds = append(kinds, node.Kind) + } + + expectedKinds := []string{ + "user_message", + "llm_call", + "tool_call", + "tool_output", + "llm_call", + "assistant_response", + } + if !reflect.DeepEqual(kinds, expectedKinds) { + t.Fatalf("expected flow kinds %v, got %v", expectedKinds, kinds) + } + + if nodes[2].Title != "get_weather" { + t.Fatalf("expected tool call title get_weather, got %q", nodes[2].Title) + } + if nodes[3].Summary != "call_weather" { + t.Fatalf("expected tool output summary to include tool_call_id, got %q", nodes[3].Summary) + } + if nodes[5].OutputPayload != "北京今天多云,25C。" { + t.Fatalf("expected final assistant response content, got %q", nodes[5].OutputPayload) + } +} + +func TestBuildAuditFlowNodesIgnoresHistoricalMessagesInFlowAndInvocationInput(t *testing.T) { + startedAt := time.Date(2026, 3, 26, 9, 0, 0, 0, time.UTC) + completedAt := startedAt.Add(2 * time.Second) + secondStartedAt := startedAt.Add(5 * time.Second) + secondCompletedAt := secondStartedAt.Add(2 * time.Second) + + invocations := []models.ModelInvocation{ + { + ID: 21, + TraceID: "trc_history_trimmed", + RequestID: "req_hist_1", + RequestedModel: "auto", + ActualProviderModel: "gpt-weather", + Status: models.ModelInvocationStatusCompleted, + RequestPayload: stringPointer(`{"messages":[{"role":"system","content":"system prompt"},{"role":"user","content":"Previous turn question"},{"role":"assistant","content":"Previous turn answer"},{"role":"user","content":"Current turn question"}],"tools":[{"type":"function","function":{"name":"get_weather"}}]}`), + ResponsePayload: stringPointer(`{"choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_current","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Shanghai\"}"}}]}}]}`), + CreatedAt: startedAt, + CompletedAt: &completedAt, + }, + { + ID: 22, + TraceID: "trc_history_trimmed", + RequestID: "req_hist_2", + RequestedModel: "auto", + ActualProviderModel: "gpt-weather", + Status: models.ModelInvocationStatusCompleted, + RequestPayload: stringPointer(`{"messages":[{"role":"system","content":"system prompt"},{"role":"user","content":"Previous turn question"},{"role":"assistant","content":"Previous turn answer"},{"role":"user","content":"Current turn question"},{"role":"assistant","tool_calls":[{"id":"call_current","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Shanghai\"}"}}]},{"role":"tool","tool_call_id":"call_current","name":"get_weather","content":"{\"temp_c\":16}"}],"tools":[{"type":"function","function":{"name":"get_weather"}}]}`), + ResponsePayload: stringPointer(`{"choices":[{"index":0,"message":{"role":"assistant","content":"Shanghai is 16C."}}]}`), + CreatedAt: secondStartedAt, + CompletedAt: &secondCompletedAt, + }, + } + + nodes := buildAuditFlowNodes(invocations) + if len(nodes) != 6 { + t.Fatalf("expected 6 flow nodes, got %d", len(nodes)) + } + if nodes[0].Kind != "user_message" || nodes[0].OutputPayload != "Current turn question" { + t.Fatalf("expected first flow node to be current user turn, got %#v", nodes[0]) + } + if strings.Contains(nodes[1].InputPayload, "Previous turn question") || strings.Contains(nodes[1].InputPayload, "system prompt") { + t.Fatalf("expected llm input payload to exclude historical messages, got %q", nodes[1].InputPayload) + } + + var llmInput map[string]any + if err := json.Unmarshal([]byte(nodes[1].InputPayload), &llmInput); err != nil { + t.Fatalf("expected llm input payload to stay valid json: %v", err) + } + rawMessages, ok := llmInput["messages"].([]any) + if !ok || len(rawMessages) != 1 { + t.Fatalf("expected sanitized llm input to keep only current-turn messages, got %#v", llmInput["messages"]) + } +} + +func stringPointer(value string) *string { + return &value +} diff --git a/backend/internal/services/audit_event_service.go b/backend/internal/services/audit_event_service.go new file mode 100644 index 0000000..71000fc --- /dev/null +++ b/backend/internal/services/audit_event_service.go @@ -0,0 +1,55 @@ +package services + +import ( + "fmt" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// AuditEventService defines application-level operations for governance audit events. +type AuditEventService interface { + RecordEvent(event *models.AuditEvent) error + ListEventsByTraceID(traceID string) ([]models.AuditEvent, error) +} + +type auditEventService struct { + repo repository.AuditEventRepository +} + +// NewAuditEventService creates a new audit event service. +func NewAuditEventService(repo repository.AuditEventRepository) AuditEventService { + return &auditEventService{repo: repo} +} + +func (s *auditEventService) RecordEvent(event *models.AuditEvent) error { + if event == nil { + return fmt.Errorf("audit event is required") + } + if strings.TrimSpace(event.TraceID) == "" { + return fmt.Errorf("trace id is required") + } + if strings.TrimSpace(event.EventType) == "" { + return fmt.Errorf("event type is required") + } + if strings.TrimSpace(event.TrafficClass) == "" { + event.TrafficClass = models.TrafficClassLLM + } + if strings.TrimSpace(event.Severity) == "" { + event.Severity = models.AuditSeverityInfo + } + if strings.TrimSpace(event.Message) == "" { + return fmt.Errorf("message is required") + } + + return s.repo.Create(event) +} + +func (s *auditEventService) ListEventsByTraceID(traceID string) ([]models.AuditEvent, error) { + items, err := s.repo.ListByTraceID(strings.TrimSpace(traceID)) + if err != nil { + return nil, fmt.Errorf("failed to list audit events by trace: %w", err) + } + return items, nil +} diff --git a/backend/internal/services/auth_service.go b/backend/internal/services/auth_service.go new file mode 100644 index 0000000..37aa413 --- /dev/null +++ b/backend/internal/services/auth_service.go @@ -0,0 +1,227 @@ +package services + +import ( + "errors" + "fmt" + "time" + + "clawreef/internal/config" + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/utils" +) + +// AuthService defines the interface for authentication operations +type AuthService interface { + Register(username, email, password string) (*models.User, error) + Login(username, password string) (*TokenPair, error) + RefreshToken(refreshToken string) (*TokenPair, error) + GetCurrentUser(userID int) (*models.User, error) + ChangePassword(userID int, currentPassword, newPassword string) error +} + +// TokenPair holds access and refresh tokens +type TokenPair struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int `json:"expires_in"` +} + +// authService implements AuthService +type authService struct { + userRepo repository.UserRepository + jwtConfig config.JWTConfig +} + +// NewAuthService creates a new auth service +func NewAuthService(userRepo repository.UserRepository, jwtConfig config.JWTConfig) AuthService { + return &authService{ + userRepo: userRepo, + jwtConfig: jwtConfig, + } +} + +// Register registers a new user +func (s *authService) Register(username, email, password string) (*models.User, error) { + // Check if username already exists + existingUser, err := s.userRepo.GetByUsername(username) + if err != nil { + return nil, fmt.Errorf("failed to check username: %w", err) + } + if existingUser != nil { + return nil, errors.New("username already exists") + } + + // Check if email already exists + existingUser, err = s.userRepo.GetByEmail(email) + if err != nil { + return nil, fmt.Errorf("failed to check email: %w", err) + } + if existingUser != nil { + return nil, errors.New("email already exists") + } + + // Hash password + passwordHash, err := utils.HashPassword(password) + if err != nil { + return nil, fmt.Errorf("failed to hash password: %w", err) + } + + // Create user + user := &models.User{ + Username: username, + Email: email, + PasswordHash: passwordHash, + Role: "user", + IsActive: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + if err := s.userRepo.Create(user); err != nil { + return nil, fmt.Errorf("failed to create user: %w", err) + } + + return user, nil +} + +// Login authenticates a user and returns tokens +func (s *authService) Login(username, password string) (*TokenPair, error) { + // Get user by username + user, err := s.userRepo.GetByUsername(username) + if err != nil { + return nil, fmt.Errorf("failed to get user: %w", err) + } + if user == nil { + return nil, errors.New("invalid username or password") + } + + // Check if user is active + if !user.IsActive { + return nil, errors.New("account is disabled") + } + + // Verify password + if !utils.VerifyPassword(password, user.PasswordHash) { + return nil, errors.New("invalid username or password") + } + + // Update last login + now := time.Now() + user.LastLogin = &now + if err := s.userRepo.Update(user); err != nil { + return nil, fmt.Errorf("failed to update last login: %w", err) + } + + // Generate tokens + tokenPair, err := s.generateTokens(user.ID) + if err != nil { + return nil, fmt.Errorf("failed to generate tokens: %w", err) + } + + return tokenPair, nil +} + +// RefreshToken refreshes the access token using a refresh token +func (s *authService) RefreshToken(refreshToken string) (*TokenPair, error) { + // Validate refresh token + claims, err := utils.ValidateToken(refreshToken, s.jwtConfig.Secret) + if err != nil { + return nil, errors.New("invalid refresh token") + } + + // Check token type + if claims.TokenType != "refresh" { + return nil, errors.New("invalid token type") + } + + // Get user + user, err := s.userRepo.GetByID(claims.UserID) + if err != nil { + return nil, fmt.Errorf("failed to get user: %w", err) + } + if user == nil { + return nil, errors.New("user not found") + } + + // Check if user is active + if !user.IsActive { + return nil, errors.New("account is disabled") + } + + // Generate new tokens + tokenPair, err := s.generateTokens(user.ID) + if err != nil { + return nil, fmt.Errorf("failed to generate tokens: %w", err) + } + + return tokenPair, nil +} + +// GetCurrentUser gets the current user by ID +func (s *authService) GetCurrentUser(userID int) (*models.User, error) { + user, err := s.userRepo.GetByID(userID) + if err != nil { + return nil, fmt.Errorf("failed to get user: %w", err) + } + if user == nil { + return nil, errors.New("user not found") + } + return user, nil +} + +// ChangePassword updates the current user's password +func (s *authService) ChangePassword(userID int, currentPassword, newPassword string) error { + user, err := s.userRepo.GetByID(userID) + if err != nil { + return fmt.Errorf("failed to get user: %w", err) + } + if user == nil { + return errors.New("user not found") + } + + if !utils.VerifyPassword(currentPassword, user.PasswordHash) { + return errors.New("current password is incorrect") + } + + passwordHash, err := utils.HashPassword(newPassword) + if err != nil { + return fmt.Errorf("failed to hash password: %w", err) + } + + user.PasswordHash = passwordHash + user.UpdatedAt = time.Now() + + if err := s.userRepo.Update(user); err != nil { + return fmt.Errorf("failed to update password: %w", err) + } + + return nil +} + +// generateTokens generates access and refresh tokens +func (s *authService) generateTokens(userID int) (*TokenPair, error) { + // Generate access token + accessToken, err := utils.GenerateToken(utils.TokenClaims{ + UserID: userID, + TokenType: "access", + }, s.jwtConfig.Secret, time.Duration(s.jwtConfig.AccessExpiry)*time.Minute) + if err != nil { + return nil, err + } + + // Generate refresh token + refreshToken, err := utils.GenerateToken(utils.TokenClaims{ + UserID: userID, + TokenType: "refresh", + }, s.jwtConfig.Secret, time.Duration(s.jwtConfig.RefreshExpiry)*time.Hour) + if err != nil { + return nil, err + } + + return &TokenPair{ + AccessToken: accessToken, + RefreshToken: refreshToken, + ExpiresIn: s.jwtConfig.AccessExpiry * 60, + }, nil +} diff --git a/backend/internal/services/chat_message_service.go b/backend/internal/services/chat_message_service.go new file mode 100644 index 0000000..6aa152b --- /dev/null +++ b/backend/internal/services/chat_message_service.go @@ -0,0 +1,110 @@ +package services + +import ( + "fmt" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// ChatMessageService defines message persistence operations. +type ChatMessageService interface { + RecordMessages(traceID, sessionID string, requestID *string, userID, instanceID, invocationID *int, messages []PersistedChatMessage) error + ListMessagesByTraceID(traceID string) ([]models.ChatMessageRecord, error) +} + +// PersistedChatMessage is the normalized write shape for chat messages. +type PersistedChatMessage struct { + Role string + Content string + SequenceNo int +} + +type chatMessageService struct { + repo repository.ChatMessageRepository +} + +// NewChatMessageService creates a new chat message service. +func NewChatMessageService(repo repository.ChatMessageRepository) ChatMessageService { + return &chatMessageService{repo: repo} +} + +func (s *chatMessageService) RecordMessages(traceID, sessionID string, requestID *string, userID, instanceID, invocationID *int, messages []PersistedChatMessage) error { + traceID = strings.TrimSpace(traceID) + sessionID = strings.TrimSpace(sessionID) + if traceID == "" || sessionID == "" || len(messages) == 0 { + return nil + } + + normalizedMessages := make([]PersistedChatMessage, 0, len(messages)) + for _, message := range messages { + role := strings.TrimSpace(message.Role) + content := strings.TrimSpace(message.Content) + if role == "" || content == "" { + continue + } + normalizedMessages = append(normalizedMessages, PersistedChatMessage{ + Role: role, + Content: content, + }) + } + if len(normalizedMessages) == 0 { + return nil + } + + existing, err := s.repo.ListByTraceID(traceID) + if err != nil { + return fmt.Errorf("failed to load existing chat messages: %w", err) + } + + commonPrefix := countCommonMessagePrefix(existing, normalizedMessages) + sequenceNo := len(existing) + for _, message := range normalizedMessages[commonPrefix:] { + sequenceNo++ + record := &models.ChatMessageRecord{ + TraceID: traceID, + SessionID: sessionID, + RequestID: requestID, + UserID: userID, + InstanceID: instanceID, + InvocationID: invocationID, + Role: message.Role, + Content: message.Content, + SequenceNo: sequenceNo, + } + if err := s.repo.Create(record); err != nil { + return fmt.Errorf("failed to record chat message: %w", err) + } + } + return nil +} + +func (s *chatMessageService) ListMessagesByTraceID(traceID string) ([]models.ChatMessageRecord, error) { + items, err := s.repo.ListByTraceID(strings.TrimSpace(traceID)) + if err != nil { + return nil, fmt.Errorf("failed to list chat messages by trace: %w", err) + } + return items, nil +} + +func countCommonMessagePrefix(existing []models.ChatMessageRecord, incoming []PersistedChatMessage) int { + limit := len(existing) + if len(incoming) < limit { + limit = len(incoming) + } + + matched := 0 + for matched < limit { + if !messagesEquivalent(existing[matched], incoming[matched]) { + break + } + matched++ + } + return matched +} + +func messagesEquivalent(existing models.ChatMessageRecord, incoming PersistedChatMessage) bool { + return strings.EqualFold(strings.TrimSpace(existing.Role), strings.TrimSpace(incoming.Role)) && + strings.TrimSpace(existing.Content) == strings.TrimSpace(incoming.Content) +} diff --git a/backend/internal/services/chat_message_service_test.go b/backend/internal/services/chat_message_service_test.go new file mode 100644 index 0000000..e533e74 --- /dev/null +++ b/backend/internal/services/chat_message_service_test.go @@ -0,0 +1,62 @@ +package services + +import ( + "testing" + + "clawreef/internal/models" +) + +type stubChatMessageRepository struct { + existing []models.ChatMessageRecord + created []models.ChatMessageRecord +} + +func (r *stubChatMessageRepository) Create(message *models.ChatMessageRecord) error { + copyRecord := *message + r.created = append(r.created, copyRecord) + r.existing = append(r.existing, copyRecord) + return nil +} + +func (r *stubChatMessageRepository) ListByTraceID(traceID string) ([]models.ChatMessageRecord, error) { + items := make([]models.ChatMessageRecord, len(r.existing)) + copy(items, r.existing) + return items, nil +} + +func TestRecordMessagesAppendsOnlyNewTranscriptSuffix(t *testing.T) { + repo := &stubChatMessageRepository{ + existing: []models.ChatMessageRecord{ + {TraceID: "trc_1", SessionID: "sess_1", Role: "user", Content: "北京今天天气怎么样", SequenceNo: 1}, + {TraceID: "trc_1", SessionID: "sess_1", Role: "assistant", Content: "tool_call get_weather({\"city\":\"Beijing\"})", SequenceNo: 2}, + }, + } + service := NewChatMessageService(repo) + + err := service.RecordMessages( + "trc_1", + "sess_1", + nil, + nil, + nil, + nil, + []PersistedChatMessage{ + {Role: "user", Content: "北京今天天气怎么样"}, + {Role: "assistant", Content: "tool_call get_weather({\"city\":\"Beijing\"})"}, + {Role: "tool", Content: "{\"temp_c\":25}"}, + }, + ) + if err != nil { + t.Fatalf("RecordMessages returned error: %v", err) + } + + if len(repo.created) != 1 { + t.Fatalf("expected exactly one new persisted message, got %d", len(repo.created)) + } + if repo.created[0].Role != "tool" { + t.Fatalf("expected appended role tool, got %q", repo.created[0].Role) + } + if repo.created[0].SequenceNo != 3 { + t.Fatalf("expected appended sequence number 3, got %d", repo.created[0].SequenceNo) + } +} diff --git a/backend/internal/services/chat_session_service.go b/backend/internal/services/chat_session_service.go new file mode 100644 index 0000000..7ef754b --- /dev/null +++ b/backend/internal/services/chat_session_service.go @@ -0,0 +1,62 @@ +package services + +import ( + "fmt" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// ChatSessionService defines session persistence operations. +type ChatSessionService interface { + GetSession(sessionID string) (*models.ChatSession, error) + EnsureSession(sessionID string, userID, instanceID *int, traceID *string, title *string) (*models.ChatSession, error) +} + +type chatSessionService struct { + repo repository.ChatSessionRepository +} + +// NewChatSessionService creates a new chat session service. +func NewChatSessionService(repo repository.ChatSessionRepository) ChatSessionService { + return &chatSessionService{repo: repo} +} + +func (s *chatSessionService) GetSession(sessionID string) (*models.ChatSession, error) { + sessionID = strings.TrimSpace(sessionID) + if sessionID == "" { + return nil, nil + } + + session, err := s.repo.GetBySessionID(sessionID) + if err != nil { + return nil, fmt.Errorf("failed to get chat session: %w", err) + } + return session, nil +} + +func (s *chatSessionService) EnsureSession(sessionID string, userID, instanceID *int, traceID *string, title *string) (*models.ChatSession, error) { + sessionID = strings.TrimSpace(sessionID) + if sessionID == "" { + return nil, fmt.Errorf("session id is required") + } + + session := &models.ChatSession{ + SessionID: sessionID, + UserID: userID, + InstanceID: instanceID, + LastTraceID: traceID, + } + if title != nil { + trimmed := strings.TrimSpace(*title) + if trimmed != "" { + session.Title = &trimmed + } + } + + if err := s.repo.Save(session); err != nil { + return nil, fmt.Errorf("failed to ensure chat session: %w", err) + } + return session, nil +} diff --git a/backend/internal/services/cluster_resource_service.go b/backend/internal/services/cluster_resource_service.go new file mode 100644 index 0000000..890fc5f --- /dev/null +++ b/backend/internal/services/cluster_resource_service.go @@ -0,0 +1,203 @@ +package services + +import ( + "context" + "fmt" + "sort" + "strings" + + "clawreef/internal/repository" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "clawreef/internal/services/k8s" +) + +type ClusterResourceService interface { + GetOverview(ctx context.Context) (*ClusterResourceOverview, error) +} + +type ClusterResourceOverview struct { + NodeCount int `json:"node_count"` + ReadyNodes int `json:"ready_nodes"` + CPU ResourceSummary `json:"cpu"` + Memory ResourceSummary `json:"memory"` + Disk ResourceSummary `json:"disk"` + Nodes []NodeResourceDetail `json:"nodes"` +} + +type ResourceSummary struct { + Capacity float64 `json:"capacity"` + Allocatable float64 `json:"allocatable"` + Requested float64 `json:"requested"` + Unit string `json:"unit"` +} + +type NodeResourceDetail struct { + Name string `json:"name"` + Ready bool `json:"ready"` + Roles []string `json:"roles"` + KubeletVersion string `json:"kubelet_version"` + InternalIP string `json:"internal_ip"` + PodCount int `json:"pod_count"` + CPU ResourceSummary `json:"cpu"` + Memory ResourceSummary `json:"memory"` + Disk ResourceSummary `json:"disk"` +} + +type clusterResourceService struct { + client *k8s.Client + instanceRepo repository.InstanceRepository +} + +func NewClusterResourceService(instanceRepo repository.InstanceRepository) ClusterResourceService { + return &clusterResourceService{ + client: k8s.GetClient(), + instanceRepo: instanceRepo, + } +} + +func (s *clusterResourceService) GetOverview(ctx context.Context) (*ClusterResourceOverview, error) { + if s.client == nil || s.client.Clientset == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + nodes, err := s.client.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list nodes: %w", err) + } + + pods, err := s.client.Clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list pods: %w", err) + } + + podByNode := make(map[string][]corev1.Pod) + for _, pod := range pods.Items { + if pod.Spec.NodeName == "" { + continue + } + if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + podByNode[pod.Spec.NodeName] = append(podByNode[pod.Spec.NodeName], pod) + } + + overview := &ClusterResourceOverview{ + NodeCount: len(nodes.Items), + CPU: ResourceSummary{Unit: "cores"}, + Memory: ResourceSummary{Unit: "GiB"}, + Disk: ResourceSummary{Unit: "GiB"}, + Nodes: make([]NodeResourceDetail, 0, len(nodes.Items)), + } + + for _, node := range nodes.Items { + ready := isNodeReady(node) + if ready { + overview.ReadyNodes++ + } + + detail := NodeResourceDetail{ + Name: node.Name, + Ready: ready, + Roles: nodeRoles(node), + KubeletVersion: node.Status.NodeInfo.KubeletVersion, + InternalIP: nodeInternalIP(node), + PodCount: len(podByNode[node.Name]), + CPU: ResourceSummary{Unit: "cores"}, + Memory: ResourceSummary{Unit: "GiB"}, + Disk: ResourceSummary{Unit: "GiB"}, + } + + detail.CPU.Capacity = cpuQuantityToCores(node.Status.Capacity[corev1.ResourceCPU]) + detail.CPU.Allocatable = cpuQuantityToCores(node.Status.Allocatable[corev1.ResourceCPU]) + detail.Memory.Capacity = bytesToGiB(node.Status.Capacity[corev1.ResourceMemory]) + detail.Memory.Allocatable = bytesToGiB(node.Status.Allocatable[corev1.ResourceMemory]) + detail.Disk.Capacity = bytesToGiB(node.Status.Capacity[corev1.ResourceEphemeralStorage]) + detail.Disk.Allocatable = bytesToGiB(node.Status.Allocatable[corev1.ResourceEphemeralStorage]) + + for _, pod := range podByNode[node.Name] { + for _, container := range pod.Spec.Containers { + detail.CPU.Requested += cpuQuantityToCores(container.Resources.Requests[corev1.ResourceCPU]) + detail.Memory.Requested += bytesToGiB(container.Resources.Requests[corev1.ResourceMemory]) + detail.Disk.Requested += bytesToGiB(container.Resources.Requests[corev1.ResourceEphemeralStorage]) + } + } + + overview.CPU.Capacity += detail.CPU.Capacity + overview.CPU.Allocatable += detail.CPU.Allocatable + overview.CPU.Requested += detail.CPU.Requested + overview.Memory.Capacity += detail.Memory.Capacity + overview.Memory.Allocatable += detail.Memory.Allocatable + overview.Memory.Requested += detail.Memory.Requested + overview.Disk.Capacity += detail.Disk.Capacity + overview.Disk.Allocatable += detail.Disk.Allocatable + overview.Disk.Requested += detail.Disk.Requested + + overview.Nodes = append(overview.Nodes, detail) + } + + if s.instanceRepo != nil { + instances, err := s.instanceRepo.GetAllRunning() + if err != nil { + return nil, fmt.Errorf("failed to list instances for storage summary: %w", err) + } + + totalAllocatedStorage := 0 + for _, instance := range instances { + totalAllocatedStorage += instance.DiskGB + } + overview.Disk.Requested = float64(totalAllocatedStorage) + } + + sort.Slice(overview.Nodes, func(i, j int) bool { + return overview.Nodes[i].Name < overview.Nodes[j].Name + }) + + return overview, nil +} + +func isNodeReady(node corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} + +func nodeInternalIP(node corev1.Node) string { + for _, address := range node.Status.Addresses { + if address.Type == corev1.NodeInternalIP { + return address.Address + } + } + return "" +} + +func nodeRoles(node corev1.Node) []string { + roles := make([]string, 0) + for key := range node.Labels { + if strings.HasPrefix(key, "node-role.kubernetes.io/") { + role := strings.TrimPrefix(key, "node-role.kubernetes.io/") + if role == "" { + role = "default" + } + roles = append(roles, role) + } + } + sort.Strings(roles) + if len(roles) == 0 { + return []string{"worker"} + } + return roles +} + +func cpuQuantityToCores(q resource.Quantity) float64 { + return float64(q.MilliValue()) / 1000 +} + +func bytesToGiB(q resource.Quantity) float64 { + return float64(q.Value()) / 1024 / 1024 / 1024 +} diff --git a/backend/internal/services/cost_record_service.go b/backend/internal/services/cost_record_service.go new file mode 100644 index 0000000..7fae299 --- /dev/null +++ b/backend/internal/services/cost_record_service.go @@ -0,0 +1,61 @@ +package services + +import ( + "fmt" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// CostRecordService defines application-level operations for token and money accounting. +type CostRecordService interface { + RecordCost(record *models.CostRecord) error + ListCostsByTraceID(traceID string) ([]models.CostRecord, error) + ListCostsByUserID(userID, limit int) ([]models.CostRecord, error) +} + +type costRecordService struct { + repo repository.CostRecordRepository +} + +// NewCostRecordService creates a new cost record service. +func NewCostRecordService(repo repository.CostRecordRepository) CostRecordService { + return &costRecordService{repo: repo} +} + +func (s *costRecordService) RecordCost(record *models.CostRecord) error { + if record == nil { + return fmt.Errorf("cost record is required") + } + if strings.TrimSpace(record.TraceID) == "" { + return fmt.Errorf("trace id is required") + } + if strings.TrimSpace(record.ProviderType) == "" { + return fmt.Errorf("provider type is required") + } + if strings.TrimSpace(record.ModelName) == "" { + return fmt.Errorf("model name is required") + } + if strings.TrimSpace(record.Currency) == "" { + record.Currency = "USD" + } + + return s.repo.Create(record) +} + +func (s *costRecordService) ListCostsByTraceID(traceID string) ([]models.CostRecord, error) { + items, err := s.repo.ListByTraceID(strings.TrimSpace(traceID)) + if err != nil { + return nil, fmt.Errorf("failed to list cost records by trace: %w", err) + } + return items, nil +} + +func (s *costRecordService) ListCostsByUserID(userID, limit int) ([]models.CostRecord, error) { + items, err := s.repo.ListByUserID(userID, limit) + if err != nil { + return nil, fmt.Errorf("failed to list cost records by user: %w", err) + } + return items, nil +} diff --git a/backend/internal/services/default_admin_repair.go b/backend/internal/services/default_admin_repair.go new file mode 100644 index 0000000..74168b8 --- /dev/null +++ b/backend/internal/services/default_admin_repair.go @@ -0,0 +1,29 @@ +package services + +import ( + "fmt" + "time" + + "clawreef/internal/repository" +) + +// RepairSeededAdminPassword upgrades shipped bad admin seed hashes to the +// expected admin123 hash without touching user-changed passwords. +func RepairSeededAdminPassword(userRepo repository.UserRepository) (bool, error) { + admin, err := userRepo.GetByUsername("admin") + if err != nil { + return false, fmt.Errorf("failed to load admin user: %w", err) + } + if admin == nil || !IsKnownBrokenAdminSeedHash(admin.PasswordHash) { + return false, nil + } + + admin.PasswordHash = DefaultAdminPasswordHash + admin.UpdatedAt = time.Now() + + if err := userRepo.Update(admin); err != nil { + return false, fmt.Errorf("failed to repair seeded admin password: %w", err) + } + + return true, nil +} diff --git a/backend/internal/services/default_passwords.go b/backend/internal/services/default_passwords.go new file mode 100644 index 0000000..1164c68 --- /dev/null +++ b/backend/internal/services/default_passwords.go @@ -0,0 +1,28 @@ +package services + +// Seeded/default passwords and hashes used by setup flows. +const ( + DefaultAdminPassword = "admin123" + DefaultUserPassword = "user123" + + // DefaultAdminPasswordHash matches DefaultAdminPassword. + DefaultAdminPasswordHash = "$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi" + + legacyBrokenAdminSeedHash = "$2a$10$N9qo8uLOickgx2ZMRZoMy.MqrzL9wGC3qD3Q.ZHqQH6t3q7l1L5uG" +) + +func DefaultPasswordForRole(role string) string { + if role == "admin" { + return DefaultAdminPassword + } + return DefaultUserPassword +} + +func IsKnownBrokenAdminSeedHash(hash string) bool { + switch hash { + case legacyBrokenAdminSeedHash: + return true + default: + return false + } +} diff --git a/backend/internal/services/default_passwords_test.go b/backend/internal/services/default_passwords_test.go new file mode 100644 index 0000000..9f538ad --- /dev/null +++ b/backend/internal/services/default_passwords_test.go @@ -0,0 +1,19 @@ +package services + +import ( + "testing" + + "clawreef/internal/utils" +) + +func TestDefaultAdminPasswordHashMatchesDocumentedPassword(t *testing.T) { + if !utils.VerifyPassword(DefaultAdminPassword, DefaultAdminPasswordHash) { + t.Fatalf("default admin password hash does not match %q", DefaultAdminPassword) + } +} + +func TestKnownBrokenAdminSeedHashesDoNotMatchDefaultPassword(t *testing.T) { + if utils.VerifyPassword(DefaultAdminPassword, legacyBrokenAdminSeedHash) { + t.Fatalf("broken admin seed hash unexpectedly matches %q", DefaultAdminPassword) + } +} diff --git a/backend/internal/services/desktop_stream_profile.go b/backend/internal/services/desktop_stream_profile.go new file mode 100644 index 0000000..9bb7e89 --- /dev/null +++ b/backend/internal/services/desktop_stream_profile.go @@ -0,0 +1,111 @@ +package services + +import "strings" + +const ( + DesktopStreamProfileLow = "low" + DesktopStreamProfileStandard = "standard" + DesktopStreamProfileHigh = "high" + + desktopStreamProfileEnvKey = "CLAWMANAGER_DESKTOP_STREAM_PROFILE" +) + +var selkiesDesktopStreamEnvKeys = []string{ + desktopStreamProfileEnvKey, + "SELKIES_ENCODER", + "SELKIES_USE_CSS_SCALING", + "SELKIES_FRAMERATE", + "SELKIES_H264_CRF", + "SELKIES_AUDIO_ENABLED", + "SELKIES_SECOND_SCREEN", +} + +func normalizeDesktopStreamProfile(profile string) (string, bool) { + switch strings.ToLower(strings.TrimSpace(profile)) { + case "": + return "", true + case DesktopStreamProfileLow: + return DesktopStreamProfileLow, true + case DesktopStreamProfileStandard: + return DesktopStreamProfileStandard, true + case DesktopStreamProfileHigh: + return DesktopStreamProfileHigh, true + default: + return "", false + } +} + +func applyDesktopStreamProfileEnv(overrides map[string]string, profile string) map[string]string { + normalized, ok := normalizeDesktopStreamProfile(profile) + if !ok || normalized == "" { + return overrides + } + + if overrides == nil { + overrides = map[string]string{} + } + + for _, key := range selkiesDesktopStreamEnvKeys { + delete(overrides, key) + } + + overrides[desktopStreamProfileEnvKey] = normalized + overrides["SELKIES_ENCODER"] = "x264enc,jpeg" + overrides["SELKIES_USE_CSS_SCALING"] = "true" + overrides["SELKIES_SECOND_SCREEN"] = "false" + overrides["SELKIES_AUDIO_ENABLED"] = "false" + + switch normalized { + case DesktopStreamProfileLow: + overrides["SELKIES_FRAMERATE"] = "30" + overrides["SELKIES_H264_CRF"] = "42" + case DesktopStreamProfileHigh: + overrides["SELKIES_FRAMERATE"] = "40" + overrides["SELKIES_H264_CRF"] = "24" + default: + overrides["SELKIES_FRAMERATE"] = "35" + overrides["SELKIES_H264_CRF"] = "34" + } + + return overrides +} + +func ensureDesktopStreamProfileEnv(overrides map[string]string, runtimeType string) map[string]string { + if normalizeInstanceRuntimeType(runtimeType) != RuntimeBackendDesktop { + return overrides + } + + profile := desktopStreamProfileFromEnv(overrides) + if profile != "" { + return applyDesktopStreamProfileEnv(overrides, profile) + } + + for _, key := range selkiesDesktopStreamEnvKeys { + if _, ok := overrides[key]; ok { + return overrides + } + } + + return applyDesktopStreamProfileEnv(overrides, DesktopStreamProfileStandard) +} + +func desktopStreamProfileFromEnv(overrides map[string]string) string { + if profile, ok := overrides[desktopStreamProfileEnvKey]; ok { + if normalized, valid := normalizeDesktopStreamProfile(profile); valid && normalized != "" { + return normalized + } + } + + framerate := strings.TrimSpace(overrides["SELKIES_FRAMERATE"]) + crf := strings.TrimSpace(overrides["SELKIES_H264_CRF"]) + switch { + case framerate == "30" && crf == "42": + return DesktopStreamProfileLow + case framerate == "40" && crf == "24": + return DesktopStreamProfileHigh + case framerate == "35" && crf == "34": + return DesktopStreamProfileStandard + default: + return "" + } +} diff --git a/backend/internal/services/desktop_stream_profile_test.go b/backend/internal/services/desktop_stream_profile_test.go new file mode 100644 index 0000000..da29d83 --- /dev/null +++ b/backend/internal/services/desktop_stream_profile_test.go @@ -0,0 +1,76 @@ +package services + +import "testing" + +func TestApplyDesktopStreamProfileEnvStandard(t *testing.T) { + overrides := applyDesktopStreamProfileEnv(map[string]string{ + "SELKIES_ENCODER": "x264enc", + "SELKIES_USE_CSS_SCALING": "false", + "SELKIES_FRAMERATE": "10", + "SELKIES_H264_CRF": "20", + }, DesktopStreamProfileStandard) + + want := map[string]string{ + "CLAWMANAGER_DESKTOP_STREAM_PROFILE": "standard", + "SELKIES_ENCODER": "x264enc,jpeg", + "SELKIES_USE_CSS_SCALING": "true", + "SELKIES_FRAMERATE": "35", + "SELKIES_H264_CRF": "34", + "SELKIES_SECOND_SCREEN": "false", + "SELKIES_AUDIO_ENABLED": "false", + } + + for key, value := range want { + if got := overrides[key]; got != value { + t.Fatalf("%s = %q, want %q", key, got, value) + } + } +} + +func TestDesktopStreamProfileFromEnvIgnoresEncoderDetails(t *testing.T) { + overrides := map[string]string{ + "SELKIES_ENCODER": "x264enc,jpeg", + "SELKIES_USE_CSS_SCALING": "true", + "SELKIES_FRAMERATE": "40", + "SELKIES_H264_CRF": "24", + } + + if got := desktopStreamProfileFromEnv(overrides); got != DesktopStreamProfileHigh { + t.Fatalf("desktopStreamProfileFromEnv() = %q, want %q", got, DesktopStreamProfileHigh) + } +} + +func TestEnsureDesktopStreamProfileEnvUpgradesSavedProfile(t *testing.T) { + overrides := ensureDesktopStreamProfileEnv(map[string]string{ + "CLAWMANAGER_DESKTOP_STREAM_PROFILE": "standard", + "SELKIES_ENCODER": "x264enc", + "SELKIES_FRAMERATE": "35", + "SELKIES_H264_CRF": "34", + }, RuntimeBackendDesktop) + + if got := overrides["SELKIES_ENCODER"]; got != "x264enc,jpeg" { + t.Fatalf("SELKIES_ENCODER = %q, want x264enc,jpeg", got) + } + if got := overrides["SELKIES_USE_CSS_SCALING"]; got != "true" { + t.Fatalf("SELKIES_USE_CSS_SCALING = %q, want true", got) + } +} + +func TestEnsureDesktopStreamProfileEnvDefaultsDesktopOnlyWhenUnset(t *testing.T) { + desktop := ensureDesktopStreamProfileEnv(nil, RuntimeBackendDesktop) + if got := desktop["CLAWMANAGER_DESKTOP_STREAM_PROFILE"]; got != DesktopStreamProfileStandard { + t.Fatalf("desktop profile = %q, want %q", got, DesktopStreamProfileStandard) + } + + custom := ensureDesktopStreamProfileEnv(map[string]string{ + "SELKIES_ENCODER": "custom", + }, RuntimeBackendDesktop) + if got := custom["SELKIES_ENCODER"]; got != "custom" { + t.Fatalf("custom encoder = %q, want custom", got) + } + + shell := ensureDesktopStreamProfileEnv(nil, RuntimeBackendShell) + if shell != nil { + t.Fatalf("shell runtime should not receive desktop stream env") + } +} diff --git a/backend/internal/services/instance_access_service.go b/backend/internal/services/instance_access_service.go new file mode 100644 index 0000000..230dd11 --- /dev/null +++ b/backend/internal/services/instance_access_service.go @@ -0,0 +1,281 @@ +package services + +import ( + "errors" + "fmt" + "os" + "sync" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +// AccessToken represents a temporary access token for instance +type AccessToken struct { + Token string `json:"token"` + InstanceID int `json:"instance_id"` + UserID int `json:"user_id"` + InstanceType string `json:"instance_type"` + TargetPort int32 `json:"target_port"` + AccessURL string `json:"access_url"` + Upstream string `json:"upstream"` + ExpiresAt time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` +} + +// InstanceAccessService manages instance access tokens +type InstanceAccessService struct { + tokens map[string]*AccessToken + mu sync.RWMutex + secret string + stopChan chan struct{} +} + +type instanceAccessClaims struct { + InstanceID int `json:"instance_id"` + UserID int `json:"user_id"` + InstanceType string `json:"instance_type"` + TargetPort int32 `json:"target_port"` + AccessURL string `json:"access_url"` + Upstream string `json:"upstream"` + TokenType string `json:"token_type"` + jwt.RegisteredClaims +} + +// NewInstanceAccessService creates a new instance access service +func NewInstanceAccessService() *InstanceAccessService { + service := &InstanceAccessService{ + tokens: make(map[string]*AccessToken), + secret: getInstanceAccessTokenSecret(), + stopChan: make(chan struct{}), + } + + // Start cleanup goroutine + go service.cleanupExpiredTokens() + + return service +} + +// GenerateToken generates a new access token for an instance +func (s *InstanceAccessService) GenerateToken(userID, instanceID int, instanceType string, accessURL string, upstream string, targetPort int32, duration time.Duration) (*AccessToken, error) { + now := time.Now() + expiresAt := now.Add(duration) + + claims := instanceAccessClaims{ + InstanceID: instanceID, + UserID: userID, + InstanceType: instanceType, + TargetPort: targetPort, + AccessURL: accessURL, + Upstream: upstream, + TokenType: "instance_access", + RegisteredClaims: jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(expiresAt), + }, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + tokenString, err := token.SignedString([]byte(s.secret)) + if err != nil { + return nil, fmt.Errorf("failed to generate token: %w", err) + } + + accessToken := &AccessToken{ + Token: tokenString, + InstanceID: instanceID, + UserID: userID, + InstanceType: instanceType, + TargetPort: targetPort, + AccessURL: accessURL, + Upstream: upstream, + ExpiresAt: expiresAt, + CreatedAt: now, + } + + return accessToken, nil +} + +// ValidateToken validates an access token +func (s *InstanceAccessService) ValidateToken(token string) (*AccessToken, error) { + accessToken, err := s.validateSignedToken(token) + if err == nil { + return accessToken, nil + } + + legacyToken, legacyErr := s.validateLegacyToken(token) + if legacyErr == nil { + return legacyToken, nil + } + + return nil, err +} + +func (s *InstanceAccessService) validateSignedToken(token string) (*AccessToken, error) { + parsed, err := jwt.ParseWithClaims(token, &instanceAccessClaims{}, func(token *jwt.Token) (interface{}, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, errors.New("unexpected signing method") + } + return []byte(s.secret), nil + }) + if err != nil { + if errors.Is(err, jwt.ErrTokenExpired) { + return nil, fmt.Errorf("token expired") + } + return nil, fmt.Errorf("invalid token") + } + + claims, ok := parsed.Claims.(*instanceAccessClaims) + if !ok || !parsed.Valid { + return nil, fmt.Errorf("invalid token") + } + + if claims.TokenType != "instance_access" || claims.AccessURL == "" { + return nil, fmt.Errorf("invalid token") + } + + expiresAt := time.Time{} + if claims.ExpiresAt != nil { + expiresAt = claims.ExpiresAt.Time + } + createdAt := time.Time{} + if claims.IssuedAt != nil { + createdAt = claims.IssuedAt.Time + } + + return &AccessToken{ + Token: token, + InstanceID: claims.InstanceID, + UserID: claims.UserID, + InstanceType: claims.InstanceType, + TargetPort: claims.TargetPort, + AccessURL: claims.AccessURL, + Upstream: claims.Upstream, + ExpiresAt: expiresAt, + CreatedAt: createdAt, + }, nil +} + +func (s *InstanceAccessService) validateLegacyToken(token string) (*AccessToken, error) { + s.mu.RLock() + accessToken, exists := s.tokens[token] + s.mu.RUnlock() + + if !exists { + return nil, fmt.Errorf("invalid token") + } + + if time.Now().After(accessToken.ExpiresAt) { + s.mu.Lock() + delete(s.tokens, token) + s.mu.Unlock() + return nil, fmt.Errorf("token expired") + } + + return accessToken, nil +} + +// RevokeToken revokes an access token +func (s *InstanceAccessService) RevokeToken(token string) { + s.mu.Lock() + delete(s.tokens, token) + s.mu.Unlock() +} + +// GetAccessURL generates access URL for an instance +func (s *InstanceAccessService) GetAccessURL(instanceID int, instanceType string, podIP string, podName string) string { + // Generate access URL based on instance type + switch instanceType { + case "openclaw": + // OpenClaw desktop typically uses VNC or web interface + if podIP != "" { + return fmt.Sprintf("https://%s:3001/", podIP) + } + case "ubuntu", "debian", "centos": + // Linux desktops typically use noVNC or similar + if podIP != "" { + return fmt.Sprintf("http://%s:6901/vnc.html", podIP) + } + default: + // Default VNC access + if podIP != "" { + return fmt.Sprintf("http://%s:6080/vnc.html", podIP) + } + } + + // Fallback to pod name based URL (for ingress/routing scenarios) + if podName != "" { + return fmt.Sprintf("/access/instance/%d", instanceID) + } + + return "" +} + +// GetAccessURLWithEndpoint generates access URL using the provided endpoint (nodeIP:port or direct IP) +func (s *InstanceAccessService) GetAccessURLWithEndpoint(instanceID int, instanceType string, endpoint string) string { + if endpoint == "" { + return "" + } + + // Generate access URL based on instance type + switch instanceType { + case "openclaw": + // OpenClaw desktop typically uses VNC or web interface + return fmt.Sprintf("https://%s/", endpoint) + case "ubuntu", "debian", "centos": + // Linux desktops typically use noVNC or similar + return fmt.Sprintf("http://%s/vnc.html", endpoint) + default: + // Default VNC access + return fmt.Sprintf("http://%s/vnc.html", endpoint) + } +} + +// GetProxyURL generates a proxied access URL +func (s *InstanceAccessService) GetProxyURL(instanceID int, token string) string { + return fmt.Sprintf("/api/v1/instances/%d/access?token=%s", instanceID, token) +} + +// cleanupExpiredTokens periodically removes expired tokens +func (s *InstanceAccessService) cleanupExpiredTokens() { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-s.stopChan: + return + case <-ticker.C: + now := time.Now() + s.mu.Lock() + for token, accessToken := range s.tokens { + if now.After(accessToken.ExpiresAt) { + delete(s.tokens, token) + } + } + s.mu.Unlock() + } + } +} + +// Stop terminates the background cleanup goroutine. +func (s *InstanceAccessService) Stop() { + close(s.stopChan) +} + +// GetActiveTokenCount returns the number of active tokens +func (s *InstanceAccessService) GetActiveTokenCount() int { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.tokens) +} + +func getInstanceAccessTokenSecret() string { + if secret := os.Getenv("INSTANCE_ACCESS_TOKEN_SECRET"); secret != "" { + return secret + } + if secret := os.Getenv("JWT_SECRET"); secret != "" { + return secret + } + return "clawreef-instance-access-secret-change-in-production" +} diff --git a/backend/internal/services/instance_access_service_test.go b/backend/internal/services/instance_access_service_test.go new file mode 100644 index 0000000..9b35794 --- /dev/null +++ b/backend/internal/services/instance_access_service_test.go @@ -0,0 +1,95 @@ +package services + +import ( + "testing" + "time" +) + +func TestInstanceAccessServiceValidatesTokenAcrossServiceInstances(t *testing.T) { + t.Setenv("INSTANCE_ACCESS_TOKEN_SECRET", "cluster-shared-secret") + + issuer := NewInstanceAccessService() + validator := NewInstanceAccessService() + + wantUpstream := "clawreef-42-demo-svc.clawmanager-user-7.svc.cluster.local:3001" + token, err := issuer.GenerateToken(7, 42, "openclaw", "/api/v1/instances/42/proxy/", wantUpstream, 3001, 5*time.Minute) + if err != nil { + t.Fatalf("GenerateToken() error = %v", err) + } + + validated, err := validator.ValidateToken(token.Token) + if err != nil { + t.Fatalf("ValidateToken() error = %v", err) + } + + if validated.InstanceID != 42 { + t.Fatalf("validated.InstanceID = %d, want 42", validated.InstanceID) + } + if validated.UserID != 7 { + t.Fatalf("validated.UserID = %d, want 7", validated.UserID) + } + if validated.InstanceType != "openclaw" { + t.Fatalf("validated.InstanceType = %q, want openclaw", validated.InstanceType) + } + if validated.Upstream != wantUpstream { + t.Fatalf("validated.Upstream = %q, want %q", validated.Upstream, wantUpstream) + } +} + +func TestInstanceAccessServiceRejectsExpiredSignedToken(t *testing.T) { + t.Setenv("INSTANCE_ACCESS_TOKEN_SECRET", "cluster-shared-secret") + + service := NewInstanceAccessService() + token, err := service.GenerateToken(7, 42, "openclaw", "/api/v1/instances/42/proxy/", "", 3001, -time.Second) + if err != nil { + t.Fatalf("GenerateToken() error = %v", err) + } + + if _, err := service.ValidateToken(token.Token); err == nil || err.Error() != "token expired" { + t.Fatalf("ValidateToken() error = %v, want token expired", err) + } +} + +func TestInstanceAccessServiceFallsBackToLegacyTokens(t *testing.T) { + t.Setenv("INSTANCE_ACCESS_TOKEN_SECRET", "cluster-shared-secret") + + service := NewInstanceAccessService() + service.tokens["legacy-token"] = &AccessToken{ + Token: "legacy-token", + InstanceID: 11, + UserID: 3, + InstanceType: "ubuntu", + TargetPort: 3001, + AccessURL: "/api/v1/instances/11/proxy/", + ExpiresAt: time.Now().Add(time.Minute), + CreatedAt: time.Now(), + } + + validated, err := service.ValidateToken("legacy-token") + if err != nil { + t.Fatalf("ValidateToken() error = %v", err) + } + + if validated.InstanceID != 11 { + t.Fatalf("validated.InstanceID = %d, want 11", validated.InstanceID) + } +} + +func TestInstanceAccessServiceStopTerminatesCleanup(t *testing.T) { + t.Setenv("INSTANCE_ACCESS_TOKEN_SECRET", "cluster-shared-secret") + + service := NewInstanceAccessService() + + // Stop should not panic and should be idempotent-safe (only called once). + service.Stop() + + // After Stop, the service should still be usable for token operations + // (only the background cleanup goroutine is stopped). + token, err := service.GenerateToken(1, 1, "openclaw", "/proxy", "", 3001, time.Minute) + if err != nil { + t.Fatalf("GenerateToken after Stop() error = %v", err) + } + if _, err := service.ValidateToken(token.Token); err != nil { + t.Fatalf("ValidateToken after Stop() error = %v", err) + } +} diff --git a/backend/internal/services/instance_agent_service.go b/backend/internal/services/instance_agent_service.go new file mode 100644 index 0000000..887bc5d --- /dev/null +++ b/backend/internal/services/instance_agent_service.go @@ -0,0 +1,421 @@ +package services + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +const ( + AgentProtocolVersionV1 = "v1" + + agentStatusOnline = "online" + agentStatusOffline = "offline" + agentStatusStale = "stale" + + openClawStatusUnknown = "unknown" +) + +type AgentRegisterRequest struct { + InstanceID int `json:"instance_id" binding:"required,min=1"` + AgentID string `json:"agent_id" binding:"required"` + AgentVersion string `json:"agent_version" binding:"required"` + ProtocolVersion string `json:"protocol_version" binding:"required"` + Capabilities []string `json:"capabilities"` + HostInfo map[string]interface{} `json:"host_info"` +} + +type AgentRegisterResponse struct { + SessionToken string `json:"session_token"` + SessionExpiresAt time.Time `json:"session_expires_at"` + HeartbeatIntervalSeconds int `json:"heartbeat_interval_seconds"` + CommandPollIntervalSeconds int `json:"command_poll_interval_seconds"` + ServerTime time.Time `json:"server_time"` +} + +type AgentHeartbeatRequest struct { + AgentID string `json:"agent_id" binding:"required"` + Timestamp time.Time `json:"timestamp"` + OpenClawStatus string `json:"openclaw_status"` + CurrentConfigRevisionID *int `json:"current_config_revision_id,omitempty"` + Summary map[string]interface{} `json:"summary"` +} + +type AgentHeartbeatResponse struct { + ServerTime time.Time `json:"server_time"` + HasPendingCommand bool `json:"has_pending_command"` + DesiredPowerState string `json:"desired_power_state"` + DesiredConfigRevisionID *int `json:"desired_config_revision_id,omitempty"` +} + +type InstanceAgentPayload struct { + AgentID string `json:"agent_id"` + AgentVersion string `json:"agent_version"` + ProtocolVersion string `json:"protocol_version"` + Status string `json:"status"` + Capabilities []string `json:"capabilities"` + HostInfo map[string]interface{} `json:"host_info,omitempty"` + LastHeartbeatAt *time.Time `json:"last_heartbeat_at,omitempty"` + LastReportedAt *time.Time `json:"last_reported_at,omitempty"` + LastSeenIP *string `json:"last_seen_ip,omitempty"` + RegisteredAt *time.Time `json:"registered_at,omitempty"` +} + +type AgentSession struct { + Instance *models.Instance + Agent *models.InstanceAgent +} + +type InstanceAgentService interface { + Register(bootstrapToken string, req AgentRegisterRequest, clientIP string) (*AgentRegisterResponse, error) + AuthenticateSession(sessionToken string) (*AgentSession, error) + Heartbeat(session *AgentSession, req AgentHeartbeatRequest, clientIP string) (*AgentHeartbeatResponse, error) + GetPayloadByInstanceID(instanceID int) (*InstanceAgentPayload, error) +} + +type instanceAgentService struct { + instanceRepo repository.InstanceRepository + agentRepo repository.InstanceAgentRepository + desiredStateRepo repository.InstanceDesiredStateRepository + runtimeRepo repository.InstanceRuntimeStatusRepository + commandRepo repository.InstanceCommandRepository +} + +func NewInstanceAgentService(instanceRepo repository.InstanceRepository, agentRepo repository.InstanceAgentRepository, desiredStateRepo repository.InstanceDesiredStateRepository, runtimeRepo repository.InstanceRuntimeStatusRepository, commandRepo repository.InstanceCommandRepository) InstanceAgentService { + return &instanceAgentService{ + instanceRepo: instanceRepo, + agentRepo: agentRepo, + desiredStateRepo: desiredStateRepo, + runtimeRepo: runtimeRepo, + commandRepo: commandRepo, + } +} + +func (s *instanceAgentService) Register(bootstrapToken string, req AgentRegisterRequest, clientIP string) (*AgentRegisterResponse, error) { + bootstrapToken = strings.TrimSpace(bootstrapToken) + if bootstrapToken == "" { + return nil, fmt.Errorf("agent bootstrap token is required") + } + if strings.TrimSpace(req.AgentID) == "" { + return nil, fmt.Errorf("agent id is required") + } + if strings.TrimSpace(req.ProtocolVersion) != AgentProtocolVersionV1 { + return nil, fmt.Errorf("unsupported agent protocol version") + } + + instance, err := s.instanceRepo.GetByAgentBootstrapToken(bootstrapToken) + if err != nil { + return nil, err + } + if instance == nil || instance.ID != req.InstanceID { + return nil, fmt.Errorf("invalid agent bootstrap token") + } + if !supportsManagedRuntimeIntegration(instance.Type) { + return nil, fmt.Errorf("agent registration is only supported for openclaw or hermes instances") + } + + now := time.Now().UTC() + sessionToken, err := generatePrefixedToken("agt_sess") + if err != nil { + return nil, fmt.Errorf("failed to generate agent session token: %w", err) + } + sessionExpiresAt := now.Add(24 * time.Hour) + + capabilitiesJSON, err := marshalJSON(req.Capabilities) + if err != nil { + return nil, fmt.Errorf("failed to encode agent capabilities: %w", err) + } + hostInfoJSON, err := marshalOptionalJSON(req.HostInfo) + if err != nil { + return nil, fmt.Errorf("failed to encode agent host info: %w", err) + } + + agent, err := s.agentRepo.GetByInstanceID(instance.ID) + if err != nil { + return nil, err + } + if agent == nil { + agent = &models.InstanceAgent{ + InstanceID: instance.ID, + AgentID: strings.TrimSpace(req.AgentID), + AgentVersion: strings.TrimSpace(req.AgentVersion), + ProtocolVersion: strings.TrimSpace(req.ProtocolVersion), + Status: agentStatusOnline, + CapabilitiesJSON: capabilitiesJSON, + HostInfoJSON: hostInfoJSON, + SessionToken: &sessionToken, + SessionExpiresAt: &sessionExpiresAt, + LastHeartbeatAt: &now, + LastReportedAt: nil, + LastSeenIP: optionalString(strings.TrimSpace(clientIP)), + RegisteredAt: &now, + } + if err := s.agentRepo.Create(agent); err != nil { + return nil, err + } + } else { + agent.AgentID = strings.TrimSpace(req.AgentID) + agent.AgentVersion = strings.TrimSpace(req.AgentVersion) + agent.ProtocolVersion = strings.TrimSpace(req.ProtocolVersion) + agent.Status = agentStatusOnline + agent.CapabilitiesJSON = capabilitiesJSON + agent.HostInfoJSON = hostInfoJSON + agent.SessionToken = &sessionToken + agent.SessionExpiresAt = &sessionExpiresAt + agent.LastHeartbeatAt = &now + agent.LastSeenIP = optionalString(strings.TrimSpace(clientIP)) + agent.RegisteredAt = &now + if err := s.agentRepo.Update(agent); err != nil { + return nil, err + } + } + + if _, err := s.ensureDesiredState(instance); err != nil { + return nil, err + } + if _, err := s.ensureRuntimeStatus(instance.ID); err != nil { + return nil, err + } + + return &AgentRegisterResponse{ + SessionToken: sessionToken, + SessionExpiresAt: sessionExpiresAt, + HeartbeatIntervalSeconds: 15, + CommandPollIntervalSeconds: 5, + ServerTime: now, + }, nil +} + +func (s *instanceAgentService) AuthenticateSession(sessionToken string) (*AgentSession, error) { + sessionToken = strings.TrimSpace(sessionToken) + if sessionToken == "" { + return nil, fmt.Errorf("agent session token is required") + } + + agent, err := s.agentRepo.GetBySessionToken(sessionToken) + if err != nil { + return nil, err + } + if agent == nil || agent.SessionExpiresAt == nil || agent.SessionExpiresAt.Before(time.Now().UTC()) { + return nil, fmt.Errorf("invalid or expired agent session token") + } + + instance, err := s.instanceRepo.GetByID(agent.InstanceID) + if err != nil { + return nil, err + } + if instance == nil { + return nil, fmt.Errorf("instance not found") + } + + return &AgentSession{Instance: instance, Agent: agent}, nil +} + +func (s *instanceAgentService) Heartbeat(session *AgentSession, req AgentHeartbeatRequest, clientIP string) (*AgentHeartbeatResponse, error) { + if session == nil || session.Agent == nil || session.Instance == nil { + return nil, fmt.Errorf("agent session is required") + } + if strings.TrimSpace(req.AgentID) == "" || strings.TrimSpace(req.AgentID) != session.Agent.AgentID { + return nil, fmt.Errorf("agent id does not match session") + } + + now := time.Now().UTC() + session.Agent.Status = agentStatusOnline + session.Agent.LastHeartbeatAt = &now + session.Agent.LastSeenIP = optionalString(strings.TrimSpace(clientIP)) + + // Renew session expiry on each successful heartbeat (rolling window). + // Without this, the 24-hour session token issued at Register() silently + // expires and the agent is permanently locked out until pod restart. + newExpiry := now.Add(24 * time.Hour) + session.Agent.SessionExpiresAt = &newExpiry + + if err := s.agentRepo.Update(session.Agent); err != nil { + return nil, err + } + + runtimeStatus, err := s.ensureRuntimeStatus(session.Instance.ID) + if err != nil { + return nil, err + } + runtimeStatus.AgentStatus = agentStatusOnline + if strings.TrimSpace(req.OpenClawStatus) != "" { + runtimeStatus.OpenClawStatus = strings.TrimSpace(req.OpenClawStatus) + } + runtimeStatus.CurrentConfigRevisionID = req.CurrentConfigRevisionID + if req.Summary != nil { + summaryJSON, err := marshalOptionalJSON(req.Summary) + if err != nil { + return nil, fmt.Errorf("failed to encode heartbeat summary: %w", err) + } + runtimeStatus.SummaryJSON = summaryJSON + if openClawPID, ok := req.Summary["openclaw_pid"].(float64); ok { + pid := int(openClawPID) + runtimeStatus.OpenClawPID = &pid + } + } + runtimeStatus.LastReportedAt = &now + if err := s.runtimeRepo.Update(runtimeStatus); err != nil { + return nil, err + } + + desiredState, err := s.ensureDesiredState(session.Instance) + if err != nil { + return nil, err + } + nextCommand, err := s.commandRepo.GetNextPendingByInstance(session.Instance.ID) + if err != nil { + return nil, err + } + + return &AgentHeartbeatResponse{ + ServerTime: now, + HasPendingCommand: nextCommand != nil, + DesiredPowerState: desiredState.DesiredPowerState, + DesiredConfigRevisionID: desiredState.DesiredConfigRevisionID, + }, nil +} + +func (s *instanceAgentService) GetPayloadByInstanceID(instanceID int) (*InstanceAgentPayload, error) { + agent, err := s.agentRepo.GetByInstanceID(instanceID) + if err != nil { + return nil, err + } + if agent == nil { + return nil, nil + } + + payload := &InstanceAgentPayload{ + AgentID: agent.AgentID, + AgentVersion: agent.AgentVersion, + ProtocolVersion: agent.ProtocolVersion, + Status: deriveAgentStatus(agent), + LastHeartbeatAt: agent.LastHeartbeatAt, + LastReportedAt: agent.LastReportedAt, + LastSeenIP: agent.LastSeenIP, + RegisteredAt: agent.RegisteredAt, + } + if err := unmarshalJSON(agent.CapabilitiesJSON, &payload.Capabilities); err != nil { + return nil, fmt.Errorf("failed to decode agent capabilities: %w", err) + } + if agent.HostInfoJSON != nil && strings.TrimSpace(*agent.HostInfoJSON) != "" { + if err := unmarshalJSON(*agent.HostInfoJSON, &payload.HostInfo); err != nil { + return nil, fmt.Errorf("failed to decode agent host info: %w", err) + } + } + return payload, nil +} + +func (s *instanceAgentService) ensureDesiredState(instance *models.Instance) (*models.InstanceDesiredState, error) { + state, err := s.desiredStateRepo.GetByInstanceID(instance.ID) + if err != nil { + return nil, err + } + if state != nil { + return state, nil + } + + desiredPowerState := "stopped" + if instance.Status == "running" || instance.Status == "creating" { + desiredPowerState = "running" + } + now := time.Now().UTC() + state = &models.InstanceDesiredState{ + InstanceID: instance.ID, + DesiredPowerState: desiredPowerState, + UpdatedAt: now, + CreatedAt: now, + } + if err := s.desiredStateRepo.Create(state); err != nil { + return nil, err + } + return state, nil +} + +func (s *instanceAgentService) ensureRuntimeStatus(instanceID int) (*models.InstanceRuntimeStatus, error) { + status, err := s.runtimeRepo.GetByInstanceID(instanceID) + if err != nil { + return nil, err + } + if status != nil { + return status, nil + } + + now := time.Now().UTC() + status = &models.InstanceRuntimeStatus{ + InstanceID: instanceID, + InfraStatus: "creating", + AgentStatus: agentStatusOffline, + OpenClawStatus: openClawStatusUnknown, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.runtimeRepo.Create(status); err != nil { + return nil, err + } + return status, nil +} + +func deriveAgentStatus(agent *models.InstanceAgent) string { + if agent == nil || agent.LastHeartbeatAt == nil { + return agentStatusOffline + } + age := time.Since(*agent.LastHeartbeatAt) + switch { + case age <= 45*time.Second: + return agentStatusOnline + case age <= 120*time.Second: + return agentStatusStale + default: + return agentStatusOffline + } +} + +func generatePrefixedToken(prefix string) (string, error) { + bytes := make([]byte, 24) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return prefix + "_" + hex.EncodeToString(bytes), nil +} + +func marshalJSON(value interface{}) (string, error) { + raw, err := json.Marshal(value) + if err != nil { + return "", err + } + return string(raw), nil +} + +func marshalOptionalJSON(value interface{}) (*string, error) { + if value == nil { + return nil, nil + } + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + encoded := string(raw) + return &encoded, nil +} + +func unmarshalJSON(raw string, target interface{}) error { + if strings.TrimSpace(raw) == "" { + return nil + } + return json.Unmarshal([]byte(raw), target) +} + +func optionalString(value string) *string { + if strings.TrimSpace(value) == "" { + return nil + } + trimmed := strings.TrimSpace(value) + return &trimmed +} diff --git a/backend/internal/services/instance_command_service.go b/backend/internal/services/instance_command_service.go new file mode 100644 index 0000000..0192b93 --- /dev/null +++ b/backend/internal/services/instance_command_service.go @@ -0,0 +1,410 @@ +package services + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +const ( + InstanceCommandTypeStartOpenClaw = "start_openclaw" + InstanceCommandTypeStopOpenClaw = "stop_openclaw" + InstanceCommandTypeRestartOpenClaw = "restart_openclaw" + InstanceCommandTypeCollectSystemInfo = "collect_system_info" + InstanceCommandTypeApplyConfigRevision = "apply_config_revision" + InstanceCommandTypeReloadConfig = "reload_config" + InstanceCommandTypeHealthCheck = "health_check" + InstanceCommandTypeInstallSkill = "install_skill" + InstanceCommandTypeUpdateSkill = "update_skill" + InstanceCommandTypeUninstallSkill = "uninstall_skill" + InstanceCommandTypeRemoveSkill = "remove_skill" + InstanceCommandTypeDisableSkill = "disable_skill" + InstanceCommandTypeQuarantineSkill = "quarantine_skill" + InstanceCommandTypeHandleSkillRisk = "handle_skill_risk" + InstanceCommandTypeSyncSkillInventory = "sync_skill_inventory" + InstanceCommandTypeRefreshSkillInventory = "refresh_skill_inventory" + InstanceCommandTypeCollectSkillPackage = "collect_skill_package" + instanceCommandStatusPending = "pending" + instanceCommandStatusDispatched = "dispatched" + instanceCommandStatusRunning = "running" + instanceCommandStatusSucceeded = "succeeded" + instanceCommandStatusFailed = "failed" +) + +type CreateInstanceCommandRequest struct { + CommandType string `json:"command_type"` + Payload map[string]interface{} `json:"payload,omitempty"` + IdempotencyKey string `json:"idempotency_key"` + TimeoutSeconds int `json:"timeout_seconds"` +} + +type InstanceCommandPayload struct { + ID int `json:"id"` + CommandType string `json:"command_type"` + Payload map[string]interface{} `json:"payload,omitempty"` + Status string `json:"status"` + IdempotencyKey string `json:"idempotency_key"` + IssuedBy *int `json:"issued_by,omitempty"` + IssuedAt time.Time `json:"issued_at"` + DispatchedAt *time.Time `json:"dispatched_at,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + TimeoutSeconds int `json:"timeout_seconds"` + Result map[string]interface{} `json:"result,omitempty"` + ErrorMessage *string `json:"error_message,omitempty"` +} + +type AgentCommandEnvelope struct { + ID int `json:"id"` + CommandType string `json:"command_type"` + Payload map[string]interface{} `json:"payload,omitempty"` + IssuedAt time.Time `json:"issued_at"` + TimeoutSeconds int `json:"timeout_seconds"` + IdempotencyKey string `json:"idempotency_key"` +} + +type AgentCommandFinishRequest struct { + AgentID string `json:"agent_id" binding:"required"` + Status string `json:"status" binding:"required"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Result map[string]interface{} `json:"result,omitempty"` + ErrorMessage string `json:"error_message"` +} + +type InstanceCommandService interface { + Create(instanceID int, issuedBy *int, req CreateInstanceCommandRequest) (*InstanceCommandPayload, error) + GetNextForAgent(session *AgentSession) (*AgentCommandEnvelope, error) + MarkStarted(session *AgentSession, commandID int, startedAt *time.Time) error + MarkFinished(session *AgentSession, commandID int, req AgentCommandFinishRequest) error + ListByInstanceID(instanceID int, limit int) ([]InstanceCommandPayload, error) +} + +type instanceCommandService struct { + commandRepo repository.InstanceCommandRepository + runtimeRepo repository.InstanceRuntimeStatusRepository + desiredStateRepo repository.InstanceDesiredStateRepository + skillRepo repository.SkillRepository +} + +func NewInstanceCommandService(commandRepo repository.InstanceCommandRepository, runtimeRepo repository.InstanceRuntimeStatusRepository, desiredStateRepo repository.InstanceDesiredStateRepository, skillRepos ...repository.SkillRepository) InstanceCommandService { + var skillRepo repository.SkillRepository + if len(skillRepos) > 0 { + skillRepo = skillRepos[0] + } + return &instanceCommandService{ + commandRepo: commandRepo, + runtimeRepo: runtimeRepo, + desiredStateRepo: desiredStateRepo, + skillRepo: skillRepo, + } +} + +func (s *instanceCommandService) Create(instanceID int, issuedBy *int, req CreateInstanceCommandRequest) (*InstanceCommandPayload, error) { + commandType := strings.TrimSpace(req.CommandType) + if !isSupportedCommandType(commandType) { + return nil, fmt.Errorf("invalid instance command type") + } + idempotencyKey := strings.TrimSpace(req.IdempotencyKey) + if idempotencyKey == "" { + idempotencyKey = fmt.Sprintf("%s-%d", commandType, time.Now().UTC().UnixNano()) + } + + existing, err := s.commandRepo.GetByInstanceIdempotencyKey(instanceID, idempotencyKey) + if err != nil { + return nil, err + } + if existing != nil { + return commandPayloadFromModel(*existing) + } + + payloadJSON, err := marshalOptionalJSON(req.Payload) + if err != nil { + return nil, fmt.Errorf("failed to encode command payload: %w", err) + } + timeoutSeconds := req.TimeoutSeconds + if timeoutSeconds <= 0 { + timeoutSeconds = 300 + } + + now := time.Now().UTC() + command := &models.InstanceCommand{ + InstanceID: instanceID, + CommandType: commandType, + PayloadJSON: payloadJSON, + Status: instanceCommandStatusPending, + IdempotencyKey: idempotencyKey, + IssuedBy: issuedBy, + IssuedAt: now, + TimeoutSeconds: timeoutSeconds, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.commandRepo.Create(command); err != nil { + return nil, err + } + + if err := s.applyDesiredStateSideEffects(instanceID, commandType, req.Payload); err != nil { + return nil, err + } + return commandPayloadFromModel(*command) +} + +func (s *instanceCommandService) GetNextForAgent(session *AgentSession) (*AgentCommandEnvelope, error) { + command, err := s.commandRepo.GetNextPendingByInstance(session.Instance.ID) + if err != nil { + return nil, err + } + if command == nil { + return nil, nil + } + + now := time.Now().UTC() + command.Status = instanceCommandStatusDispatched + command.AgentID = &session.Agent.AgentID + command.DispatchedAt = &now + if err := s.commandRepo.Update(command); err != nil { + return nil, err + } + + payload := map[string]interface{}{} + if command.PayloadJSON != nil && strings.TrimSpace(*command.PayloadJSON) != "" { + if err := json.Unmarshal([]byte(*command.PayloadJSON), &payload); err != nil { + return nil, fmt.Errorf("failed to decode command payload: %w", err) + } + } + + return &AgentCommandEnvelope{ + ID: command.ID, + CommandType: command.CommandType, + Payload: payload, + IssuedAt: command.IssuedAt, + TimeoutSeconds: command.TimeoutSeconds, + IdempotencyKey: command.IdempotencyKey, + }, nil +} + +func (s *instanceCommandService) MarkStarted(session *AgentSession, commandID int, startedAt *time.Time) error { + command, err := s.commandRepo.GetByID(commandID) + if err != nil { + return err + } + if command == nil || command.InstanceID != session.Instance.ID { + return fmt.Errorf("instance command not found") + } + + if command.Status == instanceCommandStatusRunning || command.Status == instanceCommandStatusSucceeded || command.Status == instanceCommandStatusFailed { + return nil + } + now := time.Now().UTC() + if startedAt == nil || startedAt.IsZero() { + startedAt = &now + } + command.Status = instanceCommandStatusRunning + command.AgentID = &session.Agent.AgentID + command.StartedAt = startedAt + return s.commandRepo.Update(command) +} + +func (s *instanceCommandService) MarkFinished(session *AgentSession, commandID int, req AgentCommandFinishRequest) error { + command, err := s.commandRepo.GetByID(commandID) + if err != nil { + return err + } + if command == nil || command.InstanceID != session.Instance.ID { + return fmt.Errorf("instance command not found") + } + if strings.TrimSpace(req.AgentID) != session.Agent.AgentID { + return fmt.Errorf("agent id does not match session") + } + if req.Status != instanceCommandStatusSucceeded && req.Status != instanceCommandStatusFailed { + return fmt.Errorf("invalid instance command finish status") + } + + if command.Status == instanceCommandStatusSucceeded || command.Status == instanceCommandStatusFailed { + return nil + } + + finishedAt := req.FinishedAt + if finishedAt == nil || finishedAt.IsZero() { + now := time.Now().UTC() + finishedAt = &now + } + command.Status = req.Status + command.AgentID = &session.Agent.AgentID + command.FinishedAt = finishedAt + if req.Result != nil { + resultJSON, err := marshalOptionalJSON(req.Result) + if err != nil { + return fmt.Errorf("failed to encode command result: %w", err) + } + command.ResultJSON = resultJSON + } + if strings.TrimSpace(req.ErrorMessage) != "" { + command.ErrorMessage = optionalString(strings.TrimSpace(req.ErrorMessage)) + } + if command.CommandType == InstanceCommandTypeUninstallSkill && req.Status == instanceCommandStatusSucceeded { + if err := s.markUninstalledSkillRemoved(command, *finishedAt); err != nil { + return err + } + } + return s.commandRepo.Update(command) +} + +func (s *instanceCommandService) markUninstalledSkillRemoved(command *models.InstanceCommand, observedAt time.Time) error { + if s.skillRepo == nil || command == nil || command.PayloadJSON == nil || strings.TrimSpace(*command.PayloadJSON) == "" { + return nil + } + payload := map[string]interface{}{} + if err := json.Unmarshal([]byte(*command.PayloadJSON), &payload); err != nil { + return fmt.Errorf("failed to decode uninstall skill payload: %w", err) + } + if skillID, ok := intPayloadValue(payload["skill_id"]); ok { + if err := s.skillRepo.MarkInstanceSkillRemoved(command.InstanceID, skillID, observedAt); err != nil { + return err + } + } + if skillKey, ok := stringPayloadValue(payload["target_name"]); ok { + return s.skillRepo.MarkInstanceSkillRemovedBySkillKey(command.InstanceID, skillKey, observedAt) + } + return nil +} + +func intPayloadValue(value interface{}) (int, bool) { + switch typed := value.(type) { + case float64: + if typed > 0 { + return int(typed), true + } + case int: + if typed > 0 { + return typed, true + } + } + return 0, false +} + +func stringPayloadValue(value interface{}) (string, bool) { + text, ok := value.(string) + if !ok { + return "", false + } + text = strings.TrimSpace(text) + return text, text != "" +} + +func (s *instanceCommandService) ListByInstanceID(instanceID int, limit int) ([]InstanceCommandPayload, error) { + items, err := s.commandRepo.ListByInstanceID(instanceID, limit) + if err != nil { + return nil, err + } + result := make([]InstanceCommandPayload, 0, len(items)) + for _, item := range items { + payload, err := commandPayloadFromModel(item) + if err != nil { + return nil, err + } + result = append(result, *payload) + } + return result, nil +} + +func commandPayloadFromModel(item models.InstanceCommand) (*InstanceCommandPayload, error) { + payload := &InstanceCommandPayload{ + ID: item.ID, + CommandType: item.CommandType, + Status: item.Status, + IdempotencyKey: item.IdempotencyKey, + IssuedBy: item.IssuedBy, + IssuedAt: item.IssuedAt, + DispatchedAt: item.DispatchedAt, + StartedAt: item.StartedAt, + FinishedAt: item.FinishedAt, + TimeoutSeconds: item.TimeoutSeconds, + ErrorMessage: item.ErrorMessage, + } + if item.PayloadJSON != nil && strings.TrimSpace(*item.PayloadJSON) != "" { + if err := json.Unmarshal([]byte(*item.PayloadJSON), &payload.Payload); err != nil { + return nil, fmt.Errorf("failed to decode command payload: %w", err) + } + } + if item.ResultJSON != nil && strings.TrimSpace(*item.ResultJSON) != "" { + if err := json.Unmarshal([]byte(*item.ResultJSON), &payload.Result); err != nil { + return nil, fmt.Errorf("failed to decode command result: %w", err) + } + } + return payload, nil +} + +func isSupportedCommandType(commandType string) bool { + switch strings.TrimSpace(commandType) { + case InstanceCommandTypeStartOpenClaw, + InstanceCommandTypeStopOpenClaw, + InstanceCommandTypeRestartOpenClaw, + InstanceCommandTypeCollectSystemInfo, + InstanceCommandTypeApplyConfigRevision, + InstanceCommandTypeReloadConfig, + InstanceCommandTypeHealthCheck, + InstanceCommandTypeInstallSkill, + InstanceCommandTypeUpdateSkill, + InstanceCommandTypeUninstallSkill, + InstanceCommandTypeRemoveSkill, + InstanceCommandTypeDisableSkill, + InstanceCommandTypeQuarantineSkill, + InstanceCommandTypeHandleSkillRisk, + InstanceCommandTypeSyncSkillInventory, + InstanceCommandTypeRefreshSkillInventory, + InstanceCommandTypeCollectSkillPackage: + return true + default: + return false + } +} + +func (s *instanceCommandService) applyDesiredStateSideEffects(instanceID int, commandType string, payload map[string]interface{}) error { + state, err := s.desiredStateRepo.GetByInstanceID(instanceID) + if err != nil { + return err + } + if state == nil { + return nil + } + + updated := false + switch commandType { + case InstanceCommandTypeStartOpenClaw, InstanceCommandTypeRestartOpenClaw: + if state.DesiredPowerState != "running" { + state.DesiredPowerState = "running" + updated = true + } + case InstanceCommandTypeStopOpenClaw: + if state.DesiredPowerState != "stopped" { + state.DesiredPowerState = "stopped" + updated = true + } + case InstanceCommandTypeApplyConfigRevision: + if revisionID, ok := payload["revision_id"].(float64); ok { + rev := int(revisionID) + state.DesiredConfigRevisionID = &rev + updated = true + } + } + if !updated { + return nil + } + + state.UpdatedAt = time.Now().UTC() + if err := s.desiredStateRepo.Update(state); err != nil { + return err + } + + runtime, err := s.runtimeRepo.GetByInstanceID(instanceID) + if err == nil && runtime != nil { + runtime.DesiredConfigRevisionID = state.DesiredConfigRevisionID + _ = s.runtimeRepo.Update(runtime) + } + return nil +} diff --git a/backend/internal/services/instance_config_revision_service.go b/backend/internal/services/instance_config_revision_service.go new file mode 100644 index 0000000..6e2d0de --- /dev/null +++ b/backend/internal/services/instance_config_revision_service.go @@ -0,0 +1,124 @@ +package services + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +type InstanceConfigRevisionPayload struct { + ID int `json:"id"` + InstanceID int `json:"instance_id"` + SourceSnapshotID *int `json:"source_snapshot_id,omitempty"` + SourceBundleID *int `json:"source_bundle_id,omitempty"` + RevisionNo int `json:"revision_no"` + Content json.RawMessage `json:"content"` + Checksum string `json:"checksum"` + Status string `json:"status"` + PublishedBy *int `json:"published_by,omitempty"` + PublishedAt *time.Time `json:"published_at,omitempty"` + ActivatedAt *time.Time `json:"activated_at,omitempty"` +} + +type InstanceConfigRevisionService interface { + GetByID(id int) (*InstanceConfigRevisionPayload, error) + ListByInstanceID(instanceID int, limit int) ([]InstanceConfigRevisionPayload, error) + CreateFromSnapshot(instanceID int, snapshot *models.OpenClawInjectionSnapshot, publishedBy *int) (*InstanceConfigRevisionPayload, error) +} + +type instanceConfigRevisionService struct { + repo repository.InstanceConfigRevisionRepository +} + +func NewInstanceConfigRevisionService(repo repository.InstanceConfigRevisionRepository) InstanceConfigRevisionService { + return &instanceConfigRevisionService{repo: repo} +} + +func (s *instanceConfigRevisionService) GetByID(id int) (*InstanceConfigRevisionPayload, error) { + item, err := s.repo.GetByID(id) + if err != nil { + return nil, err + } + if item == nil { + return nil, fmt.Errorf("instance config revision not found") + } + return configRevisionPayloadFromModel(*item) +} + +func (s *instanceConfigRevisionService) ListByInstanceID(instanceID int, limit int) ([]InstanceConfigRevisionPayload, error) { + items, err := s.repo.ListByInstanceID(instanceID, limit) + if err != nil { + return nil, err + } + + result := make([]InstanceConfigRevisionPayload, 0, len(items)) + for _, item := range items { + payload, err := configRevisionPayloadFromModel(item) + if err != nil { + return nil, err + } + result = append(result, *payload) + } + return result, nil +} + +func (s *instanceConfigRevisionService) CreateFromSnapshot(instanceID int, snapshot *models.OpenClawInjectionSnapshot, publishedBy *int) (*InstanceConfigRevisionPayload, error) { + if snapshot == nil { + return nil, fmt.Errorf("openclaw injection snapshot is required") + } + + latest, err := s.repo.GetLatestByInstanceID(instanceID) + if err != nil { + return nil, err + } + revisionNo := 1 + if latest != nil { + revisionNo = latest.RevisionNo + 1 + } + contentJSON := strings.TrimSpace(snapshot.RenderedManifestJSON) + if contentJSON == "" { + contentJSON = "{}" + } + sum := sha256.Sum256([]byte(contentJSON)) + checksum := "sha256:" + hex.EncodeToString(sum[:]) + now := time.Now().UTC() + revision := &models.InstanceConfigRevision{ + InstanceID: instanceID, + SourceSnapshotID: &snapshot.ID, + RevisionNo: revisionNo, + ContentJSON: contentJSON, + Checksum: checksum, + Status: "published", + PublishedBy: publishedBy, + PublishedAt: &now, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.repo.Create(revision); err != nil { + return nil, err + } + return configRevisionPayloadFromModel(*revision) +} + +func configRevisionPayloadFromModel(item models.InstanceConfigRevision) (*InstanceConfigRevisionPayload, error) { + payload := &InstanceConfigRevisionPayload{ + ID: item.ID, + InstanceID: item.InstanceID, + SourceSnapshotID: item.SourceSnapshotID, + SourceBundleID: item.SourceBundleID, + RevisionNo: item.RevisionNo, + Checksum: item.Checksum, + Status: item.Status, + PublishedBy: item.PublishedBy, + PublishedAt: item.PublishedAt, + ActivatedAt: item.ActivatedAt, + Content: json.RawMessage(item.ContentJSON), + } + return payload, nil +} diff --git a/backend/internal/services/instance_env.go b/backend/internal/services/instance_env.go new file mode 100644 index 0000000..7d3926a --- /dev/null +++ b/backend/internal/services/instance_env.go @@ -0,0 +1,150 @@ +package services + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + + "clawreef/internal/models" +) + +const ( + defaultInstanceSHMSizeGB = 1 + maxInstanceSHMSizeGB = 8 +) + +// popSHMSizeGB removes SHM_SIZE_GB from extraEnv and returns /dev/shm size in GiB. +// Desktop runtime defaults scale with instance memory. SHM_SIZE_GB=0 disables +// the custom emptyDir /dev/shm mount. +// Values above maxInstanceSHMSizeGB are clamped to protect node memory. +func popSHMSizeGB(extraEnv map[string]string, runtimeType string, memoryGB int) int { + shmSizeGB := defaultSHMSizeGB(runtimeType, memoryGB) + if shmVal, ok := extraEnv["SHM_SIZE_GB"]; ok { + if parsed, err := strconv.Atoi(strings.TrimSpace(shmVal)); err == nil { + if parsed == 0 { + shmSizeGB = 0 + } else if parsed > 0 { + shmSizeGB = parsed + if shmSizeGB > maxInstanceSHMSizeGB { + shmSizeGB = maxInstanceSHMSizeGB + } + } + } + delete(extraEnv, "SHM_SIZE_GB") + } + return shmSizeGB +} + +func defaultSHMSizeGB(runtimeType string, memoryGB int) int { + if normalizeInstanceRuntimeType(runtimeType) != "desktop" { + return defaultInstanceSHMSizeGB + } + switch { + case memoryGB >= 12: + return 4 + case memoryGB >= 8: + return 2 + default: + return defaultInstanceSHMSizeGB + } +} + +var envNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func normalizeEnvironmentOverrides(overrides map[string]string) (map[string]string, error) { + if len(overrides) == 0 { + return nil, nil + } + + normalized := make(map[string]string, len(overrides)) + for rawKey, value := range overrides { + key := strings.TrimSpace(rawKey) + if key == "" { + return nil, fmt.Errorf("environment variable name cannot be empty") + } + if !envNamePattern.MatchString(key) { + return nil, fmt.Errorf("invalid environment variable name: %s", key) + } + if _, exists := normalized[key]; exists { + return nil, fmt.Errorf("duplicate environment variable name: %s", key) + } + normalized[key] = value + } + + return normalized, nil +} + +func marshalEnvironmentOverrides(overrides map[string]string) (*string, error) { + if len(overrides) == 0 { + return nil, nil + } + + raw, err := json.Marshal(overrides) + if err != nil { + return nil, fmt.Errorf("failed to encode environment overrides: %w", err) + } + + encoded := string(raw) + return &encoded, nil +} + +func parseEnvironmentOverridesJSON(raw *string) (map[string]string, error) { + if raw == nil || strings.TrimSpace(*raw) == "" { + return nil, nil + } + + var overrides map[string]string + if err := json.Unmarshal([]byte(strings.TrimSpace(*raw)), &overrides); err != nil { + return nil, fmt.Errorf("failed to decode environment overrides: %w", err) + } + + normalized, err := normalizeEnvironmentOverrides(overrides) + if err != nil { + return nil, err + } + + return normalized, nil +} + +func buildInstancePodEnv(instance *models.Instance, runtimeEnv, gatewayEnv, agentEnv map[string]string) (map[string]string, error) { + if instance == nil { + return nil, fmt.Errorf("instance is required") + } + + overrides, err := parseEnvironmentOverridesJSON(instance.EnvironmentOverridesJSON) + if err != nil { + return nil, err + } + overrides = ensureDesktopStreamProfileEnv(overrides, instance.RuntimeType) + + resolved := mergeEnvMaps(runtimeEnv, mergeEnvMaps(gatewayEnv, agentEnv)) + resolved = withInstanceProxyEnv(instance.Type, instance.ID, resolved) + resolved["CLAWMANAGER_RUNTIME_TYPE"] = normalizeInstanceRuntimeType(instance.RuntimeType) + if normalizeInstanceRuntimeType(instance.RuntimeType) == "shell" { + resolved["CLAWMANAGER_DESKTOP_ENABLED"] = "false" + delete(resolved, "SUBFOLDER") + } + resolved = mergeEnvMaps(resolved, overrides) + + return resolved, nil +} + +func buildInstanceGatewayEnv(instance *models.Instance, gatewayEnv map[string]string) (map[string]string, error) { + if instance == nil { + return nil, fmt.Errorf("instance is required") + } + + overrides, err := parseEnvironmentOverridesJSON(instance.EnvironmentOverridesJSON) + if err != nil { + return nil, err + } + + resolved := mergeEnvMaps(gatewayEnv, nil) + resolved = withInstanceProxyEnv(instance.Type, instance.ID, resolved) + resolved["CLAWMANAGER_RUNTIME_TYPE"] = normalizeInstanceRuntimeType(instance.RuntimeType) + resolved = mergeEnvMaps(resolved, overrides) + + return resolved, nil +} diff --git a/backend/internal/services/instance_env_test.go b/backend/internal/services/instance_env_test.go new file mode 100644 index 0000000..63c6c94 --- /dev/null +++ b/backend/internal/services/instance_env_test.go @@ -0,0 +1,145 @@ +package services + +import ( + "testing" + + "clawreef/internal/models" +) + +func TestNormalizeEnvironmentOverrides(t *testing.T) { + overrides, err := normalizeEnvironmentOverrides(map[string]string{ + " FOO ": "bar", + "BAR_2": "", + }) + if err != nil { + t.Fatalf("normalizeEnvironmentOverrides returned error: %v", err) + } + + if overrides["FOO"] != "bar" { + t.Fatalf("expected trimmed key FOO to be preserved") + } + if value, ok := overrides["BAR_2"]; !ok || value != "" { + t.Fatalf("expected empty override value to be preserved") + } +} + +func TestNormalizeEnvironmentOverridesRejectsInvalidNames(t *testing.T) { + if _, err := normalizeEnvironmentOverrides(map[string]string{ + "1INVALID": "value", + }); err == nil { + t.Fatalf("expected invalid environment variable name to fail validation") + } +} + +func TestBuildInstancePodEnvAppliesOverridesAfterDefaults(t *testing.T) { + t.Setenv("CLAWMANAGER_EGRESS_PROXY_URL", "") + t.Setenv("CLAWMANAGER_SYSTEM_NAMESPACE", "") + t.Setenv("K8S_NAMESPACE", "") + + raw, err := marshalEnvironmentOverrides(map[string]string{ + "SUBFOLDER": "/custom-proxy", + "KASM_SVC_ACCEPT_CUT_TEXT": "-AcceptCutText 1", + "CUSTOM": "enabled", + }) + if err != nil { + t.Fatalf("marshalEnvironmentOverrides returned error: %v", err) + } + + instance := &models.Instance{ + ID: 42, + Type: "webtop", + EnvironmentOverridesJSON: raw, + } + + env, err := buildInstancePodEnv(instance, map[string]string{ + "TITLE": "ClawManager Webtop", + "SUBFOLDER": "/", + }, nil, nil) + if err != nil { + t.Fatalf("buildInstancePodEnv returned error: %v", err) + } + + if env["SUBFOLDER"] != "/custom-proxy" { + t.Fatalf("expected SUBFOLDER override to win, got %q", env["SUBFOLDER"]) + } + if env["KASM_SVC_ACCEPT_CUT_TEXT"] != "-AcceptCutText 1" { + t.Fatalf("expected clipboard policy override to win, got %q", env["KASM_SVC_ACCEPT_CUT_TEXT"]) + } + if env["CUSTOM"] != "enabled" { + t.Fatalf("expected custom environment variable to be merged") + } + if env["TITLE"] != "ClawManager Webtop" { + t.Fatalf("expected default environment variable to remain available") + } +} + +func TestBuildInstancePodEnvNormalizesDesktopStreamProfile(t *testing.T) { + raw, err := marshalEnvironmentOverrides(map[string]string{ + "CLAWMANAGER_DESKTOP_STREAM_PROFILE": "standard", + "SELKIES_ENCODER": "x264enc", + "SELKIES_FRAMERATE": "35", + "SELKIES_H264_CRF": "34", + }) + if err != nil { + t.Fatalf("marshalEnvironmentOverrides returned error: %v", err) + } + + env, err := buildInstancePodEnv(&models.Instance{ + ID: 42, + Type: "openclaw", + RuntimeType: RuntimeBackendDesktop, + EnvironmentOverridesJSON: raw, + }, nil, nil, nil) + if err != nil { + t.Fatalf("buildInstancePodEnv returned error: %v", err) + } + + if got := env["SELKIES_ENCODER"]; got != "x264enc,jpeg" { + t.Fatalf("SELKIES_ENCODER = %q, want x264enc,jpeg", got) + } + if got := env["SELKIES_USE_CSS_SCALING"]; got != "true" { + t.Fatalf("SELKIES_USE_CSS_SCALING = %q, want true", got) + } +} + +func TestPopSHMSizeGB(t *testing.T) { + tests := []struct { + name string + value string + hasValue bool + runtime string + memoryGB int + want int + }{ + {name: "desktop minimum preset", runtime: "desktop", memoryGB: 4, want: defaultInstanceSHMSizeGB}, + {name: "desktop medium preset", runtime: "desktop", memoryGB: 8, want: 2}, + {name: "desktop large preset", runtime: "desktop", memoryGB: 12, want: 4}, + {name: "desktop small fallback", runtime: "desktop", memoryGB: 2, want: defaultInstanceSHMSizeGB}, + {name: "shell default unchanged", runtime: "shell", memoryGB: 8, want: defaultInstanceSHMSizeGB}, + {name: "disable", value: "0", hasValue: true, runtime: "desktop", memoryGB: 8, want: 0}, + {name: "custom", value: "4", hasValue: true, runtime: "desktop", memoryGB: 4, want: 4}, + {name: "clamp", value: "128", hasValue: true, runtime: "desktop", memoryGB: 8, want: maxInstanceSHMSizeGB}, + {name: "invalid keeps dynamic desktop default", value: "nope", hasValue: true, runtime: "desktop", memoryGB: 8, want: 2}, + {name: "negative keeps dynamic desktop default", value: "-1", hasValue: true, runtime: "desktop", memoryGB: 4, want: defaultInstanceSHMSizeGB}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + extraEnv := map[string]string{"KEEP": "value"} + if tt.hasValue { + extraEnv["SHM_SIZE_GB"] = tt.value + } + + got := popSHMSizeGB(extraEnv, tt.runtime, tt.memoryGB) + if got != tt.want { + t.Fatalf("expected shm size %d, got %d", tt.want, got) + } + if _, ok := extraEnv["SHM_SIZE_GB"]; ok { + t.Fatalf("expected SHM_SIZE_GB to be removed from extra env") + } + if extraEnv["KEEP"] != "value" { + t.Fatalf("expected unrelated env to be preserved") + } + }) + } +} diff --git a/backend/internal/services/instance_external_access_service.go b/backend/internal/services/instance_external_access_service.go new file mode 100644 index 0000000..e440118 --- /dev/null +++ b/backend/internal/services/instance_external_access_service.go @@ -0,0 +1,286 @@ +package services + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +const ( + ExternalAccessModeShareLink = "share_link" + ExternalAccessModePassword = "password" + + ExternalAccessExpirationPreset = "preset" + ExternalAccessExpirationCustom = "custom" + ExternalAccessExpirationPermanent = "permanent" + + ExternalAccessPreset1Hour = "1h" + ExternalAccessPreset24Hours = "24h" + ExternalAccessPreset7Days = "7d" + ExternalAccessPreset30Days = "30d" +) + +type ExternalAccessExpirationRequest struct { + Mode string `json:"expires_mode,omitempty"` + Preset string `json:"expires_preset,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +type EnableShareLinkResult struct { + Access *models.InstanceExternalAccess `json:"access"` + ShareURL string `json:"share_url,omitempty"` +} + +type PasswordExternalAccessResult struct { + Access *models.InstanceExternalAccess `json:"access"` + Password string `json:"password"` + ShareURL string `json:"share_url,omitempty"` +} + +type InstanceExternalAccessService interface { + Get(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) + EnableShareLink(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*EnableShareLinkResult, error) + CreatePassword(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*PasswordExternalAccessResult, error) + Disable(ctx context.Context, instanceID int) error + ResolveShortLink(ctx context.Context, code string) (*models.InstanceExternalAccess, error) + ValidateShortLink(ctx context.Context, code, password string) (*models.InstanceExternalAccess, error) +} + +type instanceExternalAccessService struct { + repo repository.InstanceExternalAccessRepository +} + +func NewInstanceExternalAccessService(repo repository.InstanceExternalAccessRepository) InstanceExternalAccessService { + return &instanceExternalAccessService{repo: repo} +} + +func (s *instanceExternalAccessService) Get(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) { + if s == nil || s.repo == nil { + return nil, fmt.Errorf("instance external access repository is not configured") + } + return s.repo.GetByInstanceID(ctx, instanceID) +} + +func (s *instanceExternalAccessService) EnableShareLink(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*EnableShareLinkResult, error) { + if s == nil || s.repo == nil { + return nil, fmt.Errorf("instance external access repository is not configured") + } + code, codeHash, err := generateShortCode() + if err != nil { + return nil, err + } + expiresAt, err := resolveExternalAccessExpiresAt(expiration) + if err != nil { + return nil, err + } + access := &models.InstanceExternalAccess{ + InstanceID: instanceID, + Enabled: true, + AuthMode: ExternalAccessModeShareLink, + ShortCodeHash: &codeHash, + PublicSlug: &code, + PublicTokenHash: nil, + ExpiresAt: expiresAt, + CreatedBy: &createdBy, + PasswordHash: nil, + PasswordValue: nil, + PasswordHint: nil, + LastUsedAt: nil, + } + if err := s.repo.Upsert(ctx, access); err != nil { + return nil, err + } + saved, err := s.repo.GetByInstanceID(ctx, instanceID) + if err != nil { + return nil, err + } + return &EnableShareLinkResult{ + Access: saved, + ShareURL: shortExternalAccessURL(code), + }, nil +} + +func (s *instanceExternalAccessService) CreatePassword(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*PasswordExternalAccessResult, error) { + if s == nil || s.repo == nil { + return nil, fmt.Errorf("instance external access repository is not configured") + } + code, codeHash, err := generateShortCode() + if err != nil { + return nil, err + } + password, err := randomToken("pwd", 16) + if err != nil { + return nil, err + } + expiresAt, err := resolveExternalAccessExpiresAt(expiration) + if err != nil { + return nil, err + } + passwordHash := hashExternalSecret(password) + hint := password + if len(hint) > 12 { + hint = hint[:12] + } + access := &models.InstanceExternalAccess{ + InstanceID: instanceID, + Enabled: true, + AuthMode: ExternalAccessModePassword, + PublicSlug: &code, + ShortCodeHash: &codeHash, + PasswordHash: &passwordHash, + PasswordValue: &password, + PasswordHint: &hint, + ExpiresAt: expiresAt, + CreatedBy: &createdBy, + PublicTokenHash: nil, + LastUsedAt: nil, + } + if err := s.repo.Upsert(ctx, access); err != nil { + return nil, err + } + saved, err := s.repo.GetByInstanceID(ctx, instanceID) + if err != nil { + return nil, err + } + return &PasswordExternalAccessResult{Access: saved, Password: password, ShareURL: shortExternalAccessURL(code)}, nil +} + +func (s *instanceExternalAccessService) Disable(ctx context.Context, instanceID int) error { + if s == nil || s.repo == nil { + return fmt.Errorf("instance external access repository is not configured") + } + return s.repo.Disable(ctx, instanceID) +} + +func (s *instanceExternalAccessService) ResolveShortLink(ctx context.Context, code string) (*models.InstanceExternalAccess, error) { + if s == nil || s.repo == nil { + return nil, fmt.Errorf("instance external access repository is not configured") + } + code = strings.TrimSpace(code) + if code == "" { + return nil, fmt.Errorf("short link code is required") + } + access, err := s.repo.GetByShortCodeHash(ctx, hashExternalSecret(code)) + if err != nil { + return nil, err + } + if access == nil || !access.Enabled { + return nil, fmt.Errorf("external access is not enabled") + } + if access.ExpiresAt != nil && time.Now().UTC().After(*access.ExpiresAt) { + return nil, fmt.Errorf("external access has expired") + } + return access, nil +} + +func (s *instanceExternalAccessService) ValidateShortLink(ctx context.Context, code, password string) (*models.InstanceExternalAccess, error) { + access, err := s.ResolveShortLink(ctx, code) + if err != nil { + return nil, err + } + switch access.AuthMode { + case ExternalAccessModeShareLink: + // The short code itself is the share-link capability. A matching hash is enough. + case ExternalAccessModePassword: + if access.PasswordHash == nil || *access.PasswordHash != hashExternalSecret(strings.TrimSpace(password)) { + return nil, fmt.Errorf("invalid external access password") + } + default: + return nil, fmt.Errorf("unsupported external access mode") + } + if err := s.repo.MarkUsed(ctx, access.ID); err != nil { + return nil, err + } + return access, nil +} + +func resolveExternalAccessExpiresAt(expiration ExternalAccessExpirationRequest) (*time.Time, error) { + mode := strings.TrimSpace(expiration.Mode) + if mode == "" { + mode = ExternalAccessExpirationPreset + } + switch mode { + case ExternalAccessExpirationPermanent: + return nil, nil + case ExternalAccessExpirationCustom: + if expiration.ExpiresAt == nil { + return nil, fmt.Errorf("custom external access expiration requires expires_at") + } + value := expiration.ExpiresAt.UTC() + return &value, nil + case ExternalAccessExpirationPreset: + preset := strings.TrimSpace(expiration.Preset) + if preset == "" { + preset = ExternalAccessPreset24Hours + } + duration, err := externalAccessPresetDuration(preset) + if err != nil { + return nil, err + } + value := time.Now().UTC().Add(duration) + return &value, nil + default: + return nil, fmt.Errorf("unsupported external access expiration mode") + } +} + +func externalAccessPresetDuration(preset string) (time.Duration, error) { + switch strings.TrimSpace(preset) { + case ExternalAccessPreset1Hour: + return time.Hour, nil + case ExternalAccessPreset24Hours: + return 24 * time.Hour, nil + case ExternalAccessPreset7Days: + return 7 * 24 * time.Hour, nil + case ExternalAccessPreset30Days: + return 30 * 24 * time.Hour, nil + default: + return 0, fmt.Errorf("unsupported external access expiration preset") + } +} + +func generateShortCode() (code, codeHash string, err error) { + code, err = randomToken("sl", 12) + if err != nil { + return "", "", err + } + return code, hashExternalSecret(code), nil +} + +func shortExternalAccessURL(code string) string { + return fmt.Sprintf("/s/%s/", strings.Trim(strings.TrimSpace(code), "/")) +} + +func ExternalAccessShareURL(access *models.InstanceExternalAccess) string { + if access == nil || !access.Enabled || access.PublicSlug == nil { + return "" + } + return shortExternalAccessURL(*access.PublicSlug) +} + +func ExternalAccessPassword(access *models.InstanceExternalAccess) string { + if access == nil || !access.Enabled || access.AuthMode != ExternalAccessModePassword || access.PasswordValue == nil { + return "" + } + return *access.PasswordValue +} + +func randomToken(prefix string, bytesLen int) (string, error) { + buf := make([]byte, bytesLen) + if _, err := rand.Read(buf); err != nil { + return "", fmt.Errorf("failed to generate token: %w", err) + } + return prefix + "_" + hex.EncodeToString(buf), nil +} + +func hashExternalSecret(value string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(value))) + return hex.EncodeToString(sum[:]) +} diff --git a/backend/internal/services/instance_external_access_service_test.go b/backend/internal/services/instance_external_access_service_test.go new file mode 100644 index 0000000..5d9bb23 --- /dev/null +++ b/backend/internal/services/instance_external_access_service_test.go @@ -0,0 +1,304 @@ +package services + +import ( + "context" + "strings" + "testing" + "time" + + "clawreef/internal/models" +) + +func TestInstanceExternalAccessShareLinkValidation(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + + result, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent}) + if err != nil { + t.Fatalf("EnableShareLink failed: %v", err) + } + if !strings.HasPrefix(result.ShareURL, "/s/") || strings.Contains(result.ShareURL, "token=") || strings.Contains(result.ShareURL, "/api/v1/") || strings.Contains(result.ShareURL, "/proxy") { + t.Fatalf("unexpected short share URL: %q", result.ShareURL) + } + code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/") + if code == "" { + t.Fatalf("short code missing from URL %q", result.ShareURL) + } + if result.Access.ShortCodeHash == nil || *result.Access.ShortCodeHash == code { + t.Fatalf("expected stored short code to be hashed") + } + if result.Access.AuthMode != ExternalAccessModeShareLink { + t.Fatalf("auth mode = %q, want %q", result.Access.AuthMode, ExternalAccessModeShareLink) + } + + access, err := service.ValidateShortLink(context.Background(), code, "") + if err != nil { + t.Fatalf("ValidateShortLink failed: %v", err) + } + if access.InstanceID != 42 { + t.Fatalf("validated instance ID = %d, want 42", access.InstanceID) + } + if repo.markUsedCount != 1 { + t.Fatalf("mark used count = %d, want 1", repo.markUsedCount) + } + + if _, err := service.ValidateShortLink(context.Background(), "wrong-code", ""); err == nil { + t.Fatalf("expected wrong short code to fail") + } +} + +func TestInstanceExternalAccessShareLinkPresetExpiration(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + before := time.Now().UTC().Add(24 * time.Hour) + + result, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{ + Mode: ExternalAccessExpirationPreset, + Preset: ExternalAccessPreset24Hours, + }) + if err != nil { + t.Fatalf("EnableShareLink failed: %v", err) + } + if result.Access.ExpiresAt == nil { + t.Fatal("preset expiration must set expires_at") + } + after := time.Now().UTC().Add(24 * time.Hour) + if result.Access.ExpiresAt.Before(before) || result.Access.ExpiresAt.After(after.Add(time.Second)) { + t.Fatalf("preset expires_at = %v, want around 24h from now", result.Access.ExpiresAt) + } +} + +func TestInstanceExternalAccessShareLinkCustomAndPermanentExpiration(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + customAt := time.Now().UTC().Add(3 * time.Hour).Truncate(time.Second) + + custom, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{ + Mode: ExternalAccessExpirationCustom, + ExpiresAt: &customAt, + }) + if err != nil { + t.Fatalf("custom EnableShareLink failed: %v", err) + } + if custom.Access.ExpiresAt == nil || !custom.Access.ExpiresAt.Equal(customAt) { + t.Fatalf("custom expires_at = %v, want %v", custom.Access.ExpiresAt, customAt) + } + + permanent, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent}) + if err != nil { + t.Fatalf("permanent EnableShareLink failed: %v", err) + } + if permanent.Access.ExpiresAt != nil { + t.Fatalf("permanent expires_at = %v, want nil", permanent.Access.ExpiresAt) + } +} + +func TestInstanceExternalAccessPasswordDisable(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + + result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent}) + if err != nil { + t.Fatalf("CreatePassword failed: %v", err) + } + if !strings.HasPrefix(result.Password, "pwd_") { + t.Fatalf("unexpected password prefix: %q", result.Password) + } + if result.Access.PasswordHash == nil || *result.Access.PasswordHash == result.Password { + t.Fatalf("expected stored password to be hashed") + } + if result.Access.PasswordHint == nil || !strings.HasPrefix(result.Password, *result.Access.PasswordHint) { + t.Fatalf("stored password hint does not match generated password") + } + if result.Access.AuthMode != ExternalAccessModePassword { + t.Fatalf("auth mode = %q, want %q", result.Access.AuthMode, ExternalAccessModePassword) + } + if !strings.HasPrefix(result.ShareURL, "/s/") || strings.Contains(result.ShareURL, "token=") || strings.Contains(result.ShareURL, "/api/v1/") { + t.Fatalf("unexpected password short URL: %q", result.ShareURL) + } + code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/") + + if _, err := service.ValidateShortLink(context.Background(), code, result.Password); err != nil { + t.Fatalf("ValidateShortLink with password failed: %v", err) + } + + if err := service.Disable(context.Background(), 100); err != nil { + t.Fatalf("Disable failed: %v", err) + } + if _, err := service.ValidateShortLink(context.Background(), code, result.Password); err == nil { + t.Fatalf("expected disabled password access to fail") + } +} + +func TestInstanceExternalAccessRestoresShareURLAfterRefresh(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + + result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent}) + if err != nil { + t.Fatalf("CreatePassword failed: %v", err) + } + reloaded, err := service.Get(context.Background(), 100) + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got := ExternalAccessShareURL(reloaded); got != result.ShareURL { + t.Fatalf("ExternalAccessShareURL after reload = %q, want %q", got, result.ShareURL) + } +} + +func TestInstanceExternalAccessRestoresPasswordAfterRefresh(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + + result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent}) + if err != nil { + t.Fatalf("CreatePassword failed: %v", err) + } + reloaded, err := service.Get(context.Background(), 100) + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got := ExternalAccessPassword(reloaded); got != result.Password { + t.Fatalf("ExternalAccessPassword after reload = %q, want generated password", got) + } +} + +func TestInstanceExternalAccessExpiration(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + expiredAt := time.Now().UTC().Add(-time.Minute) + + result, err := service.EnableShareLink(context.Background(), 77, 3, ExternalAccessExpirationRequest{ + Mode: ExternalAccessExpirationCustom, + ExpiresAt: &expiredAt, + }) + if err != nil { + t.Fatalf("EnableShareLink failed: %v", err) + } + code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/") + + if _, err := service.ValidateShortLink(context.Background(), code, ""); err == nil { + t.Fatalf("expected expired public access to fail") + } +} + +type fakeExternalAccessRepo struct { + byInstance map[int]*models.InstanceExternalAccess + byCodeHash map[string]*models.InstanceExternalAccess + nextID int64 + markUsedCount int +} + +func newFakeExternalAccessRepo() *fakeExternalAccessRepo { + return &fakeExternalAccessRepo{ + byInstance: make(map[int]*models.InstanceExternalAccess), + byCodeHash: make(map[string]*models.InstanceExternalAccess), + nextID: 1, + } +} + +func (r *fakeExternalAccessRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) { + _ = ctx + return cloneExternalAccess(r.byInstance[instanceID]), nil +} + +func (r *fakeExternalAccessRepo) GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error) { + _ = ctx + return cloneExternalAccess(r.byCodeHash[codeHash]), nil +} + +func (r *fakeExternalAccessRepo) Upsert(ctx context.Context, access *models.InstanceExternalAccess) error { + _ = ctx + saved := cloneExternalAccess(access) + if saved.ID == 0 { + if existing := r.byInstance[saved.InstanceID]; existing != nil { + saved.ID = existing.ID + } else { + saved.ID = r.nextID + r.nextID++ + } + } + if existing := r.byInstance[saved.InstanceID]; existing != nil && existing.ShortCodeHash != nil { + delete(r.byCodeHash, *existing.ShortCodeHash) + } + r.byInstance[saved.InstanceID] = cloneExternalAccess(saved) + if saved.ShortCodeHash != nil { + r.byCodeHash[*saved.ShortCodeHash] = cloneExternalAccess(saved) + } + return nil +} + +func (r *fakeExternalAccessRepo) Disable(ctx context.Context, instanceID int) error { + _ = ctx + if existing := r.byInstance[instanceID]; existing != nil { + existing.Enabled = false + if existing.ShortCodeHash != nil { + if byCode := r.byCodeHash[*existing.ShortCodeHash]; byCode != nil { + byCode.Enabled = false + } + } + } + return nil +} + +func (r *fakeExternalAccessRepo) MarkUsed(ctx context.Context, id int64) error { + _ = ctx + r.markUsedCount++ + now := time.Now().UTC() + for _, access := range r.byInstance { + if access.ID == id { + access.LastUsedAt = &now + } + } + for _, access := range r.byCodeHash { + if access.ID == id { + access.LastUsedAt = &now + } + } + return nil +} + +func cloneExternalAccess(access *models.InstanceExternalAccess) *models.InstanceExternalAccess { + if access == nil { + return nil + } + clone := *access + if access.PublicSlug != nil { + value := *access.PublicSlug + clone.PublicSlug = &value + } + if access.PublicTokenHash != nil { + value := *access.PublicTokenHash + clone.PublicTokenHash = &value + } + if access.ShortCodeHash != nil { + value := *access.ShortCodeHash + clone.ShortCodeHash = &value + } + if access.PasswordHash != nil { + value := *access.PasswordHash + clone.PasswordHash = &value + } + if access.PasswordValue != nil { + value := *access.PasswordValue + clone.PasswordValue = &value + } + if access.PasswordHint != nil { + value := *access.PasswordHint + clone.PasswordHint = &value + } + if access.ExpiresAt != nil { + value := *access.ExpiresAt + clone.ExpiresAt = &value + } + if access.CreatedBy != nil { + value := *access.CreatedBy + clone.CreatedBy = &value + } + if access.LastUsedAt != nil { + value := *access.LastUsedAt + clone.LastUsedAt = &value + } + return &clone +} diff --git a/backend/internal/services/instance_proxy_service.go b/backend/internal/services/instance_proxy_service.go new file mode 100644 index 0000000..cad6a81 --- /dev/null +++ b/backend/internal/services/instance_proxy_service.go @@ -0,0 +1,947 @@ +package services + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" + + "github.com/gorilla/websocket" +) + +// InstanceProxyService handles proxying requests to instance pods +type InstanceProxyService struct { + serviceService *k8s.ServiceService + accessService *InstanceAccessService + instanceRepo repository.InstanceRepository + runtimePodRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + httpClient *http.Client + openClawGatewayToken string + openClawProxyOrigin string + serviceCache map[serviceCacheKey]serviceCacheEntry + serviceLookups map[serviceCacheKey]*serviceLookupCall + cacheMu sync.RWMutex + lookupMu sync.Mutex + serviceTTL time.Duration +} + +type serviceCacheKey struct { + userID int + instanceID int + targetPort int32 +} + +type serviceCacheEntry struct { + serviceInfo *k8s.ServiceInfo + expiresAt time.Time +} + +type serviceLookupCall struct { + done chan struct{} + serviceInfo *k8s.ServiceInfo + err error +} + +const defaultServiceCacheTTL = 30 * time.Second + +var ErrInstanceGatewayUnavailable = errors.New("instance gateway is not available") + +type InstanceProxyServiceOption func(*InstanceProxyService) + +func WithInstanceProxyRuntimeRepositories(instanceRepo repository.InstanceRepository, runtimePodRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository) InstanceProxyServiceOption { + return func(s *InstanceProxyService) { + s.instanceRepo = instanceRepo + s.runtimePodRepo = runtimePodRepo + s.bindingRepo = bindingRepo + } +} + +// NewInstanceProxyService creates a new instance proxy service +func NewInstanceProxyService(accessService *InstanceAccessService, options ...InstanceProxyServiceOption) *InstanceProxyService { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + MaxIdleConns: 256, + MaxIdleConnsPerHost: 128, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + ForceAttemptHTTP2: true, + } + + service := &InstanceProxyService{ + serviceService: k8s.NewServiceService(), + accessService: accessService, + httpClient: &http.Client{ + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // Don't follow redirects automatically, let the client handle them + return http.ErrUseLastResponse + }, + }, + openClawGatewayToken: strings.TrimSpace(os.Getenv("OPENCLAW_GATEWAY_TOKEN")), + openClawProxyOrigin: resolveOpenClawProxyOriginFromEnv(), + serviceCache: make(map[serviceCacheKey]serviceCacheEntry), + serviceLookups: make(map[serviceCacheKey]*serviceLookupCall), + serviceTTL: defaultServiceCacheTTL, + } + for _, option := range options { + if option != nil { + option(service) + } + } + return service +} + +// ProxyRequest proxies a request to an instance +func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int, token string, w http.ResponseWriter, r *http.Request) error { + // Handle CORS preflight request + if r.Method == "OPTIONS" { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With") + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.WriteHeader(http.StatusNoContent) + return nil + } + + // Validate access token + accessToken, err := s.accessService.ValidateToken(token) + if err != nil { + return fmt.Errorf("invalid token: %w", err) + } + + // Verify instance ID matches + if accessToken.InstanceID != instanceID { + return fmt.Errorf("token does not match instance") + } + + effectiveRequestPath := canonicalProxyEntryRequestPath(r.URL.Path, accessToken, instanceID) + + // Extract the actual path from the request (remove the proxy prefix) + targetPath := s.extractTargetPath(effectiveRequestPath, instanceID, accessToken.InstanceType) + targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath) + shouldRewriteHTML := s.shouldRewriteHTMLForProxy(instanceID, accessToken.InstanceType) + + // Build target URL + targetURL, err := s.resolveHTTPProxyTarget(ctx, accessToken, instanceID, targetPort, targetPath, effectiveRequestPath) + if err != nil { + return err + } + + // Copy query parameters (excluding token) + queryParams := r.URL.Query() + removeProxyAccessTokenQuery(queryParams, token) + if len(queryParams) > 0 { + targetURL.RawQuery = queryParams.Encode() + } + + // Create new request with longer timeout for streaming + proxyCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + proxyReq, err := http.NewRequestWithContext(proxyCtx, r.Method, targetURL.String(), r.Body) + if err != nil { + return fmt.Errorf("failed to create proxy request: %w", err) + } + + // Copy headers + for key, values := range r.Header { + for _, value := range values { + proxyReq.Header.Add(key, value) + } + } + + // Set X-Forwarded headers + proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr) + proxyReq.Header.Set("X-Forwarded-Host", r.Host) + proxyReq.Header.Set("X-Forwarded-Proto", requestScheme(r)) + proxyReq.Header.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)) + if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" { + proxyReq.Header.Set("Authorization", "Bearer "+token) + } + if shouldRewriteHTML { + proxyReq.Header.Del("Accept-Encoding") + } + + // Remove hop-by-hop headers + s.removeHopByHopHeaders(proxyReq.Header) + + // Execute request + resp, err := s.httpClient.Do(proxyReq) + if err != nil { + return fmt.Errorf("failed to execute proxy request: %w", err) + } + defer resp.Body.Close() + + // Add CORS headers to response + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Credentials", "true") + + if location := resp.Header.Get("Location"); location != "" { + resp.Header.Set("Location", s.rewriteRedirectLocation(instanceID, location)) + } + + if shouldRewriteHTML && strings.Contains(resp.Header.Get("Content-Type"), "text/html") { + body, readErr := io.ReadAll(resp.Body) + if readErr != nil { + return fmt.Errorf("failed to read upstream html: %w", readErr) + } + if closeErr := resp.Body.Close(); closeErr != nil { + return fmt.Errorf("failed to close upstream html body: %w", closeErr) + } + + modifiedBody := injectProxyBase(string(body), proxyBaseForRequestPath(effectiveRequestPath, instanceID)) + resp.Body = io.NopCloser(bytes.NewReader([]byte(modifiedBody))) + resp.ContentLength = int64(len(modifiedBody)) + resp.Header.Set("Content-Length", strconv.Itoa(len(modifiedBody))) + resp.Header.Del("ETag") + resp.Header.Del("Last-Modified") + } + + // Copy response headers + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + w.Header().Del("X-Frame-Options") + w.Header().Del("Content-Security-Policy") + + // Remove hop-by-hop headers from response + s.removeHopByHopHeaders(w.Header()) + + // Write status code + w.WriteHeader(resp.StatusCode) + + // Copy response body + _, err = io.Copy(w, resp.Body) + if err != nil { + return fmt.Errorf("failed to copy response body: %w", err) + } + + return nil +} + +// ProxyWebSocket handles WebSocket upgrade requests +func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID int, token string, w http.ResponseWriter, r *http.Request) error { + // Handle CORS preflight request + if r.Method == "OPTIONS" { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With") + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.WriteHeader(http.StatusNoContent) + return nil + } + + // Validate access token + accessToken, err := s.accessService.ValidateToken(token) + if err != nil { + return fmt.Errorf("invalid token: %w", err) + } + + // Verify instance ID matches + if accessToken.InstanceID != instanceID { + return fmt.Errorf("token does not match instance") + } + + // Extract the actual path from the request + targetPath := s.extractTargetPath(r.URL.Path, instanceID, accessToken.InstanceType) + targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath) + + targetURL, err := s.resolveWebSocketProxyTarget(ctx, accessToken, instanceID, targetPort, targetPath, r.URL.Path) + if err != nil { + return err + } + + // Copy query parameters (excluding token) + queryParams := r.URL.Query() + removeProxyAccessTokenQuery(queryParams, token) + if len(queryParams) > 0 { + targetURL.RawQuery = queryParams.Encode() + } + + upstreamHeader := http.Header{} + for key, values := range r.Header { + for _, value := range values { + upstreamHeader.Add(key, value) + } + } + upstreamHeader.Del("Host") + upstreamHeader.Del("Connection") + upstreamHeader.Del("Upgrade") + upstreamHeader.Del("Sec-Websocket-Key") + upstreamHeader.Del("Sec-Websocket-Version") + upstreamHeader.Del("Sec-Websocket-Extensions") + upstreamHeader.Set("X-Forwarded-For", r.RemoteAddr) + upstreamHeader.Set("X-Forwarded-Host", r.Host) + upstreamHeader.Set("X-Forwarded-Proto", requestScheme(r)) + upstreamHeader.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)) + if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" { + upstreamHeader.Set("Authorization", "Bearer "+token) + upstreamHeader.Set("Origin", s.openClawWebSocketOrigin(targetURL)) + } + + dialer := websocket.Dialer{ + Proxy: http.ProxyFromEnvironment, + HandshakeTimeout: 30 * time.Second, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + + upstreamConn, resp, err := dialer.DialContext(ctx, targetURL.String(), upstreamHeader) + if err != nil { + if resp != nil { + defer resp.Body.Close() + } + return fmt.Errorf("failed to connect upstream websocket: %w", err) + } + defer upstreamConn.Close() + + upgrader := websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { return true }, + } + + responseHeader := http.Header{} + if protocol := upstreamConn.Subprotocol(); protocol != "" { + responseHeader.Set("Sec-WebSocket-Protocol", protocol) + } + + clientConn, err := upgrader.Upgrade(w, r, responseHeader) + if err != nil { + return fmt.Errorf("failed to upgrade client websocket: %w", err) + } + defer clientConn.Close() + + errCh := make(chan error, 2) + pipe := func(dst, src *websocket.Conn) { + for { + messageType, reader, readErr := src.NextReader() + if readErr != nil { + errCh <- readErr + return + } + writer, writeErr := dst.NextWriter(messageType) + if writeErr != nil { + errCh <- writeErr + return + } + if _, copyErr := io.Copy(writer, reader); copyErr != nil { + _ = writer.Close() + errCh <- copyErr + return + } + if closeErr := writer.Close(); closeErr != nil { + errCh <- closeErr + return + } + } + } + + go pipe(upstreamConn, clientConn) + go pipe(clientConn, upstreamConn) + + select { + case <-ctx.Done(): + return nil + case <-errCh: + return nil + } +} + +// removeHopByHopHeaders removes hop-by-hop headers +func (s *InstanceProxyService) removeHopByHopHeaders(header http.Header) { + hopByHopHeaders := []string{ + "Connection", + "Keep-Alive", + "Proxy-Authenticate", + "Proxy-Authorization", + "Te", + "Trailers", + "Transfer-Encoding", + "Upgrade", + } + + for _, h := range hopByHopHeaders { + header.Del(h) + } + + // Remove headers listed in Connection header + if connections := header.Get("Connection"); connections != "" { + for _, h := range strings.Split(connections, ",") { + header.Del(strings.TrimSpace(h)) + } + } +} + +func (s *InstanceProxyService) managedRuntimeGatewayBearerToken(ctx context.Context, instanceID int, instanceType string) string { + if s == nil || s.instanceRepo == nil { + return "" + } + normalizedType, managedType := NormalizeV2RuntimeType(instanceType) + if !managedType { + return "" + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil || instance == nil { + return "" + } + if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == normalizedType { + if instance.AccessToken != nil && strings.TrimSpace(*instance.AccessToken) != "" { + return strings.TrimSpace(*instance.AccessToken) + } + } + if normalizedType == RuntimeTypeOpenClaw && s.openClawGatewayToken != "" { + return s.openClawGatewayToken + } + return "" +} + +func (s *InstanceProxyService) resolveHTTPProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPort int32, targetPath, requestPath string) (*url.URL, error) { + if targetURL, ok, err := s.resolveV2ProxyTarget(ctx, accessToken, instanceID, targetPath, requestPath, false); ok || err != nil { + return targetURL, err + } + serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort) + if err != nil { + return nil, fmt.Errorf("failed to get or create service: %w", err) + } + return &url.URL{ + Scheme: s.resolveTargetScheme(accessToken.InstanceType, false), + Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo), + Path: targetPath, + }, nil +} + +func (s *InstanceProxyService) resolveWebSocketProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPort int32, targetPath, requestPath string) (*url.URL, error) { + if targetURL, ok, err := s.resolveV2ProxyTarget(ctx, accessToken, instanceID, targetPath, requestPath, true); ok || err != nil { + return targetURL, err + } + serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort) + if err != nil { + return nil, fmt.Errorf("failed to get or create service: %w", err) + } + return &url.URL{ + Scheme: s.resolveTargetScheme(accessToken.InstanceType, true), + Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo), + Path: targetPath, + }, nil +} + +func (s *InstanceProxyService) resolveV2ProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPath, requestPath string, websocket bool) (*url.URL, bool, error) { + if s.instanceRepo == nil || s.bindingRepo == nil || s.runtimePodRepo == nil { + return nil, false, nil + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, false, fmt.Errorf("failed to get instance for proxy: %w", err) + } + if instance == nil { + return nil, false, ErrInstanceGatewayUnavailable + } + if instance.UserID != accessToken.UserID { + return nil, false, fmt.Errorf("token does not match instance owner") + } + if _, ok := v2RuntimeTypeForInstance(instance); !ok { + return nil, false, nil + } + if !strings.EqualFold(strings.TrimSpace(instance.Status), "running") { + return nil, true, ErrInstanceGatewayUnavailable + } + + binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID) + if err != nil { + return nil, true, fmt.Errorf("%w: %v", ErrInstanceGatewayUnavailable, err) + } + if binding == nil { + return nil, true, ErrInstanceGatewayUnavailable + } + if binding.Generation != instance.RuntimeGeneration { + return nil, true, ErrInstanceGatewayUnavailable + } + pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if err != nil { + return nil, true, fmt.Errorf("%w: %v", ErrInstanceGatewayUnavailable, err) + } + if pod == nil || pod.PodIP == nil || strings.TrimSpace(*pod.PodIP) == "" || binding.GatewayPort <= 0 { + return nil, true, ErrInstanceGatewayUnavailable + } + scheme := "http" + if websocket { + scheme = "ws" + } + upstreamPath := stripInstanceProxyPrefix(targetPath, instanceID) + if shouldPreserveOpenClawControlUIPath(instance) { + upstreamPath = openClawControlUIRequestPath(requestPath, instanceID) + } + return &url.URL{ + Scheme: scheme, + Host: net.JoinHostPort(strings.TrimSpace(*pod.PodIP), strconv.Itoa(binding.GatewayPort)), + Path: upstreamPath, + }, true, nil +} + +// getOrCreateService gets service info or creates the service if it doesn't exist +func (s *InstanceProxyService) getOrCreateService(ctx context.Context, userID, instanceID int, targetPort int32) (*k8s.ServiceInfo, error) { + cacheKey := serviceCacheKey{ + userID: userID, + instanceID: instanceID, + targetPort: targetPort, + } + if cached := s.getCachedService(cacheKey); cached != nil { + return cached, nil + } + + call, leader := s.getOrCreateLookup(cacheKey) + if !leader { + select { + case <-ctx.Done(): + return nil, fmt.Errorf("service lookup canceled: %w", ctx.Err()) + case <-call.done: + if call.err != nil { + return nil, call.err + } + return cloneServiceInfo(call.serviceInfo), nil + } + } + + defer s.finishLookup(cacheKey, call) + + serviceInfo, err := s.serviceService.GetServiceInfo(ctx, userID, instanceID, targetPort) + if err == nil { + s.storeCachedService(cacheKey, serviceInfo) + call.serviceInfo = cloneServiceInfo(serviceInfo) + return cloneServiceInfo(serviceInfo), nil + } + + // Try to get existing service + serviceConfig := k8s.ServiceConfig{ + InstanceID: instanceID, + InstanceName: fmt.Sprintf("instance-%d", instanceID), + UserID: userID, + ContainerPort: targetPort, + AdditionalPorts: s.getAdditionalPorts(targetPort), + } + + fmt.Printf("Service not found for instance %d, creating new service...\n", instanceID) + serviceInfo, err = s.serviceService.CreateService(ctx, serviceConfig) + if err != nil { + call.err = fmt.Errorf("failed to create service: %w", err) + return nil, call.err + } + + s.storeCachedService(cacheKey, serviceInfo) + call.serviceInfo = cloneServiceInfo(serviceInfo) + fmt.Printf("Service created successfully for instance %d (ClusterIP: %s)\n", instanceID, serviceInfo.ClusterIP) + return cloneServiceInfo(serviceInfo), nil +} + +// extractTargetPath extracts the target path from the proxy URL +// Input: /api/v1/instances/24/proxy/vnc.html +// Output: /vnc.html +func (s *InstanceProxyService) extractTargetPath(requestPath string, instanceID int, instanceType string) string { + prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + if usesWebtopImage(instanceType) { + if strings.HasPrefix(requestPath, prefix) { + path := requestPath + if path == "" { + return prefix + "/" + } + return path + } + return prefix + "/" + } + + if strings.HasPrefix(requestPath, prefix) { + path := strings.TrimPrefix(requestPath, prefix) + if path == "" { + return "/" + } + return path + } + return requestPath +} + +func stripInstanceProxyPrefix(requestPath string, instanceID int) string { + prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + if strings.HasPrefix(requestPath, prefix) { + path := strings.TrimPrefix(requestPath, prefix) + if path == "" { + return "/" + } + return path + } + return requestPath +} + +func canonicalProxyEntryRequestPath(requestPath string, accessToken *AccessToken, instanceID int) string { + prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + path := strings.TrimSpace(requestPath) + if path != prefix && path != prefix+"/" { + return requestPath + } + if accessToken == nil || strings.TrimSpace(accessToken.AccessURL) == "" { + return requestPath + } + parsed, err := url.Parse(accessToken.AccessURL) + if err != nil { + return requestPath + } + entryPath := strings.TrimSpace(parsed.Path) + if entryPath == "" || entryPath == prefix || entryPath == prefix+"/" { + return requestPath + } + if strings.HasPrefix(entryPath, prefix+"/") { + return entryPath + } + return requestPath +} + +func shouldPreserveOpenClawControlUIPath(instance *models.Instance) bool { + if instance == nil || !strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeOpenClaw) { + return false + } + _, ok := v2RuntimeTypeForInstance(instance) + return ok +} + +func openClawControlUIRequestPath(requestPath string, instanceID int) string { + prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + path := strings.TrimSpace(requestPath) + if path == "" || path == prefix { + return prefix + "/" + } + if strings.HasPrefix(path, prefix) { + return path + } + if strings.HasPrefix(path, "/") { + return prefix + path + } + return prefix + "/" + path +} + +// GetProxyURL generates a proxy URL for frontend +func (s *InstanceProxyService) GetProxyURL(instanceID int, token string) string { + return proxyURLWithPath(instanceID, "/", token) +} + +// GetProxyURLForInstance generates the best frontend entry URL for an instance. +func (s *InstanceProxyService) GetProxyURLForInstance(instance *models.Instance, token string) string { + if instance == nil { + return "" + } + if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == RuntimeTypeHermes { + return proxyURLWithPath(instance.ID, "/chat", token) + } + return proxyURLWithPath(instance.ID, "/", token) +} + +// GetTargetPortForInstance returns the service target port used by the instance type. +func (s *InstanceProxyService) GetTargetPortForInstance(instance *models.Instance) int32 { + if instance == nil { + return 3001 + } + + return buildRuntimeConfig(instance.Type, instance.OSType, instance.OSVersion, instance.ImageRegistry, instance.ImageTag).Port +} + +// ResolveUpstreamHostPort ensures the instance Service exists and returns its +// cluster-internal "host:port" target so the edge gateway can proxy directly to +// the instance without routing pixel traffic through this control-plane process. +func (s *InstanceProxyService) ResolveUpstreamHostPort(ctx context.Context, userID, instanceID int, targetPort int32) (string, error) { + serviceInfo, err := s.getOrCreateService(ctx, userID, instanceID, targetPort) + if err != nil { + return "", fmt.Errorf("failed to resolve upstream service: %w", err) + } + + return fmt.Sprintf("%s.%s.svc.cluster.local:%d", serviceInfo.Name, serviceInfo.Namespace, serviceInfo.TargetPort), nil +} + +func (s *InstanceProxyService) resolveTargetPort(instanceType string, defaultPort int32, targetPath string) int32 { + if usesWebtopImage(instanceType) { + if defaultPort == 0 { + return 3001 + } + return defaultPort + } + + if defaultPort == 0 { + defaultPort = 3000 + } + + switch { + case strings.HasPrefix(targetPath, "/websocket"), + strings.HasPrefix(targetPath, "/websockets"), + strings.HasPrefix(targetPath, "/signaling"), + strings.HasPrefix(targetPath, "/turn"): + return 8082 + default: + return defaultPort + } +} + +func (s *InstanceProxyService) getAdditionalPorts(targetPort int32) []int32 { + if targetPort == 3000 || targetPort == 8082 { + return []int32{3000, 8082} + } + + return nil +} + +func (s *InstanceProxyService) resolveTargetScheme(instanceType string, websocket bool) string { + if usesHTTPSUpstream(instanceType) { + if websocket { + return "wss" + } + return "https" + } + + if websocket { + return "ws" + } + + return "http" +} + +func usesHTTPSUpstream(instanceType string) bool { + switch instanceType { + case "ubuntu", "webtop", "hermes", "openclaw": + return true + default: + return false + } +} + +func (s *InstanceProxyService) resolveProxyHost(ctx context.Context, userID, instanceID int, serviceInfo *k8s.ServiceInfo) string { + return fmt.Sprintf("%s:%d", serviceInfo.ClusterIP, serviceInfo.TargetPort) +} + +func (s *InstanceProxyService) shouldRewriteHTML(instanceType string) bool { + return !usesWebtopImage(instanceType) +} + +func (s *InstanceProxyService) shouldRewriteHTMLForProxy(instanceID int, instanceType string) bool { + if s != nil && s.instanceRepo != nil && strings.EqualFold(strings.TrimSpace(instanceType), RuntimeTypeHermes) { + instance, err := s.instanceRepo.GetByID(instanceID) + if err == nil && instance != nil { + if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == RuntimeTypeHermes { + return true + } + } + } + return s.shouldRewriteHTML(instanceType) +} + +// IsWebtopInstanceType reports whether the instance type is served by a +// Webtop/KasmVNC desktop image (and therefore eligible for direct gateway +// proxying via SUBFOLDER-prefixed paths). +func (s *InstanceProxyService) IsWebtopInstanceType(instanceType string) bool { + return usesWebtopImage(instanceType) +} + +func (s *InstanceProxyService) getCachedService(key serviceCacheKey) *k8s.ServiceInfo { + s.cacheMu.RLock() + entry, ok := s.serviceCache[key] + s.cacheMu.RUnlock() + if !ok || time.Now().After(entry.expiresAt) { + if ok { + s.cacheMu.Lock() + delete(s.serviceCache, key) + s.cacheMu.Unlock() + } + return nil + } + + return cloneServiceInfo(entry.serviceInfo) +} + +func (s *InstanceProxyService) storeCachedService(key serviceCacheKey, serviceInfo *k8s.ServiceInfo) { + s.cacheMu.Lock() + s.serviceCache[key] = serviceCacheEntry{ + serviceInfo: cloneServiceInfo(serviceInfo), + expiresAt: time.Now().Add(s.serviceTTL), + } + s.cacheMu.Unlock() +} + +func (s *InstanceProxyService) getOrCreateLookup(key serviceCacheKey) (*serviceLookupCall, bool) { + s.lookupMu.Lock() + defer s.lookupMu.Unlock() + + if existing, ok := s.serviceLookups[key]; ok { + return existing, false + } + + call := &serviceLookupCall{ + done: make(chan struct{}), + } + s.serviceLookups[key] = call + return call, true +} + +func (s *InstanceProxyService) finishLookup(key serviceCacheKey, call *serviceLookupCall) { + s.lookupMu.Lock() + delete(s.serviceLookups, key) + close(call.done) + s.lookupMu.Unlock() +} + +func cloneServiceInfo(serviceInfo *k8s.ServiceInfo) *k8s.ServiceInfo { + if serviceInfo == nil { + return nil + } + + cloned := *serviceInfo + return &cloned +} + +func injectProxyBase(html, proxyBase string) string { + baseTag := fmt.Sprintf(``, proxyBase) + for _, tag := range []string{"", "", ""} { + if idx := strings.Index(html, tag); idx != -1 { + return html[:idx+len(tag)] + baseTag + html[idx+len(tag):] + } + } + + return baseTag + html +} + +func proxyURLWithPath(instanceID int, targetPath, token string) string { + path := strings.TrimSpace(targetPath) + if path == "" || path == "/" { + path = "/" + } else { + path = "/" + strings.TrimLeft(path, "/") + if !strings.HasSuffix(path, "/") { + path += "/" + } + } + + raw := fmt.Sprintf("/api/v1/instances/%d/proxy%s", instanceID, path) + if token == "" { + return raw + } + return fmt.Sprintf("%s?token=%s", raw, url.QueryEscape(token)) +} + +func removeProxyAccessTokenQuery(query url.Values, accessToken string) { + values := query["token"] + if len(values) == 0 { + return + } + filtered := values[:0] + for _, value := range values { + if value != accessToken { + filtered = append(filtered, value) + } + } + if len(filtered) == 0 { + query.Del("token") + return + } + query["token"] = filtered +} + +func proxyBaseForRequestPath(requestPath string, instanceID int) string { + prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + path := strings.TrimSpace(requestPath) + if strings.HasPrefix(path, prefix) { + path = strings.TrimPrefix(path, prefix) + } + if path == "" || path == "/" { + return fmt.Sprintf("%s/", prefix) + } + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if !strings.HasSuffix(path, "/") { + lastSlash := strings.LastIndex(path, "/") + if lastSlash >= 0 { + path = path[:lastSlash+1] + } else { + path = "/" + } + } + return prefix + path +} + +func websocketUpstreamOrigin(targetURL *url.URL) string { + if targetURL == nil { + return "" + } + scheme := targetURL.Scheme + switch scheme { + case "ws": + scheme = "http" + case "wss": + scheme = "https" + } + if scheme == "" || targetURL.Host == "" { + return "" + } + return scheme + "://" + targetURL.Host +} + +func (s *InstanceProxyService) openClawWebSocketOrigin(targetURL *url.URL) string { + if s != nil && s.openClawProxyOrigin != "" { + return s.openClawProxyOrigin + } + return websocketUpstreamOrigin(targetURL) +} + +func resolveOpenClawProxyOriginFromEnv() string { + for _, key := range []string{ + "OPENCLAW_PROXY_ORIGIN", + "CLAWMANAGER_TEAM_MANAGER_BASE_URL", + "CLAWMANAGER_BACKEND_URL", + } { + if origin := originFromURLString(os.Getenv(key)); origin != "" { + return origin + } + } + return "" +} + +func originFromURLString(rawURL string) string { + value := strings.TrimSpace(rawURL) + if value == "" { + return "" + } + parsed, err := url.Parse(value) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "" + } + return parsed.Scheme + "://" + parsed.Host +} + +func (s *InstanceProxyService) rewriteRedirectLocation(instanceID int, location string) string { + if strings.HasPrefix(location, "/") && !strings.HasPrefix(location, "/api/v1/instances/") { + return fmt.Sprintf("/api/v1/instances/%d/proxy%s", instanceID, location) + } + + return location +} + +func requestScheme(r *http.Request) string { + if proto := r.Header.Get("X-Forwarded-Proto"); proto != "" { + return proto + } + if r.TLS != nil { + return "https" + } + return "http" +} diff --git a/backend/internal/services/instance_proxy_service_v2_test.go b/backend/internal/services/instance_proxy_service_v2_test.go new file mode 100644 index 0000000..a73f027 --- /dev/null +++ b/backend/internal/services/instance_proxy_service_v2_test.go @@ -0,0 +1,692 @@ +package services + +import ( + "errors" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + + "github.com/gorilla/websocket" +) + +func TestInstanceProxyServiceUsesRuntimeBindingForV2(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/instances/123/proxy/apps/openclaw" { + t.Fatalf("unexpected upstream path %s", r.URL.Path) + } + if got := r.Header.Get("X-Forwarded-Prefix"); got != "/api/v1/instances/123/proxy" { + t.Fatalf("X-Forwarded-Prefix = %q", got) + } + if got := r.Header.Get("Authorization"); got != "Bearer internal-openclaw-token" { + t.Fatalf("Authorization = %q", got) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/openclaw/user-45/instance-123" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[123] = &models.Instance{ + ID: 123, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 3, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[123] = &models.InstanceRuntimeBinding{ + InstanceID: 123, + RuntimePodID: 9, + GatewayPort: gatewayPort, + State: "running", + Generation: 3, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 123, "openclaw", "/api/v1/instances/123/proxy/", "", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + service.httpClient = upstream.Client() + service.openClawGatewayToken = "internal-openclaw-token" + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/123/proxy/apps/openclaw?token="+url.QueryEscape(token.Token), nil) + rec := httptest.NewRecorder() + + if err := service.ProxyRequest(req.Context(), 123, token.Token, rec, req); err != nil { + t.Fatalf("ProxyRequest returned error: %v", err) + } + if rec.Code != http.StatusOK || rec.Body.String() != "ok" { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } +} + +func TestInstanceProxyServiceInjectsInstanceTokenForHermesLite(t *testing.T) { + instanceToken := "igt_hermes_instance" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat" { + t.Fatalf("unexpected upstream path %s", r.URL.Path) + } + if r.URL.RawQuery != "" { + t.Fatalf("unexpected upstream query %q", r.URL.RawQuery) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken { + t.Fatalf("Authorization = %q", got) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/hermes/user-45/instance-127" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[127] = &models.Instance{ + ID: 127, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + AccessToken: &instanceToken, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[127] = &models.InstanceRuntimeBinding{ + InstanceID: 127, + RuntimePodID: 10, + GatewayPort: gatewayPort, + State: "running", + Generation: 5, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 10: {ID: 10, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 127, "hermes", "/api/v1/instances/127/proxy/", "", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + service.httpClient = upstream.Client() + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/127/proxy/chat?token="+url.QueryEscape(token.Token), nil) + rec := httptest.NewRecorder() + + if err := service.ProxyRequest(req.Context(), 127, token.Token, rec, req); err != nil { + t.Fatalf("ProxyRequest returned error: %v", err) + } + if rec.Code != http.StatusOK || rec.Body.String() != "ok" { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } +} + +func TestInstanceProxyServiceUsesHermesLiteAccessURLForRootEntry(t *testing.T) { + instanceToken := "igt_hermes_instance" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat/" { + t.Fatalf("unexpected upstream path %s", r.URL.Path) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken { + t.Fatalf("Authorization = %q", got) + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("Hermesok")) + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/hermes/user-45/instance-131" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[131] = &models.Instance{ + ID: 131, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + AccessToken: &instanceToken, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[131] = &models.InstanceRuntimeBinding{ + InstanceID: 131, + RuntimePodID: 14, + GatewayPort: gatewayPort, + State: "running", + Generation: 5, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 14: {ID: 14, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 131, "hermes", "/api/v1/instances/131/proxy/chat/", "", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + service.httpClient = upstream.Client() + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/131/proxy/?token="+url.QueryEscape(token.Token), nil) + rec := httptest.NewRecorder() + + if err := service.ProxyRequest(req.Context(), 131, token.Token, rec, req); err != nil { + t.Fatalf("ProxyRequest returned error: %v", err) + } + if rec.Code != http.StatusOK { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), ``) { + t.Fatalf("expected Hermes Lite HTML to include chat proxy base, got %q", rec.Body.String()) + } +} + +func TestInstanceProxyServicePreservesHermesRuntimeQueryToken(t *testing.T) { + instanceToken := "igt_hermes_instance" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/ws" { + t.Fatalf("unexpected upstream path %s", r.URL.Path) + } + if got := r.URL.Query().Get("token"); got != "hermes-session-token" { + t.Fatalf("upstream token query = %q", got) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken { + t.Fatalf("Authorization = %q", got) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/hermes/user-45/instance-130" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[130] = &models.Instance{ + ID: 130, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + AccessToken: &instanceToken, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[130] = &models.InstanceRuntimeBinding{ + InstanceID: 130, + RuntimePodID: 13, + GatewayPort: gatewayPort, + State: "running", + Generation: 5, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 13: {ID: 13, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 130, "hermes", "/api/v1/instances/130/proxy/chat/", "", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + service.httpClient = upstream.Client() + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/130/proxy/api/ws?token=hermes-session-token", nil) + rec := httptest.NewRecorder() + + if err := service.ProxyRequest(req.Context(), 130, token.Token, rec, req); err != nil { + t.Fatalf("ProxyRequest returned error: %v", err) + } + if rec.Code != http.StatusOK || rec.Body.String() != "ok" { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } +} + +func TestInstanceProxyServiceRewritesHermesLiteHTMLBase(t *testing.T) { + instanceToken := "igt_hermes_instance" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + t.Fatalf("unexpected upstream path %s", r.URL.Path) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken { + t.Fatalf("Authorization = %q", got) + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("Hermesok")) + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/hermes/user-45/instance-128" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[128] = &models.Instance{ + ID: 128, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + AccessToken: &instanceToken, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[128] = &models.InstanceRuntimeBinding{ + InstanceID: 128, + RuntimePodID: 11, + GatewayPort: gatewayPort, + State: "running", + Generation: 5, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 11: {ID: 11, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 128, "hermes", "/api/v1/instances/128/proxy/", "", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + service.httpClient = upstream.Client() + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/128/proxy/?token="+url.QueryEscape(token.Token), nil) + rec := httptest.NewRecorder() + + if err := service.ProxyRequest(req.Context(), 128, token.Token, rec, req); err != nil { + t.Fatalf("ProxyRequest returned error: %v", err) + } + if rec.Code != http.StatusOK { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), ``) { + t.Fatalf("expected Hermes Lite HTML to include proxy base, got %q", rec.Body.String()) + } +} + +func TestInstanceProxyServiceProxiesHermesLiteWebSocket(t *testing.T) { + instanceToken := "igt_hermes_instance" + upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ws" { + t.Fatalf("unexpected upstream websocket path %s", r.URL.Path) + } + if got := r.URL.Query().Get("token"); got != "hermes-ws-token" { + t.Fatalf("upstream websocket token query = %q", got) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken { + t.Fatalf("Authorization = %q", got) + } + if got := r.Header.Get("X-Forwarded-Prefix"); got != "/api/v1/instances/129/proxy" { + t.Fatalf("X-Forwarded-Prefix = %q", got) + } + if got := r.Header.Get("X-Forwarded-Host"); got == "" { + t.Fatalf("X-Forwarded-Host is empty") + } + if got := r.Header.Get("X-Forwarded-For"); got == "" { + t.Fatalf("X-Forwarded-For is empty") + } + if got, want := r.Header.Get("Origin"), "http://"+r.Host; got != want { + t.Fatalf("Origin = %q, want %q", got, want) + } + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Fatalf("upstream websocket upgrade failed: %v", err) + } + defer conn.Close() + messageType, message, err := conn.ReadMessage() + if err != nil { + t.Fatalf("upstream websocket read failed: %v", err) + } + if string(message) != "ping" { + t.Fatalf("upstream websocket message = %q", message) + } + if err := conn.WriteMessage(messageType, []byte("pong")); err != nil { + t.Fatalf("upstream websocket write failed: %v", err) + } + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/hermes/user-45/instance-129" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[129] = &models.Instance{ + ID: 129, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + AccessToken: &instanceToken, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[129] = &models.InstanceRuntimeBinding{ + InstanceID: 129, + RuntimePodID: 12, + GatewayPort: gatewayPort, + State: "running", + Generation: 5, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 12: {ID: 12, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 129, "hermes", "/api/v1/instances/129/proxy/", "", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := service.ProxyWebSocket(r.Context(), 129, token.Token, w, r); err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + } + })) + defer proxy.Close() + + wsURL := "ws" + strings.TrimPrefix(proxy.URL, "http") + "/api/v1/instances/129/proxy/ws?token=hermes-ws-token" + clientConn, _, err := websocket.DefaultDialer.Dial(wsURL, nil) + if err != nil { + t.Fatalf("client websocket dial failed: %v", err) + } + defer clientConn.Close() + if err := clientConn.WriteMessage(websocket.TextMessage, []byte("ping")); err != nil { + t.Fatalf("client websocket write failed: %v", err) + } + _, message, err := clientConn.ReadMessage() + if err != nil { + t.Fatalf("client websocket read failed: %v", err) + } + if string(message) != "pong" { + t.Fatalf("client websocket message = %q", message) + } +} + +func TestInstanceProxyServiceUsesBaseProxyEntryForV2OpenClaw(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-123" + accessService := NewInstanceAccessService() + t.Cleanup(accessService.Stop) + service := NewInstanceProxyService(accessService) + got := service.GetProxyURLForInstance(&models.Instance{ + ID: 123, + Type: "openclaw", + RuntimeType: "gateway", + WorkspacePath: &workspacePath, + }, "token+with/slash") + + want := "/api/v1/instances/123/proxy/?token=token%2Bwith%2Fslash" + if got != want { + t.Fatalf("GetProxyURLForInstance() = %q, want %q", got, want) + } +} + +func TestInstanceProxyServiceUsesChatEntryForHermesLite(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-123" + accessService := NewInstanceAccessService() + t.Cleanup(accessService.Stop) + service := NewInstanceProxyService(accessService) + got := service.GetProxyURLForInstance(&models.Instance{ + ID: 123, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + WorkspacePath: &workspacePath, + }, "token+with/slash") + + want := "/api/v1/instances/123/proxy/chat/?token=token%2Bwith%2Fslash" + if got != want { + t.Fatalf("GetProxyURLForInstance() = %q, want %q", got, want) + } +} + +func TestProxyBaseForRequestPathUsesSubdirectory(t *testing.T) { + got := proxyBaseForRequestPath("/api/v1/instances/123/proxy/__openclaw__/canvas/", 123) + want := "/api/v1/instances/123/proxy/__openclaw__/canvas/" + if got != want { + t.Fatalf("proxyBaseForRequestPath() = %q, want %q", got, want) + } + + got = proxyBaseForRequestPath("/api/v1/instances/123/proxy/__openclaw__/canvas/app.js", 123) + if got != want { + t.Fatalf("proxyBaseForRequestPath(file) = %q, want %q", got, want) + } +} + +func TestResolveOpenClawProxyOriginFromEnvUsesInternalOrigin(t *testing.T) { + t.Setenv("OPENCLAW_PROXY_ORIGIN", "") + t.Setenv("CLAWMANAGER_BACKEND_URL", "") + t.Setenv("CLAWMANAGER_TEAM_MANAGER_BASE_URL", "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api") + + got := resolveOpenClawProxyOriginFromEnv() + want := "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + if got != want { + t.Fatalf("resolveOpenClawProxyOriginFromEnv() = %q, want %q", got, want) + } +} + +func TestOpenClawWebSocketOriginPrefersConfiguredProxyOrigin(t *testing.T) { + accessService := NewInstanceAccessService() + t.Cleanup(accessService.Stop) + service := NewInstanceProxyService(accessService) + service.openClawProxyOrigin = "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + + got := service.openClawWebSocketOrigin(&url.URL{Scheme: "ws", Host: "10.42.0.63:20000"}) + want := "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + if got != want { + t.Fatalf("openClawWebSocketOrigin() = %q, want %q", got, want) + } +} + +func TestWebSocketUpstreamOriginFallsBackToGatewayHost(t *testing.T) { + tests := []struct { + name string + url *url.URL + want string + }{ + { + name: "plain websocket", + url: &url.URL{Scheme: "ws", Host: "10.42.0.63:20000"}, + want: "http://10.42.0.63:20000", + }, + { + name: "tls websocket", + url: &url.URL{Scheme: "wss", Host: "gateway.example.test"}, + want: "https://gateway.example.test", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := websocketUpstreamOrigin(tt.url); got != tt.want { + t.Fatalf("websocketUpstreamOrigin() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestInstanceProxyServiceRejectsStoppedV2InstanceWithStaleBinding(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-125" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[125] = &models.Instance{ + ID: 125, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "stopped", + WorkspacePath: &workspacePath, + RuntimeGeneration: 4, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[125] = &models.InstanceRuntimeBinding{ + InstanceID: 125, + RuntimePodID: 9, + GatewayPort: 20025, + State: "running", + Generation: 4, + } + podIP := "10.42.0.125" + service, token := newV2ProxyTestService(t, instanceRepo, bindingRepo, &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, PodIP: &podIP}, + }, + }, 45, 125, "openclaw") + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/125/proxy/", nil) + err := service.ProxyRequest(req.Context(), 125, token.Token, httptest.NewRecorder(), req) + if !errors.Is(err, ErrInstanceGatewayUnavailable) { + t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err) + } +} + +func TestInstanceProxyServiceRejectsV2BindingGenerationMismatch(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-126" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[126] = &models.Instance{ + ID: 126, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 8, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[126] = &models.InstanceRuntimeBinding{ + InstanceID: 126, + RuntimePodID: 10, + GatewayPort: 20026, + State: "running", + Generation: 7, + } + podIP := "10.42.0.126" + service, token := newV2ProxyTestService(t, instanceRepo, bindingRepo, &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 10: {ID: 10, PodIP: &podIP}, + }, + }, 45, 126, "hermes") + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/126/proxy/", nil) + err := service.ProxyRequest(req.Context(), 126, token.Token, httptest.NewRecorder(), req) + if !errors.Is(err, ErrInstanceGatewayUnavailable) { + t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err) + } +} + +func TestInstanceProxyServiceReturnsUnavailableWhenV2BindingMissing(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-124" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[124] = &models.Instance{ + ID: 124, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 124, "hermes", "/api/v1/instances/124/proxy/", "", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = newFakeRuntimeBindingRepo() + service.runtimePodRepo = &fakeRuntimePodRepo{} + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/124/proxy/", nil) + rec := httptest.NewRecorder() + + err = service.ProxyRequest(req.Context(), 124, token.Token, rec, req) + if !errors.Is(err, ErrInstanceGatewayUnavailable) { + t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err) + } +} + +func newV2ProxyTestService(t *testing.T, instanceRepo repository.InstanceRepository, bindingRepo repository.InstanceRuntimeBindingRepository, podRepo repository.RuntimePodRepository, userID int, instanceID int, instanceType string) (*InstanceProxyService, *AccessToken) { + t.Helper() + accessService := NewInstanceAccessService() + t.Cleanup(accessService.Stop) + token, err := accessService.GenerateToken(userID, instanceID, instanceType, "/api/v1/instances/"+strconv.Itoa(instanceID)+"/proxy/", "", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + return service, token +} + +func splitURLHostPortForProxyTest(t *testing.T, rawURL string) (string, int) { + t.Helper() + parsed, err := url.Parse(rawURL) + if err != nil { + t.Fatalf("parse upstream URL: %v", err) + } + host, portText, err := net.SplitHostPort(parsed.Host) + if err != nil { + t.Fatalf("split upstream host port: %v", err) + } + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatalf("parse upstream port: %v", err) + } + return host, port +} diff --git a/backend/internal/services/instance_runtime.go b/backend/internal/services/instance_runtime.go new file mode 100644 index 0000000..aac33b2 --- /dev/null +++ b/backend/internal/services/instance_runtime.go @@ -0,0 +1,336 @@ +package services + +import ( + "fmt" + "os" + "strings" + + "clawreef/internal/services/k8s" +) + +// InstanceRuntimeConfig describes how a given instance type runs inside Kubernetes. +type InstanceRuntimeConfig struct { + Image string + Port int32 + MountPath string + Env map[string]string +} + +const ( + kasmClipboardSendDisabled = "-SendCutText 0" + kasmClipboardAcceptDisabled = "-AcceptCutText 0" +) + +func buildRuntimeConfig(instanceType, osType, osVersion string, registry, tag *string) InstanceRuntimeConfig { + if registry != nil && strings.TrimSpace(*registry) != "" && (tag == nil || strings.TrimSpace(*tag) == "") { + return InstanceRuntimeConfig{ + Image: strings.TrimSpace(*registry), + Port: defaultPortForInstanceType(instanceType), + MountPath: defaultMountPathForInstanceType(instanceType), + Env: defaultEnvForInstanceType(instanceType), + } + } + + defaultRegistry := "docker.io/clawreef" + if registry != nil && *registry != "" { + defaultRegistry = *registry + } + + defaultTag := osVersion + if tag != nil && *tag != "" { + defaultTag = *tag + } + + config := InstanceRuntimeConfig{ + Port: 3001, + MountPath: "/home/user/data", + Env: map[string]string{}, + } + + switch instanceType { + case "ubuntu": + config.Image = "lscr.io/linuxserver/webtop:ubuntu-xfce" + config.Port = 3001 + config.MountPath = "/config" + config.Env = defaultWebtopDesktopEnv("ClawManager Desktop") + case "webtop": + config.Image = "lscr.io/linuxserver/webtop:ubuntu-xfce" + config.Port = 3001 + config.MountPath = "/config" + config.Env = defaultWebtopDesktopEnv("ClawManager Webtop") + case "hermes": + config.Image = defaultSystemImageSettings["hermes"] + config.Port = 3001 + config.MountPath = "/config" + config.Env = defaultWebtopDesktopEnv("Hermes Runtime") + config.Env["HERMES_HOME"] = "/config/.hermes" + case "openclaw": + config.MountPath = "/config" + if (registry == nil || strings.TrimSpace(*registry) == "") && (tag == nil || strings.TrimSpace(*tag) == "") { + config.Image = defaultSystemImageSettings["openclaw"] + } else { + config.Image = fmt.Sprintf("%s/%s:%s", defaultRegistry, "openclaw-desktop", defaultTag) + } + config.Env = defaultWebtopDesktopEnv("ClawManager Desktop") + case "debian": + config.Image = fmt.Sprintf("%s/%s:%s", defaultRegistry, "debian-desktop", defaultTag) + case "centos": + config.Image = fmt.Sprintf("%s/%s:%s", defaultRegistry, "centos-desktop", defaultTag) + default: + config.Image = fmt.Sprintf("%s/%s:%s", defaultRegistry, fmt.Sprintf("%s-desktop", osType), defaultTag) + } + return config +} + +func defaultPortForInstanceType(instanceType string) int32 { + switch instanceType { + case "ubuntu", "webtop", "hermes": + return 3001 + default: + return 3001 + } +} + +func defaultMountPathForInstanceType(instanceType string) string { + switch instanceType { + case "ubuntu", "webtop", "openclaw": + return "/config" + case "hermes": + return "/config" + default: + return "/home/user/data" + } +} + +func defaultEnvForInstanceType(instanceType string) map[string]string { + switch instanceType { + case "ubuntu", "webtop", "openclaw": + return defaultWebtopDesktopEnv("ClawManager Desktop") + case "hermes": + env := defaultWebtopDesktopEnv("Hermes Runtime") + env["HERMES_HOME"] = "/config/.hermes" + return env + default: + return map[string]string{} + } +} + +func defaultWebtopDesktopEnv(title string) map[string]string { + return map[string]string{ + "TITLE": title, + "SUBFOLDER": "/", + "KASM_SVC_SEND_CUT_TEXT": kasmClipboardSendDisabled, + "KASM_SVC_ACCEPT_CUT_TEXT": kasmClipboardAcceptDisabled, + } +} + +func withInstanceProxyEnv(instanceType string, instanceID int, env map[string]string) map[string]string { + merged := map[string]string{} + for key, value := range env { + merged[key] = value + } + + if proxyURL, ok := defaultEgressProxyURL(); ok { + noProxy := defaultNoProxyList() + merged["HTTP_PROXY"] = proxyURL + merged["HTTPS_PROXY"] = proxyURL + merged["http_proxy"] = proxyURL + merged["https_proxy"] = proxyURL + merged["NO_PROXY"] = noProxy + merged["no_proxy"] = noProxy + } + + if usesWebtopImage(instanceType) { + merged["SUBFOLDER"] = fmt.Sprintf("/api/v1/instances/%d/proxy/", instanceID) + } + + return merged +} + +func usesWebtopImage(instanceType string) bool { + switch instanceType { + case "ubuntu", "webtop", "hermes", "openclaw": + return true + default: + return false + } +} + +// defaultImagePullPolicy returns the image pull policy to use for instance +// pods. Managed runtime instances always use IfNotPresent so local caches can +// be reused without forcing a remote registry pull during create/start flows. +func defaultImagePullPolicy() string { + return "IfNotPresent" +} + +func defaultEgressProxyURL() (string, bool) { + if override := strings.TrimSpace(os.Getenv("CLAWMANAGER_EGRESS_PROXY_URL")); override != "" { + return override, true + } + + client := k8s.GetClient() + var systemNamespace string + if overrideNamespace := strings.TrimSpace(os.Getenv("CLAWMANAGER_SYSTEM_NAMESPACE")); overrideNamespace != "" { + systemNamespace = overrideNamespace + } else if client != nil { + systemNamespace = fmt.Sprintf("%s-system", client.Namespace) + } else if baseNamespace := strings.TrimSpace(os.Getenv("K8S_NAMESPACE")); baseNamespace != "" { + systemNamespace = fmt.Sprintf("%s-system", baseNamespace) + } + + if strings.TrimSpace(systemNamespace) == "" { + return "", false + } + + serviceName := strings.TrimSpace(os.Getenv("CLAWMANAGER_EGRESS_PROXY_SERVICE_NAME")) + if serviceName == "" { + serviceName = strings.TrimSpace(os.Getenv("CLAWMANAGER_EGRESS_PROXY_SERVICE")) + } + if serviceName == "" { + serviceName = "clawmanager-egress-proxy" + } + + port := normalizePortValue( + strings.TrimSpace(os.Getenv("CLAWMANAGER_EGRESS_PROXY_SERVICE_PORT")), + strings.TrimSpace(os.Getenv("CLAWMANAGER_EGRESS_PROXY_PORT")), + ) + if port == "" { + port = "3128" + } + + return fmt.Sprintf("http://%s.%s.svc.cluster.local:%s", serviceName, systemNamespace, port), true +} + +func defaultGatewayBaseURL() (string, bool) { + if override := strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL")); override != "" { + return override, true + } + + systemNamespace := strings.TrimSpace(os.Getenv("CLAWMANAGER_SYSTEM_NAMESPACE")) + if systemNamespace == "" { + if client := k8s.GetClient(); client != nil { + systemNamespace = client.GetSystemNamespace() + } else if baseNamespace := strings.TrimSpace(os.Getenv("K8S_NAMESPACE")); baseNamespace != "" { + systemNamespace = fmt.Sprintf("%s-system", baseNamespace) + } + } + if systemNamespace == "" { + return "", false + } + + serviceName := strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GATEWAY_SERVICE_NAME")) + if serviceName == "" { + serviceName = strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GATEWAY_SERVICE")) + } + if serviceName == "" { + serviceName = "clawmanager-gateway" + } + + port := normalizePortValue( + strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GATEWAY_SERVICE_PORT")), + strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GATEWAY_PORT")), + ) + if port == "" { + port = "9001" + } + + return fmt.Sprintf("http://%s.%s.svc.cluster.local:%s/api/v1/gateway/llm", serviceName, systemNamespace, port), true +} + +func defaultAgentControlBaseURL() (string, bool) { + if override := strings.TrimSpace(os.Getenv("CLAWMANAGER_AGENT_CONTROL_BASE_URL")); override != "" { + return override, true + } + + systemNamespace := strings.TrimSpace(os.Getenv("CLAWMANAGER_SYSTEM_NAMESPACE")) + if systemNamespace == "" { + if client := k8s.GetClient(); client != nil { + systemNamespace = client.GetSystemNamespace() + } else if baseNamespace := strings.TrimSpace(os.Getenv("K8S_NAMESPACE")); baseNamespace != "" { + systemNamespace = fmt.Sprintf("%s-system", baseNamespace) + } + } + if systemNamespace == "" { + return "", false + } + + serviceName := strings.TrimSpace(os.Getenv("CLAWMANAGER_AGENT_CONTROL_SERVICE_NAME")) + if serviceName == "" { + serviceName = strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GATEWAY_SERVICE_NAME")) + } + if serviceName == "" { + serviceName = strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GATEWAY_SERVICE")) + } + if serviceName == "" { + serviceName = "clawmanager-gateway" + } + + port := normalizePortValue( + strings.TrimSpace(os.Getenv("CLAWMANAGER_AGENT_CONTROL_SERVICE_PORT")), + strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GATEWAY_SERVICE_PORT")), + strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GATEWAY_PORT")), + ) + if port == "" { + port = "9001" + } + + return fmt.Sprintf("http://%s.%s.svc.cluster.local:%s", serviceName, systemNamespace, port), true +} + +func defaultNoProxyList() string { + if override := strings.TrimSpace(os.Getenv("CLAWMANAGER_NO_PROXY")); override != "" { + return override + } + + systemNamespace := strings.TrimSpace(os.Getenv("CLAWMANAGER_SYSTEM_NAMESPACE")) + if systemNamespace == "" { + if client := k8s.GetClient(); client != nil { + systemNamespace = fmt.Sprintf("%s-system", client.Namespace) + } else if baseNamespace := strings.TrimSpace(os.Getenv("K8S_NAMESPACE")); baseNamespace != "" { + systemNamespace = fmt.Sprintf("%s-system", baseNamespace) + } + } + + serviceNames := []string{ + "localhost", + "127.0.0.1", + "clawmanager-frontend", + "clawmanager-gateway", + "clawmanager-egress-proxy", + } + + if systemNamespace != "" { + serviceNames = append(serviceNames, + fmt.Sprintf("clawmanager-frontend.%s", systemNamespace), + fmt.Sprintf("clawmanager-frontend.%s.svc", systemNamespace), + fmt.Sprintf("clawmanager-frontend.%s.svc.cluster.local", systemNamespace), + fmt.Sprintf("clawmanager-gateway.%s", systemNamespace), + fmt.Sprintf("clawmanager-gateway.%s.svc", systemNamespace), + fmt.Sprintf("clawmanager-gateway.%s.svc.cluster.local", systemNamespace), + fmt.Sprintf("clawmanager-egress-proxy.%s", systemNamespace), + fmt.Sprintf("clawmanager-egress-proxy.%s.svc", systemNamespace), + fmt.Sprintf("clawmanager-egress-proxy.%s.svc.cluster.local", systemNamespace), + ) + } + + return strings.Join(serviceNames, ",") +} + +func normalizePortValue(values ...string) string { + for _, raw := range values { + value := strings.TrimSpace(raw) + if value == "" { + continue + } + if !strings.Contains(value, "://") { + return value + } + + lastColon := strings.LastIndex(value, ":") + if lastColon >= 0 && lastColon < len(value)-1 { + return value[lastColon+1:] + } + } + + return "" +} diff --git a/backend/internal/services/instance_runtime_status_service.go b/backend/internal/services/instance_runtime_status_service.go new file mode 100644 index 0000000..25964ac --- /dev/null +++ b/backend/internal/services/instance_runtime_status_service.go @@ -0,0 +1,190 @@ +package services + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +type AgentStateReportRequest struct { + AgentID string `json:"agent_id" binding:"required"` + ReportedAt *time.Time `json:"reported_at,omitempty"` + Runtime AgentRuntimePayload `json:"runtime"` + SystemInfo map[string]interface{} `json:"system_info"` + Health map[string]interface{} `json:"health"` +} + +type AgentRuntimePayload struct { + OpenClawStatus string `json:"openclaw_status"` + OpenClawPID *int `json:"openclaw_pid,omitempty"` + OpenClawVersion string `json:"openclaw_version"` + CurrentConfigRevisionID *int `json:"current_config_revision_id,omitempty"` +} + +type InstanceRuntimeStatusPayload struct { + InstanceID int `json:"instance_id"` + InfraStatus string `json:"infra_status"` + AgentStatus string `json:"agent_status"` + OpenClawStatus string `json:"openclaw_status"` + OpenClawPID *int `json:"openclaw_pid,omitempty"` + OpenClawVersion *string `json:"openclaw_version,omitempty"` + CurrentConfigRevisionID *int `json:"current_config_revision_id,omitempty"` + DesiredConfigRevisionID *int `json:"desired_config_revision_id,omitempty"` + SystemInfo map[string]interface{} `json:"system_info,omitempty"` + Health map[string]interface{} `json:"health,omitempty"` + Summary map[string]interface{} `json:"summary,omitempty"` + LastReportedAt *time.Time `json:"last_reported_at,omitempty"` +} + +type InstanceRuntimeStatusService interface { + Report(session *AgentSession, req AgentStateReportRequest, clientIP string) error + GetByInstanceID(instanceID int) (*InstanceRuntimeStatusPayload, error) + UpsertInfraStatus(instanceID int, infraStatus string) error +} + +type instanceRuntimeStatusService struct { + runtimeRepo repository.InstanceRuntimeStatusRepository + agentRepo repository.InstanceAgentRepository + desiredStateRepo repository.InstanceDesiredStateRepository +} + +func NewInstanceRuntimeStatusService(runtimeRepo repository.InstanceRuntimeStatusRepository, agentRepo repository.InstanceAgentRepository, desiredStateRepo repository.InstanceDesiredStateRepository) InstanceRuntimeStatusService { + return &instanceRuntimeStatusService{ + runtimeRepo: runtimeRepo, + agentRepo: agentRepo, + desiredStateRepo: desiredStateRepo, + } +} + +func (s *instanceRuntimeStatusService) Report(session *AgentSession, req AgentStateReportRequest, clientIP string) error { + if session == nil || session.Agent == nil || session.Instance == nil { + return fmt.Errorf("agent session is required") + } + if strings.TrimSpace(req.AgentID) != session.Agent.AgentID { + return fmt.Errorf("agent id does not match session") + } + + status, err := s.getOrCreate(session.Instance.ID) + if err != nil { + return err + } + now := time.Now().UTC() + reportedAt := req.ReportedAt + if reportedAt == nil || reportedAt.IsZero() { + reportedAt = &now + } + + status.AgentStatus = agentStatusOnline + if strings.TrimSpace(req.Runtime.OpenClawStatus) != "" { + status.OpenClawStatus = strings.TrimSpace(req.Runtime.OpenClawStatus) + } + status.OpenClawPID = req.Runtime.OpenClawPID + if strings.TrimSpace(req.Runtime.OpenClawVersion) != "" { + version := strings.TrimSpace(req.Runtime.OpenClawVersion) + status.OpenClawVersion = &version + } + status.CurrentConfigRevisionID = req.Runtime.CurrentConfigRevisionID + status.LastReportedAt = reportedAt + + systemInfoJSON, err := marshalOptionalJSON(req.SystemInfo) + if err != nil { + return fmt.Errorf("failed to encode system info: %w", err) + } + healthJSON, err := marshalOptionalJSON(req.Health) + if err != nil { + return fmt.Errorf("failed to encode health info: %w", err) + } + status.SystemInfoJSON = systemInfoJSON + status.HealthJSON = healthJSON + if err := s.runtimeRepo.Update(status); err != nil { + return err + } + + session.Agent.Status = agentStatusOnline + session.Agent.LastHeartbeatAt = reportedAt + session.Agent.LastReportedAt = reportedAt + session.Agent.LastSeenIP = optionalString(strings.TrimSpace(clientIP)) + if err := s.agentRepo.Update(session.Agent); err != nil { + return err + } + + return nil +} + +func (s *instanceRuntimeStatusService) GetByInstanceID(instanceID int) (*InstanceRuntimeStatusPayload, error) { + status, err := s.runtimeRepo.GetByInstanceID(instanceID) + if err != nil { + return nil, err + } + if status == nil { + return nil, nil + } + payload := &InstanceRuntimeStatusPayload{ + InstanceID: status.InstanceID, + InfraStatus: status.InfraStatus, + AgentStatus: status.AgentStatus, + OpenClawStatus: status.OpenClawStatus, + OpenClawPID: status.OpenClawPID, + OpenClawVersion: status.OpenClawVersion, + CurrentConfigRevisionID: status.CurrentConfigRevisionID, + DesiredConfigRevisionID: status.DesiredConfigRevisionID, + LastReportedAt: status.LastReportedAt, + } + if status.SystemInfoJSON != nil && strings.TrimSpace(*status.SystemInfoJSON) != "" { + if err := json.Unmarshal([]byte(*status.SystemInfoJSON), &payload.SystemInfo); err != nil { + return nil, fmt.Errorf("failed to decode system info: %w", err) + } + } + if status.HealthJSON != nil && strings.TrimSpace(*status.HealthJSON) != "" { + if err := json.Unmarshal([]byte(*status.HealthJSON), &payload.Health); err != nil { + return nil, fmt.Errorf("failed to decode health info: %w", err) + } + } + if status.SummaryJSON != nil && strings.TrimSpace(*status.SummaryJSON) != "" { + if err := json.Unmarshal([]byte(*status.SummaryJSON), &payload.Summary); err != nil { + return nil, fmt.Errorf("failed to decode runtime summary: %w", err) + } + } + return payload, nil +} + +func (s *instanceRuntimeStatusService) UpsertInfraStatus(instanceID int, infraStatus string) error { + status, err := s.getOrCreate(instanceID) + if err != nil { + return err + } + status.InfraStatus = infraStatus + + desiredState, err := s.desiredStateRepo.GetByInstanceID(instanceID) + if err == nil && desiredState != nil { + status.DesiredConfigRevisionID = desiredState.DesiredConfigRevisionID + } + return s.runtimeRepo.Update(status) +} + +func (s *instanceRuntimeStatusService) getOrCreate(instanceID int) (*models.InstanceRuntimeStatus, error) { + status, err := s.runtimeRepo.GetByInstanceID(instanceID) + if err != nil { + return nil, err + } + if status != nil { + return status, nil + } + now := time.Now().UTC() + status = &models.InstanceRuntimeStatus{ + InstanceID: instanceID, + InfraStatus: "creating", + AgentStatus: agentStatusOffline, + OpenClawStatus: openClawStatusUnknown, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.runtimeRepo.Create(status); err != nil { + return nil, err + } + return status, nil +} diff --git a/backend/internal/services/instance_runtime_test.go b/backend/internal/services/instance_runtime_test.go new file mode 100644 index 0000000..5950fe2 --- /dev/null +++ b/backend/internal/services/instance_runtime_test.go @@ -0,0 +1,60 @@ +package services + +import "testing" + +func TestDefaultImagePullPolicy_Default(t *testing.T) { + t.Setenv("IMAGE_PULL_POLICY", "") + got := defaultImagePullPolicy() + if got != "IfNotPresent" { + t.Fatalf("expected IfNotPresent, got %q", got) + } +} + +func TestDefaultImagePullPolicy_IgnoresEnvOverride(t *testing.T) { + for _, envValue := range []string{"Always", "Never", "IfNotPresent", " "} { + t.Run(envValue, func(t *testing.T) { + t.Setenv("IMAGE_PULL_POLICY", envValue) + got := defaultImagePullPolicy() + if got != "IfNotPresent" { + t.Fatalf("expected IfNotPresent, got %q", got) + } + }) + } +} + +func TestBuildRuntimeConfig_HermesUsesWebtopDefaults(t *testing.T) { + config := buildRuntimeConfig("hermes", "hermes", "latest", nil, nil) + + if config.Port != 3001 { + t.Fatalf("expected Hermes port 3001, got %d", config.Port) + } + if config.MountPath != "/config" { + t.Fatalf("expected Hermes mount path /config, got %q", config.MountPath) + } + if config.Env["HERMES_HOME"] != "/config/.hermes" { + t.Fatalf("expected Hermes HERMES_HOME /config/.hermes, got %q", config.Env["HERMES_HOME"]) + } + if config.Env["SUBFOLDER"] != "/" { + t.Fatalf("expected Hermes default SUBFOLDER /, got %q", config.Env["SUBFOLDER"]) + } + if config.Env["KASM_SVC_SEND_CUT_TEXT"] != kasmClipboardSendDisabled { + t.Fatalf("expected Hermes to disable outbound clipboard sync, got %q", config.Env["KASM_SVC_SEND_CUT_TEXT"]) + } + if config.Env["KASM_SVC_ACCEPT_CUT_TEXT"] != kasmClipboardAcceptDisabled { + t.Fatalf("expected Hermes to disable inbound clipboard sync, got %q", config.Env["KASM_SVC_ACCEPT_CUT_TEXT"]) + } + if !usesWebtopImage("hermes") { + t.Fatalf("expected Hermes to use webtop proxy behavior") + } +} + +func TestBuildRuntimeConfig_OpenClawDisablesKasmClipboardSync(t *testing.T) { + config := buildRuntimeConfig("openclaw", "openclaw", "latest", nil, nil) + + if config.Env["KASM_SVC_SEND_CUT_TEXT"] != kasmClipboardSendDisabled { + t.Fatalf("expected OpenClaw to disable outbound clipboard sync, got %q", config.Env["KASM_SVC_SEND_CUT_TEXT"]) + } + if config.Env["KASM_SVC_ACCEPT_CUT_TEXT"] != kasmClipboardAcceptDisabled { + t.Fatalf("expected OpenClaw to disable inbound clipboard sync, got %q", config.Env["KASM_SVC_ACCEPT_CUT_TEXT"]) + } +} diff --git a/backend/internal/services/instance_service.go b/backend/internal/services/instance_service.go new file mode 100644 index 0000000..87f7410 --- /dev/null +++ b/backend/internal/services/instance_service.go @@ -0,0 +1,2216 @@ +package services + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path" + "strconv" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// InstanceService defines the interface for instance operations +type InstanceService interface { + Create(userID int, req CreateInstanceRequest) (*models.Instance, error) + ValidateCreateRequests(userID int, requests []CreateInstanceRequest) error + GetByID(id int) (*models.Instance, error) + GetByUserID(userID int, offset, limit int) ([]models.Instance, int, error) + GetAllInstances(offset, limit int) ([]models.Instance, int, error) + Start(instanceID int) error + Stop(instanceID int) error + Restart(instanceID int) error + Delete(instanceID int) error + Update(instanceID int, req UpdateInstanceRequest) error + GetInstanceStatus(instanceID int) (*InstanceStatus, error) + ForceSyncInstance(instanceID int) error +} + +func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateInstanceRequest) error { + if len(requests) == 0 { + return nil + } + for idx := range requests { + requests[idx].Name = strings.TrimSpace(requests[idx].Name) + if requests[idx].Name == "" { + return fmt.Errorf("instance name is required") + } + environmentOverrides, err := normalizeEnvironmentOverrides(requests[idx].EnvironmentOverrides) + if err != nil { + return err + } + if _, err := marshalEnvironmentOverrides(environmentOverrides); err != nil { + return err + } + if _, ok := normalizeDesktopStreamProfile(requests[idx].DesktopStreamProfile); !ok { + return fmt.Errorf("invalid desktop stream profile") + } + } + + quota, err := s.quotaRepo.GetByUserID(userID) + if err != nil { + return fmt.Errorf("failed to get user quota: %w", err) + } + if quota == nil { + return fmt.Errorf("user quota not found") + } + + currentCount, err := s.instanceRepo.CountByUserID(userID) + if err != nil { + return fmt.Errorf("failed to count instances: %w", err) + } + if currentCount+len(requests) > quota.MaxInstances { + return fmt.Errorf("instance limit reached: %d/%d", currentCount+len(requests), quota.MaxInstances) + } + + existingInstances, err := s.instanceRepo.GetByUserID(userID, 0, 1000) + if err != nil { + return fmt.Errorf("failed to list user instances for quota validation: %w", err) + } + + currentCPU := 0.0 + currentMemory := 0 + currentStorage := 0 + currentGPU := 0 + existingNames := map[string]struct{}{} + for _, existing := range existingInstances { + if instanceModeUsesDedicatedResources(modeForExistingInstance(&existing)) { + currentCPU += existing.CPUCores + currentMemory += existing.MemoryGB + currentStorage += existing.DiskGB + if existing.GPUEnabled { + currentGPU += existing.GPUCount + } + } + existingNames[strings.TrimSpace(strings.ToLower(existing.Name))] = struct{}{} + } + + requestedCPU := 0.0 + requestedMemory := 0 + requestedStorage := 0 + requestedGPU := 0 + requestNames := map[string]struct{}{} + for _, req := range requests { + normalizedName := strings.TrimSpace(strings.ToLower(req.Name)) + if _, exists := existingNames[normalizedName]; exists { + return fmt.Errorf("instance name already exists") + } + if _, exists := requestNames[normalizedName]; exists { + return fmt.Errorf("instance name already exists") + } + requestNames[normalizedName] = struct{}{} + if instanceModeUsesDedicatedResources(resolveCreateInstanceMode(req)) { + requestedCPU += req.CPUCores + requestedMemory += req.MemoryGB + requestedStorage += req.DiskGB + if req.GPUEnabled { + requestedGPU += req.GPUCount + } + } + } + + if currentCPU+requestedCPU > quota.MaxCPUCores { + return fmt.Errorf("CPU cores exceed quota: current %v, requested %v, max %v", currentCPU, requestedCPU, quota.MaxCPUCores) + } + if currentMemory+requestedMemory > quota.MaxMemoryGB { + return fmt.Errorf("memory exceed quota: current %dGB, requested %dGB, max %dGB", currentMemory, requestedMemory, quota.MaxMemoryGB) + } + if currentStorage+requestedStorage > quota.MaxStorageGB { + return fmt.Errorf("storage exceed quota: current %dGB, requested %dGB, max %dGB", currentStorage, requestedStorage, quota.MaxStorageGB) + } + if currentGPU+requestedGPU > quota.MaxGPUCount { + return fmt.Errorf("GPU count exceed quota: current %d, requested %d, max %d", currentGPU, requestedGPU, quota.MaxGPUCount) + } + + return nil +} + +// CreateInstanceRequest holds data for creating an instance +type CreateInstanceRequest struct { + Name string `json:"name" validate:"required,min=3,max=50"` + Description *string `json:"description,omitempty"` + Type string `json:"type" validate:"required,oneof=openclaw ubuntu debian centos custom webtop hermes"` + Mode string `json:"mode" validate:"omitempty,oneof=lite pro"` + InstanceMode string `json:"instance_mode" validate:"omitempty,oneof=lite pro"` + RuntimeType string `json:"runtime_type" validate:"omitempty,oneof=gateway desktop shell"` + DesktopStreamProfile string `json:"desktop_stream_profile,omitempty" validate:"omitempty,oneof=low standard high"` + CPUCores float64 `json:"cpu_cores" validate:"required,min=0.1,max=32"` + MemoryGB int `json:"memory_gb" validate:"required,min=1,max=128"` + DiskGB int `json:"disk_gb" validate:"required,min=10,max=1000"` + GPUEnabled bool `json:"gpu_enabled"` + GPUCount int `json:"gpu_count" validate:"min=0,max=4"` + OSType string `json:"os_type" validate:"required"` + OSVersion string `json:"os_version" validate:"required"` + ImageRegistry *string `json:"image_registry,omitempty"` + ImageTag *string `json:"image_tag,omitempty"` + EnvironmentOverrides map[string]string `json:"environment_overrides,omitempty"` + StorageClass string `json:"storage_class"` + OpenClawConfigPlan *OpenClawConfigPlan `json:"openclaw_config_plan,omitempty"` + Team *TeamInstanceConfig `json:"-"` +} + +type TeamInstanceConfig struct { + Environment map[string]string + SecretName string + SharedPVCName string + SharedMountPath string + ConfigMapName string + ConfigMountPath string + PersonaConfigKey string + SharedUID int64 + SharedGID int64 + SharedUmask string +} + +type instanceModeLimitConfig struct { + Capacity *int + MaxCPU *float64 + MaxMemoryGB *int + MaxStorageGB *int + MaxGPUCount *int +} + +// UpdateInstanceRequest holds data for updating an instance +type UpdateInstanceRequest struct { + Name *string `json:"name,omitempty" validate:"omitempty,min=3,max=50"` + Description *string `json:"description,omitempty"` + DesktopStreamProfile *string `json:"desktop_stream_profile,omitempty" validate:"omitempty,oneof=low standard high"` +} + +// InstanceStatus holds the status of an instance +type InstanceStatus struct { + InstanceID int `json:"instance_id"` + Status string `json:"status"` + Availability string `json:"availability,omitempty"` + AgentType string `json:"agent_type,omitempty"` + WorkspaceUsageBytes int64 `json:"workspace_usage_bytes,omitempty"` + PodName *string `json:"pod_name,omitempty"` + PodNamespace *string `json:"pod_namespace,omitempty"` + PodIP *string `json:"pod_ip,omitempty"` + PodStatus string `json:"pod_status,omitempty"` + CreatedAt time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at,omitempty"` +} + +// instanceService implements InstanceService +type instanceService struct { + instanceRepo repository.InstanceRepository + quotaRepo repository.QuotaRepository + llmModelRepo repository.LLMModelRepository + openClawConfigService OpenClawConfigService + allowPrivilegedPods bool + runtimePodRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + agentClient RuntimeAgentClient + workspaceRoot string + podService *k8s.PodService + deploymentService *k8s.InstanceDeploymentService + pvcService *k8s.PVCService + serviceService *k8s.ServiceService + networkPolicyService *k8s.NetworkPolicyService +} + +type gatewayModelInjection struct { + defaultModel string + modelsJSON string +} + +type InstanceServiceOption func(*instanceService) + +func WithPrivilegedInstancePods(allowed bool) InstanceServiceOption { + return func(s *instanceService) { + s.allowPrivilegedPods = allowed + } +} + +func WithV2RuntimeLifecycle(runtimePodRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, agentClient RuntimeAgentClient, workspaceRoot string) InstanceServiceOption { + return func(s *instanceService) { + s.runtimePodRepo = runtimePodRepo + s.bindingRepo = bindingRepo + s.agentClient = agentClient + if strings.TrimSpace(workspaceRoot) != "" { + s.workspaceRoot = strings.TrimSpace(workspaceRoot) + } + } +} + +// NewInstanceService creates a new instance service +func NewInstanceService(instanceRepo repository.InstanceRepository, quotaRepo repository.QuotaRepository, llmModelRepo repository.LLMModelRepository, openClawConfigService OpenClawConfigService, options ...InstanceServiceOption) InstanceService { + service := &instanceService{ + instanceRepo: instanceRepo, + quotaRepo: quotaRepo, + llmModelRepo: llmModelRepo, + openClawConfigService: openClawConfigService, + workspaceRoot: "/workspaces", + podService: k8s.NewPodService(), + deploymentService: k8s.NewInstanceDeploymentService(), + pvcService: k8s.NewPVCService(), + serviceService: k8s.NewServiceService(), + networkPolicyService: k8s.NewNetworkPolicyService(), + } + for _, option := range options { + if option != nil { + option(service) + } + } + return service +} + +// Create creates a new instance +func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models.Instance, error) { + ctx := context.Background() + req.Name = strings.TrimSpace(req.Name) + req.Type = strings.ToLower(strings.TrimSpace(req.Type)) + environmentOverrides, err := normalizeEnvironmentOverrides(req.EnvironmentOverrides) + if err != nil { + return nil, err + } + if profile, ok := normalizeDesktopStreamProfile(req.DesktopStreamProfile); !ok { + return nil, fmt.Errorf("invalid desktop stream profile") + } else if profile != "" { + environmentOverrides = applyDesktopStreamProfileEnv(environmentOverrides, profile) + } + environmentOverridesJSON, err := marshalEnvironmentOverrides(environmentOverrides) + if err != nil { + return nil, err + } + + // Check user quota + quota, err := s.quotaRepo.GetByUserID(userID) + if err != nil { + return nil, fmt.Errorf("failed to get user quota: %w", err) + } + + if quota == nil { + return nil, fmt.Errorf("user quota not found") + } + instanceMode := resolveCreateInstanceMode(req) + modeRuntimeType, _ := RuntimeTypeForInstanceMode(instanceMode) + if !hasExplicitCreateInstanceMode(req) && normalizeInstanceRuntimeType(req.RuntimeType) == RuntimeBackendShell { + modeRuntimeType = RuntimeBackendShell + } + + // Check instance count limit + currentCount, err := s.instanceRepo.CountByUserID(userID) + if err != nil { + return nil, fmt.Errorf("failed to count instances: %w", err) + } + + if currentCount >= quota.MaxInstances { + return nil, fmt.Errorf("instance limit reached: %d/%d", currentCount, quota.MaxInstances) + } + + existingInstances, err := s.instanceRepo.GetByUserID(userID, 0, 1000) + if err != nil { + return nil, fmt.Errorf("failed to list user instances for quota validation: %w", err) + } + + currentCPU := 0.0 + currentMemory := 0 + currentStorage := 0 + currentGPU := 0 + for _, existing := range existingInstances { + if instanceModeUsesDedicatedResources(modeForExistingInstance(&existing)) { + currentCPU += existing.CPUCores + currentMemory += existing.MemoryGB + currentStorage += existing.DiskGB + if existing.GPUEnabled { + currentGPU += existing.GPUCount + } + } + } + + nameExists, err := s.instanceRepo.ExistsByUserIDAndName(userID, req.Name) + if err != nil { + return nil, fmt.Errorf("failed to validate instance name: %w", err) + } + if nameExists { + return nil, fmt.Errorf("instance name already exists") + } + + requestedGPU := 0 + if req.GPUEnabled { + requestedGPU = req.GPUCount + } + if instanceModeUsesDedicatedResources(instanceMode) { + // Check CPU limit + if currentCPU+req.CPUCores > quota.MaxCPUCores { + return nil, fmt.Errorf("CPU cores exceed quota: current %v, requested %v, max %v", currentCPU, req.CPUCores, quota.MaxCPUCores) + } + + // Check memory limit + if currentMemory+req.MemoryGB > quota.MaxMemoryGB { + return nil, fmt.Errorf("memory exceed quota: current %dGB, requested %dGB, max %dGB", currentMemory, req.MemoryGB, quota.MaxMemoryGB) + } + + // Check storage limit + if currentStorage+req.DiskGB > quota.MaxStorageGB { + return nil, fmt.Errorf("storage exceed quota: current %dGB, requested %dGB, max %dGB", currentStorage, req.DiskGB, quota.MaxStorageGB) + } + + // Check GPU limit + if currentGPU+requestedGPU > quota.MaxGPUCount { + return nil, fmt.Errorf("GPU count exceed quota: current %d, requested %d, max %d", currentGPU, requestedGPU, quota.MaxGPUCount) + } + } + if err := s.enforceInstanceModeLimits(ctx, instanceMode, req.CPUCores, req.MemoryGB, req.DiskGB, requestedGPU); err != nil { + return nil, err + } + if runtimeType, isV2 := NormalizeV2RuntimeType(req.Type); isV2 && instanceMode == InstanceModeLite { + return s.createV2Instance(ctx, userID, req, runtimeType, environmentOverridesJSON) + } + + runtimeConfig := buildRuntimeConfig(req.Type, req.OSType, req.OSVersion, req.ImageRegistry, req.ImageTag) + runtimeType := normalizeInstanceRuntimeType(req.RuntimeType) + if modeRuntimeType != "" { + runtimeType = modeRuntimeType + } + if (req.ImageRegistry == nil || strings.TrimSpace(*req.ImageRegistry) == "") && (req.ImageTag == nil || strings.TrimSpace(*req.ImageTag) == "") { + if selection, ok := runtimeImageOverride(req.Type); ok { + image := selection.Image + req.ImageRegistry = &image + req.ImageTag = nil + if modeRuntimeType == "" { + runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType) + } + runtimeConfig = buildRuntimeConfig(req.Type, req.OSType, req.OSVersion, req.ImageRegistry, req.ImageTag) + } + } else if req.ImageRegistry != nil { + if selection, ok := runtimeImageOverrideForImage(req.Type, *req.ImageRegistry); ok { + if modeRuntimeType == "" { + runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType) + } + } + } + + // Check if there are any orphaned resources from previous failed creations + fmt.Printf("Checking for orphaned resources for user %d before creating new instance...\n", userID) + s.cleanupOrphanedResourcesByUser(ctx, userID) + + // Create instance record + now := time.Now() + instance := &models.Instance{ + UserID: userID, + Name: req.Name, + Description: req.Description, + Type: req.Type, + RuntimeType: runtimeType, + InstanceMode: InstanceModeForRuntimeType(runtimeType), + Status: "creating", + CPUCores: req.CPUCores, + MemoryGB: req.MemoryGB, + DiskGB: req.DiskGB, + GPUEnabled: req.GPUEnabled, + GPUCount: req.GPUCount, + OSType: req.OSType, + OSVersion: req.OSVersion, + ImageRegistry: req.ImageRegistry, + ImageTag: req.ImageTag, + EnvironmentOverridesJSON: environmentOverridesJSON, + StorageClass: req.StorageClass, + MountPath: runtimeConfig.MountPath, + CreatedAt: now, + UpdatedAt: now, + } + + if err := s.instanceRepo.Create(instance); err != nil { + return nil, fmt.Errorf("failed to create instance record: %w", err) + } + + if _, err := s.ensureGatewayToken(instance); err != nil { + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to provision instance gateway token: %w", err) + } + if _, err := s.ensureAgentBootstrapToken(instance); err != nil { + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to provision instance agent bootstrap token: %w", err) + } + + gatewayEnv, err := s.buildGatewayEnv(instance) + if err != nil { + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to build instance gateway config: %w", err) + } + agentEnv, err := s.buildAgentEnv(instance) + if err != nil { + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to build instance agent config: %w", err) + } + extraEnv, err := buildInstancePodEnv(instance, runtimeConfig.Env, gatewayEnv, agentEnv) + if err != nil { + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to resolve instance environment: %w", err) + } + if req.Team != nil { + extraEnv = mergeEnvMaps(extraEnv, req.Team.Environment) + } + + var bootstrapSnapshot *models.OpenClawInjectionSnapshot + var bootstrapSecretName string + if supportsRuntimeConfigInjection(instance.Type) && s.openClawConfigService != nil && req.OpenClawConfigPlan != nil && hasOpenClawConfigSelections(*req.OpenClawConfigPlan) { + bootstrapSnapshot, err = s.openClawConfigService.CreateSnapshotForInstance(userID, instance, req.OpenClawConfigPlan) + if err != nil { + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to compile runtime bootstrap config: %w", err) + } + if bootstrapSnapshot != nil { + instance.OpenClawConfigSnapshotID = &bootstrapSnapshot.ID + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to persist runtime snapshot reference: %w", err) + } + + bootstrapSecretName, err = s.openClawConfigService.EnsureSnapshotSecret(ctx, userID, instance, bootstrapSnapshot.ID) + if err != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to provision runtime bootstrap secret: %w", err) + } + } + } + + // Create PVC + // If storage class is not specified in request, use empty string + // PVCService will use the default from K8s client config + storageClass := req.StorageClass + + _, err = s.pvcService.CreatePVC(ctx, userID, instance.ID, req.DiskGB, storageClass) + if err != nil { + // Rollback: delete instance record + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + } + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to create PVC: %w", err) + } + + nodeSelector, err := s.pvcService.NodeSelectorForPVC(ctx, userID, instance.ID, storageClass) + if err != nil { + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + } + s.pvcService.DeletePVC(ctx, userID, instance.ID) + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to resolve PVC node selector: %w", err) + } + + // Ensure any legacy per-instance network policy is removed before creating pod. + // This keeps new pods unrestricted even if older versions created netpols. + if err := s.networkPolicyService.DeletePolicy(ctx, userID, instance.ID, instance.Name); err != nil { + s.pvcService.DeletePVC(ctx, userID, instance.ID) + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + } + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to delete network policy: %w", err) + } + + // Create Pod + shmSizeGB := popSHMSizeGB(extraEnv, runtimeType, instance.MemoryGB) + envFromSecretNames := []string{bootstrapSecretName} + extraPVCMounts := []k8s.PVCMount{} + configMapFileMounts := []k8s.ConfigMapFileMount{} + volumeOwnershipFixes := []k8s.VolumeOwnershipFix{} + var fsGroup *int64 + if req.Team != nil { + if strings.TrimSpace(req.Team.SecretName) != "" { + envFromSecretNames = append(envFromSecretNames, strings.TrimSpace(req.Team.SecretName)) + } + if strings.TrimSpace(req.Team.SharedPVCName) != "" && strings.TrimSpace(req.Team.SharedMountPath) != "" { + sharedMountPath := strings.TrimSpace(req.Team.SharedMountPath) + extraPVCMounts = append(extraPVCMounts, k8s.PVCMount{ + Name: "team-shared", + ClaimName: strings.TrimSpace(req.Team.SharedPVCName), + MountPath: sharedMountPath, + }) + sharedUID := req.Team.SharedUID + if sharedUID <= 0 { + sharedUID = 1000 + } + sharedGID := req.Team.SharedGID + if sharedGID <= 0 { + sharedGID = 1000 + } + fsGroupValue := sharedGID + fsGroup = &fsGroupValue + volumeOwnershipFixes = append(volumeOwnershipFixes, k8s.VolumeOwnershipFix{ + Name: "team-shared", + MountPath: sharedMountPath, + UID: sharedUID, + GID: sharedGID, + }) + } + if strings.TrimSpace(req.Team.ConfigMapName) != "" && strings.TrimSpace(req.Team.ConfigMountPath) != "" { + configMapFileMounts = append(configMapFileMounts, k8s.ConfigMapFileMount{ + Name: "team-config", + ConfigMapName: strings.TrimSpace(req.Team.ConfigMapName), + Key: "team.json", + MountPath: strings.TrimSpace(req.Team.ConfigMountPath), + ReadOnly: true, + AsDirectory: true, + }) + } + if strings.TrimSpace(req.Team.ConfigMapName) != "" && strings.TrimSpace(req.Team.PersonaConfigKey) != "" && strings.EqualFold(instance.Type, "hermes") { + configMapFileMounts = append(configMapFileMounts, k8s.ConfigMapFileMount{ + Name: "team-persona", + ConfigMapName: strings.TrimSpace(req.Team.ConfigMapName), + Key: strings.TrimSpace(req.Team.PersonaConfigKey), + MountPath: teamHermesSoulMountPath, + ReadOnly: true, + }) + } + } + + podConfig := k8s.PodConfig{ + InstanceID: instance.ID, + InstanceName: instance.Name, + UserID: userID, + Type: instance.Type, + RuntimeType: runtimeType, + CPUCores: instance.CPUCores, + MemoryGB: instance.MemoryGB, + GPUEnabled: instance.GPUEnabled, + GPUCount: instance.GPUCount, + Image: runtimeConfig.Image, + MountPath: runtimeConfig.MountPath, + ContainerPort: runtimeConfig.Port, + ImagePullPolicy: corev1.PullPolicy(defaultImagePullPolicy()), + ExtraEnv: extraEnv, + EnvFromSecretNames: envFromSecretNames, + ExtraPVCMounts: extraPVCMounts, + ConfigMapFileMounts: configMapFileMounts, + VolumeInitScripts: runtimeVolumeInitScripts(instance.Type, runtimeConfig.MountPath), + FSGroup: fsGroup, + NodeSelector: nodeSelector, + VolumeOwnershipFixes: volumeOwnershipFixes, + SHMSizeGB: shmSizeGB, + SecurityMode: s.securityModeForInstance(instance.Type), + } + + var workloadNamespace string + var workloadName string + if instanceUsesDesktopRuntime(instance) { + if s.deploymentService == nil { + s.pvcService.DeletePVC(ctx, userID, instance.ID) + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, fmt.Errorf("instance deployment service is not configured")) + } + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("instance deployment service is not configured") + } + deployment, err := s.deploymentService.EnsureDeployment(ctx, podConfig, 1) + if err != nil { + s.pvcService.DeletePVC(ctx, userID, instance.ID) + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + } + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to create deployment: %w", err) + } + workloadNamespace = deployment.Namespace + workloadName = deployment.Name + + // Create Service for browser desktop access. + serviceConfig := k8s.ServiceConfig{ + InstanceID: instance.ID, + InstanceName: instance.Name, + UserID: userID, + ContainerPort: runtimeConfig.Port, + AdditionalPorts: additionalServicePorts(runtimeConfig.Port), + } + + serviceInfo, err := s.serviceService.CreateService(ctx, serviceConfig) + if err != nil { + // Rollback: delete Deployment, PVC and instance record. + _ = s.deploymentService.DeleteDeployment(ctx, userID, instance.ID) + s.pvcService.DeletePVC(ctx, userID, instance.ID) + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + } + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to create service: %w", err) + } + + fmt.Printf("Instance %d: Service created successfully (ClusterIP: %s)\n", instance.ID, serviceInfo.ClusterIP) + } else { + pod, err := s.podService.CreatePod(ctx, podConfig) + if err != nil { + // Rollback: delete PVC and instance record. + s.pvcService.DeletePVC(ctx, userID, instance.ID) + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + } + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to create pod: %w", err) + } + workloadNamespace = pod.Namespace + workloadName = pod.Name + fmt.Printf("Instance %d: Shell runtime selected, skipping desktop service creation\n", instance.ID) + } + + // Update instance with initial workload info. For Pro instances this is the + // stable Deployment name; sync later records the active Pod name/IP. + podNamespace := workloadNamespace + podName := workloadName + instance.PodNamespace = &podNamespace + instance.PodName = &podName + instance.Status = "creating" + instance.StartedAt = &now + instance.UpdatedAt = now + + fmt.Printf("Instance %d created successfully, updating database with status 'creating'\n", instance.ID) + if err := s.instanceRepo.Update(instance); err != nil { + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + } + return nil, fmt.Errorf("failed to update instance with pod info: %w", err) + } + fmt.Printf("Instance %d database updated, broadcasting status via WebSocket\n", instance.ID) + + if bootstrapSnapshot != nil { + if err := s.openClawConfigService.MarkSnapshotActive(bootstrapSnapshot); err != nil { + return nil, fmt.Errorf("failed to activate runtime bootstrap snapshot: %w", err) + } + } + + // Broadcast initial creating status via WebSocket. Sync service will mark it + // running only after the pod becomes Ready. + hydrateInstanceDesktopStreamProfile(instance) + GetHub().BroadcastInstanceStatus(userID, instance) + fmt.Printf("Instance %d status broadcast complete\n", instance.ID) + + return instance, nil +} + +func (s *instanceService) createV2Instance(ctx context.Context, userID int, req CreateInstanceRequest, runtimeType string, environmentOverridesJSON *string) (*models.Instance, error) { + now := time.Now() + workspaceRoot := s.runtimeWorkspaceRoot() + instance := &models.Instance{ + UserID: userID, + Name: strings.TrimSpace(req.Name), + Description: trimOptionalString(req.Description), + Type: runtimeType, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + CPUCores: req.CPUCores, + MemoryGB: req.MemoryGB, + DiskGB: req.DiskGB, + GPUEnabled: req.GPUEnabled, + GPUCount: req.GPUCount, + OSType: req.OSType, + OSVersion: req.OSVersion, + ImageRegistry: req.ImageRegistry, + ImageTag: req.ImageTag, + EnvironmentOverridesJSON: environmentOverridesJSON, + StorageClass: strings.TrimSpace(req.StorageClass), + MountPath: workspaceRoot, + RuntimeGeneration: 1, + CreatedAt: now, + UpdatedAt: now, + StartedAt: &now, + } + + if err := s.instanceRepo.Create(instance); err != nil { + return nil, fmt.Errorf("failed to create instance record: %w", err) + } + + if _, err := s.ensureGatewayToken(instance); err != nil { + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to provision lite gateway token: %w", err) + } + if _, err := s.ensureAgentBootstrapToken(instance); err != nil { + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to provision lite agent bootstrap token: %w", err) + } + + if supportsRuntimeConfigInjection(instance.Type) && s.openClawConfigService != nil && req.OpenClawConfigPlan != nil && hasOpenClawConfigSelections(*req.OpenClawConfigPlan) { + bootstrapSnapshot, err := s.openClawConfigService.CreateSnapshotForInstance(userID, instance, req.OpenClawConfigPlan) + if err != nil { + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to compile lite runtime bootstrap config: %w", err) + } + if bootstrapSnapshot != nil { + instance.OpenClawConfigSnapshotID = &bootstrapSnapshot.ID + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to persist lite runtime snapshot reference: %w", err) + } + if err := s.openClawConfigService.MarkSnapshotActive(bootstrapSnapshot); err != nil { + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to activate lite runtime bootstrap snapshot: %w", err) + } + } + } + + workspacePath, err := ensureRuntimeWorkspaceDirectories(workspaceRoot, runtimeType, userID, instance.ID) + if err != nil { + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to create instance workspace: %w", err) + } + if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil { + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to persist instance workspace path: %w", err) + } + instance.WorkspacePath = &workspacePath + + GetHub().BroadcastInstanceStatus(userID, instance) + return instance, nil +} + +// GetByID gets an instance by ID +func (s *instanceService) GetByID(id int) (*models.Instance, error) { + instance, err := s.instanceRepo.GetByID(id) + if err != nil { + return nil, err + } + hydrateInstanceDesktopStreamProfile(instance) + return instance, nil +} + +// GetByUserID gets instances by user ID with pagination +func (s *instanceService) GetByUserID(userID int, offset, limit int) ([]models.Instance, int, error) { + instances, err := s.instanceRepo.GetByUserID(userID, offset, limit) + if err != nil { + return nil, 0, err + } + hydrateInstancesDesktopStreamProfile(instances) + + total, err := s.instanceRepo.CountByUserID(userID) + if err != nil { + return nil, 0, err + } + + return instances, total, nil +} + +func (s *instanceService) GetAllInstances(offset, limit int) ([]models.Instance, int, error) { + instances, err := s.instanceRepo.GetAll(offset, limit) + if err != nil { + return nil, 0, err + } + hydrateInstancesDesktopStreamProfile(instances) + + total, err := s.instanceRepo.CountAll() + if err != nil { + return nil, 0, err + } + + return instances, total, nil +} + +func hydrateInstancesDesktopStreamProfile(instances []models.Instance) { + for idx := range instances { + hydrateInstanceDesktopStreamProfile(&instances[idx]) + } +} + +func hydrateInstanceDesktopStreamProfile(instance *models.Instance) { + if instance == nil { + return + } + environmentOverrides, err := parseEnvironmentOverridesJSON(instance.EnvironmentOverridesJSON) + if err != nil { + return + } + instance.DesktopStreamProfile = desktopStreamProfileFromEnv(environmentOverrides) +} + +// Start starts an instance +func (s *instanceService) Start(instanceID int) error { + ctx := context.Background() + + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return fmt.Errorf("failed to get instance: %w", err) + } + + if instance == nil { + return fmt.Errorf("instance not found") + } + + if instance.Status == "running" { + return fmt.Errorf("instance is already running") + } + if err := s.enforceInstanceModeLimits(ctx, modeForExistingInstance(instance), instance.CPUCores, instance.MemoryGB, instance.DiskGB, instance.GPUCount); err != nil { + return err + } + + if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok { + return s.startV2Instance(ctx, instance, runtimeType) + } + + if _, err := s.ensureGatewayToken(instance); err != nil { + return fmt.Errorf("failed to provision instance gateway token: %w", err) + } + if _, err := s.ensureAgentBootstrapToken(instance); err != nil { + return fmt.Errorf("failed to provision instance agent bootstrap token: %w", err) + } + + gatewayEnv, err := s.buildGatewayEnv(instance) + if err != nil { + return fmt.Errorf("failed to build instance gateway config: %w", err) + } + agentEnv, err := s.buildAgentEnv(instance) + if err != nil { + return fmt.Errorf("failed to build instance agent config: %w", err) + } + runtimeConfig := buildRuntimeConfig(instance.Type, instance.OSType, instance.OSVersion, instance.ImageRegistry, instance.ImageTag) + mountPath := persistentVolumeMountPath(instance) + instance.MountPath = mountPath + extraEnv, err := buildInstancePodEnv(instance, runtimeConfig.Env, gatewayEnv, agentEnv) + if err != nil { + return fmt.Errorf("failed to resolve instance environment: %w", err) + } + + bootstrapSecretName := "" + if supportsRuntimeConfigInjection(instance.Type) && s.openClawConfigService != nil && instance.OpenClawConfigSnapshotID != nil && *instance.OpenClawConfigSnapshotID > 0 { + bootstrapSecretName, err = s.openClawConfigService.EnsureSnapshotSecret(ctx, instance.UserID, instance, *instance.OpenClawConfigSnapshotID) + if err != nil { + return fmt.Errorf("failed to restore runtime bootstrap secret: %w", err) + } + } + + // Remove legacy per-instance network policy before starting pod. + if err := s.networkPolicyService.DeletePolicy(ctx, instance.UserID, instance.ID, instance.Name); err != nil { + return fmt.Errorf("failed to delete network policy: %w", err) + } + + runtimeType := normalizeInstanceRuntimeType(instance.RuntimeType) + shmSizeGB := popSHMSizeGB(extraEnv, runtimeType, instance.MemoryGB) + nodeSelector, err := s.pvcService.NodeSelectorForPVC(ctx, instance.UserID, instance.ID, instance.StorageClass) + if err != nil { + return fmt.Errorf("failed to resolve PVC node selector: %w", err) + } + podConfig := k8s.PodConfig{ + InstanceID: instance.ID, + InstanceName: instance.Name, + UserID: instance.UserID, + Type: instance.Type, + RuntimeType: runtimeType, + CPUCores: instance.CPUCores, + MemoryGB: instance.MemoryGB, + GPUEnabled: instance.GPUEnabled, + GPUCount: instance.GPUCount, + Image: runtimeConfig.Image, + MountPath: mountPath, + ContainerPort: runtimeConfig.Port, + ImagePullPolicy: corev1.PullPolicy(defaultImagePullPolicy()), + ExtraEnv: extraEnv, + EnvFromSecretNames: []string{bootstrapSecretName}, + VolumeInitScripts: runtimeVolumeInitScripts(instance.Type, mountPath), + NodeSelector: nodeSelector, + SHMSizeGB: shmSizeGB, + SecurityMode: s.securityModeForInstance(instance.Type), + } + + var workloadNamespace string + var workloadName string + if instanceUsesDesktopRuntime(instance) { + if s.deploymentService == nil { + return fmt.Errorf("instance deployment service is not configured") + } + deployment, err := s.deploymentService.EnsureDeployment(ctx, podConfig, 1) + if err != nil { + return fmt.Errorf("failed to ensure deployment: %w", err) + } + workloadNamespace = deployment.Namespace + workloadName = deployment.Name + + // Ensure Service exists (create if not exists) + serviceExists, _ := s.serviceService.ServiceExists(ctx, instance.UserID, instance.ID) + if !serviceExists { + serviceConfig := k8s.ServiceConfig{ + InstanceID: instance.ID, + InstanceName: instance.Name, + UserID: instance.UserID, + ContainerPort: runtimeConfig.Port, + AdditionalPorts: additionalServicePorts(runtimeConfig.Port), + } + _, err = s.serviceService.CreateService(ctx, serviceConfig) + if err != nil { + fmt.Printf("Warning: failed to create service for instance %d: %v\n", instance.ID, err) + // Don't fail if service creation fails, pod is already running + } + } + } else { + pod, err := s.podService.CreatePod(ctx, podConfig) + if err != nil { + return fmt.Errorf("failed to create pod: %w", err) + } + workloadNamespace = pod.Namespace + workloadName = pod.Name + } + + // Update instance status + now := time.Now() + podNamespace := workloadNamespace + podName := workloadName + instance.PodNamespace = &podNamespace + instance.PodName = &podName + instance.Status = "creating" + instance.StartedAt = &now + instance.UpdatedAt = now + + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to update instance status: %w", err) + } + + // Broadcast status update via WebSocket + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + + return nil +} + +func (s *instanceService) securityModeForInstance(instanceType string) k8s.PodSecurityMode { + if s != nil && s.allowPrivilegedPods { + return k8s.PodSecurityPrivileged + } + if strings.EqualFold(strings.TrimSpace(instanceType), "openclaw") { + return k8s.PodSecurityChromiumCompat + } + return k8s.PodSecurityDefault +} + +func (s *instanceService) ensureGatewayToken(instance *models.Instance) (string, error) { + if instance.AccessToken != nil && strings.TrimSpace(*instance.AccessToken) != "" { + return strings.TrimSpace(*instance.AccessToken), nil + } + + tokenBytes := make([]byte, 32) + if _, err := rand.Read(tokenBytes); err != nil { + return "", fmt.Errorf("failed to generate instance gateway token: %w", err) + } + + token := "igt_" + hex.EncodeToString(tokenBytes) + instance.AccessToken = &token + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + return "", fmt.Errorf("failed to persist instance gateway token: %w", err) + } + + return token, nil +} + +func (s *instanceService) buildGatewayEnv(instance *models.Instance) (map[string]string, error) { + if instance == nil || instance.AccessToken == nil || strings.TrimSpace(*instance.AccessToken) == "" { + return map[string]string{}, nil + } + if !supportsManagedRuntimeIntegration(instance.Type) { + return map[string]string{}, nil + } + + baseURL, ok := defaultGatewayBaseURL() + if !ok { + return nil, fmt.Errorf("gateway base URL is not configured") + } + + modelInjection, err := s.resolveGatewayModelInjection() + if err != nil { + return nil, err + } + + token := strings.TrimSpace(*instance.AccessToken) + return map[string]string{ + "CLAWMANAGER_LLM_BASE_URL": baseURL, + "CLAWMANAGER_LLM_API_KEY": token, + "CLAWMANAGER_LLM_MODEL": modelInjection.modelsJSON, + "CLAWMANAGER_LLM_PROVIDER": "openai-compatible", + "CLAWMANAGER_INSTANCE_TOKEN": token, + "OPENAI_BASE_URL": baseURL, + "OPENAI_API_BASE": baseURL, + "OPENAI_API_KEY": token, + "OPENAI_MODEL": modelInjection.defaultModel, + }, nil +} + +func (s *instanceService) BuildGatewayEnv(instance *models.Instance) (map[string]string, error) { + if instance == nil || !supportsManagedRuntimeIntegration(instance.Type) { + return s.buildGatewayEnv(instance) + } + if instance.AccessToken == nil || strings.TrimSpace(*instance.AccessToken) == "" { + if s == nil || s.instanceRepo == nil { + return nil, fmt.Errorf("instance repository is not configured") + } + if _, err := s.ensureGatewayToken(instance); err != nil { + return nil, err + } + } + gatewayEnv, err := s.buildGatewayEnv(instance) + if err != nil { + return nil, err + } + + bootstrapEnv, err := s.runtimeBootstrapEnv(instance) + if err != nil { + return nil, err + } + + agentEnv := map[string]string{} + if _, ok := defaultAgentControlBaseURL(); ok { + if instance.AgentBootstrapToken == nil || strings.TrimSpace(*instance.AgentBootstrapToken) == "" { + if s == nil || s.instanceRepo == nil { + return nil, fmt.Errorf("instance repository is not configured") + } + if _, err := s.ensureAgentBootstrapToken(instance); err != nil { + return nil, err + } + } + agentEnv, err = s.buildAgentEnv(instance) + if err != nil { + return nil, err + } + } + + merged := mergeEnvMaps(gatewayEnv, bootstrapEnv) + merged = mergeEnvMaps(merged, agentEnv) + return buildInstanceGatewayEnv(instance, merged) +} + +func (s *instanceService) runtimeBootstrapEnv(instance *models.Instance) (map[string]string, error) { + if instance == nil || s == nil || s.openClawConfigService == nil || instance.OpenClawConfigSnapshotID == nil || *instance.OpenClawConfigSnapshotID <= 0 { + return map[string]string{}, nil + } + provider, ok := s.openClawConfigService.(interface { + RuntimeEnvForSnapshot(userID int, instanceType string, snapshotID int) (map[string]string, error) + }) + if !ok { + return map[string]string{}, nil + } + return provider.RuntimeEnvForSnapshot(instance.UserID, instance.Type, *instance.OpenClawConfigSnapshotID) +} +func (s *instanceService) ensureAgentBootstrapToken(instance *models.Instance) (string, error) { + if instance.AgentBootstrapToken != nil && strings.TrimSpace(*instance.AgentBootstrapToken) != "" { + return strings.TrimSpace(*instance.AgentBootstrapToken), nil + } + + token, err := generatePrefixedToken("agt_boot") + if err != nil { + return "", fmt.Errorf("failed to generate instance agent bootstrap token: %w", err) + } + instance.AgentBootstrapToken = &token + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + return "", fmt.Errorf("failed to persist instance agent bootstrap token: %w", err) + } + return token, nil +} + +func (s *instanceService) buildAgentEnv(instance *models.Instance) (map[string]string, error) { + if instance == nil || !supportsManagedRuntimeIntegration(instance.Type) { + return map[string]string{}, nil + } + if instance.AgentBootstrapToken == nil || strings.TrimSpace(*instance.AgentBootstrapToken) == "" { + return nil, fmt.Errorf("instance agent bootstrap token is not configured") + } + + baseURL, ok := defaultAgentControlBaseURL() + if !ok { + return nil, fmt.Errorf("agent control base URL is not configured") + } + + diskLimitBytes := int64(instance.DiskGB) * 1024 * 1024 * 1024 + + return map[string]string{ + "CLAWMANAGER_AGENT_ENABLED": "true", + "CLAWMANAGER_AGENT_BASE_URL": baseURL, + "CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN": strings.TrimSpace(*instance.AgentBootstrapToken), + "CLAWMANAGER_AGENT_DISK_LIMIT_BYTES": strconv.FormatInt(diskLimitBytes, 10), + "CLAWMANAGER_AGENT_INSTANCE_ID": fmt.Sprintf("%d", instance.ID), + "CLAWMANAGER_AGENT_PERSISTENT_DIR": managedRuntimePersistentDir(instance), + "CLAWMANAGER_AGENT_PROTOCOL_VERSION": AgentProtocolVersionV1, + }, nil +} + +func supportsManagedRuntimeIntegration(instanceType string) bool { + switch strings.ToLower(strings.TrimSpace(instanceType)) { + case "openclaw", "hermes": + return true + default: + return false + } +} + +func supportsRuntimeConfigInjection(instanceType string) bool { + switch strings.ToLower(strings.TrimSpace(instanceType)) { + case "openclaw", "hermes": + return true + default: + return false + } +} + +func managedRuntimePersistentDir(instance *models.Instance) string { + if instance == nil { + return "/config" + } + if isLiteRuntimeInstance(instance) && instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != "" { + workspacePath := strings.TrimSpace(*instance.WorkspacePath) + if strings.EqualFold(instance.Type, "hermes") { + return path.Join(workspacePath, "home", ".hermes") + } + return path.Join(workspacePath, "home", ".openclaw") + } + if strings.EqualFold(instance.Type, "hermes") { + return "/config/.hermes" + } + return persistentVolumeMountPath(instance) +} +func persistentVolumeMountPath(instance *models.Instance) string { + if instance == nil { + return "/config" + } + if defaultPath := defaultMountPathForInstanceType(instance.Type); defaultPath == "/config" { + return defaultPath + } + if strings.TrimSpace(instance.MountPath) != "" { + return strings.TrimSpace(instance.MountPath) + } + return defaultMountPathForInstanceType(instance.Type) +} + +func runtimeVolumeInitScripts(instanceType, mountPath string) []k8s.VolumeInitScript { + if !strings.EqualFold(strings.TrimSpace(instanceType), "hermes") || strings.TrimSpace(mountPath) != "/config" { + return nil + } + return []k8s.VolumeInitScript{ + { + Name: "data", + MountPath: "/config", + Script: `set -eu +base="${CLAWMANAGER_VOLUME_PATH:-/config}" +target="$base/.hermes" +if [ ! -d "$target" ]; then + legacy_found=0 + for name in hermes-agent skills channels.json session.json bootstrap inventory.json; do + if [ -e "$base/$name" ]; then legacy_found=1; fi + done + mkdir -p "$target" + if [ "$legacy_found" = "1" ]; then + for entry in "$base"/* "$base"/.[!.]* "$base"/..?*; do + [ -e "$entry" ] || continue + name="${entry##*/}" + case "$name" in .|..|.hermes|Desktop|Downloads|lost+found) continue;; esac + mv "$entry" "$target"/ + done + fi +fi +chown -R 1000:1000 "$target" || true`, + }, + } +} + +func (s *instanceService) resolveGatewayModelInjection() (*gatewayModelInjection, error) { + if s.llmModelRepo == nil { + return nil, fmt.Errorf("llm model repository not configured") + } + + items, err := s.llmModelRepo.ListActive() + if err != nil { + return nil, fmt.Errorf("failed to list active models: %w", err) + } + if len(items) == 0 { + return nil, fmt.Errorf("no active models are configured") + } + + modelsForInjection := []string{"auto"} + seen := map[string]struct{}{ + "auto": {}, + } + + for _, item := range items { + displayName := strings.TrimSpace(item.DisplayName) + if displayName == "" { + displayName = strings.TrimSpace(item.ProviderModelName) + } + if displayName == "" { + continue + } + + normalizedName := strings.ToLower(displayName) + if _, exists := seen[normalizedName]; exists { + continue + } + seen[normalizedName] = struct{}{} + modelsForInjection = append(modelsForInjection, displayName) + } + + rawModels, err := json.Marshal(modelsForInjection) + if err != nil { + return nil, fmt.Errorf("failed to encode gateway model list: %w", err) + } + + return &gatewayModelInjection{ + defaultModel: "auto", + modelsJSON: string(rawModels), + }, nil +} + +func mergeEnvMaps(base map[string]string, overlay map[string]string) map[string]string { + merged := map[string]string{} + for key, value := range base { + merged[key] = value + } + for key, value := range overlay { + merged[key] = value + } + return merged +} + +func (s *instanceService) startV2Instance(ctx context.Context, instance *models.Instance, runtimeType string) error { + if err := s.ensureV2Workspace(ctx, instance, runtimeType); err != nil { + return err + } + nextGeneration := instance.RuntimeGeneration + 1 + if nextGeneration <= 0 { + nextGeneration = 1 + } + if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "creating", nextGeneration, nil); err != nil { + return fmt.Errorf("failed to mark v2 instance creating: %w", err) + } + instance.Status = "creating" + instance.RuntimeGeneration = nextGeneration + instance.RuntimeErrorMessage = nil + now := time.Now() + instance.StartedAt = &now + instance.UpdatedAt = now + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + return nil +} + +func (s *instanceService) stopV2Instance(ctx context.Context, instance *models.Instance) error { + if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "stopped", instance.RuntimeGeneration, nil); err != nil { + return fmt.Errorf("failed to mark v2 instance stopped: %w", err) + } + now := time.Now() + instance.Status = "stopped" + instance.StoppedAt = &now + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = now + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + return s.cleanupV2GatewayBinding(ctx, instance) +} + +func (s *instanceService) deleteV2Instance(ctx context.Context, instance *models.Instance) error { + if instance.Status != "deleting" { + now := time.Now() + instance.Status = "deleting" + instance.UpdatedAt = now + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to mark v2 instance as deleting: %w", err) + } + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + + cleanupErr := s.cleanupV2GatewayBinding(ctx, instance) + if cleanupErr != nil { + return cleanupErr + } + if err := s.instanceRepo.Delete(instance.ID); err != nil { + return fmt.Errorf("failed to delete v2 instance record: %w", err) + } + return nil +} + +func (s *instanceService) cleanupV2GatewayBinding(ctx context.Context, instance *models.Instance) error { + if s.bindingRepo == nil { + return nil + } + binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID) + if err != nil { + return fmt.Errorf("failed to get v2 runtime binding: %w", err) + } + if binding == nil { + return nil + } + + if s.runtimePodRepo != nil { + pod, podErr := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if podErr != nil { + return fmt.Errorf("failed to get runtime pod %d for v2 cleanup: %w", binding.RuntimePodID, podErr) + } else if pod == nil { + return fmt.Errorf("runtime pod %d is not available for v2 cleanup", binding.RuntimePodID) + } else if pod != nil && pod.AgentEndpoint != nil && strings.TrimSpace(*pod.AgentEndpoint) != "" && s.agentClient != nil && binding.GatewayID != "" { + if err := s.agentClient.DeleteGateway(ctx, strings.TrimSpace(*pod.AgentEndpoint), binding.GatewayID); err != nil { + return fmt.Errorf("failed to delete v2 gateway: %w", err) + } + } + } + + if err := s.bindingRepo.DeleteByInstanceIDAndReleaseSlot(ctx, instance.ID, binding.RuntimePodID); err != nil { + return fmt.Errorf("failed to delete v2 runtime binding and release slot: %w", err) + } + return nil +} + +func (s *instanceService) ensureV2Workspace(ctx context.Context, instance *models.Instance, runtimeType string) error { + if instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != "" { + return nil + } + workspacePath, err := ensureRuntimeWorkspaceDirectories(s.runtimeWorkspaceRoot(), runtimeType, instance.UserID, instance.ID) + if err != nil { + return fmt.Errorf("failed to create instance workspace: %w", err) + } + if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil { + return fmt.Errorf("failed to persist instance workspace path: %w", err) + } + instance.WorkspacePath = &workspacePath + return nil +} + +func ensureRuntimeWorkspaceDirectories(root, runtimeType string, userID, instanceID int) (string, error) { + workspacePath := RuntimeWorkspacePathWithRoot(root, runtimeType, userID, instanceID) + if err := os.MkdirAll(workspacePath, 0750); err != nil { + return "", err + } + + // Allow the isolated gateway UID to traverse to its own workspace without + // granting read/list access to sibling user or instance directories. + userRoot := path.Dir(workspacePath) + runtimeRoot := path.Dir(userRoot) + for _, dir := range []string{runtimeRoot, userRoot} { + if err := os.Chmod(dir, 0711); err != nil { + return "", err + } + } + if err := os.Chmod(workspacePath, 0750); err != nil { + return "", err + } + return workspacePath, nil +} + +// Stop stops an instance +func (s *instanceService) Stop(instanceID int) error { + ctx := context.Background() + + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return fmt.Errorf("failed to get instance: %w", err) + } + + if instance == nil { + return fmt.Errorf("instance not found") + } + + if _, ok := v2RuntimeTypeForInstance(instance); ok { + return s.stopV2Instance(ctx, instance) + } + + if instance.Status != "running" { + return fmt.Errorf("instance is not running") + } + + if instanceUsesDesktopRuntime(instance) { + if s.deploymentService == nil { + return fmt.Errorf("instance deployment service is not configured") + } + if err := s.deploymentService.ScaleDeployment(ctx, instance.UserID, instance.ID, 0); err != nil { + fmt.Printf("Warning: failed to stop deployment for instance %d, falling back to pod delete: %v\n", instance.ID, err) + if podErr := s.podService.DeletePod(ctx, instance.UserID, instance.ID); podErr != nil { + return fmt.Errorf("failed to stop deployment: %w", err) + } + } + } else { + // Delete shell pod + if err := s.podService.DeletePod(ctx, instance.UserID, instance.ID); err != nil { + return fmt.Errorf("failed to delete pod: %w", err) + } + } + + // Update instance status + now := time.Now() + instance.Status = "stopped" + instance.StoppedAt = &now + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = now + + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to update instance status: %w", err) + } + + // Broadcast status update via WebSocket + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + + return nil +} + +// Restart restarts an instance +func (s *instanceService) Restart(instanceID int) error { + ctx := context.Background() + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return fmt.Errorf("failed to get instance: %w", err) + } + if instance == nil { + return fmt.Errorf("instance not found") + } + + _, isV2 := v2RuntimeTypeForInstance(instance) + waitForDesktopPods := !isV2 && instanceUsesDesktopRuntime(instance) + + if err := s.Stop(instanceID); err != nil { + return fmt.Errorf("failed to stop instance: %w", err) + } + + if waitForDesktopPods { + if s.deploymentService == nil { + return fmt.Errorf("instance deployment service is not configured") + } + if err := s.deploymentService.WaitForDeploymentPodsDeleted(ctx, instance.UserID, instance.ID); err != nil { + return fmt.Errorf("failed waiting for desktop pods to stop: %w", err) + } + } + + if err := s.Start(instanceID); err != nil { + return fmt.Errorf("failed to start instance: %w", err) + } + + return nil +} + +// Delete starts deleting an instance and all associated K8s resources. +func (s *instanceService) Delete(instanceID int) error { + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return fmt.Errorf("failed to get instance: %w", err) + } + + if instance == nil { + return fmt.Errorf("instance not found") + } + + if _, ok := v2RuntimeTypeForInstance(instance); ok { + return s.deleteV2Instance(context.Background(), instance) + } + + if instance.Status != "deleting" { + now := time.Now() + instance.Status = "deleting" + instance.UpdatedAt = now + + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to mark instance as deleting: %w", err) + } + + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + + go s.completeDeletion(instance.UserID, instance.ID) + + return nil +} + +func (s *instanceService) completeDeletion(userID, instanceID int) { + ctx := context.Background() + + fmt.Printf("Starting background deletion of instance %d (user %d)\n", instanceID, userID) + + // Use CleanupService to delete ALL resources for this instance (including duplicates) + cleanupService := k8s.NewCleanupService() + if err := cleanupService.DeleteAllInstanceResources(ctx, userID, instanceID); err != nil { + fmt.Printf("Warning: error during resource cleanup for instance %d: %v\n", instanceID, err) + } + + // Delete instance record from database after background cleanup finishes. + fmt.Printf("Deleting instance %d from database...\n", instanceID) + if err := s.instanceRepo.Delete(instanceID); err != nil { + fmt.Printf("Error: failed to delete instance %d record: %v\n", instanceID, err) + return + } + + fmt.Printf("Instance %d deleted successfully\n", instanceID) +} + +// cleanupOrphanedResources cleans up any orphaned K8s resources for an instance +func (s *instanceService) cleanupOrphanedResources(ctx context.Context, userID, instanceID int) error { + namespace := s.pvcService.GetClient().GetNamespace(userID) + instanceLabel := fmt.Sprintf("%d", instanceID) + client := s.pvcService.GetClient().Clientset + + // Check if namespace has other instances' pods + allPods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "managed-by=clawreef", + }) + if err == nil { + otheInstanceCount := 0 + for _, pod := range allPods.Items { + if pod.Labels["instance-id"] != instanceLabel { + otheInstanceCount++ + } + } + fmt.Printf("Namespace %s has %d other instance(s), will not delete namespace\n", namespace, otheInstanceCount) + } + + deployments, err := client.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s", instanceLabel), + }) + if err == nil && len(deployments.Items) > 0 { + for _, deployment := range deployments.Items { + fmt.Printf("Deleting orphaned Deployment %s\n", deployment.Name) + propagation := metav1.DeletePropagationForeground + client.AppsV1().Deployments(namespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{ + PropagationPolicy: &propagation, + }) + } + } + + // List and delete ConfigMaps with instance label + configMaps, err := client.CoreV1().ConfigMaps(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s", instanceLabel), + }) + if err == nil && len(configMaps.Items) > 0 { + for _, cm := range configMaps.Items { + fmt.Printf("Deleting orphaned ConfigMap %s\n", cm.Name) + client.CoreV1().ConfigMaps(namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{}) + } + } + + // List and delete Secrets with instance label + secrets, err := client.CoreV1().Secrets(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s", instanceLabel), + }) + if err == nil && len(secrets.Items) > 0 { + for _, secret := range secrets.Items { + fmt.Printf("Deleting orphaned Secret %s\n", secret.Name) + client.CoreV1().Secrets(namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}) + } + } + + // List and delete Services with instance label + services, err := client.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s", instanceLabel), + }) + if err == nil && len(services.Items) > 0 { + for _, svc := range services.Items { + fmt.Printf("Deleting orphaned Service %s\n", svc.Name) + client.CoreV1().Services(namespace).Delete(ctx, svc.Name, metav1.DeleteOptions{}) + } + } + + return nil +} + +// cleanupOrphanedResourcesByUser cleans up any orphaned resources for a user that don't have corresponding DB records +func (s *instanceService) cleanupOrphanedResourcesByUser(ctx context.Context, userID int) { + namespace := s.pvcService.GetClient().GetNamespace(userID) + client := s.pvcService.GetClient().Clientset + + deployments, err := client.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "managed-by=clawreef", + }) + if err != nil { + fmt.Printf("Warning: failed to list deployments in namespace %s: %v\n", namespace, err) + } else { + for _, deployment := range deployments.Items { + instanceIDStr := deployment.Labels["instance-id"] + if instanceIDStr == "" { + continue + } + + instanceID := 0 + fmt.Sscanf(instanceIDStr, "%d", &instanceID) + + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil || instance == nil { + fmt.Printf("Found orphaned deployment %s (instance-id: %s), deleting...\n", deployment.Name, instanceIDStr) + propagation := metav1.DeletePropagationForeground + if err := client.AppsV1().Deployments(namespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{ + PropagationPolicy: &propagation, + }); err != nil { + fmt.Printf("Warning: failed to delete orphaned deployment %s: %v\n", deployment.Name, err) + } + } + } + } + + // Get all pods in the namespace with clawreef label + pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "managed-by=clawreef", + }) + if err != nil { + fmt.Printf("Warning: failed to list pods in namespace %s: %v\n", namespace, err) + return + } + + // For each pod, check if corresponding instance exists in DB + for _, pod := range pods.Items { + instanceIDStr := pod.Labels["instance-id"] + if instanceIDStr == "" { + continue + } + + instanceID := 0 + fmt.Sscanf(instanceIDStr, "%d", &instanceID) + + // Check if instance exists in DB + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil || instance == nil { + // Instance doesn't exist, this is an orphaned pod + fmt.Printf("Found orphaned pod %s (instance-id: %s), deleting...\n", pod.Name, instanceIDStr) + if err := client.CoreV1().Pods(namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}); err != nil { + fmt.Printf("Warning: failed to delete orphaned pod %s: %v\n", pod.Name, err) + } + } + } + + // Also check PVCs + pvcs, err := client.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "managed-by=clawreef", + }) + if err != nil { + fmt.Printf("Warning: failed to list PVCs in namespace %s: %v\n", namespace, err) + return + } + + for _, pvc := range pvcs.Items { + instanceIDStr := pvc.Labels["instance-id"] + if instanceIDStr == "" { + continue + } + + instanceID := 0 + fmt.Sscanf(instanceIDStr, "%d", &instanceID) + + // Check if instance exists in DB + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil || instance == nil { + // Instance doesn't exist, this is an orphaned PVC + fmt.Printf("Found orphaned PVC %s (instance-id: %s), deleting...\n", pvc.Name, instanceIDStr) + if err := client.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvc.Name, metav1.DeleteOptions{}); err != nil { + fmt.Printf("Warning: failed to delete orphaned PVC %s: %v\n", pvc.Name, err) + } + // Also try to delete the associated PV + if pvc.Spec.VolumeName != "" { + client.CoreV1().PersistentVolumes().Delete(ctx, pvc.Spec.VolumeName, metav1.DeleteOptions{}) + } + } + } + + networkPolicies, err := client.NetworkingV1().NetworkPolicies(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "managed-by=clawreef", + }) + if err != nil { + fmt.Printf("Warning: failed to list network policies in namespace %s: %v\n", namespace, err) + return + } + + for _, policy := range networkPolicies.Items { + instanceIDStr := policy.Labels["instance-id"] + if instanceIDStr == "" { + continue + } + + instanceID := 0 + fmt.Sscanf(instanceIDStr, "%d", &instanceID) + + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil || instance == nil { + fmt.Printf("Found orphaned NetworkPolicy %s (instance-id: %s), deleting...\n", policy.Name, instanceIDStr) + if err := client.NetworkingV1().NetworkPolicies(namespace).Delete(ctx, policy.Name, metav1.DeleteOptions{}); err != nil { + fmt.Printf("Warning: failed to delete orphaned NetworkPolicy %s: %v\n", policy.Name, err) + } + } + } +} + +// Update updates an instance +func (s *instanceService) Update(instanceID int, req UpdateInstanceRequest) error { + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return fmt.Errorf("failed to get instance: %w", err) + } + + if instance == nil { + return fmt.Errorf("instance not found") + } + + // Update fields + if req.Name != nil { + instance.Name = *req.Name + } + if req.Description != nil { + instance.Description = req.Description + } + if req.DesktopStreamProfile != nil { + profile, ok := normalizeDesktopStreamProfile(*req.DesktopStreamProfile) + if !ok || profile == "" { + return fmt.Errorf("invalid desktop stream profile") + } + environmentOverrides, err := parseEnvironmentOverridesJSON(instance.EnvironmentOverridesJSON) + if err != nil { + return err + } + environmentOverrides = applyDesktopStreamProfileEnv(environmentOverrides, profile) + environmentOverridesJSON, err := marshalEnvironmentOverrides(environmentOverrides) + if err != nil { + return err + } + instance.EnvironmentOverridesJSON = environmentOverridesJSON + } + + instance.UpdatedAt = time.Now() + + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to update instance: %w", err) + } + + return nil +} + +// GetInstanceStatus gets the detailed status of an instance +func (s *instanceService) GetInstanceStatus(instanceID int) (*InstanceStatus, error) { + ctx := context.Background() + + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, fmt.Errorf("failed to get instance: %w", err) + } + + if instance == nil { + return nil, fmt.Errorf("instance not found") + } + + if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok { + return &InstanceStatus{ + InstanceID: instance.ID, + Status: instance.Status, + Availability: s.v2InstanceAvailability(ctx, instance), + AgentType: runtimeType, + WorkspaceUsageBytes: instance.WorkspaceUsageBytes, + CreatedAt: instance.CreatedAt, + StartedAt: instance.StartedAt, + }, nil + } + + status := &InstanceStatus{ + InstanceID: instance.ID, + Status: instance.Status, + PodName: instance.PodName, + PodNamespace: instance.PodNamespace, + PodIP: instance.PodIP, + CreatedAt: instance.CreatedAt, + StartedAt: instance.StartedAt, + } + + // Get pod status if running + if instance.Status == "running" || instance.Status == "creating" { + podStatus, err := s.podService.GetPodStatus(ctx, instance.UserID, instance.ID) + if err == nil && podStatus != nil { + status.PodStatus = string(podStatus.Phase) + } + } + + return status, nil +} + +// ForceSyncInstance forces a status sync for a single instance +func (s *instanceService) ForceSyncInstance(instanceID int) error { + ctx := context.Background() + + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return fmt.Errorf("failed to get instance: %w", err) + } + + if instance == nil { + return fmt.Errorf("instance not found") + } + + if _, ok := v2RuntimeTypeForInstance(instance); ok { + return nil + } + if instanceUsesDesktopRuntime(instance) { + return s.forceSyncDeploymentInstance(ctx, instance) + } + + fmt.Printf("Force syncing instance %d (current status: %s, user: %d)\n", instanceID, instance.Status, instance.UserID) + + // First try direct lookup by instance ID + pod, err := s.podService.GetPod(ctx, instance.UserID, instance.ID) + if err != nil { + // Pod not found by instance ID, try to find by namespace scan + fmt.Printf("Instance %d: Pod not found by ID, scanning namespace for any matching pods...\n", instanceID) + + namespace := s.pvcService.GetClient().GetNamespace(instance.UserID) + pods, listErr := s.pvcService.GetClient().Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "managed-by=clawreef", + }) + + if listErr == nil && len(pods.Items) > 0 { + // Try to find a pod that might belong to this instance by name pattern + for _, p := range pods.Items { + // Check if pod name contains instance ID + if p.Labels["instance-id"] == fmt.Sprintf("%d", instanceID) { + fmt.Printf("Instance %d: Found matching pod %s by label scan\n", instanceID, p.Name) + pod = &p + err = nil + break + } + } + } + } + + if err != nil { + fmt.Printf("Instance %d: Pod not found in K8s: %v\n", instanceID, err) + + deploymentExists, deploymentErr := s.podService.DeploymentExists(ctx, instance.UserID, instance.ID) + if deploymentErr != nil { + fmt.Printf("Instance %d: failed to check deployment while pod was missing: %v\n", instanceID, deploymentErr) + } + if deploymentExists { + fmt.Printf("Instance %d: Deployment exists but no pod is available yet, updating to creating\n", instanceID) + if instance.Status != "creating" { + instance.Status = "creating" + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = time.Now() + + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to update instance status: %w", err) + } + + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + return nil + } + + // If instance thinks it's running or creating but pod doesn't exist, update to stopped + if instance.Status == "running" || instance.Status == "creating" { + fmt.Printf("Instance %d: Updating status from %s to stopped\n", instanceID, instance.Status) + instance.Status = "stopped" + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = time.Now() + + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to update instance status: %w", err) + } + + // Broadcast status update + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + return nil + } + + // Pod exists, sync status + fmt.Printf("Instance %d: Pod found - %s (Status: %s, IP: %s)\n", + instanceID, pod.Name, pod.Status.Phase, pod.Status.PodIP) + + needsUpdate := false + + // Check pod status + if pod.Status.Phase == "Running" && instance.Status != "running" { + fmt.Printf("Instance %d: Status mismatch - Pod Running but instance %s, updating\n", instanceID, instance.Status) + instance.Status = "running" + needsUpdate = true + } else if pod.Status.Phase == "Pending" && instance.Status != "creating" { + fmt.Printf("Instance %d: Status mismatch - Pod Pending but instance %s, updating\n", instanceID, instance.Status) + instance.Status = "creating" + needsUpdate = true + } + + // Update Pod info if changed + if instance.PodName == nil || *instance.PodName != pod.Name { + instance.PodName = &pod.Name + needsUpdate = true + } + if instance.PodNamespace == nil || *instance.PodNamespace != pod.Namespace { + instance.PodNamespace = &pod.Namespace + needsUpdate = true + } + if pod.Status.PodIP != "" && (instance.PodIP == nil || *instance.PodIP != pod.Status.PodIP) { + instance.PodIP = &pod.Status.PodIP + needsUpdate = true + } + + if needsUpdate { + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to update instance: %w", err) + } + + fmt.Printf("Instance %d: Status updated to %s, broadcasting\n", instanceID, instance.Status) + // Broadcast status update + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } else { + fmt.Printf("Instance %d: Status already in sync (%s)\n", instanceID, instance.Status) + } + + return nil +} + +func (s *instanceService) forceSyncDeploymentInstance(ctx context.Context, instance *models.Instance) error { + if s.deploymentService == nil { + return fmt.Errorf("instance deployment service is not configured") + } + deployment, err := s.deploymentService.GetDeployment(ctx, instance.UserID, instance.ID) + if err != nil { + if instance.Status == "running" || instance.Status == "creating" { + nextStatus := "stopped" + if instance.Status == "creating" { + nextStatus = "error" + } + instance.Status = nextStatus + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to update instance status: %w", err) + } + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + return nil + } + + needsUpdate := false + desiredStatus := mapDeploymentToInstanceStatus(deployment) + if instance.Status != desiredStatus { + instance.Status = desiredStatus + needsUpdate = true + } + if pod, podErr := s.deploymentService.GetActivePod(ctx, instance.UserID, instance.ID); podErr == nil && pod != nil { + if pod.Status.PodIP != "" && (instance.PodIP == nil || *instance.PodIP != pod.Status.PodIP) { + instance.PodIP = &pod.Status.PodIP + needsUpdate = true + } + if instance.PodName == nil || *instance.PodName != pod.Name { + instance.PodName = &pod.Name + needsUpdate = true + } + if instance.PodNamespace == nil || *instance.PodNamespace != pod.Namespace { + instance.PodNamespace = &pod.Namespace + needsUpdate = true + } + } + if needsUpdate { + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to update instance: %w", err) + } + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + return nil +} + +func additionalServicePorts(primaryPort int32) []int32 { + if primaryPort == 3000 || primaryPort == 8082 { + return []int32{3000, 8082} + } + + return nil +} + +func normalizeInstanceRuntimeType(runtimeType string) string { + switch strings.ToLower(strings.TrimSpace(runtimeType)) { + case RuntimeBackendGateway: + return RuntimeBackendGateway + case "shell": + return RuntimeBackendShell + default: + return RuntimeBackendDesktop + } +} + +func instanceUsesDesktopRuntime(instance *models.Instance) bool { + if instance == nil { + return true + } + return normalizeInstanceRuntimeType(instance.RuntimeType) == RuntimeBackendDesktop +} + +func resolveCreateInstanceMode(req CreateInstanceRequest) string { + if mode, ok := NormalizeInstanceMode(req.Mode); ok { + return mode + } + if mode, ok := NormalizeInstanceMode(req.InstanceMode); ok { + return mode + } + if strings.TrimSpace(req.RuntimeType) == "" { + return InstanceModeLite + } + return InstanceModeForRuntimeType(normalizeInstanceRuntimeType(req.RuntimeType)) +} + +func hasExplicitCreateInstanceMode(req CreateInstanceRequest) bool { + if _, ok := NormalizeInstanceMode(req.Mode); ok { + return true + } + if _, ok := NormalizeInstanceMode(req.InstanceMode); ok { + return true + } + return false +} + +func modeForExistingInstance(instance *models.Instance) string { + if instance == nil { + return InstanceModeLite + } + if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok { + return mode + } + return InstanceModeForRuntimeType(normalizeInstanceRuntimeType(instance.RuntimeType)) +} + +func instanceModeUsesDedicatedResources(mode string) bool { + normalized, ok := NormalizeInstanceMode(mode) + return ok && normalized == InstanceModePro +} + +func (s *instanceService) enforceInstanceModeLimits(ctx context.Context, mode string, cpuCores float64, memoryGB, storageGB, gpuCount int) error { + normalizedMode, ok := NormalizeInstanceMode(mode) + if !ok { + return fmt.Errorf("unsupported instance mode %q", mode) + } + limits := loadInstanceModeLimitConfig(normalizedMode) + if limits.Capacity != nil { + if *limits.Capacity <= 0 { + return fmt.Errorf("%s instance mode is disabled", normalizedMode) + } + if s == nil || s.instanceRepo == nil { + return fmt.Errorf("instance repository is not configured") + } + count, err := s.instanceRepo.CountActiveByMode(ctx, normalizedMode) + if err != nil { + return err + } + if count >= *limits.Capacity { + return fmt.Errorf("%s instance capacity reached: %d/%d", normalizedMode, count, *limits.Capacity) + } + } + if !instanceModeUsesDedicatedResources(normalizedMode) { + return nil + } + if limits.MaxCPU != nil && cpuCores > *limits.MaxCPU { + return fmt.Errorf("%s CPU cores exceed mode limit: requested %g, max %g", normalizedMode, cpuCores, *limits.MaxCPU) + } + if limits.MaxMemoryGB != nil && memoryGB > *limits.MaxMemoryGB { + return fmt.Errorf("%s memory exceeds mode limit: requested %dGB, max %dGB", normalizedMode, memoryGB, *limits.MaxMemoryGB) + } + if limits.MaxStorageGB != nil && storageGB > *limits.MaxStorageGB { + return fmt.Errorf("%s storage exceeds mode limit: requested %dGB, max %dGB", normalizedMode, storageGB, *limits.MaxStorageGB) + } + if limits.MaxGPUCount != nil && gpuCount > *limits.MaxGPUCount { + return fmt.Errorf("%s GPU count exceeds mode limit: requested %d, max %d", normalizedMode, gpuCount, *limits.MaxGPUCount) + } + return nil +} + +func loadInstanceModeLimitConfig(mode string) instanceModeLimitConfig { + prefix := "CLAWMANAGER_" + strings.ToUpper(mode) + "_" + return instanceModeLimitConfig{ + Capacity: optionalIntEnv(prefix + "CAPACITY"), + MaxCPU: optionalFloatEnv(prefix + "MAX_CPU_CORES"), + MaxMemoryGB: optionalIntEnv(prefix + "MAX_MEMORY_GB"), + MaxStorageGB: optionalIntEnv(prefix + "MAX_STORAGE_GB"), + MaxGPUCount: optionalIntEnv(prefix + "MAX_GPU_COUNT"), + } +} + +func optionalIntEnv(key string) *int { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return nil + } + parsed, err := strconv.Atoi(value) + if err != nil { + return nil + } + return &parsed +} + +func optionalFloatEnv(key string) *float64 { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return nil + } + parsed, err := strconv.ParseFloat(value, 64) + if err != nil { + return nil + } + return &parsed +} + +func (s *instanceService) runtimeWorkspaceRoot() string { + if s != nil && strings.TrimSpace(s.workspaceRoot) != "" { + return strings.TrimSpace(s.workspaceRoot) + } + return "/workspaces" +} + +func v2RuntimeTypeForInstance(instance *models.Instance) (string, bool) { + if instance == nil { + return "", false + } + runtimeType, ok := NormalizeV2RuntimeType(instance.Type) + if !ok { + return "", false + } + if strings.EqualFold(strings.TrimSpace(instance.RuntimeType), RuntimeBackendGateway) { + return runtimeType, true + } + if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok && mode == InstanceModeLite { + return runtimeType, true + } + return "", false +} + +func availabilityForStatus(status string) string { + switch strings.ToLower(strings.TrimSpace(status)) { + case "running": + return "available" + case "creating": + return "starting" + default: + return "unavailable" + } +} + +func (s *instanceService) v2InstanceAvailability(ctx context.Context, instance *models.Instance) string { + base := availabilityForStatus(instance.Status) + if base != "available" { + return base + } + if s == nil || s.bindingRepo == nil || s.runtimePodRepo == nil { + return "unavailable" + } + binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID) + if err != nil || binding == nil || binding.GatewayPort <= 0 { + return "unavailable" + } + pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if err != nil || pod == nil || pod.PodIP == nil || strings.TrimSpace(*pod.PodIP) == "" { + return "unavailable" + } + return "available" +} + +func trimOptionalString(value *string) *string { + if value == nil { + return nil + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return nil + } + return &trimmed +} diff --git a/backend/internal/services/instance_service_test.go b/backend/internal/services/instance_service_test.go new file mode 100644 index 0000000..1064c0d --- /dev/null +++ b/backend/internal/services/instance_service_test.go @@ -0,0 +1,284 @@ +package services + +import ( + "strings" + "testing" + + "clawreef/internal/models" +) + +type stubLLMModelRepository struct { + active []models.LLMModel +} + +func (r *stubLLMModelRepository) List() ([]models.LLMModel, error) { + items := make([]models.LLMModel, len(r.active)) + copy(items, r.active) + return items, nil +} + +func (r *stubLLMModelRepository) ListActive() ([]models.LLMModel, error) { + items := make([]models.LLMModel, len(r.active)) + copy(items, r.active) + return items, nil +} + +func (r *stubLLMModelRepository) GetByID(id int) (*models.LLMModel, error) { + return nil, nil +} + +func (r *stubLLMModelRepository) GetByDisplayName(displayName string) (*models.LLMModel, error) { + return nil, nil +} + +func (r *stubLLMModelRepository) Save(model *models.LLMModel) error { + return nil +} + +func (r *stubLLMModelRepository) Delete(id int) error { + return nil +} + +func TestBuildGatewayEnvInjectsGatewayModelCatalog(t *testing.T) { + t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm") + + token := "igt_test_token" + for _, instanceType := range []string{"openclaw", "hermes"} { + t.Run(instanceType, func(t *testing.T) { + service := &instanceService{ + llmModelRepo: &stubLLMModelRepository{ + active: []models.LLMModel{ + {DisplayName: "GPT-4.1"}, + {DisplayName: "Claude 3.7 Sonnet"}, + {DisplayName: "auto"}, + {ProviderModelName: "deepseek-r1"}, + }, + }, + } + + env, err := service.buildGatewayEnv(&models.Instance{ + Type: instanceType, + AccessToken: &token, + }) + if err != nil { + t.Fatalf("buildGatewayEnv returned error: %v", err) + } + + if env["CLAWMANAGER_LLM_BASE_URL"] != "http://gateway.example/api/v1/gateway/llm" { + t.Fatalf("expected CLAWMANAGER_LLM_BASE_URL to use gateway base URL, got %q", env["CLAWMANAGER_LLM_BASE_URL"]) + } + if env["CLAWMANAGER_LLM_MODEL"] != `["auto","GPT-4.1","Claude 3.7 Sonnet","deepseek-r1"]` { + t.Fatalf("expected CLAWMANAGER_LLM_MODEL to contain injected model catalog JSON, got %q", env["CLAWMANAGER_LLM_MODEL"]) + } + if env["OPENAI_MODEL"] != "auto" { + t.Fatalf("expected OPENAI_MODEL to remain the default gateway alias, got %q", env["OPENAI_MODEL"]) + } + if env["CLAWMANAGER_LLM_API_KEY"] != token || env["OPENAI_API_KEY"] != token { + t.Fatalf("expected gateway token aliases to be preserved") + } + }) + } +} + +func TestBuildGatewayEnvEnsuresMissingGatewayToken(t *testing.T) { + t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm") + + instanceRepo := &stubGatewayEnvInstanceRepository{fakeRuntimeInstanceRepo: newFakeRuntimeInstanceRepo()} + service := &instanceService{ + instanceRepo: instanceRepo, + llmModelRepo: &stubLLMModelRepository{ + active: []models.LLMModel{{DisplayName: "auto"}}, + }, + } + instance := &models.Instance{ + ID: 68, + UserID: 1, + Type: "openclaw", + } + + env, err := service.BuildGatewayEnv(instance) + if err != nil { + t.Fatalf("BuildGatewayEnv returned error: %v", err) + } + if instance.AccessToken == nil || *instance.AccessToken == "" { + t.Fatal("BuildGatewayEnv did not provision instance access token") + } + if env["CLAWMANAGER_LLM_API_KEY"] != *instance.AccessToken { + t.Fatalf("CLAWMANAGER_LLM_API_KEY = %q, want provisioned token %q", env["CLAWMANAGER_LLM_API_KEY"], *instance.AccessToken) + } + if got := instanceRepo.updated[68]; got == nil || got.AccessToken == nil || *got.AccessToken != *instance.AccessToken { + t.Fatalf("repository update did not persist provisioned token: %#v", instanceRepo.updated[68]) + } +} + +func TestBuildGatewayEnvMergesEnvironmentOverrides(t *testing.T) { + t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm") + token := "igt_test_token" + raw, err := marshalEnvironmentOverrides(map[string]string{ + "CLAWMANAGER_TEAM_ENABLED": "true", + "CLAWMANAGER_TEAM_MEMBER_ID": "lite-worker", + "CUSTOM_GATEWAY_ENV": "enabled", + }) + if err != nil { + t.Fatalf("marshalEnvironmentOverrides returned error: %v", err) + } + service := &instanceService{ + llmModelRepo: &stubLLMModelRepository{ + active: []models.LLMModel{{DisplayName: "auto"}}, + }, + } + + env, err := service.BuildGatewayEnv(&models.Instance{ + ID: 88, + Type: "openclaw", + RuntimeType: RuntimeBackendGateway, + AccessToken: &token, + EnvironmentOverridesJSON: raw, + }) + if err != nil { + t.Fatalf("BuildGatewayEnv returned error: %v", err) + } + + if env["CLAWMANAGER_TEAM_ENABLED"] != "true" || env["CLAWMANAGER_TEAM_MEMBER_ID"] != "lite-worker" { + t.Fatalf("expected Team environment overrides to be merged into gateway env, got %#v", env) + } + if env["CUSTOM_GATEWAY_ENV"] != "enabled" { + t.Fatalf("expected custom gateway environment override to be merged, got %#v", env) + } + if env["CLAWMANAGER_LLM_API_KEY"] != token { + t.Fatalf("expected gateway token env to remain available") + } + if env["CLAWMANAGER_RUNTIME_TYPE"] != RuntimeBackendGateway { + t.Fatalf("expected runtime type marker %q, got %q", RuntimeBackendGateway, env["CLAWMANAGER_RUNTIME_TYPE"]) + } +} + +func TestBuildGatewayEnvSkipsUnmanagedRuntime(t *testing.T) { + token := "igt_test_token" + service := &instanceService{} + + env, err := service.buildGatewayEnv(&models.Instance{ + Type: "ubuntu", + AccessToken: &token, + }) + if err != nil { + t.Fatalf("buildGatewayEnv returned error: %v", err) + } + if len(env) != 0 { + t.Fatalf("expected unmanaged runtime to receive no gateway env, got %#v", env) + } +} + +type stubGatewayEnvInstanceRepository struct { + *fakeRuntimeInstanceRepo + updated map[int]*models.Instance +} + +func (r *stubGatewayEnvInstanceRepository) Update(instance *models.Instance) error { + if r.updated == nil { + r.updated = map[int]*models.Instance{} + } + copy := *instance + r.updated[instance.ID] = © + return nil +} + +func TestBuildAgentEnvInjectsHermesAgentConfig(t *testing.T) { + t.Setenv("CLAWMANAGER_AGENT_CONTROL_BASE_URL", "http://agent-control.example") + + token := "agt_boot_test_token" + service := &instanceService{} + + env, err := service.buildAgentEnv(&models.Instance{ + ID: 24, + Type: "hermes", + DiskGB: 20, + AgentBootstrapToken: &token, + }) + if err != nil { + t.Fatalf("buildAgentEnv returned error: %v", err) + } + + if env["CLAWMANAGER_AGENT_ENABLED"] != "true" { + t.Fatalf("expected Hermes agent to be enabled") + } + if env["CLAWMANAGER_AGENT_BASE_URL"] != "http://agent-control.example" { + t.Fatalf("expected Hermes agent base URL to be injected, got %q", env["CLAWMANAGER_AGENT_BASE_URL"]) + } + if env["CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN"] != token { + t.Fatalf("expected Hermes agent bootstrap token to be injected") + } + if env["CLAWMANAGER_AGENT_INSTANCE_ID"] != "24" { + t.Fatalf("expected Hermes instance id to be injected, got %q", env["CLAWMANAGER_AGENT_INSTANCE_ID"]) + } + if env["CLAWMANAGER_AGENT_PERSISTENT_DIR"] != "/config/.hermes" { + t.Fatalf("expected Hermes persistent dir /config/.hermes, got %q", env["CLAWMANAGER_AGENT_PERSISTENT_DIR"]) + } + if env["CLAWMANAGER_AGENT_DISK_LIMIT_BYTES"] != "21474836480" { + t.Fatalf("expected Hermes disk limit bytes to be injected, got %q", env["CLAWMANAGER_AGENT_DISK_LIMIT_BYTES"]) + } +} + +func TestPersistentVolumeMountPathNormalizesManagedDesktopRuntimes(t *testing.T) { + for _, instanceType := range []string{"openclaw", "ubuntu", "webtop", "hermes"} { + t.Run(instanceType, func(t *testing.T) { + got := persistentVolumeMountPath(&models.Instance{ + Type: instanceType, + MountPath: "/data", + }) + if got != "/config" { + t.Fatalf("expected %s PVC mount path /config, got %q", instanceType, got) + } + }) + } +} + +func TestManagedRuntimePersistentDirKeepsHermesSubdirectory(t *testing.T) { + got := managedRuntimePersistentDir(&models.Instance{ + Type: "hermes", + MountPath: "/config", + }) + if got != "/config/.hermes" { + t.Fatalf("expected Hermes persistent dir /config/.hermes, got %q", got) + } +} + +func TestRuntimeVolumeInitScriptsAddsHermesLayoutMigration(t *testing.T) { + scripts := runtimeVolumeInitScripts("hermes", "/config") + if len(scripts) != 1 { + t.Fatalf("expected one Hermes volume init script, got %d", len(scripts)) + } + if scripts[0].Name != "data" || scripts[0].MountPath != "/config" { + t.Fatalf("unexpected Hermes volume init script: %#v", scripts[0]) + } + if !strings.Contains(scripts[0].Script, `target="$base/.hermes"`) { + t.Fatalf("expected Hermes init script to target /config/.hermes, got %s", scripts[0].Script) + } +} + +func TestResolveGatewayModelInjectionRequiresActiveModels(t *testing.T) { + service := &instanceService{ + llmModelRepo: &stubLLMModelRepository{}, + } + + injection, err := service.resolveGatewayModelInjection() + if err == nil { + t.Fatalf("expected resolveGatewayModelInjection to fail when no active models exist, got %#v", injection) + } +} + +func TestSecurityModeForInstance(t *testing.T) { + service := &instanceService{} + + if got := service.securityModeForInstance("openclaw"); got != "chromium-compat" { + t.Fatalf("expected openclaw to use chromium compat mode, got %q", got) + } + if got := service.securityModeForInstance("ubuntu"); got != "default" { + t.Fatalf("expected ubuntu to use default security mode, got %q", got) + } + + service.allowPrivilegedPods = true + if got := service.securityModeForInstance("openclaw"); got != "privileged" { + t.Fatalf("expected explicit privileged override to win, got %q", got) + } +} diff --git a/backend/internal/services/instance_service_v2_test.go b/backend/internal/services/instance_service_v2_test.go new file mode 100644 index 0000000..7ba7d70 --- /dev/null +++ b/backend/internal/services/instance_service_v2_test.go @@ -0,0 +1,910 @@ +package services + +import ( + "context" + "errors" + "os" + "path" + "strings" + "testing" + + "clawreef/internal/models" +) + +func TestInstanceServiceCreateV2CreatesWorkspaceOnly(t *testing.T) { + workspaceRoot := strings.ReplaceAll(t.TempDir(), "\\", "/") + instanceRepo := newV2LifecycleInstanceRepo() + service := &instanceService{ + instanceRepo: instanceRepo, + quotaRepo: v2LifecycleQuotaRepo{}, + workspaceRoot: workspaceRoot, + } + + instance, err := service.Create(45, CreateInstanceRequest{ + Name: "OpenClaw Dev", + Type: " OpenClaw ", + CPUCores: 2, + MemoryGB: 4, + DiskGB: 20, + OSType: "openclaw", + OSVersion: "latest", + }) + if err != nil { + t.Fatalf("Create returned error: %v", err) + } + + expectedWorkspacePath := RuntimeWorkspacePathWithRoot(workspaceRoot, "openclaw", 45, instance.ID) + if instance.Type != "openclaw" { + t.Fatalf("instance type = %q, want openclaw", instance.Type) + } + if instance.RuntimeType != "gateway" { + t.Fatalf("runtime type = %q, want gateway", instance.RuntimeType) + } + if instance.InstanceMode != InstanceModeLite { + t.Fatalf("instance mode = %q, want lite", instance.InstanceMode) + } + if instance.Status != "creating" { + t.Fatalf("status = %q, want creating", instance.Status) + } + if instance.RuntimeGeneration != 1 { + t.Fatalf("runtime generation = %d, want 1", instance.RuntimeGeneration) + } + if instance.WorkspacePath == nil || *instance.WorkspacePath != expectedWorkspacePath { + t.Fatalf("workspace path = %v, want %q", instance.WorkspacePath, expectedWorkspacePath) + } + if _, err := os.Stat(expectedWorkspacePath); err != nil { + t.Fatalf("expected workspace directory to exist: %v", err) + } + if os.PathSeparator == '/' { + assertDirMode(t, path.Dir(path.Dir(expectedWorkspacePath)), 0711) + assertDirMode(t, path.Dir(expectedWorkspacePath), 0711) + assertDirMode(t, expectedWorkspacePath, 0750) + } + if instance.PodName != nil || instance.PodNamespace != nil || instance.PodIP != nil { + t.Fatalf("V2 create must not set per-instance pod fields: %#v", instance) + } + if len(instanceRepo.created) != 1 { + t.Fatalf("created instance records = %d, want 1", len(instanceRepo.created)) + } +} + +func TestInstanceServiceCreateV2PersistsAgentTokensAndSnapshot(t *testing.T) { + workspaceRoot := strings.ReplaceAll(t.TempDir(), "\\", "/") + instanceRepo := newV2LifecycleInstanceRepo() + configService := &liteOpenClawConfigStub{ + snapshot: &models.OpenClawInjectionSnapshot{ID: 99, UserID: 45}, + } + service := &instanceService{ + instanceRepo: instanceRepo, + quotaRepo: v2LifecycleQuotaRepo{}, + workspaceRoot: workspaceRoot, + openClawConfigService: configService, + } + + instance, err := service.Create(45, CreateInstanceRequest{ + Name: "Lite With Skills", + Type: "openclaw", + Mode: InstanceModeLite, + CPUCores: 2, + MemoryGB: 4, + DiskGB: 20, + OSType: "openclaw", + OSVersion: "latest", + OpenClawConfigPlan: &OpenClawConfigPlan{ + Mode: OpenClawConfigPlanModeManual, + ResourceIDs: []int{7}, + }, + }) + if err != nil { + t.Fatalf("Create returned error: %v", err) + } + + if instance.AccessToken == nil || strings.TrimSpace(*instance.AccessToken) == "" { + t.Fatalf("expected lite instance gateway token to be provisioned") + } + if instance.AgentBootstrapToken == nil || strings.TrimSpace(*instance.AgentBootstrapToken) == "" { + t.Fatalf("expected lite instance agent bootstrap token to be provisioned") + } + if instance.OpenClawConfigSnapshotID == nil || *instance.OpenClawConfigSnapshotID != 99 { + t.Fatalf("snapshot id = %v, want 99", instance.OpenClawConfigSnapshotID) + } + if !configService.activated { + t.Fatalf("expected lite bootstrap snapshot to be marked active") + } + persisted := instanceRepo.byID[instance.ID] + if persisted == nil || persisted.AgentBootstrapToken == nil || persisted.OpenClawConfigSnapshotID == nil { + t.Fatalf("expected token and snapshot reference to be persisted, got %#v", persisted) + } +} + +func TestBuildGatewayEnvInjectsLiteAgentAndBootstrapEnv(t *testing.T) { + t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm") + t.Setenv("CLAWMANAGER_AGENT_CONTROL_BASE_URL", "http://agent-control.example") + + accessToken := "igt_test_token" + agentToken := "agt_boot_test_token" + snapshotID := 12 + workspacePath := "/workspaces/openclaw/user-45/instance-88" + service := &instanceService{ + llmModelRepo: &stubLLMModelRepository{active: []models.LLMModel{{DisplayName: "auto"}}}, + openClawConfigService: &liteOpenClawConfigStub{runtimeEnv: map[string]string{ + OpenClawSkillsEnv: `{"schemaVersion":1,"items":[{"key":"paper-ranker"}]}`, + }}, + } + + env, err := service.BuildGatewayEnv(&models.Instance{ + ID: 88, + UserID: 45, + Type: "openclaw", + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + AccessToken: &accessToken, + AgentBootstrapToken: &agentToken, + OpenClawConfigSnapshotID: &snapshotID, + WorkspacePath: &workspacePath, + DiskGB: 20, + }) + if err != nil { + t.Fatalf("BuildGatewayEnv returned error: %v", err) + } + + if env["CLAWMANAGER_AGENT_ENABLED"] != "true" || env["CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN"] != agentToken { + t.Fatalf("expected lite agent env to be injected, got %#v", env) + } + wantPersistentDir := path.Join(workspacePath, "home", ".openclaw") + if env["CLAWMANAGER_AGENT_PERSISTENT_DIR"] != wantPersistentDir { + t.Fatalf("persistent dir = %q, want %q", env["CLAWMANAGER_AGENT_PERSISTENT_DIR"], wantPersistentDir) + } + if !strings.Contains(env[OpenClawSkillsEnv], "paper-ranker") { + t.Fatalf("expected bootstrap skill env to be injected, got %q", env[OpenClawSkillsEnv]) + } +} +func TestBuildGatewayEnvInjectsHermesLitePersistentDir(t *testing.T) { + t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm") + t.Setenv("CLAWMANAGER_AGENT_CONTROL_BASE_URL", "http://agent-control.example") + + accessToken := "igt_test_token" + agentToken := "agt_boot_test_token" + workspacePath := "/workspaces/hermes/user-45/instance-90" + service := &instanceService{ + llmModelRepo: &stubLLMModelRepository{active: []models.LLMModel{{DisplayName: "auto"}}}, + openClawConfigService: &liteOpenClawConfigStub{}, + } + + env, err := service.BuildGatewayEnv(&models.Instance{ + ID: 90, + UserID: 45, + Type: "hermes", + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + AccessToken: &accessToken, + AgentBootstrapToken: &agentToken, + WorkspacePath: &workspacePath, + DiskGB: 20, + }) + if err != nil { + t.Fatalf("BuildGatewayEnv returned error: %v", err) + } + + wantPersistentDir := path.Join(workspacePath, "home", ".hermes") + if env["CLAWMANAGER_AGENT_PERSISTENT_DIR"] != wantPersistentDir { + t.Fatalf("persistent dir = %q, want %q", env["CLAWMANAGER_AGENT_PERSISTENT_DIR"], wantPersistentDir) + } +} +func TestInstanceServiceCreateLiteSkipsPerInstanceResourceQuota(t *testing.T) { + workspaceRoot := strings.ReplaceAll(t.TempDir(), "\\", "/") + instanceRepo := newV2LifecycleInstanceRepo() + service := &instanceService{ + instanceRepo: instanceRepo, + quotaRepo: fixedV2QuotaRepo{cpu: 1, memory: 1, storage: 10, gpu: 0}, + workspaceRoot: workspaceRoot, + } + + instance, err := service.Create(45, CreateInstanceRequest{ + Name: "Lite Gateway", + Type: "openclaw", + Mode: InstanceModeLite, + RuntimeType: RuntimeBackendGateway, + CPUCores: 8, + MemoryGB: 32, + DiskGB: 200, + GPUEnabled: true, + GPUCount: 1, + OSType: "openclaw", + OSVersion: "latest", + StorageClass: "manual", + }) + if err != nil { + t.Fatalf("Create lite returned error: %v", err) + } + if instance.InstanceMode != InstanceModeLite || instance.RuntimeType != RuntimeBackendGateway { + t.Fatalf("created runtime = mode %q type %q, want lite gateway", instance.InstanceMode, instance.RuntimeType) + } +} + +func TestInstanceServiceCreateProEnforcesPerInstanceResourceQuota(t *testing.T) { + instanceRepo := newV2LifecycleInstanceRepo() + service := &instanceService{ + instanceRepo: instanceRepo, + quotaRepo: fixedV2QuotaRepo{cpu: 1, memory: 1, storage: 10, gpu: 0}, + pvcService: nil, + deploymentService: nil, + serviceService: nil, + networkPolicyService: nil, + openClawConfigService: nil, + } + + _, err := service.Create(45, CreateInstanceRequest{ + Name: "Pro Desktop", + Type: "openclaw", + Mode: InstanceModePro, + RuntimeType: RuntimeBackendDesktop, + CPUCores: 8, + MemoryGB: 32, + DiskGB: 200, + OSType: "openclaw", + OSVersion: "latest", + }) + if err == nil || !strings.Contains(err.Error(), "CPU cores exceed quota") { + t.Fatalf("Create pro error = %v, want CPU quota failure", err) + } +} + +func assertDirMode(t *testing.T, dir string, want os.FileMode) { + t.Helper() + info, err := os.Stat(dir) + if err != nil { + t.Fatalf("stat %s: %v", dir, err) + } + if got := info.Mode().Perm(); got != want { + t.Fatalf("mode %s = %04o, want %04o", dir, got, want) + } +} + +func TestInstanceServiceStartV2MarksCreatingWithNextGeneration(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-77" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[77] = &models.Instance{ + ID: 77, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + Status: "stopped", + WorkspacePath: &workspacePath, + RuntimeGeneration: 4, + } + service := &instanceService{instanceRepo: instanceRepo} + + if err := service.Start(77); err != nil { + t.Fatalf("Start returned error: %v", err) + } + + state := instanceRepo.runtimeStates[77] + if state.status != "creating" || state.generation != 5 { + t.Fatalf("runtime state = %#v, want creating generation 5", state) + } +} + +func TestInstanceServiceStopV2DeletesGatewayBindingAndReleasesSlot(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-88" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[88] = &models.Instance{ + ID: 88, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 3, + } + endpoint := "http://agent.local:19090" + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, AgentEndpoint: &endpoint}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[88] = &models.InstanceRuntimeBinding{ + InstanceID: 88, + RuntimePodID: 9, + GatewayID: "gw-88", + GatewayPort: 20018, + State: "running", + Generation: 3, + } + agent := &fakeRuntimeAgentClient{} + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: podRepo, + bindingRepo: bindingRepo, + agentClient: agent, + workspaceRoot: "/workspaces", + } + + if err := service.Stop(88); err != nil { + t.Fatalf("Stop returned error: %v", err) + } + + if len(agent.deleteRequests) != 1 || agent.deleteRequests[0].endpoint != endpoint || agent.deleteRequests[0].gatewayID != "gw-88" { + t.Fatalf("delete gateway requests = %#v", agent.deleteRequests) + } + if bindingRepo.deleteAndReleaseCalls[88] != 1 { + t.Fatalf("binding delete and release calls = %d, want 1", bindingRepo.deleteAndReleaseCalls[88]) + } + state := instanceRepo.runtimeStates[88] + if state.status != "stopped" || state.generation != 3 { + t.Fatalf("runtime state = %#v, want stopped generation 3", state) + } +} + +func TestInstanceServiceStopV2KeepsBindingWhenAgentDeleteFails(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-188" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[188] = &models.Instance{ + ID: 188, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 6, + } + endpoint := "http://agent.local:19090" + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 19: {ID: 19, AgentEndpoint: &endpoint}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[188] = &models.InstanceRuntimeBinding{ + InstanceID: 188, + RuntimePodID: 19, + GatewayID: "gw-188", + GatewayPort: 20018, + State: "running", + Generation: 6, + } + agent := &fakeRuntimeAgentClient{deleteErr: errors.New("agent delete failed")} + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: podRepo, + bindingRepo: bindingRepo, + agentClient: agent, + } + + err := service.Stop(188) + if err == nil || !strings.Contains(err.Error(), "failed to delete v2 gateway") { + t.Fatalf("Stop error = %v, want gateway delete failure", err) + } + if bindingRepo.bindings[188] == nil { + t.Fatal("binding was deleted after gateway delete failed") + } + if bindingRepo.deleteAndReleaseCalls[188] != 0 { + t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[188]) + } + state := instanceRepo.runtimeStates[188] + if state.status != "stopped" || state.generation != 6 { + t.Fatalf("runtime state = %#v, want stopped generation 6", state) + } +} + +func TestInstanceServiceStopV2KeepsBindingWhenRuntimePodLookupFails(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-189" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[189] = &models.Instance{ + ID: 189, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 6, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[189] = &models.InstanceRuntimeBinding{ + InstanceID: 189, + RuntimePodID: 29, + GatewayID: "gw-189", + GatewayPort: 20019, + State: "running", + Generation: 6, + } + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: &fakeRuntimePodRepo{getErr: errors.New("pod lookup failed")}, + bindingRepo: bindingRepo, + agentClient: &fakeRuntimeAgentClient{}, + } + + err := service.Stop(189) + if err == nil || !strings.Contains(err.Error(), "failed to get runtime pod") { + t.Fatalf("Stop error = %v, want pod lookup failure", err) + } + if bindingRepo.bindings[189] == nil { + t.Fatal("binding was deleted after pod lookup failed") + } + if bindingRepo.deleteAndReleaseCalls[189] != 0 { + t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[189]) + } + state := instanceRepo.runtimeStates[189] + if state.status != "stopped" || state.generation != 6 { + t.Fatalf("runtime state = %#v, want stopped generation 6", state) + } +} + +func TestInstanceServiceRestartV2RecreatesGatewayViaScheduler(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-89" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[89] = &models.Instance{ + ID: 89, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 8, + } + endpoint := "http://agent.local:19090" + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 11: {ID: 11, AgentEndpoint: &endpoint}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[89] = &models.InstanceRuntimeBinding{ + InstanceID: 89, + RuntimePodID: 11, + GatewayID: "gw-89", + GatewayPort: 20019, + State: "running", + Generation: 8, + } + agent := &fakeRuntimeAgentClient{} + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: podRepo, + bindingRepo: bindingRepo, + agentClient: agent, + } + + if err := service.Restart(89); err != nil { + t.Fatalf("Restart returned error: %v", err) + } + + if len(agent.deleteRequests) != 1 || agent.deleteRequests[0].gatewayID != "gw-89" { + t.Fatalf("delete gateway requests = %#v", agent.deleteRequests) + } + if bindingRepo.deleteAndReleaseCalls[89] != 1 { + t.Fatalf("binding delete and release calls = %d, want 1", bindingRepo.deleteAndReleaseCalls[89]) + } + state := instanceRepo.runtimeStates[89] + if state.status != "creating" || state.generation != 9 { + t.Fatalf("runtime state = %#v, want creating generation 9", state) + } +} + +func TestInstanceServiceDeleteV2MissingBindingStillDeletesInstance(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-90" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[90] = &models.Instance{ + ID: 90, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + Status: "stopped", + WorkspacePath: &workspacePath, + RuntimeGeneration: 2, + } + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: &fakeRuntimePodRepo{}, + bindingRepo: newFakeRuntimeBindingRepo(), + agentClient: &fakeRuntimeAgentClient{}, + } + + if err := service.Delete(90); err != nil { + t.Fatalf("Delete returned error: %v", err) + } + if len(instanceRepo.deleted) != 1 || instanceRepo.deleted[0] != 90 { + t.Fatalf("deleted instances = %#v, want [90]", instanceRepo.deleted) + } +} + +func TestInstanceServiceDeleteV2KeepsInstanceAndBindingWhenAgentDeleteFails(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-190" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[190] = &models.Instance{ + ID: 190, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 2, + } + endpoint := "http://agent.local:19090" + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 39: {ID: 39, AgentEndpoint: &endpoint}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[190] = &models.InstanceRuntimeBinding{ + InstanceID: 190, + RuntimePodID: 39, + GatewayID: "gw-190", + GatewayPort: 20090, + State: "running", + Generation: 2, + } + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: podRepo, + bindingRepo: bindingRepo, + agentClient: &fakeRuntimeAgentClient{deleteErr: errors.New("agent delete failed")}, + } + + err := service.Delete(190) + if err == nil || !strings.Contains(err.Error(), "failed to delete v2 gateway") { + t.Fatalf("Delete error = %v, want gateway delete failure", err) + } + if len(instanceRepo.deleted) != 0 { + t.Fatalf("deleted instances = %#v, want none", instanceRepo.deleted) + } + if instanceRepo.byID[190] == nil { + t.Fatal("instance was removed after cleanup failure") + } + if bindingRepo.bindings[190] == nil { + t.Fatal("binding was removed after cleanup failure") + } + if bindingRepo.deleteAndReleaseCalls[190] != 0 { + t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[190]) + } +} + +func TestInstanceServiceV2StatusRequiresRunningBindingAndPodIP(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-92" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[92] = &models.Instance{ + ID: 92, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + WorkspaceUsageBytes: 2048, + RuntimeGeneration: 1, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[92] = &models.InstanceRuntimeBinding{ + InstanceID: 92, + RuntimePodID: 12, + GatewayPort: 20020, + State: "running", + } + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{12: {ID: 12}}}, + bindingRepo: bindingRepo, + } + + status, err := service.GetInstanceStatus(92) + if err != nil { + t.Fatalf("GetInstanceStatus returned error: %v", err) + } + if status.Availability != "unavailable" { + t.Fatalf("availability without pod IP = %q, want unavailable", status.Availability) + } + + podIP := "10.42.0.92" + service.runtimePodRepo = &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{12: {ID: 12, PodIP: &podIP}}} + status, err = service.GetInstanceStatus(92) + if err != nil { + t.Fatalf("GetInstanceStatus returned error after pod IP: %v", err) + } + if status.Availability != "available" { + t.Fatalf("availability with running binding and pod IP = %q, want available", status.Availability) + } +} + +func TestSyncInstanceSkipsV2RuntimeGatewayInstances(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-91" + instanceRepo := newV2LifecycleInstanceRepo() + service := &SyncService{instanceRepo: instanceRepo} + instance := &models.Instance{ + ID: 91, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + WorkspacePath: &workspacePath, + } + + service.syncInstance(context.Background(), instance) + + if len(instanceRepo.updated) != 0 { + t.Fatalf("sync should not update V2 gateway instance via pod status, got %#v", instanceRepo.updated) + } +} + +func TestV2RuntimeTypeForInstanceRequiresLiteGatewayBoundary(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-191" + lite := &models.Instance{ + ID: 191, + Type: "openclaw", + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: &workspacePath, + } + if runtimeType, ok := v2RuntimeTypeForInstance(lite); !ok || runtimeType != RuntimeTypeOpenClaw { + t.Fatalf("lite gateway instance was not recognized: %q %v", runtimeType, ok) + } + + pro := &models.Instance{ + ID: 192, + Type: "openclaw", + RuntimeType: RuntimeBackendDesktop, + InstanceMode: InstanceModePro, + WorkspacePath: &workspacePath, + } + if runtimeType, ok := v2RuntimeTypeForInstance(pro); ok { + t.Fatalf("pro desktop instance must not be scheduler-managed, got %q", runtimeType) + } +} + +func TestInstanceModeCapacityCanDisablePro(t *testing.T) { + t.Setenv("CLAWMANAGER_PRO_CAPACITY", "0") + service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()} + + err := service.enforceInstanceModeLimits(context.Background(), InstanceModePro, 2, 4, 20, 0) + if err == nil || !strings.Contains(err.Error(), "pro instance mode is disabled") { + t.Fatalf("capacity error = %v, want pro disabled", err) + } +} + +func TestInstanceModeResourceLimitRejectsOversizedPro(t *testing.T) { + t.Setenv("CLAWMANAGER_PRO_MAX_CPU_CORES", "1.5") + service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()} + + err := service.enforceInstanceModeLimits(context.Background(), InstanceModePro, 2, 4, 20, 0) + if err == nil || !strings.Contains(err.Error(), "pro CPU cores exceed mode limit") { + t.Fatalf("resource limit error = %v, want pro CPU limit", err) + } +} + +func TestInstanceModeResourceLimitAllowsOversizedLite(t *testing.T) { + t.Setenv("CLAWMANAGER_LITE_MAX_CPU_CORES", "1.5") + service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()} + + err := service.enforceInstanceModeLimits(context.Background(), InstanceModeLite, 2, 4, 20, 0) + if err != nil { + t.Fatalf("resource limit error = %v, want lite to skip dedicated resource limits", err) + } +} + +type liteOpenClawConfigStub struct { + OpenClawConfigService + snapshot *models.OpenClawInjectionSnapshot + runtimeEnv map[string]string + activated bool + failed bool +} + +func (s *liteOpenClawConfigStub) CreateSnapshotForInstance(userID int, instance *models.Instance, plan *OpenClawConfigPlan) (*models.OpenClawInjectionSnapshot, error) { + if s.snapshot == nil { + return nil, nil + } + return s.snapshot, nil +} + +func (s *liteOpenClawConfigStub) MarkSnapshotActive(snapshot *models.OpenClawInjectionSnapshot) error { + s.activated = true + if snapshot != nil { + snapshot.Status = openClawActiveSnapshotStatus + } + return nil +} + +func (s *liteOpenClawConfigStub) MarkSnapshotFailed(snapshot *models.OpenClawInjectionSnapshot, err error) error { + s.failed = true + return nil +} + +func (s *liteOpenClawConfigStub) RuntimeEnvForSnapshot(userID int, instanceType string, snapshotID int) (map[string]string, error) { + result := map[string]string{} + for key, value := range s.runtimeEnv { + result[key] = value + } + return result, nil +} + +type v2LifecycleInstanceRepo struct { + byID map[int]*models.Instance + created []models.Instance + updated []models.Instance + deleted []int + workspacePath map[int]string + runtimeStates map[int]v2RuntimeStateRecord + nextID int +} + +type v2RuntimeStateRecord struct { + status string + generation int + message *string +} + +func newV2LifecycleInstanceRepo() *v2LifecycleInstanceRepo { + return &v2LifecycleInstanceRepo{ + byID: map[int]*models.Instance{}, + workspacePath: map[int]string{}, + runtimeStates: map[int]v2RuntimeStateRecord{}, + nextID: 1, + } +} + +func (r *v2LifecycleInstanceRepo) Create(instance *models.Instance) error { + if instance.ID == 0 { + instance.ID = r.nextID + r.nextID++ + } + copy := *instance + r.byID[instance.ID] = © + r.created = append(r.created, copy) + return nil +} + +func (r *v2LifecycleInstanceRepo) GetByID(id int) (*models.Instance, error) { + return r.byID[id], nil +} + +func (r *v2LifecycleInstanceRepo) GetByAccessToken(string) (*models.Instance, error) { + return nil, nil +} + +func (r *v2LifecycleInstanceRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) { + return nil, nil +} + +func (r *v2LifecycleInstanceRepo) GetAll(offset, limit int) ([]models.Instance, error) { + return nil, nil +} + +func (r *v2LifecycleInstanceRepo) CountAll() (int, error) { return len(r.byID), nil } + +func (r *v2LifecycleInstanceRepo) GetByUserID(userID int, offset, limit int) ([]models.Instance, error) { + instances := make([]models.Instance, 0) + for _, instance := range r.byID { + if instance.UserID == userID { + instances = append(instances, *instance) + } + } + return instances, nil +} + +func (r *v2LifecycleInstanceRepo) CountByUserID(userID int) (int, error) { + count := 0 + for _, instance := range r.byID { + if instance.UserID == userID { + count++ + } + } + return count, nil +} + +func (r *v2LifecycleInstanceRepo) CountActiveByMode(ctx context.Context, mode string) (int, error) { + count := 0 + for _, instance := range r.byID { + if instance.InstanceMode == mode && (instance.Status == "creating" || instance.Status == "running") { + count++ + } + } + return count, nil +} + +func (r *v2LifecycleInstanceRepo) ExistsByUserIDAndName(userID int, name string) (bool, error) { + normalized := strings.TrimSpace(strings.ToLower(name)) + for _, instance := range r.byID { + if instance.UserID == userID && strings.TrimSpace(strings.ToLower(instance.Name)) == normalized { + return true, nil + } + } + return false, nil +} + +func (r *v2LifecycleInstanceRepo) GetAllRunning() ([]models.Instance, error) { + instances := make([]models.Instance, 0, len(r.byID)) + for _, instance := range r.byID { + instances = append(instances, *instance) + } + return instances, nil +} + +func (r *v2LifecycleInstanceRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) { + return nil, nil +} + +func (r *v2LifecycleInstanceRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) { + return nil, nil +} + +func (r *v2LifecycleInstanceRepo) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error { + instance := r.byID[id] + if instance != nil { + instance.Status = status + instance.RuntimeGeneration = generation + instance.RuntimeErrorMessage = message + } + r.runtimeStates[id] = v2RuntimeStateRecord{status: status, generation: generation, message: message} + return nil +} + +func (r *v2LifecycleInstanceRepo) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error { + instance := r.byID[id] + if instance != nil { + instance.WorkspacePath = &workspacePath + } + r.workspacePath[id] = workspacePath + return nil +} + +func (r *v2LifecycleInstanceRepo) UpdateWorkspaceUsage(context.Context, int, int64) error { + return nil +} + +func (r *v2LifecycleInstanceRepo) Update(instance *models.Instance) error { + copy := *instance + r.byID[instance.ID] = © + r.updated = append(r.updated, copy) + return nil +} + +func (r *v2LifecycleInstanceRepo) Delete(id int) error { + delete(r.byID, id) + r.deleted = append(r.deleted, id) + return nil +} + +type v2LifecycleQuotaRepo struct{} + +func (v2LifecycleQuotaRepo) Create(*models.UserQuota) error { return nil } +func (v2LifecycleQuotaRepo) GetByUserID(userID int) (*models.UserQuota, error) { + return &models.UserQuota{ + UserID: userID, + MaxInstances: 100, + MaxCPUCores: 100, + MaxMemoryGB: 1000, + MaxStorageGB: 1000, + MaxGPUCount: 8, + }, nil +} +func (v2LifecycleQuotaRepo) Update(*models.UserQuota) error { return nil } +func (v2LifecycleQuotaRepo) DeleteByUserID(int) error { return nil } +func (v2LifecycleQuotaRepo) CreateDefaultQuota(userID int) (*models.UserQuota, error) { + return v2LifecycleQuotaRepo{}.GetByUserID(userID) +} + +type fixedV2QuotaRepo struct { + cpu float64 + memory int + storage int + gpu int +} + +func (fixedV2QuotaRepo) Create(*models.UserQuota) error { return nil } +func (r fixedV2QuotaRepo) GetByUserID(userID int) (*models.UserQuota, error) { + return &models.UserQuota{ + UserID: userID, + MaxInstances: 100, + MaxCPUCores: r.cpu, + MaxMemoryGB: r.memory, + MaxStorageGB: r.storage, + MaxGPUCount: r.gpu, + }, nil +} +func (fixedV2QuotaRepo) Update(*models.UserQuota) error { return nil } +func (fixedV2QuotaRepo) DeleteByUserID(int) error { return nil } +func (r fixedV2QuotaRepo) CreateDefaultQuota(userID int) (*models.UserQuota, error) { + return r.GetByUserID(userID) +} diff --git a/backend/internal/services/instance_shell_service.go b/backend/internal/services/instance_shell_service.go new file mode 100644 index 0000000..20c1814 --- /dev/null +++ b/backend/internal/services/instance_shell_service.go @@ -0,0 +1,238 @@ +package services + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sync" + + "clawreef/internal/services/k8s" + + "github.com/gorilla/websocket" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" +) + +// InstanceShellService streams an interactive shell into an instance pod. +type InstanceShellService struct { + podService *k8s.PodService + upgrader websocket.Upgrader +} + +type shellClientMessage struct { + Type string `json:"type"` + Data string `json:"data,omitempty"` + Cols uint16 `json:"cols,omitempty"` + Rows uint16 `json:"rows,omitempty"` +} + +type shellTerminalSizeQueue struct { + sizes chan remotecommand.TerminalSize + mu sync.Mutex + closed bool +} + +type shellWebSocketWriter struct { + conn *websocket.Conn + mu *sync.Mutex + done <-chan struct{} +} + +func NewInstanceShellService() *InstanceShellService { + return &InstanceShellService{ + podService: k8s.NewPodService(), + upgrader: websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { return true }, + }, + } +} + +func (s *InstanceShellService) Stream(ctx context.Context, userID, instanceID int, w http.ResponseWriter, r *http.Request) error { + if s.podService == nil || s.podService.GetClient() == nil || s.podService.GetClient().Clientset == nil { + return fmt.Errorf("k8s client not initialized") + } + + conn, err := s.upgrader.Upgrade(w, r, nil) + if err != nil { + return fmt.Errorf("failed to upgrade shell websocket: %w", err) + } + + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + + writeMu := &sync.Mutex{} + done := make(chan struct{}) + defer close(done) + defer conn.Close() + + stdinReader, stdinWriter := io.Pipe() + defer stdinReader.Close() + defer stdinWriter.Close() + + sizeQueue := newShellTerminalSizeQueue() + defer sizeQueue.close() + sizeQueue.push(120, 32) + + go readShellWebSocket(streamCtx, cancel, conn, stdinWriter, sizeQueue) + + pod, err := s.podService.GetPod(streamCtx, userID, instanceID) + if err != nil { + _ = writeShellBytes(conn, writeMu, done, []byte(fmt.Sprintf("\r\nfailed to get pod: %v\r\n", err))) + return fmt.Errorf("failed to get pod: %w", err) + } + + req := s.podService.GetClient().Clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("exec") + + req.VersionedParams(&corev1.PodExecOptions{ + Container: "desktop", + Command: defaultShellCommand(), + Stdin: true, + Stdout: true, + Stderr: false, + TTY: true, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(s.podService.GetClient().Config, "POST", req.URL()) + if err != nil { + _ = writeShellBytes(conn, writeMu, done, []byte(fmt.Sprintf("\r\nfailed to initialize shell: %v\r\n", err))) + return fmt.Errorf("failed to initialize shell stream: %w", err) + } + + writer := &shellWebSocketWriter{conn: conn, mu: writeMu, done: done} + if err := exec.StreamWithContext(streamCtx, remotecommand.StreamOptions{ + Stdin: stdinReader, + Stdout: writer, + Tty: true, + TerminalSizeQueue: sizeQueue, + }); err != nil && streamCtx.Err() == nil { + _ = writeShellBytes(conn, writeMu, done, []byte(fmt.Sprintf("\r\nshell closed: %v\r\n", err))) + return fmt.Errorf("shell stream failed: %w", err) + } + + return nil +} + +func defaultShellCommand() []string { + return []string{ + "sh", + "-lc", + `export TERM="${TERM:-xterm-256color}" COLORTERM="${COLORTERM:-truecolor}"; if command -v tmux >/dev/null 2>&1; then session="${CLAWMANAGER_TMUX_SESSION:-openclaw-shell}"; exec tmux new-session -A -s "$session"; fi; if command -v bash >/dev/null 2>&1; then exec bash -l; fi; exec sh`, + } +} + +func newShellTerminalSizeQueue() *shellTerminalSizeQueue { + return &shellTerminalSizeQueue{ + sizes: make(chan remotecommand.TerminalSize, 4), + } +} + +func (q *shellTerminalSizeQueue) Next() *remotecommand.TerminalSize { + size, ok := <-q.sizes + if !ok { + return nil + } + return &size +} + +func (q *shellTerminalSizeQueue) push(cols, rows uint16) { + if q == nil || cols == 0 || rows == 0 { + return + } + q.mu.Lock() + defer q.mu.Unlock() + if q.closed { + return + } + size := remotecommand.TerminalSize{Width: cols, Height: rows} + select { + case q.sizes <- size: + default: + select { + case <-q.sizes: + default: + } + select { + case q.sizes <- size: + default: + } + } +} + +func (q *shellTerminalSizeQueue) close() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + if !q.closed { + q.closed = true + close(q.sizes) + } +} + +func readShellWebSocket(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, stdin *io.PipeWriter, sizeQueue *shellTerminalSizeQueue) { + defer cancel() + defer stdin.Close() + + for { + select { + case <-ctx.Done(): + return + default: + } + + messageType, payload, err := conn.ReadMessage() + if err != nil { + _ = stdin.CloseWithError(err) + return + } + switch messageType { + case websocket.BinaryMessage: + if len(payload) > 0 { + _, _ = stdin.Write(payload) + } + case websocket.TextMessage: + var message shellClientMessage + if json.Unmarshal(payload, &message) == nil && message.Type != "" { + switch message.Type { + case "input": + if message.Data != "" { + _, _ = io.WriteString(stdin, message.Data) + } + case "resize": + sizeQueue.push(message.Cols, message.Rows) + } + continue + } + + if len(payload) > 0 { + _, _ = stdin.Write(payload) + } + } + } +} + +func (w *shellWebSocketWriter) Write(payload []byte) (int, error) { + if err := writeShellBytes(w.conn, w.mu, w.done, payload); err != nil { + return 0, err + } + return len(payload), nil +} + +func writeShellBytes(conn *websocket.Conn, mu *sync.Mutex, done <-chan struct{}, payload []byte) error { + select { + case <-done: + return io.ErrClosedPipe + default: + } + mu.Lock() + defer mu.Unlock() + return conn.WriteMessage(websocket.BinaryMessage, payload) +} diff --git a/backend/internal/services/instance_visibility_test.go b/backend/internal/services/instance_visibility_test.go new file mode 100644 index 0000000..7dd9981 --- /dev/null +++ b/backend/internal/services/instance_visibility_test.go @@ -0,0 +1,178 @@ +package services + +import ( + "context" + "testing" + + "clawreef/internal/models" +) + +// stubInstanceVisibilityRepo is a minimal InstanceRepository used by the +// visibility tests below. It only implements the read paths exercised by +// GetByUserID / GetAllInstances; every other method panics so that any +// accidental call surfaces as a test failure instead of a silent stub. +type stubInstanceVisibilityRepo struct { + all []models.Instance +} + +func (r *stubInstanceVisibilityRepo) byUser(userID int) []models.Instance { + out := make([]models.Instance, 0, len(r.all)) + for _, inst := range r.all { + if inst.UserID == userID { + out = append(out, inst) + } + } + return out +} + +func (r *stubInstanceVisibilityRepo) Create(*models.Instance) error { panic("not used") } +func (r *stubInstanceVisibilityRepo) GetByID(int) (*models.Instance, error) { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) GetByAccessToken(string) (*models.Instance, error) { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) GetAll(offset, limit int) ([]models.Instance, error) { + return paginate(r.all, offset, limit), nil +} +func (r *stubInstanceVisibilityRepo) CountAll() (int, error) { return len(r.all), nil } +func (r *stubInstanceVisibilityRepo) GetByUserID(userID, offset, limit int) ([]models.Instance, error) { + return paginate(r.byUser(userID), offset, limit), nil +} +func (r *stubInstanceVisibilityRepo) CountByUserID(userID int) (int, error) { + return len(r.byUser(userID)), nil +} +func (r *stubInstanceVisibilityRepo) CountActiveByMode(context.Context, string) (int, error) { + return 0, nil +} +func (r *stubInstanceVisibilityRepo) ExistsByUserIDAndName(int, string) (bool, error) { + return false, nil +} +func (r *stubInstanceVisibilityRepo) GetAllRunning() ([]models.Instance, error) { + return nil, nil +} +func (r *stubInstanceVisibilityRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) UpdateRuntimeState(context.Context, int, string, int, *string) error { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) SetWorkspacePath(context.Context, int, string) error { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) UpdateWorkspaceUsage(context.Context, int, int64) error { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) Update(*models.Instance) error { panic("not used") } +func (r *stubInstanceVisibilityRepo) Delete(int) error { panic("not used") } + +func paginate(items []models.Instance, offset, limit int) []models.Instance { + if offset >= len(items) { + return []models.Instance{} + } + end := offset + limit + if end > len(items) { + end = len(items) + } + out := make([]models.Instance, end-offset) + copy(out, items[offset:end]) + return out +} + +// TestGetByUserIDFiltersByCaller locks in the workspace-view contract: the +// caller-scoped listing must return only the caller's own instances, +// regardless of any role the caller may hold elsewhere. Regression guard +// for the admin-cross-user-leakage bug in which role=admin widened this +// endpoint to every user's instances. +func TestGetByUserIDFiltersByCaller(t *testing.T) { + t.Parallel() + + repo := &stubInstanceVisibilityRepo{ + all: []models.Instance{ + {ID: 1, UserID: 10, Name: "alice-1"}, + {ID: 2, UserID: 10, Name: "alice-2"}, + {ID: 3, UserID: 20, Name: "bob-1"}, + }, + } + svc := &instanceService{instanceRepo: repo} + + instances, total, err := svc.GetByUserID(10, 0, 50) + if err != nil { + t.Fatalf("GetByUserID returned error: %v", err) + } + if total != 2 { + t.Fatalf("expected total=2 for alice, got %d", total) + } + if len(instances) != 2 || instances[0].Name != "alice-1" || instances[1].Name != "alice-2" { + t.Fatalf("expected alice's instances only, got %+v", instances) + } + + instances, total, err = svc.GetByUserID(20, 0, 50) + if err != nil { + t.Fatalf("GetByUserID returned error: %v", err) + } + if total != 1 { + t.Fatalf("expected total=1 for bob, got %d", total) + } + if len(instances) != 1 || instances[0].Name != "bob-1" { + t.Fatalf("expected bob's instance only, got %+v", instances) + } + + // A user with no instances sees an empty list, never someone else's. + instances, total, err = svc.GetByUserID(99, 0, 50) + if err != nil { + t.Fatalf("GetByUserID returned error: %v", err) + } + if total != 0 || len(instances) != 0 { + t.Fatalf("expected empty listing for user 99, got total=%d instances=%+v", total, instances) + } +} + +// TestGetAllInstancesReturnsEveryUser verifies the admin-console surface: +// cross-user listing, with pagination, independent of caller identity (the +// admin middleware gates reachability at the route layer, not here). +func TestGetAllInstancesReturnsEveryUser(t *testing.T) { + t.Parallel() + + repo := &stubInstanceVisibilityRepo{ + all: []models.Instance{ + {ID: 1, UserID: 10, Name: "alice-1"}, + {ID: 2, UserID: 10, Name: "alice-2"}, + {ID: 3, UserID: 20, Name: "bob-1"}, + }, + } + svc := &instanceService{instanceRepo: repo} + + instances, total, err := svc.GetAllInstances(0, 50) + if err != nil { + t.Fatalf("GetAllInstances returned error: %v", err) + } + if total != 3 { + t.Fatalf("expected total=3 across users, got %d", total) + } + if len(instances) != 3 { + t.Fatalf("expected 3 instances, got %d", len(instances)) + } + + // Pagination still respected. + page1, _, err := svc.GetAllInstances(0, 2) + if err != nil { + t.Fatalf("GetAllInstances page1 error: %v", err) + } + if len(page1) != 2 { + t.Fatalf("expected page1 size=2, got %d", len(page1)) + } + page2, _, err := svc.GetAllInstances(2, 2) + if err != nil { + t.Fatalf("GetAllInstances page2 error: %v", err) + } + if len(page2) != 1 || page2[0].Name != "bob-1" { + t.Fatalf("expected page2=[bob-1], got %+v", page2) + } +} diff --git a/backend/internal/services/k8s/cleanup_service.go b/backend/internal/services/k8s/cleanup_service.go new file mode 100644 index 0000000..a2ef983 --- /dev/null +++ b/backend/internal/services/k8s/cleanup_service.go @@ -0,0 +1,399 @@ +package k8s + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CleanupService handles cleanup of K8s resources +type CleanupService struct { + client *Client +} + +// NewCleanupService creates a new cleanup service +func NewCleanupService() *CleanupService { + return &CleanupService{ + client: globalClient, + } +} + +// DeleteAllInstanceResources deletes all K8s resources for an instance (including duplicates) +// If userID is 0, it will search all clawreef namespaces +func (s *CleanupService) DeleteAllInstanceResources(ctx context.Context, userID, instanceID int) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + instanceLabel := fmt.Sprintf("%d", instanceID) + + // If userID is 0, find all namespaces with this instance + var namespaces []string + if userID == 0 { + nsList, err := s.findNamespacesWithInstance(ctx, instanceLabel) + if err != nil { + fmt.Printf("Warning: failed to find namespaces: %v\n", err) + } + namespaces = nsList + fmt.Printf("Found instance %d in %d namespace(s): %v\n", instanceID, len(namespaces), namespaces) + } else { + namespaces = []string{s.client.GetNamespace(userID)} + } + + // Delete resources from all found namespaces + for _, namespace := range namespaces { + fmt.Printf("Deleting resources for instance %d in namespace %s\n", instanceID, namespace) + + // 1. Delete all Deployments first so managed Pods are not recreated. + if err := s.deleteAllDeployments(ctx, namespace, instanceLabel); err != nil { + fmt.Printf("Warning: error deleting deployments: %v\n", err) + } + + // 2. Delete all Pods with matching instance-id label + if err := s.deleteAllPods(ctx, namespace, instanceLabel); err != nil { + fmt.Printf("Warning: error deleting pods: %v\n", err) + } + + // 3. Delete all PVCs with matching instance-id label + if err := s.deleteAllPVCs(ctx, namespace, instanceLabel); err != nil { + fmt.Printf("Warning: error deleting PVCs: %v\n", err) + } + + // 4. Delete all Services with matching instance-id label + if err := s.deleteAllServices(ctx, namespace, instanceLabel); err != nil { + fmt.Printf("Warning: error deleting services: %v\n", err) + } + + // 5. Delete all NetworkPolicies with matching instance-id label + if err := s.deleteAllNetworkPolicies(ctx, namespace, instanceLabel); err != nil { + fmt.Printf("Warning: error deleting network policies: %v\n", err) + } + + // 6. Delete all ConfigMaps with matching instance-id label + if err := s.deleteAllConfigMaps(ctx, namespace, instanceLabel); err != nil { + fmt.Printf("Warning: error deleting configmaps: %v\n", err) + } + + // 7. Delete all Secrets with matching instance-id label + if err := s.deleteAllSecrets(ctx, namespace, instanceLabel); err != nil { + fmt.Printf("Warning: error deleting secrets: %v\n", err) + } + } + + // 8. Delete all PVs associated with this instance (PVs are not namespaced) + if err := s.deleteAllPVs(ctx, userID, instanceID); err != nil { + fmt.Printf("Warning: error deleting PVs: %v\n", err) + } + + // Wait for all resources to be actually deleted (with 30 second timeout) + if err := s.WaitForResourceDeletion(ctx, namespaces, instanceLabel, 30*time.Second); err != nil { + fmt.Printf("Warning: %v\n", err) + } + + return nil +} + +// findNamespacesWithInstance finds all namespaces containing resources for this instance +func (s *CleanupService) findNamespacesWithInstance(ctx context.Context, instanceLabel string) ([]string, error) { + // List all namespaces with clawreef label + namespaces, err := s.client.Clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{ + LabelSelector: "managed-by=clawreef", + }) + if err != nil { + return nil, err + } + + var result []string + for _, ns := range namespaces.Items { + deployments, err := s.client.Clientset.AppsV1().Deployments(ns.Name).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err == nil && len(deployments.Items) > 0 { + result = append(result, ns.Name) + continue + } + + // Check if this namespace has any pods for this instance + pods, err := s.client.Clientset.CoreV1().Pods(ns.Name).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err == nil && len(pods.Items) > 0 { + result = append(result, ns.Name) + continue + } + + // Check if this namespace has any PVCs for this instance + pvcs, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(ns.Name).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err == nil && len(pvcs.Items) > 0 { + result = append(result, ns.Name) + continue + } + + networkPolicies, err := s.client.Clientset.NetworkingV1().NetworkPolicies(ns.Name).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err == nil && len(networkPolicies.Items) > 0 { + result = append(result, ns.Name) + } + } + + return result, nil +} + +// deleteAllDeployments deletes all deployments with matching instance-id label. +func (s *CleanupService) deleteAllDeployments(ctx context.Context, namespace, instanceLabel string) error { + deployments, err := s.client.Clientset.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err != nil { + return fmt.Errorf("failed to list deployments: %w", err) + } + + if len(deployments.Items) == 0 { + fmt.Printf("No deployments found for instance %s\n", instanceLabel) + return nil + } + + for _, deployment := range deployments.Items { + fmt.Printf("Deleting deployment: %s\n", deployment.Name) + propagation := metav1.DeletePropagationForeground + if err := s.client.Clientset.AppsV1().Deployments(namespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{ + PropagationPolicy: &propagation, + }); err != nil && !errors.IsNotFound(err) { + fmt.Printf("Warning: failed to delete deployment %s: %v\n", deployment.Name, err) + } + } + + return nil +} + +// deleteAllPods deletes all pods with matching instance-id label +func (s *CleanupService) deleteAllPods(ctx context.Context, namespace, instanceLabel string) error { + pods, err := s.client.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err != nil { + return fmt.Errorf("failed to list pods: %w", err) + } + + if len(pods.Items) == 0 { + fmt.Printf("No pods found for instance %s\n", instanceLabel) + return nil + } + + for _, pod := range pods.Items { + fmt.Printf("Deleting pod: %s\n", pod.Name) + if err := s.client.Clientset.CoreV1().Pods(namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}); err != nil { + fmt.Printf("Warning: failed to delete pod %s: %v\n", pod.Name, err) + } + } + + return nil +} + +// deleteAllPVCs deletes all PVCs with matching instance-id label +func (s *CleanupService) deleteAllPVCs(ctx context.Context, namespace, instanceLabel string) error { + pvcs, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err != nil { + return fmt.Errorf("failed to list PVCs: %w", err) + } + + if len(pvcs.Items) == 0 { + fmt.Printf("No PVCs found for instance %s\n", instanceLabel) + return nil + } + + for _, pvc := range pvcs.Items { + fmt.Printf("Deleting PVC: %s\n", pvc.Name) + if err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvc.Name, metav1.DeleteOptions{}); err != nil { + fmt.Printf("Warning: failed to delete PVC %s: %v\n", pvc.Name, err) + } + } + + return nil +} + +// deleteAllPVs deletes all PVs for an instance +func (s *CleanupService) deleteAllPVs(ctx context.Context, userID, instanceID int) error { + instanceLabel := fmt.Sprintf("%d", instanceID) + + // If userID is specified, try to delete the predictable PV name first + if userID > 0 { + pvName := s.client.GetInstancePVName(userID, instanceID) + fmt.Printf("Deleting PV: %s\n", pvName) + if err := s.client.Clientset.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil { + // Ignore not found error + if !errors.IsNotFound(err) { + fmt.Printf("Warning: failed to delete PV %s: %v\n", pvName, err) + } + } + } + + // Search for any PVs with matching instance-id label + pvs, err := s.client.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err != nil { + return fmt.Errorf("failed to list PVs: %w", err) + } + + for _, pv := range pvs.Items { + fmt.Printf("Deleting PV: %s\n", pv.Name) + if err := s.client.Clientset.CoreV1().PersistentVolumes().Delete(ctx, pv.Name, metav1.DeleteOptions{}); err != nil { + fmt.Printf("Warning: failed to delete PV %s: %v\n", pv.Name, err) + } + } + + return nil +} + +// deleteAllServices deletes all services with matching instance-id label +func (s *CleanupService) deleteAllServices(ctx context.Context, namespace, instanceLabel string) error { + services, err := s.client.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err != nil { + return fmt.Errorf("failed to list services: %w", err) + } + + for _, svc := range services.Items { + fmt.Printf("Deleting service: %s\n", svc.Name) + if err := s.client.Clientset.CoreV1().Services(namespace).Delete(ctx, svc.Name, metav1.DeleteOptions{}); err != nil { + fmt.Printf("Warning: failed to delete service %s: %v\n", svc.Name, err) + } + } + + return nil +} + +// deleteAllNetworkPolicies deletes all network policies with matching instance-id label. +func (s *CleanupService) deleteAllNetworkPolicies(ctx context.Context, namespace, instanceLabel string) error { + networkPolicies, err := s.client.Clientset.NetworkingV1().NetworkPolicies(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err != nil { + return fmt.Errorf("failed to list network policies: %w", err) + } + + for _, policy := range networkPolicies.Items { + fmt.Printf("Deleting network policy: %s\n", policy.Name) + if err := s.client.Clientset.NetworkingV1().NetworkPolicies(namespace).Delete(ctx, policy.Name, metav1.DeleteOptions{}); err != nil { + fmt.Printf("Warning: failed to delete network policy %s: %v\n", policy.Name, err) + } + } + + return nil +} + +// deleteAllConfigMaps deletes all configmaps with matching instance-id label +func (s *CleanupService) deleteAllConfigMaps(ctx context.Context, namespace, instanceLabel string) error { + configMaps, err := s.client.Clientset.CoreV1().ConfigMaps(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err != nil { + return fmt.Errorf("failed to list configmaps: %w", err) + } + + for _, cm := range configMaps.Items { + fmt.Printf("Deleting configmap: %s\n", cm.Name) + if err := s.client.Clientset.CoreV1().ConfigMaps(namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{}); err != nil { + fmt.Printf("Warning: failed to delete configmap %s: %v\n", cm.Name, err) + } + } + + return nil +} + +// deleteAllSecrets deletes all secrets with matching instance-id label +func (s *CleanupService) deleteAllSecrets(ctx context.Context, namespace, instanceLabel string) error { + secrets, err := s.client.Clientset.CoreV1().Secrets(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err != nil { + return fmt.Errorf("failed to list secrets: %w", err) + } + + for _, secret := range secrets.Items { + fmt.Printf("Deleting secret: %s\n", secret.Name) + if err := s.client.Clientset.CoreV1().Secrets(namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}); err != nil { + fmt.Printf("Warning: failed to delete secret %s: %v\n", secret.Name, err) + } + } + + return nil +} + +// WaitForResourceDeletion waits for all resources to be deleted with timeout +func (s *CleanupService) WaitForResourceDeletion(ctx context.Context, namespaces []string, instanceLabel string, timeout time.Duration) error { + fmt.Printf("Waiting up to %v for resources to be deleted...\n", timeout) + + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + timeoutChan := time.After(timeout) + + for { + select { + case <-timeoutChan: + return fmt.Errorf("timeout waiting for resources to be deleted") + case <-ticker.C: + allDeleted := true + + for _, namespace := range namespaces { + deployments, err := s.client.Clientset.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err == nil && len(deployments.Items) > 0 { + allDeleted = false + fmt.Printf(" Still waiting: %d deployment(s) in %s\n", len(deployments.Items), namespace) + } + + // Check pods + pods, err := s.client.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err == nil && len(pods.Items) > 0 { + allDeleted = false + fmt.Printf(" Still waiting: %d pod(s) in %s\n", len(pods.Items), namespace) + } + + // Check PVCs + pvcs, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err == nil && len(pvcs.Items) > 0 { + allDeleted = false + fmt.Printf(" Still waiting: %d PVC(s) in %s\n", len(pvcs.Items), namespace) + } + + networkPolicies, err := s.client.Clientset.NetworkingV1().NetworkPolicies(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err == nil && len(networkPolicies.Items) > 0 { + allDeleted = false + fmt.Printf(" Still waiting: %d network policy(s) in %s\n", len(networkPolicies.Items), namespace) + } + } + + // Check PVs + pvs, err := s.client.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%s,managed-by=clawreef", instanceLabel), + }) + if err == nil && len(pvs.Items) > 0 { + allDeleted = false + fmt.Printf(" Still waiting: %d PV(s)\n", len(pvs.Items)) + } + + if allDeleted { + fmt.Println("All resources deleted successfully") + return nil + } + } + } +} diff --git a/backend/internal/services/k8s/client.go b/backend/internal/services/k8s/client.go new file mode 100644 index 0000000..003a243 --- /dev/null +++ b/backend/internal/services/k8s/client.go @@ -0,0 +1,328 @@ +package k8s + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "clawreef/internal/config" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" +) + +// ConnectionMode defines how to connect to Kubernetes +type ConnectionMode string + +const ( + // ModeAuto automatically detects in-cluster or out-of-cluster + ModeAuto ConnectionMode = "auto" + // ModeInCluster forces in-cluster configuration + ModeInCluster ConnectionMode = "incluster" + // ModeOutOfCluster forces out-of-cluster configuration using kubeconfig + ModeOutOfCluster ConnectionMode = "outofcluster" +) + +// Client wraps the Kubernetes client +type Client struct { + Clientset kubernetes.Interface + Config *rest.Config + Namespace string + StorageClass string + StorageProfile string + HostPathFallbackEnabled bool + PVCBindTimeout time.Duration + ControlPlaneStorageClass string + InstanceStorageClass string + WorkspaceStorageClass string + WorkspaceAccessMode string + HostPathPrefix string + WorkspaceRoot string + WorkspacePVCClaimName string + WorkspaceNFSServer string + WorkspaceNFSPath string + Mode ConnectionMode +} + +var ( + // globalClient is the global K8s client instance + globalClient *Client + k8sNameInvalidChars = regexp.MustCompile(`[^a-z0-9-]+`) + k8sNameExtraDashes = regexp.MustCompile(`-+`) +) + +// Initialize initializes the Kubernetes client with support for both in-cluster and out-of-cluster modes +// Configuration priority: +// 1. configs/k8s.yaml - Main configuration file +// 2. Environment variables - Override config file settings +func Initialize(cfg *config.Config) error { + mode := ConnectionMode(cfg.GetMode()) + if mode == "" { + mode = ModeAuto + } + + var restConfig *rest.Config + var err error + var detectedMode ConnectionMode + + switch mode { + case ModeInCluster: + restConfig, err = buildInClusterConfig(cfg) + if err != nil { + return fmt.Errorf("failed to initialize in-cluster config: %w", err) + } + detectedMode = ModeInCluster + + case ModeOutOfCluster: + restConfig, err = buildOutOfClusterConfig(cfg) + if err != nil { + return fmt.Errorf("failed to initialize out-of-cluster config: %w", err) + } + detectedMode = ModeOutOfCluster + + case ModeAuto: + // Try in-cluster first + restConfig, err = buildInClusterConfig(cfg) + if err == nil { + detectedMode = ModeInCluster + } else { + // Fall back to out-of-cluster + restConfig, err = buildOutOfClusterConfig(cfg) + if err != nil { + return fmt.Errorf("failed to initialize K8s client (both in-cluster and out-of-cluster failed): %w", err) + } + detectedMode = ModeOutOfCluster + } + + default: + return fmt.Errorf("invalid K8S_MODE: %s (must be 'auto', 'incluster', or 'outofcluster')", mode) + } + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to create kubernetes clientset: %w", err) + } + + globalClient = &Client{ + Clientset: clientset, + Config: restConfig, + Namespace: cfg.GetNamespace(), + StorageClass: cfg.GetStorageClass(), + StorageProfile: cfg.GetStorageProfile(), + HostPathFallbackEnabled: cfg.Storage.HostPathFallbackEnabled, + PVCBindTimeout: cfg.GetPVCBindTimeout(), + ControlPlaneStorageClass: cfg.GetControlPlaneStorageClass(), + InstanceStorageClass: cfg.GetInstanceStorageClass(), + WorkspaceStorageClass: cfg.GetWorkspaceStorageClass(), + WorkspaceAccessMode: cfg.GetWorkspaceAccessMode(), + HostPathPrefix: cfg.GetHostPathPrefix(), + WorkspaceRoot: cfg.Runtime.WorkspaceRoot, + WorkspacePVCClaimName: cfg.Runtime.WorkspacePVCClaimName, + WorkspaceNFSServer: cfg.Runtime.WorkspaceNFSServer, + WorkspaceNFSPath: cfg.Runtime.WorkspaceNFSPath, + Mode: detectedMode, + } + + return nil +} + +// buildInClusterConfig builds REST config for in-cluster mode +func buildInClusterConfig(cfg *config.Config) (*rest.Config, error) { + // Use standard in-cluster configuration + restConfig, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + // Apply timeout from config + if cfg.Kubernetes.Common.Timeout > 0 { + restConfig.Timeout = time.Duration(cfg.Kubernetes.Common.Timeout) * time.Second + } + + return restConfig, nil +} + +// buildOutOfClusterConfig builds REST config from kubeconfig file +func buildOutOfClusterConfig(cfg *config.Config) (*rest.Config, error) { + kubeconfig := cfg.GetKubeconfigPath() + + // Priority for kubeconfig location: + // 1. Config file setting (configs/k8s.yaml -> kubernetes.outOfCluster.kubeconfig) + // 2. KUBECONFIG environment variable + // 3. K8S_KUBECONFIG environment variable + // 4. Default location (~/.kube/config) + if kubeconfig == "" { + if kubeconfig = os.Getenv("KUBECONFIG"); kubeconfig == "" { + if kubeconfig = os.Getenv("K8S_KUBECONFIG"); kubeconfig == "" { + home := homedir.HomeDir() + kubeconfig = filepath.Join(home, ".kube", "config") + } + } + } + + // Check if kubeconfig file exists + if _, err := os.Stat(kubeconfig); os.IsNotExist(err) { + return nil, fmt.Errorf("kubeconfig file not found at %s", kubeconfig) + } + + // Build config from kubeconfig file + loadingRules := &clientcmd.ClientConfigLoadingRules{ + ExplicitPath: kubeconfig, + } + + configOverrides := &clientcmd.ConfigOverrides{} + + // Override context if specified + if cfg.Kubernetes.OutOfCluster.Context != "" { + configOverrides.CurrentContext = cfg.Kubernetes.OutOfCluster.Context + } + + // Override API server if specified + if cfg.Kubernetes.OutOfCluster.APIServer != "" { + configOverrides.ClusterInfo.Server = cfg.Kubernetes.OutOfCluster.APIServer + } + + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + loadingRules, + configOverrides, + ) + + restConfig, err := kubeConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("failed to build config from kubeconfig %s: %w", kubeconfig, err) + } + + // Apply TLS settings + if cfg.Kubernetes.OutOfCluster.TLS.InsecureSkipVerify { + restConfig.Insecure = true + restConfig.TLSClientConfig.Insecure = true + } + + if cfg.Kubernetes.OutOfCluster.TLS.CAFile != "" { + restConfig.TLSClientConfig.CAFile = cfg.Kubernetes.OutOfCluster.TLS.CAFile + } + + if cfg.Kubernetes.OutOfCluster.TLS.CertFile != "" { + restConfig.TLSClientConfig.CertFile = cfg.Kubernetes.OutOfCluster.TLS.CertFile + } + + if cfg.Kubernetes.OutOfCluster.TLS.KeyFile != "" { + restConfig.TLSClientConfig.KeyFile = cfg.Kubernetes.OutOfCluster.TLS.KeyFile + } + + // Apply timeout from config + if cfg.Kubernetes.Common.Timeout > 0 { + restConfig.Timeout = time.Duration(cfg.Kubernetes.Common.Timeout) * time.Second + } + + return restConfig, nil +} + +// GetClient returns the global Kubernetes client +func GetClient() *Client { + return globalClient +} + +// IsInCluster returns true if running in-cluster +func (c *Client) IsInCluster() bool { + return c.Mode == ModeInCluster +} + +// IsOutOfCluster returns true if running out-of-cluster +func (c *Client) IsOutOfCluster() bool { + return c.Mode == ModeOutOfCluster +} + +// GetConnectionMode returns the current connection mode +func (c *Client) GetConnectionMode() ConnectionMode { + return c.Mode +} + +// GetNamespace returns the namespace for an instance +func (c *Client) GetNamespace(userID int) string { + return sanitizeK8sName(fmt.Sprintf("%s-user-%d", c.Namespace, userID)) +} + +// GetSystemNamespace returns the control-plane namespace used by ClawManager itself. +func (c *Client) GetSystemNamespace() string { + return sanitizeK8sName(fmt.Sprintf("%s-system", c.Namespace)) +} + +// GetPodName returns the pod name for an instance +func (c *Client) GetPodName(instanceID int, instanceName string) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-%d-%s", instanceID, instanceName)) +} + +// GetDeploymentName returns the deployment name for an instance. +func (c *Client) GetDeploymentName(instanceID int, instanceName string) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-%d-%s", instanceID, instanceName)) +} + +// GetPVCName returns the PVC name for an instance +func (c *Client) GetPVCName(instanceID int) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-%d-pvc", instanceID)) +} + +// GetTeamSharedPVCName returns the PVC name used for a Team shared workspace. +func (c *Client) GetTeamSharedPVCName(teamID int) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-team-%d-shared", teamID)) +} + +// GetInstancePVName returns the cluster-scoped PV name for an instance. +func (c *Client) GetInstancePVName(userID, instanceID int) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-pv-%s-instance-%d", c.GetNamespace(userID), instanceID)) +} + +// GetTeamSharedPVName returns the cluster-scoped PV name for a Team shared workspace. +func (c *Client) GetTeamSharedPVName(userID, teamID int) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-pv-%s-team-%d-shared", c.GetNamespace(userID), teamID)) +} + +// GetTeamSecretName returns the Secret name used for Team Redis URL and token env. +func (c *Client) GetTeamSecretName(teamID int) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-team-%d-bus", teamID)) +} + +// GetTeamConfigMapName returns the ConfigMap name used for Team roster/config. +func (c *Client) GetTeamConfigMapName(teamID int) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-team-%d-config", teamID)) +} + +// GetServiceName returns the service name for an instance +func (c *Client) GetServiceName(instanceID int, instanceName string) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-%d-%s-svc", instanceID, instanceName)) +} + +// GetOpenClawBootstrapSecretName returns the secret name used to store rendered OpenClaw bootstrap payloads. +func (c *Client) GetOpenClawBootstrapSecretName(instanceID int, instanceName string) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-%d-%s-openclaw-bootstrap", instanceID, instanceName)) +} + +// GetNetworkPolicyName returns the default network policy name for an instance. +func (c *Client) GetNetworkPolicyName(instanceID int, instanceName string) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-%d-%s-netpol", instanceID, instanceName)) +} + +func sanitizeK8sName(name string) string { + sanitized := strings.ToLower(name) + sanitized = k8sNameInvalidChars.ReplaceAllString(sanitized, "-") + sanitized = k8sNameExtraDashes.ReplaceAllString(sanitized, "-") + sanitized = strings.Trim(sanitized, "-") + + if sanitized == "" { + return "clawreef" + } + + if len(sanitized) > 63 { + sanitized = strings.Trim(sanitized[:63], "-") + if sanitized == "" { + return "clawreef" + } + } + + return sanitized +} diff --git a/backend/internal/services/k8s/configmap_service.go b/backend/internal/services/k8s/configmap_service.go new file mode 100644 index 0000000..28c71bb --- /dev/null +++ b/backend/internal/services/k8s/configmap_service.go @@ -0,0 +1,83 @@ +package k8s + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ConfigMapService handles Kubernetes ConfigMap writes used for non-sensitive runtime config. +type ConfigMapService struct { + client *Client + namespaceService *NamespaceService +} + +// NewConfigMapService creates a new ConfigMap service. +func NewConfigMapService() *ConfigMapService { + return &ConfigMapService{ + client: globalClient, + namespaceService: NewNamespaceService(), + } +} + +// UpsertConfigMap creates or updates a ConfigMap in the user's namespace. +func (s *ConfigMapService) UpsertConfigMap(ctx context.Context, userID int, name string, data map[string]string, labels map[string]string) error { + if s.client == nil || s.client.Clientset == nil { + return fmt.Errorf("k8s client not initialized") + } + + if _, err := s.namespaceService.EnsureNamespace(ctx, userID); err != nil { + return fmt.Errorf("failed to ensure namespace: %w", err) + } + + namespace := s.client.GetNamespace(userID) + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Data: data, + } + + existing, err := s.client.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{}) + if err == nil && existing != nil { + existing.Data = data + if existing.Labels == nil { + existing.Labels = map[string]string{} + } + for key, value := range labels { + existing.Labels[key] = value + } + if _, err := s.client.Clientset.CoreV1().ConfigMaps(namespace).Update(ctx, existing, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update configmap %s/%s: %w", namespace, name, err) + } + return nil + } + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to inspect configmap %s/%s: %w", namespace, name, err) + } + + if _, err := s.client.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create configmap %s/%s: %w", namespace, name, err) + } + return nil +} + +// DeleteConfigMap deletes a ConfigMap from the user's namespace. Missing maps are treated as already deleted. +func (s *ConfigMapService) DeleteConfigMap(ctx context.Context, userID int, name string) error { + if s.client == nil || s.client.Clientset == nil { + return fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + if err := s.client.Clientset.CoreV1().ConfigMaps(namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + if errors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to delete configmap %s/%s: %w", namespace, name, err) + } + return nil +} diff --git a/backend/internal/services/k8s/instance_deployment_service.go b/backend/internal/services/k8s/instance_deployment_service.go new file mode 100644 index 0000000..34a7ee0 --- /dev/null +++ b/backend/internal/services/k8s/instance_deployment_service.go @@ -0,0 +1,440 @@ +package k8s + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const instanceDeploymentRestartedAtAnnotation = "clawmanager.io/restarted-at" + +type InstanceDeploymentService struct { + client *Client + namespaceService *NamespaceService +} + +func NewInstanceDeploymentService() *InstanceDeploymentService { + return &InstanceDeploymentService{ + client: globalClient, + namespaceService: NewNamespaceService(), + } +} + +func InstanceDeploymentName(instanceID int) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-%d-deployment", instanceID)) +} + +func (s *InstanceDeploymentService) EnsureDeployment(ctx context.Context, config PodConfig, replicas int32) (*appsv1.Deployment, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + if _, err := s.namespaceService.EnsureNamespace(ctx, config.UserID); err != nil { + return nil, fmt.Errorf("failed to ensure namespace: %w", err) + } + + desired := BuildInstanceDeployment(s.client, config, replicas) + deployments := s.client.Clientset.AppsV1().Deployments(desired.Namespace) + if err := s.deleteLegacyDeployments(ctx, desired.Namespace, config.InstanceID, desired.Name); err != nil { + return nil, err + } + existing, err := deployments.Get(ctx, desired.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + created, createErr := deployments.Create(ctx, desired, metav1.CreateOptions{}) + if createErr != nil { + return nil, fmt.Errorf("failed to create instance deployment %s/%s: %w", desired.Namespace, desired.Name, createErr) + } + return created, nil + } + if err != nil { + return nil, fmt.Errorf("failed to get instance deployment %s/%s: %w", desired.Namespace, desired.Name, err) + } + if !reflect.DeepEqual(existing.Spec.Selector, desired.Spec.Selector) { + return nil, fmt.Errorf("instance deployment %s/%s selector mismatch; delete and recreate the deployment to change immutable selector", desired.Namespace, desired.Name) + } + + updated := existing.DeepCopy() + if updated.Labels == nil { + updated.Labels = map[string]string{} + } + for key, value := range desired.Labels { + updated.Labels[key] = value + } + updated.Spec.Replicas = desired.Spec.Replicas + updated.Spec.Template = desired.Spec.Template + + result, updateErr := deployments.Update(ctx, updated, metav1.UpdateOptions{}) + if updateErr != nil { + return nil, fmt.Errorf("failed to update instance deployment %s/%s: %w", desired.Namespace, desired.Name, updateErr) + } + return result, nil +} + +func (s *InstanceDeploymentService) deleteLegacyDeployments(ctx context.Context, namespace string, instanceID int, expectedName string) error { + deployments := s.client.Clientset.AppsV1().Deployments(namespace) + items, err := deployments.List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("instance-id=%d", instanceID), + }) + if err != nil { + return fmt.Errorf("failed to list instance deployments %s/instance-id=%d: %w", namespace, instanceID, err) + } + + for _, item := range items.Items { + if item.Name == expectedName { + continue + } + propagation := metav1.DeletePropagationForeground + if err := deployments.Delete(ctx, item.Name, metav1.DeleteOptions{PropagationPolicy: &propagation}); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete legacy instance deployment %s/%s: %w", namespace, item.Name, err) + } + } + return nil +} + +func (s *InstanceDeploymentService) ScaleDeployment(ctx context.Context, userID, instanceID int, replicas int32) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + name := InstanceDeploymentName(instanceID) + patch, err := json.Marshal(map[string]any{ + "spec": map[string]any{"replicas": replicas}, + }) + if err != nil { + return fmt.Errorf("failed to build instance deployment scale patch: %w", err) + } + if _, err := s.client.Clientset.AppsV1().Deployments(namespace).Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to scale instance deployment %s/%s: %w", namespace, name, err) + } + return nil +} + +func (s *InstanceDeploymentService) WaitForDeploymentPodsDeleted(ctx context.Context, userID, instanceID int) error { + return s.waitForDeploymentPodsDeleted(ctx, userID, instanceID, podDeletionTimeout) +} + +func (s *InstanceDeploymentService) waitForDeploymentPodsDeleted(ctx context.Context, userID, instanceID int, timeout time.Duration) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + if timeout <= 0 { + timeout = podDeletionTimeout + } + + namespace := s.client.GetNamespace(userID) + selector := fmt.Sprintf("instance-id=%d,app=clawreef", instanceID) + deadline := time.NewTimer(timeout) + defer deadline.Stop() + + ticker := time.NewTicker(podDeletionPollInterval) + defer ticker.Stop() + + for { + pods, err := s.client.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return fmt.Errorf("failed to list instance deployment pods: %w", err) + } + if len(pods.Items) == 0 { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + return fmt.Errorf("timed out waiting for instance deployment pods to be deleted") + case <-ticker.C: + } + } +} + +func (s *InstanceDeploymentService) RestartDeployment(ctx context.Context, userID, instanceID int) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + name := InstanceDeploymentName(instanceID) + patch, err := json.Marshal(map[string]any{ + "spec": map[string]any{ + "template": map[string]any{ + "metadata": map[string]any{ + "annotations": map[string]string{ + instanceDeploymentRestartedAtAnnotation: time.Now().UTC().Format(time.RFC3339Nano), + }, + }, + }, + }, + }) + if err != nil { + return fmt.Errorf("failed to build instance deployment restart patch: %w", err) + } + if _, err := s.client.Clientset.AppsV1().Deployments(namespace).Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to restart instance deployment %s/%s: %w", namespace, name, err) + } + return nil +} + +func (s *InstanceDeploymentService) DeleteDeployment(ctx context.Context, userID, instanceID int) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + name := InstanceDeploymentName(instanceID) + err := s.client.Clientset.AppsV1().Deployments(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete instance deployment %s/%s: %w", namespace, name, err) + } + return nil +} + +func (s *InstanceDeploymentService) GetDeployment(ctx context.Context, userID, instanceID int) (*appsv1.Deployment, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + name := InstanceDeploymentName(instanceID) + deployment, err := s.client.Clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get instance deployment %s/%s: %w", namespace, name, err) + } + return deployment, nil +} + +func (s *InstanceDeploymentService) GetActivePod(ctx context.Context, userID, instanceID int) (*corev1.Pod, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + selector := fmt.Sprintf("instance-id=%d,app=clawreef", instanceID) + pods, err := s.client.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, fmt.Errorf("failed to list instance deployment pods: %w", err) + } + if len(pods.Items) == 0 { + return nil, fmt.Errorf("pod not found for instance %d", instanceID) + } + for _, pod := range pods.Items { + if pod.DeletionTimestamp == nil && pod.Status.Phase == corev1.PodRunning && podReady(&pod) { + selected := pod + return &selected, nil + } + } + selected := pods.Items[0] + return &selected, nil +} + +func BuildInstanceDeployment(client *Client, config PodConfig, replicas int32) *appsv1.Deployment { + namespace := client.GetNamespace(config.UserID) + name := InstanceDeploymentName(config.InstanceID) + runtimeType := normalizePodRuntimeType(config.RuntimeType) + labels := map[string]string{ + "app": "clawreef", + "instance-id": fmt.Sprintf("%d", config.InstanceID), + "instance-name": config.InstanceName, + "user-id": fmt.Sprintf("%d", config.UserID), + "instance-type": config.Type, + "runtime-type": runtimeType, + "managed-by": "clawreef", + } + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: copyStringMap(labels), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "clawreef", + "instance-id": fmt.Sprintf("%d", config.InstanceID), + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: instanceDeploymentAnnotations(config), + Labels: labels, + }, + Spec: buildInstanceDeploymentPodSpec(client, config, runtimeType), + }, + }, + } +} + +func instanceDeploymentAnnotations(config PodConfig) map[string]string { + if config.SecurityMode == PodSecurityChromiumCompat { + return map[string]string{"container.apparmor.security.beta.kubernetes.io/desktop": "unconfined"} + } + return nil +} + +func buildInstanceDeploymentPodSpec(client *Client, config PodConfig, runtimeType string) corev1.PodSpec { + pvcName := client.GetPVCName(config.InstanceID) + if config.ContainerPort == 0 { + config.ContainerPort = 3001 + } + pullPolicy := config.ImagePullPolicy + if pullPolicy == "" { + pullPolicy = corev1.PullIfNotPresent + } + + resources := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%g", config.CPUCores)), + corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", config.MemoryGB)), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%g", config.CPUCores)), + corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", config.MemoryGB)), + }, + } + if config.GPUEnabled && config.GPUCount > 0 { + resources.Limits["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", config.GPUCount)) + resources.Requests["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", config.GPUCount)) + } + + container := corev1.Container{ + Name: "desktop", + Image: config.Image, + ImagePullPolicy: pullPolicy, + Ports: []corev1.ContainerPort{{ + ContainerPort: config.ContainerPort, + Name: "http", + }}, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}}, + FailureThreshold: 30, + PeriodSeconds: 5, + TimeoutSeconds: 2, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}}, + InitialDelaySeconds: 3, + PeriodSeconds: 5, + TimeoutSeconds: 2, + FailureThreshold: 6, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}}, + InitialDelaySeconds: 15, + PeriodSeconds: 10, + TimeoutSeconds: 2, + FailureThreshold: 3, + }, + SecurityContext: buildContainerSecurityContext(config.SecurityMode), + Resources: resources, + VolumeMounts: []corev1.VolumeMount{{ + Name: "data", + MountPath: config.MountPath, + }}, + Env: []corev1.EnvVar{ + {Name: "INSTANCE_ID", Value: fmt.Sprintf("%d", config.InstanceID)}, + {Name: "USER_ID", Value: fmt.Sprintf("%d", config.UserID)}, + }, + } + if runtimeType == "shell" { + container.Ports = nil + container.StartupProbe = nil + container.ReadinessProbe = nil + container.LivenessProbe = nil + } + for key, value := range config.ExtraEnv { + container.Env = append(container.Env, corev1.EnvVar{Name: key, Value: value}) + } + for _, secretName := range config.EnvFromSecretNames { + if secretName == "" { + continue + } + container.EnvFrom = append(container.EnvFrom, corev1.EnvFromSource{ + SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: secretName}}, + }) + } + + spec := corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + SecurityContext: buildPodSecurityContext(config.FSGroup), + NodeSelector: copyStringMap(config.NodeSelector), + Containers: []corev1.Container{container}, + Volumes: []corev1.Volume{{ + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName}, + }, + }}, + } + + for _, mount := range config.ExtraPVCMounts { + if mount.Name == "" || mount.ClaimName == "" || mount.MountPath == "" { + continue + } + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: mount.Name, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: mount.ClaimName, ReadOnly: mount.ReadOnly}, + }, + }) + spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: mount.Name, + MountPath: mount.MountPath, + ReadOnly: mount.ReadOnly, + }) + } + + for index, fix := range config.VolumeOwnershipFixes { + if fix.Name == "" || fix.MountPath == "" || fix.UID < 0 || fix.GID < 0 { + continue + } + spec.InitContainers = append(spec.InitContainers, buildVolumeOwnershipInitContainer(index, config.Image, pullPolicy, fix)) + } + + for _, mount := range config.ConfigMapFileMounts { + if mount.Name == "" || mount.ConfigMapName == "" || mount.Key == "" || mount.MountPath == "" { + continue + } + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: mount.Name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: mount.ConfigMapName}, + Items: []corev1.KeyToPath{{Key: mount.Key, Path: mount.Key}}, + }, + }, + }) + volumeMount := corev1.VolumeMount{Name: mount.Name, MountPath: mount.MountPath, ReadOnly: true} + if !mount.AsDirectory { + volumeMount.SubPath = mount.Key + } + spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, volumeMount) + } + + if config.SHMSizeGB > 0 { + shmLimit := resource.MustParse(fmt.Sprintf("%dGi", config.SHMSizeGB)) + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: "shm", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: &shmLimit}, + }, + }) + spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "shm", MountPath: "/dev/shm"}) + } + + return spec +} + +func podReady(pod *corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} diff --git a/backend/internal/services/k8s/instance_deployment_service_test.go b/backend/internal/services/k8s/instance_deployment_service_test.go new file mode 100644 index 0000000..8d518fe --- /dev/null +++ b/backend/internal/services/k8s/instance_deployment_service_test.go @@ -0,0 +1,205 @@ +package k8s + +import ( + "context" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestBuildInstanceDeploymentUsesStableIdentityAndPVC(t *testing.T) { + client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"} + deployment := BuildInstanceDeployment(client, PodConfig{ + InstanceID: 42, + InstanceName: "Pro Desktop", + UserID: 7, + Type: "openclaw", + RuntimeType: "desktop", + CPUCores: 2, + MemoryGB: 4, + Image: "registry/openclaw:pro", + MountPath: "/config", + ContainerPort: 3001, + ImagePullPolicy: corev1.PullIfNotPresent, + }, 1) + + if deployment.Name != "clawreef-42-deployment" { + t.Fatalf("deployment name = %q, want stable instance deployment name", deployment.Name) + } + if deployment.Namespace != "clawreef-user-7" { + t.Fatalf("namespace = %q, want user namespace", deployment.Namespace) + } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 { + t.Fatalf("replicas = %#v, want 1", deployment.Spec.Replicas) + } + if got := deployment.Spec.Selector.MatchLabels["instance-id"]; got != "42" { + t.Fatalf("selector instance-id = %q, want 42", got) + } + template := deployment.Spec.Template + if got := template.Labels["runtime-type"]; got != "desktop" { + t.Fatalf("template runtime-type = %q, want desktop", got) + } + container := template.Spec.Containers[0] + if container.Name != "desktop" || container.Image != "registry/openclaw:pro" { + t.Fatalf("unexpected container: %#v", container) + } + if template.Spec.RestartPolicy != corev1.RestartPolicyAlways { + t.Fatalf("restart policy = %q, want Always", template.Spec.RestartPolicy) + } + if len(template.Spec.Volumes) == 0 || template.Spec.Volumes[0].PersistentVolumeClaim == nil { + t.Fatalf("expected PVC data volume, got %#v", template.Spec.Volumes) + } + if got := template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName; got != "clawreef-42-pvc" { + t.Fatalf("PVC name = %q, want clawreef-42-pvc", got) + } +} + +func TestBuildInstanceDeploymentAppliesNodeSelector(t *testing.T) { + client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"} + deployment := BuildInstanceDeployment(client, PodConfig{ + InstanceID: 44, + InstanceName: "Pro Desktop", + UserID: 7, + Type: "openclaw", + RuntimeType: "desktop", + CPUCores: 2, + MemoryGB: 4, + Image: "registry/openclaw:pro", + MountPath: "/config", + ContainerPort: 3001, + NodeSelector: map[string]string{ + "kubernetes.io/hostname": "node125", + }, + }, 1) + + if got := deployment.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"]; got != "node125" { + t.Fatalf("node selector hostname = %q, want node125", got) + } +} + +func TestInstanceDeploymentServiceEnsureAndScale(t *testing.T) { + client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"} + service := &InstanceDeploymentService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + config := PodConfig{ + InstanceID: 43, + InstanceName: "Pro Desktop", + UserID: 8, + Type: "openclaw", + RuntimeType: "desktop", + CPUCores: 2, + MemoryGB: 4, + Image: "registry/openclaw:v1", + MountPath: "/config", + ContainerPort: 3001, + } + + if _, err := service.EnsureDeployment(context.Background(), config, 1); err != nil { + t.Fatalf("EnsureDeployment create returned error: %v", err) + } + deployment, err := client.Clientset.AppsV1().Deployments("clawreef-user-8").Get(context.Background(), "clawreef-43-deployment", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get deployment: %v", err) + } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 { + t.Fatalf("created replicas = %#v, want 1", deployment.Spec.Replicas) + } + + if err := service.ScaleDeployment(context.Background(), 8, 43, 0); err != nil { + t.Fatalf("ScaleDeployment returned error: %v", err) + } + scaled, err := client.Clientset.AppsV1().Deployments("clawreef-user-8").Get(context.Background(), "clawreef-43-deployment", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get scaled deployment: %v", err) + } + if scaled.Spec.Replicas == nil || *scaled.Spec.Replicas != 0 { + t.Fatalf("scaled replicas = %#v, want 0", scaled.Spec.Replicas) + } +} + +func TestInstanceDeploymentServiceWaitForDeploymentPodsDeleted(t *testing.T) { + client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"} + service := &InstanceDeploymentService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + ctx := context.Background() + + if err := service.waitForDeploymentPodsDeleted(ctx, 8, 43, 10*time.Millisecond); err != nil { + t.Fatalf("wait with no pods returned error: %v", err) + } + + if _, err := client.Clientset.CoreV1().Pods("clawreef-user-8").Create(ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clawreef-43-desktop", + Namespace: "clawreef-user-8", + Labels: map[string]string{ + "app": "clawreef", + "instance-id": "43", + }, + }, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("create pod: %v", err) + } + + if err := service.waitForDeploymentPodsDeleted(ctx, 8, 43, 10*time.Millisecond); err == nil { + t.Fatalf("expected timeout while pod still exists") + } +} + +func TestInstanceDeploymentServiceEnsureDeletesLegacyDeployments(t *testing.T) { + client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"} + service := &InstanceDeploymentService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + ctx := context.Background() + namespace := "clawreef-user-9" + if _, err := client.Clientset.AppsV1().Deployments(namespace).Create(ctx, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clawreef-44-old-name", + Namespace: namespace, + Labels: map[string]string{ + "app": "clawreef", + "instance-id": "44", + "managed-by": "clawreef", + }, + }, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("create legacy deployment: %v", err) + } + + if _, err := service.EnsureDeployment(ctx, PodConfig{ + InstanceID: 44, + InstanceName: "Pro Desktop", + UserID: 9, + Type: "openclaw", + RuntimeType: "desktop", + CPUCores: 2, + MemoryGB: 4, + Image: "registry/openclaw:v2", + MountPath: "/config", + ContainerPort: 3001, + }, 1); err != nil { + t.Fatalf("EnsureDeployment returned error: %v", err) + } + + deployments, err := client.Clientset.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "instance-id=44", + }) + if err != nil { + t.Fatalf("list deployments: %v", err) + } + if len(deployments.Items) != 1 { + t.Fatalf("deployment count = %d, want 1: %#v", len(deployments.Items), deployments.Items) + } + if got := deployments.Items[0].Name; got != "clawreef-44-deployment" { + t.Fatalf("remaining deployment = %q, want stable deployment", got) + } +} diff --git a/backend/internal/services/k8s/namespace_service.go b/backend/internal/services/k8s/namespace_service.go new file mode 100644 index 0000000..9d89af3 --- /dev/null +++ b/backend/internal/services/k8s/namespace_service.go @@ -0,0 +1,96 @@ +package k8s + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NamespaceService handles namespace operations +type NamespaceService struct { + client *Client +} + +// NewNamespaceService creates a new namespace service +func NewNamespaceService() *NamespaceService { + return &NamespaceService{ + client: globalClient, + } +} + +// EnsureNamespace ensures a namespace exists, creates it if not +func (s *NamespaceService) EnsureNamespace(ctx context.Context, userID int) (*corev1.Namespace, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + namespace := s.client.GetNamespace(userID) + + // Try to get the namespace + ns, err := s.client.Clientset.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if err == nil { + // Namespace already exists + return ns, nil + } + + // If not found, create it + if errors.IsNotFound(err) { + newNs := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Labels: map[string]string{ + "app": "clawreef", + "user-id": fmt.Sprintf("%d", userID), + "managed-by": "clawreef", + }, + }, + } + + createdNs, err := s.client.Clientset.CoreV1().Namespaces().Create(ctx, newNs, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to create namespace %s: %w", namespace, err) + } + + return createdNs, nil + } + + return nil, fmt.Errorf("failed to get namespace %s: %w", namespace, err) +} + +// DeleteNamespace deletes a namespace +func (s *NamespaceService) DeleteNamespace(ctx context.Context, userID int) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + namespace := s.client.GetNamespace(userID) + + err := s.client.Clientset.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete namespace %s: %w", namespace, err) + } + + return nil +} + +// NamespaceExists checks if a namespace exists +func (s *NamespaceService) NamespaceExists(ctx context.Context, userID int) (bool, error) { + if s.client == nil { + return false, fmt.Errorf("k8s client not initialized") + } + + namespace := s.client.GetNamespace(userID) + + _, err := s.client.Clientset.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("failed to check namespace %s: %w", namespace, err) + } + + return true, nil +} diff --git a/backend/internal/services/k8s/network_policy_service.go b/backend/internal/services/k8s/network_policy_service.go new file mode 100644 index 0000000..b0e4021 --- /dev/null +++ b/backend/internal/services/k8s/network_policy_service.go @@ -0,0 +1,165 @@ +package k8s + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// NetworkPolicyService handles default network policy management for instances. +type NetworkPolicyService struct { + client *Client + namespaceService *NamespaceService +} + +// NewNetworkPolicyService creates a new network policy service. +func NewNetworkPolicyService() *NetworkPolicyService { + return &NetworkPolicyService{ + client: globalClient, + namespaceService: NewNamespaceService(), + } +} + +// EnsureDefaultPolicy creates or updates the default egress restriction policy for an instance. +func (s *NetworkPolicyService) EnsureDefaultPolicy(ctx context.Context, userID, instanceID int, instanceName string) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + if _, err := s.namespaceService.EnsureNamespace(ctx, userID); err != nil { + return fmt.Errorf("failed to ensure namespace: %w", err) + } + + namespace := s.client.GetNamespace(userID) + policyName := s.client.GetNetworkPolicyName(instanceID, instanceName) + instanceLabel := fmt.Sprintf("%d", instanceID) + + policy := &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: policyName, + Namespace: namespace, + Labels: map[string]string{ + "app": "clawreef", + "instance-id": instanceLabel, + "instance-name": instanceName, + "user-id": fmt.Sprintf("%d", userID), + "managed-by": "clawreef", + "policy-role": "instance-default-egress", + }, + }, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "clawreef", + "instance-id": instanceLabel, + "managed-by": "clawreef", + }, + }, + PolicyTypes: []networkingv1.PolicyType{ + networkingv1.PolicyTypeEgress, + }, + Egress: []networkingv1.NetworkPolicyEgressRule{ + { + To: []networkingv1.NetworkPolicyPeer{ + { + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": "kube-system", + }, + }, + }, + }, + Ports: []networkingv1.NetworkPolicyPort{ + { + Protocol: protocolPtr(corev1.ProtocolUDP), + Port: intstrPtr(53), + }, + { + Protocol: protocolPtr(corev1.ProtocolTCP), + Port: intstrPtr(53), + }, + }, + }, + { + To: []networkingv1.NetworkPolicyPeer{ + { + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": s.client.GetSystemNamespace(), + }, + }, + }, + }, + Ports: []networkingv1.NetworkPolicyPort{ + { + Protocol: protocolPtr(corev1.ProtocolTCP), + Port: intstrPtr(80), + }, + { + Protocol: protocolPtr(corev1.ProtocolTCP), + Port: intstrPtr(443), + }, + { + Protocol: protocolPtr(corev1.ProtocolTCP), + Port: intstrPtr(8443), + }, + { + Protocol: protocolPtr(corev1.ProtocolTCP), + Port: intstrPtr(9001), + }, + { + Protocol: protocolPtr(corev1.ProtocolTCP), + Port: intstrPtr(3128), + }, + }, + }, + }, + }, + } + + existing, err := s.client.Clientset.NetworkingV1().NetworkPolicies(namespace).Get(ctx, policyName, metav1.GetOptions{}) + if err == nil && existing != nil { + policy.ResourceVersion = existing.ResourceVersion + if _, err := s.client.Clientset.NetworkingV1().NetworkPolicies(namespace).Update(ctx, policy, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update network policy %s: %w", policyName, err) + } + return nil + } + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to inspect network policy %s: %w", policyName, err) + } + + if _, err := s.client.Clientset.NetworkingV1().NetworkPolicies(namespace).Create(ctx, policy, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create network policy %s: %w", policyName, err) + } + + return nil +} + +// DeletePolicy deletes the default instance network policy if it exists. +func (s *NetworkPolicyService) DeletePolicy(ctx context.Context, userID, instanceID int, instanceName string) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + namespace := s.client.GetNamespace(userID) + policyName := s.client.GetNetworkPolicyName(instanceID, instanceName) + if err := s.client.Clientset.NetworkingV1().NetworkPolicies(namespace).Delete(ctx, policyName, metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete network policy %s: %w", policyName, err) + } + return nil +} + +func protocolPtr(protocol corev1.Protocol) *corev1.Protocol { + return &protocol +} + +func intstrPtr(port int) *intstr.IntOrString { + value := intstr.FromInt(port) + return &value +} diff --git a/backend/internal/services/k8s/pod_service.go b/backend/internal/services/k8s/pod_service.go new file mode 100644 index 0000000..8872e23 --- /dev/null +++ b/backend/internal/services/k8s/pod_service.go @@ -0,0 +1,790 @@ +package k8s + +import ( + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// PodService handles Pod operations +type PodService struct { + client *Client +} + +type PodSecurityMode string + +const ( + podDeletionPollInterval = 500 * time.Millisecond + podDeletionTimeout = 60 * time.Second + deploymentDeletionTimeout = 60 * time.Second + + PodSecurityDefault PodSecurityMode = "default" + PodSecurityChromiumCompat PodSecurityMode = "chromium-compat" + PodSecurityPrivileged PodSecurityMode = "privileged" +) + +// NewPodService creates a new Pod service +func NewPodService() *PodService { + return &PodService{ + client: globalClient, + } +} + +// GetClient returns the k8s client +func (s *PodService) GetClient() *Client { + return s.client +} + +// PodConfig holds configuration for creating a pod +type PodConfig struct { + InstanceID int + InstanceName string + UserID int + Type string + RuntimeType string + CPUCores float64 + MemoryGB int + GPUEnabled bool + GPUCount int + Image string + MountPath string + ContainerPort int32 + ImagePullPolicy corev1.PullPolicy + ExtraEnv map[string]string + EnvFromSecretNames []string + ExtraPVCMounts []PVCMount + ConfigMapFileMounts []ConfigMapFileMount + VolumeInitScripts []VolumeInitScript + FSGroup *int64 + NodeSelector map[string]string + VolumeOwnershipFixes []VolumeOwnershipFix + SHMSizeGB int + SecurityMode PodSecurityMode +} + +type PVCMount struct { + Name string + ClaimName string + MountPath string + ReadOnly bool +} + +type ConfigMapFileMount struct { + Name string + ConfigMapName string + Key string + MountPath string + ReadOnly bool + AsDirectory bool +} + +type VolumeOwnershipFix struct { + Name string + MountPath string + UID int64 + GID int64 +} + +type VolumeInitScript struct { + Name string + MountPath string + Script string +} + +// CreatePod creates a single-replica Deployment for an instance and returns the +// pod template that will be used for managed Pods. Callers should use GetPod to +// resolve the current real Pod after the Deployment controller creates it. +func (s *PodService) CreatePod(ctx context.Context, config PodConfig) (*corev1.Pod, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + deploymentName := s.client.GetDeploymentName(config.InstanceID, config.InstanceName) + namespace := s.client.GetNamespace(config.UserID) + pvcName := s.client.GetPVCName(config.InstanceID) + runtimeType := normalizePodRuntimeType(config.RuntimeType) + + // Build resource requirements + resources := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%g", config.CPUCores)), + corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", config.MemoryGB)), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%g", config.CPUCores)), + corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", config.MemoryGB)), + }, + } + + // Add GPU resources if enabled + if config.GPUEnabled && config.GPUCount > 0 { + resources.Limits["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", config.GPUCount)) + resources.Requests["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", config.GPUCount)) + } + + // Default container port + if config.ContainerPort == 0 { + config.ContainerPort = 3001 + } + + // Default image pull policy to IfNotPresent so that air-gapped and + // enterprise environments can use locally cached images without being + // forced to pull from a remote registry (fixes #94). + pullPolicy := config.ImagePullPolicy + if pullPolicy == "" { + pullPolicy = corev1.PullIfNotPresent + } + + annotations := map[string]string{} + if config.SecurityMode == PodSecurityChromiumCompat { + annotations["container.apparmor.security.beta.kubernetes.io/desktop"] = "unconfined" + } + + labels := map[string]string{ + "app": "clawreef", + "instance-id": fmt.Sprintf("%d", config.InstanceID), + "instance-name": config.InstanceName, + "user-id": fmt.Sprintf("%d", config.UserID), + "instance-type": config.Type, + "runtime-type": runtimeType, + "managed-by": "clawreef", + } + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentName, + Namespace: namespace, + Annotations: copyStringMap(annotations), + Labels: copyStringMap(labels), + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + SecurityContext: buildPodSecurityContext(config.FSGroup), + NodeSelector: copyStringMap(config.NodeSelector), + Containers: []corev1.Container{ + { + Name: "desktop", + Image: config.Image, + ImagePullPolicy: pullPolicy, + Ports: []corev1.ContainerPort{ + { + ContainerPort: config.ContainerPort, + Name: "http", + }, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstrFromInt32(config.ContainerPort), + }, + }, + FailureThreshold: 30, + PeriodSeconds: 5, + TimeoutSeconds: 2, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstrFromInt32(config.ContainerPort), + }, + }, + InitialDelaySeconds: 3, + PeriodSeconds: 5, + TimeoutSeconds: 2, + FailureThreshold: 6, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstrFromInt32(config.ContainerPort), + }, + }, + InitialDelaySeconds: 15, + PeriodSeconds: 10, + TimeoutSeconds: 2, + FailureThreshold: 3, + }, + SecurityContext: buildContainerSecurityContext(config.SecurityMode), + Resources: resources, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: config.MountPath, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "INSTANCE_ID", + Value: fmt.Sprintf("%d", config.InstanceID), + }, + { + Name: "USER_ID", + Value: fmt.Sprintf("%d", config.UserID), + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + }, + }, + }, + }, + }, + } + + if runtimeType == "shell" { + pod.Spec.Containers[0].Ports = nil + pod.Spec.Containers[0].StartupProbe = nil + pod.Spec.Containers[0].ReadinessProbe = nil + pod.Spec.Containers[0].LivenessProbe = nil + } + + for key, value := range config.ExtraEnv { + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ + Name: key, + Value: value, + }) + } + + for _, mount := range config.ExtraPVCMounts { + if mount.Name == "" || mount.ClaimName == "" || mount.MountPath == "" { + continue + } + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: mount.Name, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: mount.ClaimName, + ReadOnly: mount.ReadOnly, + }, + }, + }) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: mount.Name, + MountPath: mount.MountPath, + ReadOnly: mount.ReadOnly, + }) + } + + for index, initScript := range config.VolumeInitScripts { + if initScript.Name == "" || initScript.MountPath == "" || initScript.Script == "" { + continue + } + pod.Spec.InitContainers = append(pod.Spec.InitContainers, buildVolumeInitScriptContainer(index, config.Image, pullPolicy, initScript)) + } + + for index, fix := range config.VolumeOwnershipFixes { + if fix.Name == "" || fix.MountPath == "" || fix.UID < 0 || fix.GID < 0 { + continue + } + pod.Spec.InitContainers = append(pod.Spec.InitContainers, buildVolumeOwnershipInitContainer(index, config.Image, pullPolicy, fix)) + } + + for _, mount := range config.ConfigMapFileMounts { + if mount.Name == "" || mount.ConfigMapName == "" || mount.Key == "" || mount.MountPath == "" { + continue + } + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: mount.Name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: mount.ConfigMapName}, + Items: []corev1.KeyToPath{ + { + Key: mount.Key, + Path: mount.Key, + }, + }, + }, + }, + }) + volumeMount := corev1.VolumeMount{ + Name: mount.Name, + MountPath: mount.MountPath, + ReadOnly: true, + } + if !mount.AsDirectory { + volumeMount.SubPath = mount.Key + } + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, volumeMount) + } + + if config.SHMSizeGB > 0 { + shmLimit := resource.MustParse(fmt.Sprintf("%dGi", config.SHMSizeGB)) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "shm", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + SizeLimit: &shmLimit, + }, + }, + }) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "shm", + MountPath: "/dev/shm", + }) + } + + for _, secretName := range config.EnvFromSecretNames { + if secretName == "" { + continue + } + pod.Spec.Containers[0].EnvFrom = append(pod.Spec.Containers[0].EnvFrom, corev1.EnvFromSource{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, + }, + }) + } + + replicas := int32(1) + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentName, + Namespace: namespace, + Labels: copyStringMap(labels), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "clawreef", + "instance-id": fmt.Sprintf("%d", config.InstanceID), + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: copyStringMap(annotations), + Labels: copyStringMap(labels), + }, + Spec: pod.Spec, + }, + }, + } + + createdDeployment, err := s.client.Clientset.AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{}) + if err != nil { + if errors.IsAlreadyExists(err) { + existingDeployment, getErr := s.client.Clientset.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if getErr == nil && existingDeployment != nil { + if existingDeployment.DeletionTimestamp == nil { + return podFromDeployment(existingDeployment), nil + } + + if waitErr := s.waitForDeploymentDeletion(ctx, namespace, existingDeployment.Name); waitErr != nil { + return nil, fmt.Errorf("failed waiting for deployment deletion %s: %w", existingDeployment.Name, waitErr) + } + + createdDeployment, err = s.client.Clientset.AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to create deployment after deletion %s: %w", deploymentName, err) + } + return podFromDeployment(createdDeployment), nil + } + } + return nil, fmt.Errorf("failed to create deployment %s: %w", deploymentName, err) + } + + return podFromDeployment(createdDeployment), nil +} + +func buildPodSecurityContext(fsGroup *int64) *corev1.PodSecurityContext { + if fsGroup == nil || *fsGroup <= 0 { + return nil + } + fsGroupValue := *fsGroup + policy := corev1.FSGroupChangeOnRootMismatch + return &corev1.PodSecurityContext{ + FSGroup: &fsGroupValue, + FSGroupChangePolicy: &policy, + } +} + +func buildContainerSecurityContext(mode PodSecurityMode) *corev1.SecurityContext { + switch mode { + case PodSecurityChromiumCompat: + allowPrivilegeEscalation := true + return &corev1.SecurityContext{ + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeUnconfined, + }, + } + case PodSecurityPrivileged: + privileged := true + return &corev1.SecurityContext{ + Privileged: &privileged, + } + default: + return nil + } +} + +func buildVolumeOwnershipInitContainer(index int, image string, pullPolicy corev1.PullPolicy, fix VolumeOwnershipFix) corev1.Container { + rootUser := int64(0) + return corev1.Container{ + Name: fmt.Sprintf("fix-volume-permissions-%d", index+1), + Image: image, + ImagePullPolicy: pullPolicy, + Command: []string{ + "sh", + "-c", + `set -eu +target="${CLAWMANAGER_FIX_VOLUME_PATH}" +uid="${CLAWMANAGER_FIX_VOLUME_UID}" +gid="${CLAWMANAGER_FIX_VOLUME_GID}" +if [ -d "$target" ]; then + chown -R "${uid}:${gid}" "$target" || true + chmod -R ug+rwX "$target" || true + find "$target" -type d -exec chmod g+s {} \; || true +fi`, + }, + Env: []corev1.EnvVar{ + {Name: "CLAWMANAGER_FIX_VOLUME_PATH", Value: fix.MountPath}, + {Name: "CLAWMANAGER_FIX_VOLUME_UID", Value: fmt.Sprintf("%d", fix.UID)}, + {Name: "CLAWMANAGER_FIX_VOLUME_GID", Value: fmt.Sprintf("%d", fix.GID)}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &rootUser, + RunAsGroup: &rootUser, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: fix.Name, + MountPath: fix.MountPath, + }, + }, + } +} + +func buildVolumeInitScriptContainer(index int, image string, pullPolicy corev1.PullPolicy, initScript VolumeInitScript) corev1.Container { + rootUser := int64(0) + return corev1.Container{ + Name: fmt.Sprintf("init-volume-layout-%d", index+1), + Image: image, + ImagePullPolicy: pullPolicy, + Command: []string{ + "sh", + "-c", + initScript.Script, + }, + Env: []corev1.EnvVar{ + {Name: "CLAWMANAGER_VOLUME_PATH", Value: initScript.MountPath}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &rootUser, + RunAsGroup: &rootUser, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: initScript.Name, + MountPath: initScript.MountPath, + }, + }, + } +} + +func normalizePodRuntimeType(runtimeType string) string { + if runtimeType == "shell" { + return "shell" + } + return "desktop" +} + +func intstrFromInt32(port int32) intstr.IntOrString { + return intstr.FromInt32(port) +} + +func podFromDeployment(deployment *appsv1.Deployment) *corev1.Pod { + if deployment == nil { + return nil + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: deployment.Name, + Namespace: deployment.Namespace, + Labels: copyStringMap(deployment.Spec.Template.Labels), + Annotations: copyStringMap(deployment.Spec.Template.Annotations), + }, + Spec: deployment.Spec.Template.Spec, + } +} + +func copyStringMap(values map[string]string) map[string]string { + if len(values) == 0 { + return map[string]string{} + } + copied := make(map[string]string, len(values)) + for key, value := range values { + copied[key] = value + } + return copied +} + +// GetPod gets a pod by instance ID +func (s *PodService) GetPod(ctx context.Context, userID, instanceID int) (*corev1.Pod, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + // List pods with instance-id label + namespace := s.client.GetNamespace(userID) + selector := fmt.Sprintf("instance-id=%d", instanceID) + + pods, err := s.client.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + return nil, fmt.Errorf("failed to list pods: %w", err) + } + + if len(pods.Items) == 0 { + return nil, fmt.Errorf("pod not found for instance %d", instanceID) + } + + return selectCurrentPod(pods.Items), nil +} + +func selectCurrentPod(pods []corev1.Pod) *corev1.Pod { + var selected *corev1.Pod + selectedRank := -1 + for i := range pods { + pod := &pods[i] + rank := podSelectionRank(pod) + if rank > selectedRank { + selected = pod + selectedRank = rank + } + } + if selected != nil { + return selected + } + return &pods[0] +} + +func podSelectionRank(pod *corev1.Pod) int { + if pod == nil { + return 0 + } + if pod.DeletionTimestamp != nil { + return 1 + } + switch pod.Status.Phase { + case corev1.PodRunning: + if isPodReadyForSelection(pod) { + return 6 + } + return 5 + case corev1.PodPending: + return 4 + case corev1.PodUnknown: + return 3 + case corev1.PodSucceeded: + return 2 + case corev1.PodFailed: + return 2 + default: + return 3 + } +} + +func isPodReadyForSelection(pod *corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} + +// DeletePod deletes the instance Deployment so managed Pods are not recreated. +func (s *PodService) DeletePod(ctx context.Context, userID, instanceID int) error { + return s.DeleteDeployment(ctx, userID, instanceID) +} + +// DeleteDeployment deletes all Deployments for an instance. +func (s *PodService) DeleteDeployment(ctx context.Context, userID, instanceID int) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + namespace := s.client.GetNamespace(userID) + selector := fmt.Sprintf("instance-id=%d", instanceID) + + deployments, err := s.client.Clientset.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + if isNotFoundError(err) { + return nil + } + return fmt.Errorf("failed to list deployments: %w", err) + } + + for _, deployment := range deployments.Items { + propagation := metav1.DeletePropagationForeground + err = s.client.Clientset.AppsV1().Deployments(namespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{ + PropagationPolicy: &propagation, + }) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete deployment %s: %w", deployment.Name, err) + } + + if err := s.waitForDeploymentDeletion(ctx, namespace, deployment.Name); err != nil { + return fmt.Errorf("failed waiting for deployment %s to be deleted: %w", deployment.Name, err) + } + } + + pods, err := s.client.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + return fmt.Errorf("failed to list pods after deployment deletion: %w", err) + } + for _, pod := range pods.Items { + err = s.client.Clientset.CoreV1().Pods(namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete pod %s: %w", pod.Name, err) + } + if err := s.waitForPodDeletion(ctx, namespace, pod.Name); err != nil { + return fmt.Errorf("failed waiting for pod %s to be deleted: %w", pod.Name, err) + } + } + + return nil +} + +// GetPodStatus gets the status of a pod +func (s *PodService) GetPodStatus(ctx context.Context, userID, instanceID int) (*corev1.PodStatus, error) { + pod, err := s.GetPod(ctx, userID, instanceID) + if err != nil { + return nil, err + } + return &pod.Status, nil +} + +// GetPodIP gets the pod IP +func (s *PodService) GetPodIP(ctx context.Context, userID, instanceID int) (string, error) { + pod, err := s.GetPod(ctx, userID, instanceID) + if err != nil { + return "", err + } + return pod.Status.PodIP, nil +} + +// PodExists checks if a pod exists +func (s *PodService) PodExists(ctx context.Context, userID, instanceID int) (bool, error) { + _, err := s.GetPod(ctx, userID, instanceID) + if err != nil { + if isNotFoundError(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// DeploymentExists checks if an instance Deployment exists. +func (s *PodService) DeploymentExists(ctx context.Context, userID, instanceID int) (bool, error) { + if s.client == nil { + return false, fmt.Errorf("k8s client not initialized") + } + + namespace := s.client.GetNamespace(userID) + selector := fmt.Sprintf("instance-id=%d", instanceID) + deployments, err := s.client.Clientset.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("failed to list deployments: %w", err) + } + + return len(deployments.Items) > 0, nil +} + +func isNotFoundError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return containsSubstring(errStr, "not found") || + containsSubstring(errStr, "NotFound") +} + +func containsSubstring(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} + +func (s *PodService) waitForPodDeletion(ctx context.Context, namespace, podName string) error { + waitCtx, cancel := context.WithTimeout(ctx, podDeletionTimeout) + defer cancel() + + ticker := time.NewTicker(podDeletionPollInterval) + defer ticker.Stop() + + for { + _, err := s.client.Clientset.CoreV1().Pods(namespace).Get(waitCtx, podName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("failed to check pod %s: %w", podName, err) + } + + select { + case <-waitCtx.Done(): + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("timed out waiting for pod %s deletion", podName) + case <-ticker.C: + } + } +} + +func (s *PodService) waitForDeploymentDeletion(ctx context.Context, namespace, deploymentName string) error { + waitCtx, cancel := context.WithTimeout(ctx, deploymentDeletionTimeout) + defer cancel() + + ticker := time.NewTicker(podDeletionPollInterval) + defer ticker.Stop() + + for { + _, err := s.client.Clientset.AppsV1().Deployments(namespace).Get(waitCtx, deploymentName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("failed to check deployment %s: %w", deploymentName, err) + } + + select { + case <-waitCtx.Done(): + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("timed out waiting for deployment %s deletion", deploymentName) + case <-ticker.C: + } + } +} diff --git a/backend/internal/services/k8s/pod_service_test.go b/backend/internal/services/k8s/pod_service_test.go new file mode 100644 index 0000000..c08ebb8 --- /dev/null +++ b/backend/internal/services/k8s/pod_service_test.go @@ -0,0 +1,275 @@ +package k8s + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestBuildContainerSecurityContext(t *testing.T) { + if got := buildContainerSecurityContext(PodSecurityDefault); got != nil { + t.Fatalf("expected default security mode to leave securityContext unset, got %#v", got) + } + + chromium := buildContainerSecurityContext(PodSecurityChromiumCompat) + if chromium == nil { + t.Fatalf("expected chromium compat security context") + } + if chromium.Privileged != nil && *chromium.Privileged { + t.Fatalf("chromium compat mode must not enable privileged") + } + if chromium.AllowPrivilegeEscalation == nil || !*chromium.AllowPrivilegeEscalation { + t.Fatalf("expected chromium compat mode to allow privilege escalation for browser sandbox helpers") + } + if chromium.SeccompProfile == nil || chromium.SeccompProfile.Type != "Unconfined" { + t.Fatalf("expected chromium compat mode to use unconfined seccomp profile, got %#v", chromium.SeccompProfile) + } + + privileged := buildContainerSecurityContext(PodSecurityPrivileged) + if privileged == nil || privileged.Privileged == nil || !*privileged.Privileged { + t.Fatalf("expected privileged mode to enable privileged security context") + } +} + +func TestCreatePodAppliesSecurityModes(t *testing.T) { + previousClient := globalClient + t.Cleanup(func() { + globalClient = previousClient + }) + + globalClient = &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawreef", + StorageClass: "standard", + } + + service := NewPodService() + pod, err := service.CreatePod(context.Background(), PodConfig{ + InstanceID: 42, + InstanceName: "openclaw-test", + UserID: 7, + Type: "openclaw", + CPUCores: 1, + MemoryGB: 2, + Image: "openclaw:test", + MountPath: "/config", + ContainerPort: 3001, + SHMSizeGB: 1, + SecurityMode: PodSecurityChromiumCompat, + }) + if err != nil { + t.Fatalf("CreatePod returned error: %v", err) + } + + deployment := mustGetDeployment(t, service, "clawreef-user-7", "clawreef-42-openclaw-test") + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 { + t.Fatalf("expected single-replica deployment, got %#v", deployment.Spec.Replicas) + } + if deployment.Spec.Template.Spec.RestartPolicy != corev1.RestartPolicyAlways { + t.Fatalf("expected deployment-managed pod restartPolicy Always, got %s", deployment.Spec.Template.Spec.RestartPolicy) + } + + container := pod.Spec.Containers[0] + if container.SecurityContext == nil { + t.Fatalf("expected chromium compat security context") + } + if container.SecurityContext.Privileged != nil && *container.SecurityContext.Privileged { + t.Fatalf("chromium compat pod must not be privileged") + } + if pod.Annotations["container.apparmor.security.beta.kubernetes.io/desktop"] != "unconfined" { + t.Fatalf("expected apparmor unconfined annotation for chromium compat mode, got %#v", pod.Annotations) + } + if len(pod.Spec.Volumes) != 2 { + t.Fatalf("expected data and shm volumes, got %d", len(pod.Spec.Volumes)) + } +} + +func TestCreatePodShellRuntimeSkipsDesktopNetworkProbes(t *testing.T) { + previousClient := globalClient + t.Cleanup(func() { + globalClient = previousClient + }) + + globalClient = &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawreef", + StorageClass: "standard", + } + + service := NewPodService() + pod, err := service.CreatePod(context.Background(), PodConfig{ + InstanceID: 43, + InstanceName: "shell-test", + UserID: 7, + Type: "openclaw", + RuntimeType: "shell", + CPUCores: 1, + MemoryGB: 2, + Image: "openclaw:test", + MountPath: "/config", + ContainerPort: 3001, + }) + if err != nil { + t.Fatalf("CreatePod returned error: %v", err) + } + + deployment := mustGetDeployment(t, service, "clawreef-user-7", "clawreef-43-shell-test") + if len(deployment.Spec.Template.Spec.Containers[0].Ports) != 0 { + t.Fatalf("expected shell deployment template to skip desktop ports") + } + + container := pod.Spec.Containers[0] + if got := pod.Labels["runtime-type"]; got != "shell" { + t.Fatalf("expected runtime-type shell label, got %q", got) + } + if len(container.Ports) != 0 { + t.Fatalf("expected shell runtime to skip desktop ports, got %d", len(container.Ports)) + } + if container.StartupProbe != nil || container.ReadinessProbe != nil || container.LivenessProbe != nil { + t.Fatalf("expected shell runtime to skip desktop TCP probes") + } +} + +func TestCreatePodAppliesExtraPVCMountsAndSecretEnv(t *testing.T) { + previousClient := globalClient + t.Cleanup(func() { + globalClient = previousClient + }) + + globalClient = &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawreef", + StorageClass: "standard", + } + + service := NewPodService() + pod, err := service.CreatePod(context.Background(), PodConfig{ + InstanceID: 43, + InstanceName: "openclaw-team", + UserID: 7, + Type: "openclaw", + CPUCores: 1, + MemoryGB: 2, + Image: "openclaw:test", + MountPath: "/config", + ContainerPort: 3001, + EnvFromSecretNames: []string{"clawreef-team-1-bus"}, + ExtraPVCMounts: []PVCMount{ + {Name: "team-shared", ClaimName: "clawreef-team-1-shared", MountPath: "/team"}, + }, + ConfigMapFileMounts: []ConfigMapFileMount{ + {Name: "team-config", ConfigMapName: "clawreef-team-1-config", Key: "team.json", MountPath: "/etc/clawmanager/team", ReadOnly: true, AsDirectory: true}, + }, + FSGroup: int64Ptr(1000), + VolumeOwnershipFixes: []VolumeOwnershipFix{ + {Name: "team-shared", MountPath: "/team", UID: 1000, GID: 1000}, + }, + }) + if err != nil { + t.Fatalf("CreatePod returned error: %v", err) + } + + deployment := mustGetDeployment(t, service, "clawreef-user-7", "clawreef-43-openclaw-team") + if deployment.Spec.Template.Labels["instance-id"] != "43" { + t.Fatalf("expected deployment template instance-id label, got %#v", deployment.Spec.Template.Labels) + } + + container := pod.Spec.Containers[0] + if len(container.EnvFrom) != 1 || container.EnvFrom[0].SecretRef == nil || container.EnvFrom[0].SecretRef.Name != "clawreef-team-1-bus" { + t.Fatalf("expected Team secret envFrom, got %#v", container.EnvFrom) + } + + foundMount := false + for _, mount := range container.VolumeMounts { + if mount.Name == "team-shared" && mount.MountPath == "/team" { + foundMount = true + } + } + if !foundMount { + t.Fatalf("expected /team shared PVC mount, got %#v", container.VolumeMounts) + } + + if pod.Spec.SecurityContext == nil || pod.Spec.SecurityContext.FSGroup == nil || *pod.Spec.SecurityContext.FSGroup != 1000 { + t.Fatalf("expected pod fsGroup 1000, got %#v", pod.Spec.SecurityContext) + } + if pod.Spec.SecurityContext.FSGroupChangePolicy == nil || *pod.Spec.SecurityContext.FSGroupChangePolicy != corev1.FSGroupChangeOnRootMismatch { + t.Fatalf("expected OnRootMismatch fsGroup change policy, got %#v", pod.Spec.SecurityContext) + } + + if len(pod.Spec.InitContainers) != 1 { + t.Fatalf("expected one Team shared permission initContainer, got %#v", pod.Spec.InitContainers) + } + initContainer := pod.Spec.InitContainers[0] + if initContainer.SecurityContext == nil || initContainer.SecurityContext.RunAsUser == nil || *initContainer.SecurityContext.RunAsUser != 0 { + t.Fatalf("expected permission initContainer to run as root, got %#v", initContainer.SecurityContext) + } + foundInitMount := false + for _, mount := range initContainer.VolumeMounts { + if mount.Name == "team-shared" && mount.MountPath == "/team" { + foundInitMount = true + } + } + if !foundInitMount { + t.Fatalf("expected permission initContainer to mount /team, got %#v", initContainer.VolumeMounts) + } + + foundConfig := false + for _, mount := range container.VolumeMounts { + if mount.Name == "team-config" && mount.MountPath == "/etc/clawmanager/team" && mount.SubPath == "" && mount.ReadOnly { + foundConfig = true + } + } + if !foundConfig { + t.Fatalf("expected Team ConfigMap directory mount, got %#v", container.VolumeMounts) + } +} + +func TestSelectCurrentPodPrefersRecoveringDeploymentPod(t *testing.T) { + pods := []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "old-evicted"}, + Status: corev1.PodStatus{Phase: corev1.PodFailed}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "new-pending"}, + Status: corev1.PodStatus{Phase: corev1.PodPending}, + }, + } + + selected := selectCurrentPod(pods) + if selected == nil || selected.Name != "new-pending" { + t.Fatalf("expected recovering pending pod to win over failed pod, got %#v", selected) + } + + pods = append(pods, corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "ready"}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }, + }) + + selected = selectCurrentPod(pods) + if selected == nil || selected.Name != "ready" { + t.Fatalf("expected ready running pod to win, got %#v", selected) + } +} + +func int64Ptr(value int64) *int64 { + return &value +} + +func mustGetDeployment(t *testing.T, service *PodService, namespace, name string) *appsv1.Deployment { + t.Helper() + deployment, err := service.GetClient().Clientset.AppsV1().Deployments(namespace).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected deployment %s/%s to exist: %v", namespace, name, err) + } + return deployment +} diff --git a/backend/internal/services/k8s/pvc_service.go b/backend/internal/services/k8s/pvc_service.go new file mode 100644 index 0000000..7d8b6ef --- /dev/null +++ b/backend/internal/services/k8s/pvc_service.go @@ -0,0 +1,1159 @@ +package k8s + +import ( + "context" + "fmt" + "net" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PVCService handles PersistentVolumeClaim operations +type PVCService struct { + client *Client + namespaceService *NamespaceService +} + +const ( + nodeHostnameLabel = "kubernetes.io/hostname" + noProvisionerName = "kubernetes.io/no-provisioner" + storageProfileLabel = "clawmanager.io/storage-profile" + storageProfileSingle = "single-node" + storageProfileLegacyNFS = "legacy-nfs" + defaultPVCBindTimeout = 2 * time.Minute + // os.FileMode keeps setgid outside the Unix permission-bit range, so + // 0o2775 alone does not set it when passed to os.Chmod. + teamSharedDirectoryMode = os.FileMode(0o775) | os.ModeSetgid +) + +// NewPVCService creates a new PVC service +func NewPVCService() *PVCService { + return &PVCService{ + client: globalClient, + namespaceService: NewNamespaceService(), + } +} + +// GetClient returns the k8s client +func (s *PVCService) GetClient() *Client { + return s.client +} + +// CreatePVC creates a new PVC for an instance +func (s *PVCService) CreatePVC(ctx context.Context, userID, instanceID int, storageSizeGB int, storageClass string) (*corev1.PersistentVolumeClaim, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + // Ensure namespace exists + if _, err := s.namespaceService.EnsureNamespace(ctx, userID); err != nil { + return nil, fmt.Errorf("failed to ensure namespace: %w", err) + } + + pvcName := s.client.GetPVCName(instanceID) + namespace := s.client.GetNamespace(userID) + + // Use default storage class if not specified + if storageClass == "" { + storageClass = firstNonEmpty(s.client.InstanceStorageClass, s.client.StorageClass) + } + + fmt.Printf("Creating PVC %s in namespace %s with storageClass: %s\n", pvcName, namespace, storageClass) + + storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB)) + + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + Labels: map[string]string{ + "app": "clawreef", + "instance-id": fmt.Sprintf("%d", instanceID), + "managed-by": "clawreef", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storageSize, + }, + }, + StorageClassName: &storageClass, + }, + } + + createdPVC, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) + if err != nil { + // Check if PVC already exists + if errors.IsAlreadyExists(err) { + // Try to get the existing PVC + existingPVC, getErr := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if getErr == nil && existingPVC != nil { + // Check if PVC belongs to the same instance + if existingPVC.Labels["instance-id"] == fmt.Sprintf("%d", instanceID) { + // PVC exists and belongs to this instance, return it + return existingPVC, nil + } + // PVC exists but belongs to a different instance, delete it and recreate + deleteErr := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}) + if deleteErr != nil && !errors.IsNotFound(deleteErr) { + return nil, fmt.Errorf("failed to delete existing PVC %s: %w", pvcName, deleteErr) + } + // Wait a moment for deletion to complete + select { + case <-time.After(2 * time.Second): + } + // Retry creation + createdPVC, err = s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to create PVC after deletion %s: %w", pvcName, err) + } + return createdPVC, nil + } + } + return nil, fmt.Errorf("failed to create PVC %s: %w", pvcName, err) + } + + if s.shouldUseManualHostPathFallback(storageClass) { + fmt.Printf("PVC %s created, scheduling manual hostPath binding monitor...\n", pvcName) + go s.monitorPVCBinding(context.Background(), namespace, pvcName, userID, instanceID, storageSizeGB, storageClass, s.pvcBindTimeout()) + } else if usesManualHostPathFallback(storageClass) { + fmt.Printf("PVC %s created with manual storageClass, but hostPath fallback is disabled for storage profile %q\n", pvcName, s.storageProfile()) + } else { + fmt.Printf("PVC %s created with dynamic storageClass %s, leaving binding to the provisioner\n", pvcName, storageClass) + } + + return createdPVC, nil +} + +// CreateTeamSharedPVC creates the RWX PVC mounted by every member of a Team. +func (s *PVCService) CreateTeamSharedPVC(ctx context.Context, userID, teamID, storageSizeGB int, storageClass string) (*corev1.PersistentVolumeClaim, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + if storageSizeGB <= 0 { + storageSizeGB = 10 + } + + if _, err := s.namespaceService.EnsureNamespace(ctx, userID); err != nil { + return nil, fmt.Errorf("failed to ensure namespace: %w", err) + } + + pvcName := s.client.GetTeamSharedPVCName(teamID) + namespace := s.client.GetNamespace(userID) + storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB)) + useWorkspaceNFS := strings.TrimSpace(s.client.WorkspaceNFSServer) != "" + storageClass = s.teamSharedPVCStorageClass(storageClass, useWorkspaceNFS) + workspaceAccessMode := workspacePVCAccessMode(s.client.WorkspaceAccessMode) + if useWorkspaceNFS { + if err := s.ensureTeamSharedWorkspaceDirectory(userID, teamID); err != nil { + return nil, err + } + } + + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + Labels: map[string]string{ + "app": "clawreef", + "team-id": fmt.Sprintf("%d", teamID), + "managed-by": "clawreef", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + workspaceAccessMode, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storageSize, + }, + }, + StorageClassName: &storageClass, + }, + } + + createdPVC, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) + if err != nil { + if errors.IsAlreadyExists(err) { + existingPVC, getErr := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if getErr == nil && existingPVC != nil && existingPVC.Labels["team-id"] == fmt.Sprintf("%d", teamID) { + if useWorkspaceNFS { + if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, existingPVC, userID, teamID, storageSizeGB, storageClass, s.client.GetTeamSharedPVName(userID, teamID)); err != nil { + return nil, err + } + } else if existingPVC.Status.Phase != corev1.ClaimBound && s.shouldUseManualHostPathFallback(storageClass) { + go s.monitorTeamSharedPVCBinding(context.Background(), namespace, pvcName, userID, teamID, storageSizeGB, storageClass, s.pvcBindTimeout()) + } + return existingPVC, nil + } + } + return nil, fmt.Errorf("failed to create Team shared PVC %s: %w", pvcName, err) + } + + if useWorkspaceNFS { + if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, createdPVC, userID, teamID, storageSizeGB, storageClass, s.client.GetTeamSharedPVName(userID, teamID)); err != nil { + return nil, err + } + } else if s.shouldUseManualHostPathFallback(storageClass) { + go s.monitorTeamSharedPVCBinding(context.Background(), namespace, pvcName, userID, teamID, storageSizeGB, storageClass, s.pvcBindTimeout()) + } + return createdPVC, nil +} + +func usesManualHostPathFallback(storageClass string) bool { + return strings.EqualFold(strings.TrimSpace(storageClass), "manual") +} + +func (s *PVCService) teamSharedPVCStorageClass(requested string, useWorkspaceNFS bool) string { + requested = strings.TrimSpace(requested) + if !useWorkspaceNFS { + return firstNonEmpty(requested, s.client.WorkspaceStorageClass, s.client.StorageClass) + } + workspaceClass := strings.TrimSpace(s.client.WorkspaceStorageClass) + globalClass := strings.TrimSpace(s.client.StorageClass) + instanceClass := strings.TrimSpace(s.client.InstanceStorageClass) + if requested == "" || + strings.EqualFold(requested, globalClass) || + strings.EqualFold(requested, instanceClass) { + return firstNonEmpty(workspaceClass, "manual") + } + return requested +} + +func (s *PVCService) shouldUseManualHostPathFallback(storageClass string) bool { + return s.hostPathFallbackEnabled() && usesManualHostPathFallback(storageClass) +} + +func (s *PVCService) monitorTeamSharedPVCBinding(ctx context.Context, namespace, pvcName string, userID, teamID, storageSizeGB int, storageClass string, timeout time.Duration) { + if _, err := s.waitForTeamSharedPVCBinding(ctx, namespace, pvcName, userID, teamID, storageSizeGB, storageClass, timeout); err != nil { + fmt.Printf("Async Team shared PVC binding monitor failed for %s: %v\n", pvcName, err) + } +} + +func (s *PVCService) waitForTeamSharedPVCBinding(ctx context.Context, namespace, pvcName string, userID, teamID, storageSizeGB int, storageClass string, timeout time.Duration) (*corev1.PersistentVolumeClaim, error) { + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get Team shared PVC %s: %w", pvcName, err) + } + if pvc.Status.Phase == corev1.ClaimBound { + return pvc, nil + } + + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + timeoutChan := time.After(timeout) + + for { + select { + case <-timeoutChan: + if usesManualHostPathFallback(storageClass) && !s.hostPathFallbackEnabled() { + return nil, fmt.Errorf("Team shared PVC %s was not bound after %s and hostPath fallback is disabled for storage profile %q", pvcName, timeout, s.storageProfile()) + } + if !usesManualHostPathFallback(storageClass) { + fmt.Printf("Team shared PVC %s binding timeout with dynamic storageClass %s, leaving binding to the provisioner\n", pvcName, storageClass) + return pvc, nil + } + fmt.Printf("Team shared PVC %s binding timeout, creating hostPath RWX PV manually\n", pvcName) + return s.createPVForTeamSharedPVC(ctx, namespace, pvcName, userID, teamID, storageSizeGB, storageClass) + case <-ticker.C: + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get Team shared PVC %s during wait: %w", pvcName, err) + } + if pvc.Status.Phase == corev1.ClaimBound { + fmt.Printf("Team shared PVC %s bound successfully to %s\n", pvcName, pvc.Spec.VolumeName) + return pvc, nil + } + fmt.Printf("Waiting for Team shared PVC %s binding, current status: %s\n", pvcName, pvc.Status.Phase) + } + } +} + +func (s *PVCService) createPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, userID, teamID, storageSizeGB int, storageClass string) (*corev1.PersistentVolumeClaim, error) { + pvName := s.client.GetTeamSharedPVName(userID, teamID) + + existingPV, err := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err == nil && existingPV != nil { + if existingPV.Status.Phase == corev1.VolumeReleased { + if deleteErr := s.client.Clientset.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); deleteErr != nil && !errors.IsNotFound(deleteErr) { + return nil, fmt.Errorf("failed to delete released Team shared PV %s: %w", pvName, deleteErr) + } + time.Sleep(3 * time.Second) + } else if existingPV.Spec.ClaimRef != nil && + existingPV.Spec.ClaimRef.Namespace == namespace && + existingPV.Spec.ClaimRef.Name == pvcName { + if strings.TrimSpace(s.client.WorkspaceNFSServer) != "" { + nfsPath := teamSharedWorkspaceNFSPath(s.client.WorkspaceNFSPath, userID, teamID) + nfsServer, nfsErr := s.workspaceNFSServerForPV(ctx) + if nfsErr != nil { + return nil, nfsErr + } + if existingPV.Spec.NFS == nil || + strings.TrimSpace(existingPV.Spec.NFS.Server) != nfsServer || + strings.TrimSpace(existingPV.Spec.NFS.Path) != nfsPath { + return nil, fmt.Errorf("Team shared PV %s already exists for %s/%s but is not workspace NFS-backed", pvName, namespace, pvcName) + } + } + time.Sleep(2 * time.Second) + return s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + } else if existingPV.Spec.ClaimRef != nil { + return nil, fmt.Errorf("Team shared PV %s already belongs to %s/%s", pvName, existingPV.Spec.ClaimRef.Namespace, existingPV.Spec.ClaimRef.Name) + } + } + + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get Team shared PVC %s for UID: %w", pvcName, err) + } + + if strings.TrimSpace(s.client.WorkspaceNFSServer) != "" { + return s.createWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, pvc, userID, teamID, storageSizeGB, storageClass, pvName) + } + + hostPathPrefix := "/data/clawreef" + if s.client != nil && s.client.HostPathPrefix != "" { + hostPathPrefix = s.client.HostPathPrefix + } + hostPath := fmt.Sprintf("%s/user-%d/team-%d-shared", hostPathPrefix, userID, teamID) + storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB)) + nodeAffinity, err := s.hostPathPVNodeAffinity(ctx) + if err != nil { + return nil, fmt.Errorf("failed to select node for Team shared hostPath PV %s: %w", pvName, err) + } + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + Labels: map[string]string{ + "app": "clawreef", + "user-id": fmt.Sprintf("%d", userID), + "team-id": fmt.Sprintf("%d", teamID), + "managed-by": "clawreef", + storageProfileLabel: storageProfileSingle, + }, + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: storageSize, + }, + AccessModes: []corev1.PersistentVolumeAccessMode{ + workspacePVCAccessMode(s.client.WorkspaceAccessMode), + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: storageClass, + NodeAffinity: nodeAffinity, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPath, + Type: func() *corev1.HostPathType { + t := corev1.HostPathDirectoryOrCreate + return &t + }(), + }, + }, + ClaimRef: &corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + Namespace: namespace, + Name: pvcName, + UID: pvc.UID, + ResourceVersion: pvc.ResourceVersion, + }, + }, + } + if _, err := s.client.Clientset.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil && !errors.IsAlreadyExists(err) { + return nil, fmt.Errorf("failed to create Team shared PV %s: %w", pvName, err) + } + + time.Sleep(3 * time.Second) + pvc, err = s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get Team shared PVC after PV creation: %w", err) + } + if pvc.Status.Phase != corev1.ClaimBound { + return nil, fmt.Errorf("Team shared PVC %s is still not bound after PV creation, status: %s", pvcName, pvc.Status.Phase) + } + fmt.Printf("Team shared PVC %s successfully bound to PV %s\n", pvcName, pvName) + return pvc, nil +} + +func (s *PVCService) createWorkspaceNFSPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, pvc *corev1.PersistentVolumeClaim, userID, teamID, storageSizeGB int, storageClass, pvName string) (*corev1.PersistentVolumeClaim, error) { + if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, pvc, userID, teamID, storageSizeGB, storageClass, pvName); err != nil { + return nil, err + } + + time.Sleep(3 * time.Second) + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get Team shared PVC after NFS PV creation: %w", err) + } + if pvc.Status.Phase != corev1.ClaimBound { + return nil, fmt.Errorf("Team shared PVC %s is still not bound after NFS PV creation, status: %s", pvcName, pvc.Status.Phase) + } + fmt.Printf("Team shared PVC %s successfully bound to NFS PV %s (%s:%s)\n", pvcName, pvName, s.client.WorkspaceNFSServer, teamSharedWorkspaceNFSPath(s.client.WorkspaceNFSPath, userID, teamID)) + return pvc, nil +} + +func (s *PVCService) ensureWorkspaceNFSPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, pvc *corev1.PersistentVolumeClaim, userID, teamID, storageSizeGB int, storageClass, pvName string) error { + if err := s.ensureTeamSharedWorkspaceDirectory(userID, teamID); err != nil { + return err + } + + storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB)) + nfsPath := teamSharedWorkspaceNFSPath(s.client.WorkspaceNFSPath, userID, teamID) + nfsServer, err := s.workspaceNFSServerForPV(ctx) + if err != nil { + return err + } + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + Labels: map[string]string{ + "app": "clawreef", + "user-id": fmt.Sprintf("%d", userID), + "team-id": fmt.Sprintf("%d", teamID), + "managed-by": "clawreef", + storageProfileLabel: storageProfileLegacyNFS, + }, + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: storageSize, + }, + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: storageClass, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + NFS: &corev1.NFSVolumeSource{ + Server: nfsServer, + Path: nfsPath, + }, + }, + ClaimRef: &corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + Namespace: namespace, + Name: pvcName, + UID: pvc.UID, + ResourceVersion: pvc.ResourceVersion, + }, + }, + } + if _, err := s.client.Clientset.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil { + if errors.IsAlreadyExists(err) { + existing, getErr := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if getErr != nil { + return fmt.Errorf("failed to get existing Team shared NFS PV %s: %w", pvName, getErr) + } + if existing.Spec.ClaimRef != nil && existing.Spec.ClaimRef.Namespace == namespace && existing.Spec.ClaimRef.Name == pvcName { + if existing.Spec.NFS != nil && + strings.TrimSpace(existing.Spec.NFS.Server) == nfsServer && + strings.TrimSpace(existing.Spec.NFS.Path) == nfsPath { + return nil + } + return fmt.Errorf("Team shared PV %s already exists for %s/%s but is not workspace NFS-backed", pvName, namespace, pvcName) + } + } + return fmt.Errorf("failed to create Team shared NFS PV %s: %w", pvName, err) + } + return nil +} + +func (s *PVCService) workspaceNFSServerForPV(ctx context.Context) (string, error) { + server := strings.TrimSpace(s.client.WorkspaceNFSServer) + if server == "" || s.client.Clientset == nil { + return server, nil + } + serviceName, namespace, ok := workspaceNFSServiceRef(server, s.client.Namespace) + if !ok { + return server, nil + } + svc, err := s.client.Clientset.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return server, nil + } + return "", fmt.Errorf("failed to resolve workspace NFS service %s/%s: %w", namespace, serviceName, err) + } + clusterIP := strings.TrimSpace(svc.Spec.ClusterIP) + if clusterIP == "" || strings.EqualFold(clusterIP, corev1.ClusterIPNone) { + return server, nil + } + return clusterIP, nil +} + +func workspaceNFSServiceRef(server, defaultNamespace string) (string, string, bool) { + host := strings.TrimSuffix(strings.TrimSpace(server), ".") + if host == "" || net.ParseIP(host) != nil { + return "", "", false + } + parts := strings.Split(strings.ToLower(host), ".") + if len(parts) == 1 { + namespace := strings.TrimSpace(defaultNamespace) + if namespace == "" { + namespace = "default" + } + return parts[0], namespace, true + } + if len(parts) >= 3 && parts[2] == "svc" && parts[0] != "" && parts[1] != "" { + return parts[0], parts[1], true + } + return "", "", false +} + +func (s *PVCService) ensureTeamSharedWorkspaceDirectory(userID, teamID int) error { + root := strings.TrimSpace(s.client.WorkspaceRoot) + if root == "" { + return nil + } + dir := filepath.Join(root, filepath.FromSlash(TeamSharedWorkspaceRelativePath(userID, teamID))) + dirs := append([]string{dir}, teamSharedRuntimeSubdirectories(dir)...) + for _, target := range dirs { + if err := os.MkdirAll(target, teamSharedDirectoryMode); err != nil { + return fmt.Errorf("failed to create Team shared runtime workspace directory %s: %w", target, err) + } + _ = os.Chown(target, 1000, 1000) + if err := os.Chmod(target, teamSharedDirectoryMode); err != nil { + return fmt.Errorf("failed to chmod Team shared runtime workspace directory %s: %w", target, err) + } + } + return nil +} + +func teamSharedRuntimeSubdirectories(root string) []string { + return []string{ + filepath.Join(root, "status"), + filepath.Join(root, "inbox"), + filepath.Join(root, "results"), + filepath.Join(root, "tasks"), + filepath.Join(root, ".openclaw-redis-team"), + filepath.Join(root, ".openclaw-redis-team", "tasks"), + } +} + +func TeamSharedWorkspaceRelativePath(userID, teamID int) string { + return fmt.Sprintf("teams/user-%d/team-%d-shared", userID, teamID) +} + +func TeamSharedWorkspacePath(workspaceRoot string, userID, teamID int) string { + root := strings.TrimRight(strings.TrimSpace(workspaceRoot), "/") + if root == "" { + root = "/workspaces" + } + return root + "/" + TeamSharedWorkspaceRelativePath(userID, teamID) +} + +func teamSharedWorkspaceNFSPath(basePath string, userID, teamID int) string { + basePath = strings.TrimSpace(basePath) + if basePath == "" { + basePath = "/" + } + relativePath := TeamSharedWorkspaceRelativePath(userID, teamID) + if basePath == "/" { + return "/" + relativePath + } + return path.Join(basePath, relativePath) +} + +func (s *PVCService) hostPathPVNodeHostname(ctx context.Context) (string, error) { + if s == nil || s.client == nil { + return "", fmt.Errorf("k8s client not initialized") + } + nodes, err := s.client.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return "", err + } + type candidate struct { + name string + hostname string + } + candidates := []candidate{} + for _, node := range nodes.Items { + if !isHostPathPVNodeCandidate(node) { + continue + } + hostname := strings.TrimSpace(node.Labels[nodeHostnameLabel]) + if hostname == "" { + hostname = strings.TrimSpace(node.Name) + } + if hostname == "" { + continue + } + candidates = append(candidates, candidate{name: node.Name, hostname: hostname}) + } + if len(candidates) == 0 { + return "", fmt.Errorf("no ready node found for hostPath PV") + } + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].name < candidates[j].name + }) + return candidates[0].hostname, nil +} + +func (s *PVCService) hostPathPVNodeSelector(ctx context.Context) (map[string]string, error) { + hostname, err := s.hostPathPVNodeHostname(ctx) + if err != nil { + return nil, err + } + return nodeSelectorForHostname(hostname), nil +} + +func (s *PVCService) hostPathPVNodeAffinity(ctx context.Context) (*corev1.VolumeNodeAffinity, error) { + hostname, err := s.hostPathPVNodeHostname(ctx) + if err != nil { + return nil, err + } + return hostPathPVNodeAffinityForHostname(hostname), nil +} + +func hostPathPVNodeAffinityForHostname(hostname string) *corev1.VolumeNodeAffinity { + return &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: nodeHostnameLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{hostname}, + }, + }, + }, + }, + }, + } +} + +func isStorageNodeReady(node corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} + +func isHostPathPVNodeCandidate(node corev1.Node) bool { + if node.Spec.Unschedulable || !isStorageNodeReady(node) { + return false + } + for _, taint := range node.Spec.Taints { + if taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute { + return false + } + } + return true +} + +func (s *PVCService) monitorPVCBinding(ctx context.Context, namespace, pvcName string, userID, instanceID, storageSizeGB int, storageClass string, timeout time.Duration) { + if _, err := s.waitForPVCBinding(ctx, namespace, pvcName, userID, instanceID, storageSizeGB, storageClass, timeout); err != nil { + fmt.Printf("Async PVC binding monitor failed for %s: %v\n", pvcName, err) + } +} + +// waitForPVCBinding waits for PVC to be bound, if timeout creates PV manually +func (s *PVCService) waitForPVCBinding(ctx context.Context, namespace, pvcName string, userID, instanceID, storageSizeGB int, storageClass string, timeout time.Duration) (*corev1.PersistentVolumeClaim, error) { + // Check if PVC is already bound + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PVC %s: %w", pvcName, err) + } + + if pvc.Status.Phase == corev1.ClaimBound { + fmt.Printf("PVC %s is already bound to %s\n", pvcName, pvc.Spec.VolumeName) + return pvc, nil + } + + // Wait for binding with timeout + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + timeoutChan := time.After(timeout) + + for { + select { + case <-timeoutChan: + if usesManualHostPathFallback(storageClass) && !s.hostPathFallbackEnabled() { + return nil, fmt.Errorf("PVC %s was not bound after %s and hostPath fallback is disabled for storage profile %q", pvcName, timeout, s.storageProfile()) + } + if !usesManualHostPathFallback(storageClass) { + fmt.Printf("PVC %s binding timeout with dynamic storageClass %s, leaving binding to the provisioner\n", pvcName, storageClass) + return pvc, nil + } + // Timeout, try to create PV manually. + fmt.Printf("PVC %s binding timeout, creating PV manually\n", pvcName) + return s.createPVForPVC(ctx, namespace, pvcName, userID, instanceID, storageSizeGB, storageClass) + case <-ticker.C: + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PVC %s during wait: %w", pvcName, err) + } + if pvc.Status.Phase == corev1.ClaimBound { + fmt.Printf("PVC %s bound successfully to %s\n", pvcName, pvc.Spec.VolumeName) + return pvc, nil + } + fmt.Printf("Waiting for PVC %s binding, current status: %s\n", pvcName, pvc.Status.Phase) + } + } +} + +// createPVForPVC creates a PV manually to bind to the PVC +func (s *PVCService) createPVForPVC(ctx context.Context, namespace, pvcName string, userID, instanceID, storageSizeGB int, storageClass string) (*corev1.PersistentVolumeClaim, error) { + pvName := s.client.GetInstancePVName(userID, instanceID) + // Use configurable host path prefix for persistent storage + hostPathPrefix := "/data/clawreef" + if s.client != nil && s.client.HostPathPrefix != "" { + hostPathPrefix = s.client.HostPathPrefix + } + hostPath := fmt.Sprintf("%s/user-%d/instance-%d", hostPathPrefix, userID, instanceID) + + fmt.Printf("Creating PV %s with hostPath %s for PVC %s\n", pvName, hostPath, pvcName) + + // Check if PV already exists + existingPV, err := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err == nil && existingPV != nil { + fmt.Printf("PV %s already exists (status: %s), checking if it's bound to our PVC\n", pvName, existingPV.Status.Phase) + + // Check if PV is in Released state - if so, we need to delete and recreate it + if existingPV.Status.Phase == corev1.VolumeReleased { + fmt.Printf("PV %s is in Released state, deleting it to recreate\n", pvName) + deleteErr := s.client.Clientset.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}) + if deleteErr != nil && !errors.IsNotFound(deleteErr) { + return nil, fmt.Errorf("failed to delete released PV %s: %w", pvName, deleteErr) + } + // Wait for PV deletion + time.Sleep(3 * time.Second) + } else if existingPV.Spec.ClaimRef != nil && + existingPV.Spec.ClaimRef.Namespace == namespace && + existingPV.Spec.ClaimRef.Name == pvcName { + // PV is already bound to our PVC, wait a bit for binding to complete + time.Sleep(2 * time.Second) + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PVC after existing PV check: %w", err) + } + return pvc, nil + } else if existingPV.Spec.ClaimRef != nil { + // PV exists but bound to different PVC, delete it + fmt.Printf("PV %s exists but bound to different claim (%s/%s), deleting it\n", + pvName, existingPV.Spec.ClaimRef.Namespace, existingPV.Spec.ClaimRef.Name) + deleteErr := s.client.Clientset.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}) + if deleteErr != nil && !errors.IsNotFound(deleteErr) { + return nil, fmt.Errorf("failed to delete existing PV %s: %w", pvName, deleteErr) + } + // Wait for PV deletion + time.Sleep(3 * time.Second) + } + } + + // Get PVC to get its UID + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PVC %s for UID: %w", pvcName, err) + } + + storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB)) + nodeAffinity, err := s.hostPathPVNodeAffinity(ctx) + if err != nil { + return nil, fmt.Errorf("failed to select node for hostPath PV %s: %w", pvName, err) + } + + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + Labels: map[string]string{ + "app": "clawreef", + "user-id": fmt.Sprintf("%d", userID), + "instance-id": fmt.Sprintf("%d", instanceID), + "managed-by": "clawreef", + storageProfileLabel: storageProfileSingle, + }, + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: storageSize, + }, + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: storageClass, + NodeAffinity: nodeAffinity, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPath, + Type: func() *corev1.HostPathType { + t := corev1.HostPathDirectoryOrCreate + return &t + }(), + }, + }, + ClaimRef: &corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + Namespace: namespace, + Name: pvcName, + UID: pvc.UID, + ResourceVersion: pvc.ResourceVersion, + }, + }, + } + + _, err = s.client.Clientset.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + if err != nil { + if errors.IsAlreadyExists(err) { + // PV was created by another process, wait for it to bind + fmt.Printf("PV %s was created by another process, waiting for binding\n", pvName) + time.Sleep(3 * time.Second) + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PVC after PV creation: %w", err) + } + return pvc, nil + } + return nil, fmt.Errorf("failed to create PV %s: %w", pvName, err) + } + + fmt.Printf("PV %s created successfully, waiting for binding\n", pvName) + + // Wait for PVC to be bound + time.Sleep(3 * time.Second) + pvc, err = s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PVC after PV creation: %w", err) + } + + if pvc.Status.Phase != corev1.ClaimBound { + return nil, fmt.Errorf("PVC %s is still not bound after PV creation, status: %s", pvcName, pvc.Status.Phase) + } + + fmt.Printf("PVC %s successfully bound to PV %s\n", pvcName, pvName) + return pvc, nil +} + +// GetPVC gets a PVC by user ID and instance ID +func (s *PVCService) GetPVC(ctx context.Context, userID, instanceID int) (*corev1.PersistentVolumeClaim, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + pvcName := s.client.GetPVCName(instanceID) + namespace := s.client.GetNamespace(userID) + + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PVC %s: %w", pvcName, err) + } + + return pvc, nil +} + +func (s *PVCService) NodeSelectorForPVC(ctx context.Context, userID, instanceID int, storageClass string) (map[string]string, error) { + if s == nil || s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + pvcName := s.client.GetPVCName(instanceID) + namespace := s.client.GetNamespace(userID) + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PVC %s: %w", pvcName, err) + } + + storageClass = pvcStorageClassName(pvc, firstNonEmpty(storageClass, s.client.InstanceStorageClass, s.client.StorageClass)) + + if strings.TrimSpace(pvc.Spec.VolumeName) != "" { + pv, err := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get bound PV %s for PVC %s: %w", pvc.Spec.VolumeName, pvcName, err) + } + return nodeSelectorFromPVNodeAffinity(pv.Spec.NodeAffinity), nil + } + + return s.nodeSelectorForStorageClass(ctx, storageClass) +} + +func (s *PVCService) nodeSelectorForStorageClass(ctx context.Context, storageClass string) (map[string]string, error) { + if s == nil || s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + storageClass = strings.TrimSpace(storageClass) + if storageClass == "" { + storageClass = firstNonEmpty(s.client.InstanceStorageClass, s.client.StorageClass) + } + if storageClass == "" { + return nil, nil + } + + sc, err := s.client.Clientset.StorageV1().StorageClasses().Get(ctx, storageClass, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to get storage class %s: %w", storageClass, err) + } + if sc.Provisioner != noProvisionerName { + return nil, nil + } + return s.hostPathPVNodeSelector(ctx) +} + +func pvcStorageClassName(pvc *corev1.PersistentVolumeClaim, fallback string) string { + storageClass := strings.TrimSpace(fallback) + if storageClass == "" && pvc != nil && pvc.Spec.StorageClassName != nil { + storageClass = strings.TrimSpace(*pvc.Spec.StorageClassName) + } + return storageClass +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if trimmed := strings.TrimSpace(value); trimmed != "" { + return trimmed + } + } + return "" +} + +func workspacePVCAccessMode(value string) corev1.PersistentVolumeAccessMode { + switch strings.ToLower(strings.TrimSpace(value)) { + case "readwriteonce", "rwo": + return corev1.ReadWriteOnce + case "readonlymany", "rox": + return corev1.ReadOnlyMany + case "readwritemany", "rwx", "": + return corev1.ReadWriteMany + default: + return corev1.ReadWriteMany + } +} + +func (s *PVCService) pvcBindTimeout() time.Duration { + if s != nil && s.client != nil && s.client.PVCBindTimeout > 0 { + return s.client.PVCBindTimeout + } + return defaultPVCBindTimeout +} + +func (s *PVCService) storageProfile() string { + if s == nil || s.client == nil { + return "" + } + return strings.ToLower(strings.TrimSpace(s.client.StorageProfile)) +} + +func (s *PVCService) hostPathFallbackEnabled() bool { + if s == nil || s.client == nil || !s.client.HostPathFallbackEnabled { + return false + } + profile := s.storageProfile() + return profile == "" || profile == storageProfileSingle +} + +func isClawManagerHostPathPV(pv *corev1.PersistentVolume) bool { + if pv == nil || pv.Spec.HostPath == nil { + return false + } + if pv.Labels["managed-by"] != "clawreef" { + return false + } + if strings.EqualFold(pv.Labels[storageProfileLabel], storageProfileSingle) { + return true + } + return strings.HasPrefix(pv.Name, "clawreef-pv-user-") +} + +func nodeSelectorForHostname(hostname string) map[string]string { + hostname = strings.TrimSpace(hostname) + if hostname == "" { + return nil + } + return map[string]string{nodeHostnameLabel: hostname} +} + +func nodeSelectorFromPVNodeAffinity(affinity *corev1.VolumeNodeAffinity) map[string]string { + if affinity == nil || affinity.Required == nil { + return nil + } + for _, term := range affinity.Required.NodeSelectorTerms { + for _, expression := range term.MatchExpressions { + if expression.Key != nodeHostnameLabel || expression.Operator != corev1.NodeSelectorOpIn { + continue + } + if len(expression.Values) != 1 { + continue + } + return nodeSelectorForHostname(expression.Values[0]) + } + } + return nil +} + +// DeletePVC deletes a PVC and associated PV +func (s *PVCService) DeletePVC(ctx context.Context, userID, instanceID int) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + pvcName := s.client.GetPVCName(instanceID) + namespace := s.client.GetNamespace(userID) + pvName := s.client.GetInstancePVName(userID, instanceID) + + // Get PVC first to find the associated PV + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // PVC doesn't exist, still try to delete the PV directly + fmt.Printf("PVC %s not found, trying to delete PV %s directly\n", pvcName, pvName) + s.deleteManagedHostPathPV(ctx, pvName) + return nil + } + return fmt.Errorf("failed to get PVC %s: %w", pvcName, err) + } + + // Store the PV name before deleting PVC + boundPVName := pvc.Spec.VolumeName + + // Delete the PVC first + err = s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // PVC already deleted, continue to delete PV + fmt.Printf("PVC %s already deleted\n", pvcName) + } else { + return fmt.Errorf("failed to delete PVC %s: %w", pvcName, err) + } + } else { + fmt.Printf("PVC %s deleted successfully\n", pvcName) + } + + // Delete the manually created PV + // First try the predictable name + s.deleteManagedHostPathPV(ctx, pvName) + + // Also try to delete the bound PV if it exists and is different + if boundPVName != "" && boundPVName != pvName { + s.deleteManagedHostPathPV(ctx, boundPVName) + } + + return nil +} + +func (s *PVCService) deleteManagedHostPathPV(ctx context.Context, pvName string) { + pvName = strings.TrimSpace(pvName) + if pvName == "" || s == nil || s.client == nil || s.client.Clientset == nil { + return + } + pv, err := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + fmt.Printf("PV %s not found or already deleted\n", pvName) + } else { + fmt.Printf("Warning: failed to get PV %s before delete: %v\n", pvName, err) + } + return + } + if !isClawManagerHostPathPV(pv) { + fmt.Printf("Skipping PV %s delete because it is not a ClawManager-managed hostPath PV\n", pvName) + return + } + if err := s.client.Clientset.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + fmt.Printf("Warning: failed to delete PV %s: %v\n", pvName, err) + } + return + } + fmt.Printf("PV %s deleted successfully\n", pvName) +} + +func (s *PVCService) deleteManagedTeamSharedPV(ctx context.Context, pvName string, userID, teamID int) { + pvName = strings.TrimSpace(pvName) + if pvName == "" || s == nil || s.client == nil || s.client.Clientset == nil { + return + } + pv, err := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + fmt.Printf("Team shared PV %s not found or already deleted\n", pvName) + } else { + fmt.Printf("Warning: failed to get Team shared PV %s before delete: %v\n", pvName, err) + } + return + } + if pv.Labels["managed-by"] != "clawreef" || + pv.Labels["user-id"] != fmt.Sprintf("%d", userID) || + pv.Labels["team-id"] != fmt.Sprintf("%d", teamID) { + fmt.Printf("Skipping Team shared PV %s delete because it is not owned by user %d team %d\n", pvName, userID, teamID) + return + } + if pv.Spec.HostPath == nil && pv.Spec.NFS == nil { + fmt.Printf("Skipping Team shared PV %s delete because it is not a ClawManager hostPath/NFS PV\n", pvName) + return + } + if err := s.client.Clientset.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + fmt.Printf("Warning: failed to delete Team shared PV %s: %v\n", pvName, err) + } + return + } + fmt.Printf("Team shared PV %s deleted successfully\n", pvName) +} + +// DeleteTeamSharedPVC deletes a Team shared PVC and the predictable Team PV +// managed by ClawManager. It only deletes a bound PV when that PV is explicitly +// labeled as belonging to this Team, avoiding accidental deletion of external +// provisioner volumes. +func (s *PVCService) DeleteTeamSharedPVC(ctx context.Context, userID, teamID int) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + pvcName := s.client.GetTeamSharedPVCName(teamID) + namespace := s.client.GetNamespace(userID) + pvName := s.client.GetTeamSharedPVName(userID, teamID) + boundPVName := "" + + if pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}); err == nil && pvc != nil { + boundPVName = pvc.Spec.VolumeName + } else if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to get Team shared PVC %s/%s before delete: %w", namespace, pvcName, err) + } + + if err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete Team shared PVC %s/%s: %w", namespace, pvcName, err) + } + } + s.deleteManagedTeamSharedPV(ctx, pvName, userID, teamID) + if boundPVName != "" && boundPVName != pvName { + s.deleteManagedTeamSharedPV(ctx, boundPVName, userID, teamID) + } + return nil +} + +// PVCExists checks if a PVC exists +func (s *PVCService) PVCExists(ctx context.Context, userID, instanceID int) (bool, error) { + _, err := s.GetPVC(ctx, userID, instanceID) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/backend/internal/services/k8s/pvc_service_test.go b/backend/internal/services/k8s/pvc_service_test.go new file mode 100644 index 0000000..9950478 --- /dev/null +++ b/backend/internal/services/k8s/pvc_service_test.go @@ -0,0 +1,685 @@ +package k8s + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" +) + +func TestHostPathPVNodeAffinitySelectsReadyNode(t *testing.T) { + service := &PVCService{ + client: &Client{ + Clientset: fake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-b", + Labels: map[string]string{ + "kubernetes.io/hostname": "node-b-host", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-a", + Labels: map[string]string{ + "kubernetes.io/hostname": "node-a-host", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + }, + ), + }, + } + + affinity, err := service.hostPathPVNodeAffinity(context.Background()) + if err != nil { + t.Fatalf("hostPathPVNodeAffinity returned error: %v", err) + } + requireHostnameAffinity(t, affinity, "node-a-host") +} + +func TestHostPathPVNodeAffinitySkipsUnschedulableAndNotReadyNodes(t *testing.T) { + service := &PVCService{ + client: &Client{ + Clientset: fake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-a", + Labels: map[string]string{ + "kubernetes.io/hostname": "node-a-host", + }, + }, + Spec: corev1.NodeSpec{Unschedulable: true}, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-b", + Labels: map[string]string{ + "kubernetes.io/hostname": "node-b-host", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionFalse}, + }, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-c", + Labels: map[string]string{ + "kubernetes.io/hostname": "node-c-host", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + }, + ), + }, + } + + affinity, err := service.hostPathPVNodeAffinity(context.Background()) + if err != nil { + t.Fatalf("hostPathPVNodeAffinity returned error: %v", err) + } + requireHostnameAffinity(t, affinity, "node-c-host") +} + +func TestCreatePVForTeamSharedPVCUsesWorkspaceNFSWhenConfigured(t *testing.T) { + ctx := context.Background() + workspaceRoot := t.TempDir() + namespace := "clawmanager-user-1" + pvcName := "clawreef-team-28-shared" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + UID: types.UID("pvc-uid"), + ResourceVersion: "1", + }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + clientset := fake.NewSimpleClientset(pvc) + service := &PVCService{ + client: &Client{ + Clientset: clientset, + Namespace: "clawmanager", + WorkspaceRoot: workspaceRoot, + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + }, + } + + if _, err := service.createPVForTeamSharedPVC(ctx, namespace, pvcName, 1, 28, 10, "manual"); err != nil { + t.Fatalf("createPVForTeamSharedPVC returned error: %v", err) + } + + pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-clawmanager-user-1-team-28-shared", metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected Team shared PV: %v", err) + } + if pv.Spec.PersistentVolumeSource.NFS == nil { + t.Fatalf("expected NFS PV source, got %#v", pv.Spec.PersistentVolumeSource) + } + if pv.Spec.PersistentVolumeSource.NFS.Server != "workspace-store.clawmanager-system.svc.cluster.local" { + t.Fatalf("unexpected NFS server: %#v", pv.Spec.PersistentVolumeSource.NFS) + } + if pv.Spec.PersistentVolumeSource.NFS.Path != "/teams/user-1/team-28-shared" { + t.Fatalf("unexpected NFS path: %#v", pv.Spec.PersistentVolumeSource.NFS) + } + if pv.Spec.NodeAffinity != nil { + t.Fatalf("NFS-backed Team PV must not pin runtime sharing to one node: %#v", pv.Spec.NodeAffinity) + } + if _, err := os.Stat(filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared")); err != nil { + t.Fatalf("expected runtime workspace team directory to be created: %v", err) + } +} + +func TestCreatePVForTeamSharedPVCResolvesWorkspaceServiceDNS(t *testing.T) { + ctx := context.Background() + namespace := "clawmanager-user-1" + pvcName := "clawreef-team-28-shared" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + UID: types.UID("pvc-uid"), + ResourceVersion: "1", + }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "workspace-store", + Namespace: "clawmanager-system", + }, + Spec: corev1.ServiceSpec{ClusterIP: "10.96.0.44"}, + } + clientset := fake.NewSimpleClientset(pvc, service) + pvcService := &PVCService{ + client: &Client{ + Clientset: clientset, + Namespace: "clawmanager", + WorkspaceRoot: t.TempDir(), + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + }, + } + + if _, err := pvcService.createPVForTeamSharedPVC(ctx, namespace, pvcName, 1, 28, 10, "manual"); err != nil { + t.Fatalf("createPVForTeamSharedPVC returned error: %v", err) + } + + pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-clawmanager-user-1-team-28-shared", metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected Team shared PV: %v", err) + } + if pv.Spec.NFS == nil || pv.Spec.NFS.Server != "10.96.0.44" { + t.Fatalf("expected resolved ClusterIP NFS server, got %#v", pv.Spec.PersistentVolumeSource) + } +} + +func TestCreateTeamSharedPVCCreatesWorkspaceDirectoryBeforeReturning(t *testing.T) { + ctx := context.Background() + workspaceRoot := t.TempDir() + client := &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawmanager", + StorageClass: "manual", + WorkspaceRoot: workspaceRoot, + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil { + t.Fatalf("CreateTeamSharedPVC returned error: %v", err) + } + + if _, err := os.Stat(filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared")); err != nil { + t.Fatalf("expected Team shared runtime workspace directory before Lite gateways start: %v", err) + } +} + +func TestCreateTeamSharedPVCCreatesWritableRuntimeSubdirectories(t *testing.T) { + ctx := context.Background() + workspaceRoot := t.TempDir() + client := &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawmanager", + StorageClass: "manual", + WorkspaceRoot: workspaceRoot, + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil { + t.Fatalf("CreateTeamSharedPVC returned error: %v", err) + } + + for _, name := range []string{"status", "inbox", "results", "tasks", ".openclaw-redis-team", filepath.Join(".openclaw-redis-team", "tasks")} { + dir := filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared", name) + info, err := os.Stat(dir) + if err != nil { + t.Fatalf("expected Team shared runtime subdirectory %s before gateways start: %v", name, err) + } + if !info.IsDir() { + t.Fatalf("expected %s to be a directory", dir) + } + if runtime.GOOS != "windows" { + mode := info.Mode() + if mode&os.ModeSetgid == 0 || mode.Perm()&0o020 == 0 { + t.Fatalf("expected %s to preserve shared group-write/setgid permissions, got mode %v", dir, mode) + } + } + } +} + +func TestCreateTeamSharedPVCCreatesWorkspaceNFSPVBeforeReturning(t *testing.T) { + ctx := context.Background() + client := &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawmanager", + StorageClass: "manual", + WorkspaceRoot: t.TempDir(), + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil { + t.Fatalf("CreateTeamSharedPVC returned error: %v", err) + } + + pv, err := client.Clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-clawmanager-user-1-team-28-shared", metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected Team shared NFS PV before Lite gateways start: %v", err) + } + if pv.Spec.NFS == nil || pv.Spec.NFS.Path != "/teams/user-1/team-28-shared" { + t.Fatalf("unexpected Team shared PV source: %#v", pv.Spec.PersistentVolumeSource) + } +} + +func TestCreateTeamSharedPVCUsesWorkspaceNFSInsteadOfInstanceStorageClass(t *testing.T) { + ctx := context.Background() + client := &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawmanager", + StorageClass: "longhorn", + InstanceStorageClass: "longhorn", + WorkspaceRoot: t.TempDir(), + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + pvc, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "") + if err != nil { + t.Fatalf("CreateTeamSharedPVC returned error: %v", err) + } + if pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != "manual" { + t.Fatalf("Team shared PVC storage class = %#v, want manual static NFS binding", pvc.Spec.StorageClassName) + } + pv, err := client.Clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-clawmanager-user-1-team-28-shared", metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected Team shared NFS PV: %v", err) + } + if pv.Spec.NFS == nil { + t.Fatalf("Team shared PV should use workspace NFS, got %#v", pv.Spec.PersistentVolumeSource) + } +} + +func TestCreateTeamSharedPVCRejectsExistingNonWorkspaceNFSPV(t *testing.T) { + ctx := context.Background() + namespace := "clawmanager-user-1" + pvcName := "clawreef-team-28-shared" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + Labels: map[string]string{"team-id": "28"}, + }, + } + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "clawreef-pv-clawmanager-user-1-team-28-shared"}, + Spec: corev1.PersistentVolumeSpec{ + ClaimRef: &corev1.ObjectReference{Namespace: namespace, Name: pvcName}, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + HostPath: &corev1.HostPathVolumeSource{Path: "/data/clawreef/user-1/team-28-shared"}, + }, + }, + } + client := &Client{ + Clientset: fake.NewSimpleClientset(pvc, pv), + Namespace: "clawmanager", + StorageClass: "manual", + WorkspaceRoot: t.TempDir(), + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual") + if err == nil || !strings.Contains(err.Error(), "workspace NFS") { + t.Fatalf("expected existing hostPath Team PV to be rejected, got %v", err) + } +} + +func TestCreatePVCUsesInstanceStorageClassDefault(t *testing.T) { + ctx := context.Background() + client := &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawmanager", + StorageClass: "standard", + InstanceStorageClass: "longhorn", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + pvc, err := service.CreatePVC(ctx, 1, 371, 10, "") + if err != nil { + t.Fatalf("CreatePVC returned error: %v", err) + } + if pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != "longhorn" { + t.Fatalf("PVC storage class = %#v, want longhorn", pvc.Spec.StorageClassName) + } +} + +func TestCreateTeamSharedPVCUsesWorkspaceStorageDefaults(t *testing.T) { + ctx := context.Background() + client := &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawmanager", + StorageClass: "standard", + WorkspaceStorageClass: "longhorn-rwx", + WorkspaceAccessMode: "ReadWriteMany", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + pvc, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "") + if err != nil { + t.Fatalf("CreateTeamSharedPVC returned error: %v", err) + } + if pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != "longhorn-rwx" { + t.Fatalf("PVC storage class = %#v, want longhorn-rwx", pvc.Spec.StorageClassName) + } + if len(pvc.Spec.AccessModes) != 1 || pvc.Spec.AccessModes[0] != corev1.ReadWriteMany { + t.Fatalf("PVC access modes = %#v, want ReadWriteMany", pvc.Spec.AccessModes) + } +} + +func TestWaitForPVCBindingDoesNotCreateHostPathPVWhenFallbackDisabled(t *testing.T) { + ctx := context.Background() + namespace := "clawmanager-user-162" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clawreef-371-pvc", + Namespace: namespace, + }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, + } + clientset := fake.NewSimpleClientset(pvc) + service := &PVCService{ + client: &Client{ + Clientset: clientset, + Namespace: "clawmanager", + StorageProfile: "cluster", + HostPathFallbackEnabled: false, + }, + } + + _, err := service.waitForPVCBinding(ctx, namespace, pvc.Name, 162, 371, 10, "manual", time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "hostPath fallback is disabled") { + t.Fatalf("expected disabled fallback error, got %v", err) + } + pvs, listErr := clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) + if listErr != nil { + t.Fatalf("failed to list PVs: %v", listErr) + } + if len(pvs.Items) != 0 { + t.Fatalf("expected no hostPath PV to be created, got %#v", pvs.Items) + } +} + +func TestDeletePVCSkipsDynamicProvisionerPV(t *testing.T) { + ctx := context.Background() + namespace := "clawmanager-user-162" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clawreef-371-pvc", + Namespace: namespace, + Labels: map[string]string{"instance-id": "371"}, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "pvc-dynamic-123", + }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-dynamic-123", + Labels: map[string]string{ + "managed-by": "external-provisioner", + }, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{Driver: "driver.longhorn.io"}, + }, + }, + } + client := &Client{ + Clientset: fake.NewSimpleClientset(pvc, pv), + Namespace: "clawmanager", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + if err := service.DeletePVC(ctx, 162, 371); err != nil { + t.Fatalf("DeletePVC returned error: %v", err) + } + if _, err := client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvc.Name, metav1.GetOptions{}); !errors.IsNotFound(err) { + t.Fatalf("expected PVC to be deleted, got err=%v", err) + } + if _, err := client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pv.Name, metav1.GetOptions{}); err != nil { + t.Fatalf("dynamic provisioner PV must not be deleted, got %v", err) + } +} + +func TestHostPathPVNodeAffinitySkipsHardTaintedNodes(t *testing.T) { + service := &PVCService{ + client: &Client{ + Clientset: fake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "k8s-master", + Labels: map[string]string{ + "kubernetes.io/hostname": "k8s-master", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + {Key: "node-role.kubernetes.io/control-plane", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "k8s-worker1", + Labels: map[string]string{ + "kubernetes.io/hostname": "k8s-worker1", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + }, + ), + }, + } + + affinity, err := service.hostPathPVNodeAffinity(context.Background()) + if err != nil { + t.Fatalf("hostPathPVNodeAffinity returned error: %v", err) + } + requireHostnameAffinity(t, affinity, "k8s-worker1") +} + +func TestNodeSelectorForPVCUsesBoundPVNodeAffinity(t *testing.T) { + ctx := context.Background() + namespace := "clawmanager-user-162" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clawreef-371-pvc", + Namespace: namespace, + Labels: map[string]string{"instance-id": "371"}, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "clawreef-pv-user-162-instance-371", + }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "clawreef-pv-user-162-instance-371"}, + Spec: corev1.PersistentVolumeSpec{ + NodeAffinity: hostPathPVNodeAffinityForHostname("node125"), + }, + } + client := &Client{ + Clientset: fake.NewSimpleClientset(pvc, pv), + Namespace: "clawmanager", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + selector, err := service.NodeSelectorForPVC(ctx, 162, 371, "manual") + if err != nil { + t.Fatalf("NodeSelectorForPVC returned error: %v", err) + } + if selector["kubernetes.io/hostname"] != "node125" { + t.Fatalf("selector = %#v, want hostname node125", selector) + } +} + +func TestNodeSelectorForPVCFallsBackForNoProvisionerStorageClass(t *testing.T) { + ctx := context.Background() + namespace := "clawmanager-user-162" + storageClassName := "manual" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clawreef-371-pvc", + Namespace: namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: &storageClassName, + }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, + } + storageClass := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "manual"}, + Provisioner: "kubernetes.io/no-provisioner", + } + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node125", + Labels: map[string]string{"kubernetes.io/hostname": "node125"}, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, + }, + } + client := &Client{ + Clientset: fake.NewSimpleClientset(pvc, storageClass, node), + Namespace: "clawmanager", + StorageClass: "manual", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + selector, err := service.NodeSelectorForPVC(ctx, 162, 371, "") + if err != nil { + t.Fatalf("NodeSelectorForPVC returned error: %v", err) + } + if selector["kubernetes.io/hostname"] != "node125" { + t.Fatalf("selector = %#v, want hostname node125", selector) + } +} + +func TestWaitForPVCBindingLeavesDynamicStorageClassToProvisioner(t *testing.T) { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clawreef-114-pvc", + Namespace: "clawmanager-user-155", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: stringPtr("local-path"), + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimPending, + }, + } + service := &PVCService{ + client: &Client{ + Clientset: fake.NewSimpleClientset(pvc), + }, + } + + got, err := service.waitForPVCBinding(context.Background(), pvc.Namespace, pvc.Name, 155, 114, 100, "local-path", time.Nanosecond) + if err != nil { + t.Fatalf("waitForPVCBinding returned error: %v", err) + } + if got.Name != pvc.Name { + t.Fatalf("expected PVC %q, got %q", pvc.Name, got.Name) + } + pvs, err := service.client.Clientset.CoreV1().PersistentVolumes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("failed to list PVs: %v", err) + } + if len(pvs.Items) != 0 { + t.Fatalf("expected no manual PVs for dynamic storageClass, got %#v", pvs.Items) + } +} + +func stringPtr(value string) *string { + return &value +} + +func requireHostnameAffinity(t *testing.T, affinity *corev1.VolumeNodeAffinity, hostname string) { + t.Helper() + if affinity == nil || affinity.Required == nil || len(affinity.Required.NodeSelectorTerms) != 1 { + t.Fatalf("unexpected node affinity: %#v", affinity) + } + expressions := affinity.Required.NodeSelectorTerms[0].MatchExpressions + if len(expressions) != 1 { + t.Fatalf("unexpected node affinity expressions: %#v", expressions) + } + expression := expressions[0] + if expression.Key != "kubernetes.io/hostname" || expression.Operator != corev1.NodeSelectorOpIn { + t.Fatalf("unexpected node affinity expression: %#v", expression) + } + if len(expression.Values) != 1 || expression.Values[0] != hostname { + t.Fatalf("expected hostname %q, got %#v", hostname, expression.Values) + } +} diff --git a/backend/internal/services/k8s/runtime_deployment_service.go b/backend/internal/services/k8s/runtime_deployment_service.go new file mode 100644 index 0000000..9ff060d --- /dev/null +++ b/backend/internal/services/k8s/runtime_deployment_service.go @@ -0,0 +1,473 @@ +package k8s + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strconv" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" +) + +const ( + runtimeAgentPort int32 = 19090 + runtimeWorkspaceVolume = "workspaces" + defaultWorkspaceMount = "/workspaces" + defaultGatewayPortStart = 20000 + defaultGatewayPortEnd = 20099 + hermesTUIDir = "/usr/local/lib/hermes-agent/ui-tui" +) + +type RuntimeDeploymentSpec struct { + Name string + Namespace string + RuntimeType string + Image string + Replicas int32 + WorkspacePVCClaimName string + WorkspaceNFSServer string + WorkspaceNFSPath string + WorkspaceMountPath string + GatewayPortStart int + GatewayPortEnd int + AgentControlToken string + AgentReportToken string + BackendURL string + TrustedProxyCIDRs string +} + +type RuntimeDeploymentPod struct { + RuntimeType string + Namespace string + DeploymentName string + PodName string + PodIP *string + NodeName *string + ImageRef string + State string +} + +type RuntimeDeploymentService interface { + Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error + Scale(ctx context.Context, namespace, name string, replicas int32) error + RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error + ListPods(ctx context.Context, namespace, runtimeType string) ([]RuntimeDeploymentPod, error) +} + +type runtimeDeploymentService struct { + client kubernetes.Interface +} + +func NewRuntimeDeploymentService(client kubernetes.Interface) RuntimeDeploymentService { + return &runtimeDeploymentService{client: client} +} + +func BuildRuntimeDeployment(spec RuntimeDeploymentSpec) *appsv1.Deployment { + labels := map[string]string{ + "app": spec.Name, + "clawmanager.io/runtime-type": spec.RuntimeType, + } + replicas := spec.Replicas + workspaceMountPath := runtimeWorkspaceMountPath(spec.WorkspaceMountPath) + gatewayPortStart := runtimeGatewayPortValue(spec.GatewayPortStart, defaultGatewayPortStart) + gatewayPortEnd := runtimeGatewayPortValue(spec.GatewayPortEnd, defaultGatewayPortEnd) + env := buildRuntimeAgentEnv(spec, workspaceMountPath, gatewayPortStart, gatewayPortEnd) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: spec.Name, + Namespace: spec.Namespace, + Labels: copyStringMap(labels), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: copyStringMap(labels), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: copyStringMap(labels), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "runtime", + Image: spec.Image, + Ports: []corev1.ContainerPort{ + { + Name: "agent", + ContainerPort: runtimeAgentPort, + }, + }, + Env: env, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: runtimeWorkspaceVolume, + MountPath: workspaceMountPath, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: runtimeWorkspaceVolume, + VolumeSource: runtimeWorkspaceVolumeSource(spec), + }, + }, + }, + }, + }, + } +} + +func runtimeWorkspaceVolumeSource(spec RuntimeDeploymentSpec) corev1.VolumeSource { + if claimName := strings.TrimSpace(spec.WorkspacePVCClaimName); claimName != "" { + return corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: claimName, + }, + } + } + if server := strings.TrimSpace(spec.WorkspaceNFSServer); server != "" { + nfsPath := strings.TrimSpace(spec.WorkspaceNFSPath) + if nfsPath == "" { + nfsPath = "/" + } + return corev1.VolumeSource{ + NFS: &corev1.NFSVolumeSource{ + Server: server, + Path: nfsPath, + }, + } + } + return corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} +} + +func buildRuntimeAgentEnv(spec RuntimeDeploymentSpec, workspaceRoot string, gatewayPortStart, gatewayPortEnd int) []corev1.EnvVar { + env := []corev1.EnvVar{ + {Name: "CLAWMANAGER_RUNTIME_TYPE", Value: spec.RuntimeType}, + {Name: "CLAWMANAGER_BACKEND_URL", Value: runtimeBackendURL(spec)}, + {Name: "CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME", Value: spec.Name}, + {Name: "CLAWMANAGER_RUNTIME_IMAGE_REF", Value: spec.Image}, + {Name: "CLAWMANAGER_AGENT_PORT", Value: "19090"}, + {Name: "CLAWMANAGER_GATEWAY_PORT_START", Value: strconv.Itoa(gatewayPortStart)}, + {Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: strconv.Itoa(gatewayPortEnd)}, + {Name: "CLAWMANAGER_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken}, + {Name: "CLAWMANAGER_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken}, + {Name: "RUNTIME_WORKSPACE_ROOT", Value: workspaceRoot}, + {Name: "RUNTIME_AGENT_LISTEN_ADDR", Value: "0.0.0.0:19090"}, + {Name: "RUNTIME_AGENT_PUBLIC_PORT", Value: "19090"}, + {Name: "RUNTIME_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken}, + {Name: "RUNTIME_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken}, + {Name: "RUNTIME_GATEWAY_PORT_START", Value: strconv.Itoa(gatewayPortStart)}, + {Name: "RUNTIME_GATEWAY_PORT_END", Value: strconv.Itoa(gatewayPortEnd)}, + } + if strings.EqualFold(spec.RuntimeType, "hermes") { + env = append(env, corev1.EnvVar{Name: "HERMES_TUI_DIR", Value: hermesTUIDir}) + } + env = append(env, + fieldRefEnv("POD_NAME", "metadata.name"), + fieldRefEnv("POD_NAMESPACE", "metadata.namespace"), + fieldRefEnv("POD_IP", "status.podIP"), + fieldRefEnv("NODE_NAME", "spec.nodeName"), + ) + if spec.TrustedProxyCIDRs != "" { + env = append(env, corev1.EnvVar{Name: "CLAWMANAGER_TRUSTED_PROXY_CIDRS", Value: spec.TrustedProxyCIDRs}) + } + return env +} + +func runtimeBackendURL(spec RuntimeDeploymentSpec) string { + if spec.BackendURL != "" { + return spec.BackendURL + } + namespace := spec.Namespace + if namespace == "" { + namespace = "clawmanager-system" + } + return fmt.Sprintf("http://clawmanager-gateway.%s.svc.cluster.local:9001", namespace) +} + +func fieldRefEnv(name, fieldPath string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: fieldPath}, + }, + } +} + +func runtimeWorkspaceMountPath(value string) string { + if value == "" { + return defaultWorkspaceMount + } + return value +} + +func runtimeGatewayPortValue(value, defaultValue int) int { + if value == 0 { + return defaultValue + } + return value +} + +func (s *runtimeDeploymentService) Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error { + if s == nil || s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + desired := BuildRuntimeDeployment(spec) + deployments := s.client.AppsV1().Deployments(spec.Namespace) + existing, err := deployments.Get(ctx, spec.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + _, createErr := deployments.Create(ctx, desired, metav1.CreateOptions{}) + if createErr != nil { + return fmt.Errorf("failed to create runtime deployment %s/%s: %w", spec.Namespace, spec.Name, createErr) + } + return nil + } + if err != nil { + return fmt.Errorf("failed to get runtime deployment %s/%s: %w", spec.Namespace, spec.Name, err) + } + + if !reflect.DeepEqual(existing.Spec.Selector, desired.Spec.Selector) { + return fmt.Errorf("runtime deployment %s/%s selector mismatch; delete and recreate the deployment to change immutable selector", spec.Namespace, spec.Name) + } + + updated := existing.DeepCopy() + if updated.Labels == nil { + updated.Labels = map[string]string{} + } + for key, value := range desired.Labels { + updated.Labels[key] = value + } + updated.Spec.Replicas = desired.Spec.Replicas + if updated.Spec.Template.Labels == nil { + updated.Spec.Template.Labels = map[string]string{} + } + for key, value := range desired.Spec.Template.Labels { + updated.Spec.Template.Labels[key] = value + } + updated.Spec.Template.Spec = desired.Spec.Template.Spec + + _, err = deployments.Update(ctx, updated, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update runtime deployment %s/%s: %w", spec.Namespace, spec.Name, err) + } + return nil +} + +func (s *runtimeDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error { + if s == nil || s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + deployments := s.client.AppsV1().Deployments(namespace) + patch, err := json.Marshal(map[string]interface{}{ + "spec": map[string]interface{}{ + "replicas": replicas, + }, + }) + if err != nil { + return fmt.Errorf("failed to build runtime deployment scale patch %s/%s: %w", namespace, name, err) + } + if _, err := deployments.Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to scale runtime deployment %s/%s: %w", namespace, name, err) + } + return nil +} + +func (s *runtimeDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error { + if s == nil || s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + image = strings.TrimSpace(image) + if image == "" { + return fmt.Errorf("runtime rollout image is required") + } + + deployments := s.client.AppsV1().Deployments(namespace) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + existing, err := deployments.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + + updated := existing.DeepCopy() + containerIndex := -1 + for index, container := range updated.Spec.Template.Spec.Containers { + if container.Name == "runtime" { + containerIndex = index + break + } + } + if containerIndex < 0 { + return fmt.Errorf("runtime deployment %s/%s has no runtime container", namespace, name) + } + + container := &updated.Spec.Template.Spec.Containers[containerIndex] + container.Image = image + upsertEnvVar(container, "CLAWMANAGER_RUNTIME_IMAGE_REF", image) + + updated.Spec.Strategy.Type = appsv1.RollingUpdateDeploymentStrategyType + updated.Spec.Strategy.RollingUpdate = &appsv1.RollingUpdateDeployment{ + MaxUnavailable: intOrStringPtr(positiveRolloutInt(maxUnavailable)), + MaxSurge: intOrStringPtr(positiveRolloutInt(maxSurge)), + } + + _, err = deployments.Update(ctx, updated, metav1.UpdateOptions{}) + return err + }) + if err != nil { + return fmt.Errorf("failed to update runtime deployment %s/%s image: %w", namespace, name, err) + } + return nil +} + +func (s *runtimeDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]RuntimeDeploymentPod, error) { + if s == nil || s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + namespace = strings.TrimSpace(namespace) + if namespace == "" { + return nil, fmt.Errorf("runtime namespace is required") + } + runtimeType = strings.ToLower(strings.TrimSpace(runtimeType)) + + listOptions := metav1.ListOptions{} + if runtimeType != "" { + listOptions.LabelSelector = labels.Set{"clawmanager.io/runtime-type": runtimeType}.String() + } + deploymentList, err := s.client.AppsV1().Deployments(namespace).List(ctx, listOptions) + if err != nil { + return nil, fmt.Errorf("failed to list runtime deployments in %s: %w", namespace, err) + } + + var pods []RuntimeDeploymentPod + for _, deployment := range deploymentList.Items { + deploymentRuntimeType := strings.ToLower(strings.TrimSpace(deployment.Labels["clawmanager.io/runtime-type"])) + if deploymentRuntimeType == "" { + continue + } + if runtimeType != "" && deploymentRuntimeType != runtimeType { + continue + } + if deployment.Spec.Selector == nil { + return nil, fmt.Errorf("runtime deployment %s/%s has no selector", deployment.Namespace, deployment.Name) + } + selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector) + if err != nil { + return nil, fmt.Errorf("runtime deployment %s/%s has invalid selector: %w", deployment.Namespace, deployment.Name, err) + } + podList, err := s.client.CoreV1().Pods(deployment.Namespace).List(ctx, metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + return nil, fmt.Errorf("failed to list runtime deployment pods %s/%s: %w", deployment.Namespace, deployment.Name, err) + } + deploymentImage := runtimeContainerImage(deployment.Spec.Template.Spec.Containers) + for _, pod := range podList.Items { + image := runtimeContainerImage(pod.Spec.Containers) + if image == "" { + image = deploymentImage + } + pods = append(pods, RuntimeDeploymentPod{ + RuntimeType: deploymentRuntimeType, + Namespace: pod.Namespace, + DeploymentName: deployment.Name, + PodName: pod.Name, + PodIP: stringPtrIfNotEmpty(pod.Status.PodIP), + NodeName: stringPtrIfNotEmpty(pod.Spec.NodeName), + ImageRef: image, + State: runtimeK8sPodState(pod), + }) + } + } + return pods, nil +} + +func runtimeContainerImage(containers []corev1.Container) string { + for _, container := range containers { + if container.Name == "runtime" { + return strings.TrimSpace(container.Image) + } + } + if len(containers) == 1 { + return strings.TrimSpace(containers[0].Image) + } + return "" +} + +func runtimeK8sPodState(pod corev1.Pod) string { + if pod.DeletionTimestamp != nil { + return "deleted" + } + switch pod.Status.Phase { + case corev1.PodPending: + return "pending" + case corev1.PodRunning: + if runtimeK8sPodReady(pod) { + return "ready" + } + return "pending" + case corev1.PodSucceeded, corev1.PodFailed: + return "unhealthy" + default: + return "pending" + } +} + +func runtimeK8sPodReady(pod corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +func stringPtrIfNotEmpty(value string) *string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + return &value +} + +func upsertEnvVar(container *corev1.Container, name, value string) { + for index := range container.Env { + if container.Env[index].Name == name { + container.Env[index].Value = value + container.Env[index].ValueFrom = nil + return + } + } + container.Env = append(container.Env, corev1.EnvVar{Name: name, Value: value}) +} + +func positiveRolloutInt(value int) int { + if value <= 0 { + return 1 + } + return value +} + +func intOrStringPtr(value int) *intstr.IntOrString { + result := intstr.FromInt(value) + return &result +} diff --git a/backend/internal/services/k8s/runtime_deployment_service_test.go b/backend/internal/services/k8s/runtime_deployment_service_test.go new file mode 100644 index 0000000..8e18e67 --- /dev/null +++ b/backend/internal/services/k8s/runtime_deployment_service_test.go @@ -0,0 +1,508 @@ +package k8s + +import ( + "context" + "fmt" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func TestBuildRuntimeDeployment(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:latest", + Replicas: 2, + WorkspaceNFSServer: "10.0.0.9", + WorkspaceNFSPath: "/exports/workspaces", + AgentControlToken: "control-secret", + AgentReportToken: "report-secret", + }) + + if deployment.Name != "runtime-openclaw" || deployment.Namespace != "runtime-system" { + t.Fatalf("unexpected deployment identity: %s/%s", deployment.Namespace, deployment.Name) + } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 2 { + t.Fatalf("expected replicas 2, got %#v", deployment.Spec.Replicas) + } + if got := deployment.Labels["clawmanager.io/runtime-type"]; got != "openclaw" { + t.Fatalf("expected runtime-type label openclaw, got %q", got) + } + + if len(deployment.Spec.Template.Spec.Containers) != 1 { + t.Fatalf("expected one container, got %d", len(deployment.Spec.Template.Spec.Containers)) + } + container := deployment.Spec.Template.Spec.Containers[0] + if container.Name != "runtime" || container.Image != "registry/openclaw:latest" { + t.Fatalf("unexpected runtime container: %#v", container) + } + requireContainerPort(t, container, "agent", 19090) + requireVolumeMount(t, container, "workspaces", "/workspaces") + requireEnv(t, container, "CLAWMANAGER_RUNTIME_TYPE", "openclaw") + requireEnv(t, container, "CLAWMANAGER_AGENT_PORT", "19090") + requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_START", "20000") + requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "20099") + requireEnv(t, container, "CLAWMANAGER_AGENT_CONTROL_TOKEN", "control-secret") + requireEnv(t, container, "CLAWMANAGER_AGENT_REPORT_TOKEN", "report-secret") + + if got := container.Resources.Requests[corev1.ResourceCPU]; got.Cmp(resource.MustParse("500m")) != 0 { + t.Fatalf("expected CPU request 500m, got %s", got.String()) + } + if got := container.Resources.Requests[corev1.ResourceMemory]; got.Cmp(resource.MustParse("1Gi")) != 0 { + t.Fatalf("expected memory request 1Gi, got %s", got.String()) + } + + if len(deployment.Spec.Template.Spec.Volumes) != 1 { + t.Fatalf("expected one volume, got %#v", deployment.Spec.Template.Spec.Volumes) + } + volume := deployment.Spec.Template.Spec.Volumes[0] + if volume.Name != "workspaces" || volume.NFS == nil { + t.Fatalf("expected workspaces NFS volume, got %#v", volume) + } + if volume.NFS.Server != "10.0.0.9" || volume.NFS.Path != "/exports/workspaces" { + t.Fatalf("unexpected NFS source: %#v", volume.NFS) + } +} + +func TestBuildRuntimeDeploymentUsesWorkspacePVCWhenClaimConfigured(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:latest", + Replicas: 2, + WorkspacePVCClaimName: "clawmanager-workspaces", + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/exports/workspaces", + }) + + if len(deployment.Spec.Template.Spec.Volumes) != 1 { + t.Fatalf("expected one volume, got %#v", deployment.Spec.Template.Spec.Volumes) + } + volume := deployment.Spec.Template.Spec.Volumes[0] + if volume.Name != "workspaces" || volume.PersistentVolumeClaim == nil { + t.Fatalf("expected workspaces PVC volume, got %#v", volume) + } + if got, want := volume.PersistentVolumeClaim.ClaimName, "clawmanager-workspaces"; got != want { + t.Fatalf("PVC claim name = %q, want %q", got, want) + } + if volume.NFS != nil { + t.Fatalf("PVC workspace volume must not also include NFS source: %#v", volume.NFS) + } +} + +func TestBuildRuntimeDeploymentKeepsCapacityLimitOutOfRuntimeEnv(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:latest", + Replicas: 2, + WorkspaceNFSServer: "10.0.0.9", + WorkspaceNFSPath: "/exports/workspaces", + WorkspaceMountPath: "/runtime-workspaces", + GatewayPortStart: 21000, + GatewayPortEnd: 21049, + }) + + container := deployment.Spec.Template.Spec.Containers[0] + requireVolumeMount(t, container, "workspaces", "/runtime-workspaces") + requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_START", "21000") + requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "21049") + requireEnvAbsent(t, container, "CLAWMANAGER_MAX_GATEWAYS_PER_POD") + requireEnvAbsent(t, container, "RUNTIME_MAX_GATEWAYS_PER_POD") +} + +func TestBuildRuntimeDeploymentInjectsAgentV2Environment(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-hermes", + Namespace: "runtime-system", + RuntimeType: "hermes", + Image: "registry/hermes:v2", + Replicas: 1, + WorkspaceNFSServer: "10.0.0.9", + WorkspaceNFSPath: "/exports/workspaces", + WorkspaceMountPath: "/workspaces", + GatewayPortStart: 21000, + GatewayPortEnd: 21099, + AgentControlToken: "control-secret", + AgentReportToken: "report-secret", + BackendURL: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001", + TrustedProxyCIDRs: "10.42.0.0/16,10.43.0.0/16", + }) + + container := deployment.Spec.Template.Spec.Containers[0] + requireEnv(t, container, "CLAWMANAGER_RUNTIME_TYPE", "hermes") + requireEnv(t, container, "CLAWMANAGER_BACKEND_URL", "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001") + requireEnv(t, container, "CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME", "runtime-hermes") + requireEnv(t, container, "CLAWMANAGER_RUNTIME_IMAGE_REF", "registry/hermes:v2") + requireEnv(t, container, "RUNTIME_WORKSPACE_ROOT", "/workspaces") + requireEnv(t, container, "RUNTIME_AGENT_LISTEN_ADDR", "0.0.0.0:19090") + requireEnv(t, container, "RUNTIME_AGENT_PUBLIC_PORT", "19090") + requireEnv(t, container, "RUNTIME_AGENT_CONTROL_TOKEN", "control-secret") + requireEnv(t, container, "RUNTIME_AGENT_REPORT_TOKEN", "report-secret") + requireEnv(t, container, "RUNTIME_GATEWAY_PORT_START", "21000") + requireEnv(t, container, "RUNTIME_GATEWAY_PORT_END", "21099") + requireEnv(t, container, "HERMES_TUI_DIR", "/usr/local/lib/hermes-agent/ui-tui") + requireEnv(t, container, "CLAWMANAGER_TRUSTED_PROXY_CIDRS", "10.42.0.0/16,10.43.0.0/16") + requireEnvFieldRef(t, container, "POD_NAME", "metadata.name") + requireEnvFieldRef(t, container, "POD_NAMESPACE", "metadata.namespace") + requireEnvFieldRef(t, container, "POD_IP", "status.podIP") + requireEnvFieldRef(t, container, "NODE_NAME", "spec.nodeName") +} + +func TestRuntimeDeploymentServiceEnsureCreatesAndUpdates(t *testing.T) { + client := fake.NewSimpleClientset() + service := NewRuntimeDeploymentService(client) + spec := RuntimeDeploymentSpec{ + Name: "runtime-hermes", + Namespace: "runtime-system", + RuntimeType: "hermes", + Image: "registry/hermes:v1", + Replicas: 1, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + } + + if err := service.Ensure(context.Background(), spec); err != nil { + t.Fatalf("Ensure create returned error: %v", err) + } + created, err := client.AppsV1().Deployments(spec.Namespace).Get(context.Background(), spec.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get created deployment: %v", err) + } + if created.Spec.Replicas == nil || *created.Spec.Replicas != 1 { + t.Fatalf("expected created replicas 1, got %#v", created.Spec.Replicas) + } + + spec.Image = "registry/hermes:v2" + spec.Replicas = 3 + if err := service.Ensure(context.Background(), spec); err != nil { + t.Fatalf("Ensure update returned error: %v", err) + } + updated, err := client.AppsV1().Deployments(spec.Namespace).Get(context.Background(), spec.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get updated deployment: %v", err) + } + if updated.Spec.Template.Spec.Containers[0].Image != "registry/hermes:v2" { + t.Fatalf("expected updated image, got %q", updated.Spec.Template.Spec.Containers[0].Image) + } + if updated.Spec.Replicas == nil || *updated.Spec.Replicas != 3 { + t.Fatalf("expected updated replicas 3, got %#v", updated.Spec.Replicas) + } +} + +func TestRuntimeDeploymentServiceEnsurePreservesUnownedMetadata(t *testing.T) { + existing := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:v1", + Replicas: 1, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }) + existing.Labels["admission.example.com/injected"] = "keep" + existing.Annotations = map[string]string{"admission.example.com/sidecar": "enabled"} + existing.Finalizers = []string{"protect.example.com/runtime"} + existing.Spec.Template.Labels["admission.example.com/template-label"] = "keep" + existing.Spec.Template.Annotations = map[string]string{"admission.example.com/template-annotation": "enabled"} + client := fake.NewSimpleClientset(existing) + service := NewRuntimeDeploymentService(client) + + if err := service.Ensure(context.Background(), RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:v2", + Replicas: 3, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }); err != nil { + t.Fatalf("Ensure update returned error: %v", err) + } + + updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-openclaw", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get updated deployment: %v", err) + } + if updated.Labels["admission.example.com/injected"] != "keep" { + t.Fatalf("expected unrelated label to survive, got %#v", updated.Labels) + } + if updated.Annotations["admission.example.com/sidecar"] != "enabled" { + t.Fatalf("expected annotation to survive, got %#v", updated.Annotations) + } + if len(updated.Finalizers) != 1 || updated.Finalizers[0] != "protect.example.com/runtime" { + t.Fatalf("expected finalizer to survive, got %#v", updated.Finalizers) + } + if updated.Labels["clawmanager.io/runtime-type"] != "openclaw" { + t.Fatalf("expected owned runtime label to be present, got %#v", updated.Labels) + } + if updated.Spec.Template.Labels["admission.example.com/template-label"] != "keep" { + t.Fatalf("expected unrelated template label to survive, got %#v", updated.Spec.Template.Labels) + } + if updated.Spec.Template.Annotations["admission.example.com/template-annotation"] != "enabled" { + t.Fatalf("expected template annotation to survive, got %#v", updated.Spec.Template.Annotations) + } + if updated.Spec.Template.Labels["clawmanager.io/runtime-type"] != "openclaw" { + t.Fatalf("expected owned template runtime label to be present, got %#v", updated.Spec.Template.Labels) + } + if updated.Spec.Template.Spec.Containers[0].Image != "registry/openclaw:v2" { + t.Fatalf("expected image update while preserving template metadata, got %q", updated.Spec.Template.Spec.Containers[0].Image) + } +} + +func TestRuntimeDeploymentServiceEnsureRejectsSelectorChange(t *testing.T) { + existing := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "runtime-openclaw", Namespace: "runtime-system"}, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "different"}}, + }, + } + client := fake.NewSimpleClientset(existing) + service := NewRuntimeDeploymentService(client) + + err := service.Ensure(context.Background(), RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:latest", + Replicas: 1, + }) + if err == nil { + t.Fatalf("expected selector mismatch error") + } +} + +func TestRuntimeDeploymentServiceScale(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:latest", + Replicas: 1, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }) + deployment.Spec.Template.Annotations = map[string]string{"admission.example.com/template": "keep"} + client := fake.NewSimpleClientset(deployment) + service := NewRuntimeDeploymentService(client) + + if err := service.Scale(context.Background(), "runtime-system", "runtime-openclaw", 4); err != nil { + t.Fatalf("Scale returned error: %v", err) + } + scaled, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-openclaw", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get scaled deployment: %v", err) + } + if scaled.Spec.Replicas == nil || *scaled.Spec.Replicas != 4 { + t.Fatalf("expected replicas 4, got %#v", scaled.Spec.Replicas) + } + if scaled.Spec.Template.Spec.Containers[0].Image != "registry/openclaw:latest" { + t.Fatalf("expected Scale to preserve image, got %q", scaled.Spec.Template.Spec.Containers[0].Image) + } + if scaled.Spec.Template.Annotations["admission.example.com/template"] != "keep" { + t.Fatalf("expected Scale to preserve template metadata, got %#v", scaled.Spec.Template.Annotations) + } +} + +func TestRuntimeDeploymentServiceRolloutImage(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-hermes", + Namespace: "runtime-system", + RuntimeType: "hermes", + Image: "registry/hermes:v1", + Replicas: 3, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }) + client := fake.NewSimpleClientset(deployment) + service := NewRuntimeDeploymentService(client) + + if err := service.RolloutImage(context.Background(), "runtime-system", "runtime-hermes", "registry/hermes:v2", 1, 2); err != nil { + t.Fatalf("RolloutImage returned error: %v", err) + } + + updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-hermes", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get updated deployment: %v", err) + } + container := updated.Spec.Template.Spec.Containers[0] + if container.Image != "registry/hermes:v2" { + t.Fatalf("expected updated image, got %q", container.Image) + } + requireEnv(t, container, "CLAWMANAGER_RUNTIME_IMAGE_REF", "registry/hermes:v2") + if updated.Spec.Strategy.Type != appsv1.RollingUpdateDeploymentStrategyType { + t.Fatalf("expected RollingUpdate strategy, got %q", updated.Spec.Strategy.Type) + } + if updated.Spec.Strategy.RollingUpdate == nil { + t.Fatal("expected rolling update strategy") + } + if got := updated.Spec.Strategy.RollingUpdate.MaxUnavailable.IntValue(); got != 1 { + t.Fatalf("maxUnavailable = %d, want 1", got) + } + if got := updated.Spec.Strategy.RollingUpdate.MaxSurge.IntValue(); got != 2 { + t.Fatalf("maxSurge = %d, want 2", got) + } +} + +func TestRuntimeDeploymentServiceRolloutImageRetriesConflict(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-hermes", + Namespace: "runtime-system", + RuntimeType: "hermes", + Image: "registry/hermes:v1", + Replicas: 1, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }) + client := fake.NewSimpleClientset(deployment) + updateAttempts := 0 + client.Fake.PrependReactor("update", "deployments", func(action k8stesting.Action) (bool, runtime.Object, error) { + updateAttempts++ + if updateAttempts == 1 { + return true, nil, apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "deployments"}, "runtime-hermes", fmt.Errorf("stale resource version")) + } + return false, nil, nil + }) + service := NewRuntimeDeploymentService(client) + + if err := service.RolloutImage(context.Background(), "runtime-system", "runtime-hermes", "registry/hermes:v2", 1, 1); err != nil { + t.Fatalf("RolloutImage returned error: %v", err) + } + if updateAttempts < 2 { + t.Fatalf("update attempts = %d, want retry after conflict", updateAttempts) + } + + updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-hermes", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get updated deployment: %v", err) + } + if got := updated.Spec.Template.Spec.Containers[0].Image; got != "registry/hermes:v2" { + t.Fatalf("image = %q, want registry/hermes:v2", got) + } +} + +func TestRuntimeDeploymentServiceListPodsReturnsRuntimeDeploymentPods(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "openclaw-runtime", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw-lite:final2", + Replicas: 1, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }) + podIP := "10.42.0.12" + nodeName := "node-a" + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "openclaw-runtime-abc", + Namespace: "runtime-system", + Labels: map[string]string{ + "app": "openclaw-runtime", + "clawmanager.io/runtime-type": "openclaw", + }, + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + Containers: []corev1.Container{{ + Name: "runtime", + Image: "registry/openclaw-lite:final2", + }}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + PodIP: podIP, + Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }}, + }, + } + client := fake.NewSimpleClientset(deployment, pod) + service := NewRuntimeDeploymentService(client) + + pods, err := service.ListPods(context.Background(), "runtime-system", "openclaw") + if err != nil { + t.Fatalf("ListPods returned error: %v", err) + } + if len(pods) != 1 { + t.Fatalf("pods length = %d, want 1: %#v", len(pods), pods) + } + got := pods[0] + if got.RuntimeType != "openclaw" || got.Namespace != "runtime-system" || got.DeploymentName != "openclaw-runtime" || got.PodName != "openclaw-runtime-abc" { + t.Fatalf("runtime pod identity = %+v, want openclaw runtime pod", got) + } + if got.ImageRef != "registry/openclaw-lite:final2" { + t.Fatalf("image = %q, want registry/openclaw-lite:final2", got.ImageRef) + } + if got.State != "ready" { + t.Fatalf("state = %q, want ready", got.State) + } + if got.PodIP == nil || *got.PodIP != podIP || got.NodeName == nil || *got.NodeName != nodeName { + t.Fatalf("pod network fields = ip:%v node:%v, want %s/%s", got.PodIP, got.NodeName, podIP, nodeName) + } +} + +func requireContainerPort(t *testing.T, container corev1.Container, name string, port int32) { + t.Helper() + for _, got := range container.Ports { + if got.Name == name && got.ContainerPort == port { + return + } + } + t.Fatalf("expected container port %s=%d, got %#v", name, port, container.Ports) +} + +func requireVolumeMount(t *testing.T, container corev1.Container, name, mountPath string) { + t.Helper() + for _, got := range container.VolumeMounts { + if got.Name == name && got.MountPath == mountPath { + return + } + } + t.Fatalf("expected volume mount %s at %s, got %#v", name, mountPath, container.VolumeMounts) +} + +func requireEnv(t *testing.T, container corev1.Container, name, value string) { + t.Helper() + for _, got := range container.Env { + if got.Name == name && got.Value == value { + return + } + } + t.Fatalf("expected env %s=%q, got %#v", name, value, container.Env) +} + +func requireEnvAbsent(t *testing.T, container corev1.Container, name string) { + t.Helper() + for _, got := range container.Env { + if got.Name == name { + t.Fatalf("expected env %s to be absent, got %#v", name, container.Env) + } + } +} + +func requireEnvFieldRef(t *testing.T, container corev1.Container, name, fieldPath string) { + t.Helper() + for _, got := range container.Env { + if got.Name != name || got.ValueFrom == nil || got.ValueFrom.FieldRef == nil { + continue + } + if got.ValueFrom.FieldRef.FieldPath == fieldPath { + return + } + } + t.Fatalf("expected env %s from fieldRef %q, got %#v", name, fieldPath, container.Env) +} diff --git a/backend/internal/services/k8s/secret_service.go b/backend/internal/services/k8s/secret_service.go new file mode 100644 index 0000000..8a93c33 --- /dev/null +++ b/backend/internal/services/k8s/secret_service.go @@ -0,0 +1,108 @@ +package k8s + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SecretService handles Kubernetes Secret reads. +type SecretService struct { + client *Client + namespaceService *NamespaceService +} + +// NewSecretService creates a new secret service. +func NewSecretService() *SecretService { + return &SecretService{ + client: globalClient, + namespaceService: NewNamespaceService(), + } +} + +// GetSecretValue returns a decoded string value from a Kubernetes Secret. +func (s *SecretService) GetSecretValue(ctx context.Context, namespace, name, key string) (string, error) { + if s.client == nil || s.client.Clientset == nil { + return "", fmt.Errorf("k8s client not initialized") + } + + secret, err := s.client.Clientset.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("failed to get secret %s/%s: %w", namespace, name, err) + } + + value, ok := secret.Data[key] + if !ok { + return "", fmt.Errorf("secret key %s not found in %s/%s", key, namespace, name) + } + + return string(value), nil +} + +// UpsertSecret creates or updates a secret in the user's namespace. +func (s *SecretService) UpsertSecret(ctx context.Context, userID int, name string, data map[string]string, labels map[string]string) error { + if s.client == nil || s.client.Clientset == nil { + return fmt.Errorf("k8s client not initialized") + } + + if _, err := s.namespaceService.EnsureNamespace(ctx, userID); err != nil { + return fmt.Errorf("failed to ensure namespace: %w", err) + } + + namespace := s.client.GetNamespace(userID) + secretData := map[string][]byte{} + for key, value := range data { + secretData[key] = []byte(value) + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Type: corev1.SecretTypeOpaque, + Data: secretData, + } + + existing, err := s.client.Clientset.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) + if err == nil && existing != nil { + existing.Data = secretData + if existing.Labels == nil { + existing.Labels = map[string]string{} + } + for key, value := range labels { + existing.Labels[key] = value + } + if _, err := s.client.Clientset.CoreV1().Secrets(namespace).Update(ctx, existing, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update secret %s/%s: %w", namespace, name, err) + } + return nil + } + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to inspect secret %s/%s: %w", namespace, name, err) + } + + if _, err := s.client.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create secret %s/%s: %w", namespace, name, err) + } + return nil +} + +// DeleteSecret deletes a Secret from the user's namespace. Missing secrets are treated as already deleted. +func (s *SecretService) DeleteSecret(ctx context.Context, userID int, name string) error { + if s.client == nil || s.client.Clientset == nil { + return fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + if err := s.client.Clientset.CoreV1().Secrets(namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + if errors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to delete secret %s/%s: %w", namespace, name, err) + } + return nil +} diff --git a/backend/internal/services/k8s/service_service.go b/backend/internal/services/k8s/service_service.go new file mode 100644 index 0000000..ea3cccb --- /dev/null +++ b/backend/internal/services/k8s/service_service.go @@ -0,0 +1,289 @@ +package k8s + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// ServiceService handles Kubernetes Service operations +type ServiceService struct { + client *Client + namespaceService *NamespaceService +} + +// NewServiceService creates a new Service service +func NewServiceService() *ServiceService { + return &ServiceService{ + client: globalClient, + namespaceService: NewNamespaceService(), + } +} + +// ServiceConfig holds configuration for creating a service +type ServiceConfig struct { + InstanceID int + InstanceName string + UserID int + ContainerPort int32 + AdditionalPorts []int32 +} + +// ServiceInfo holds information about a created service +type ServiceInfo struct { + Name string + Namespace string + ClusterIP string + NodePort int32 + TargetPort int32 +} + +// CreateService creates a service for an instance. +func (s *ServiceService) CreateService(ctx context.Context, config ServiceConfig) (*ServiceInfo, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + serviceName := s.client.GetServiceName(config.InstanceID, config.InstanceName) + namespace := s.client.GetNamespace(config.UserID) + + // Ensure namespace exists + if _, err := s.namespaceService.EnsureNamespace(ctx, config.UserID); err != nil { + return nil, fmt.Errorf("failed to ensure namespace: %w", err) + } + + // Default container port + targetPort := config.ContainerPort + if targetPort == 0 { + targetPort = 3001 + } + + servicePorts := []corev1.ServicePort{ + { + Name: "http", + Port: targetPort, + TargetPort: intstr.FromInt(int(targetPort)), + Protocol: corev1.ProtocolTCP, + }, + } + + for _, additionalPort := range config.AdditionalPorts { + if additionalPort == 0 || additionalPort == targetPort { + continue + } + + servicePorts = append(servicePorts, corev1.ServicePort{ + Name: fmt.Sprintf("tcp-%d", additionalPort), + Port: additionalPort, + TargetPort: intstr.FromInt(int(additionalPort)), + Protocol: corev1.ProtocolTCP, + }) + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: namespace, + Labels: map[string]string{ + "app": "clawreef", + "instance-id": fmt.Sprintf("%d", config.InstanceID), + "instance-name": config.InstanceName, + "user-id": fmt.Sprintf("%d", config.UserID), + "managed-by": "clawreef", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + "instance-id": fmt.Sprintf("%d", config.InstanceID), + "app": "clawreef", + }, + Ports: servicePorts, + }, + } + + createdService, err := s.client.Clientset.CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{}) + if err != nil { + if errors.IsAlreadyExists(err) { + // Service already exists, get the existing one + existingService, getErr := s.GetService(ctx, config.UserID, config.InstanceID) + if getErr == nil && existingService != nil { + return s.extractServiceInfo(existingService, targetPort), nil + } + } + return nil, fmt.Errorf("failed to create service %s: %w", serviceName, err) + } + + return &ServiceInfo{ + Name: createdService.Name, + Namespace: createdService.Namespace, + ClusterIP: createdService.Spec.ClusterIP, + NodePort: 0, + TargetPort: targetPort, + }, nil +} + +// GetService gets a service by instance ID +func (s *ServiceService) GetService(ctx context.Context, userID, instanceID int) (*corev1.Service, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + namespace := s.client.GetNamespace(userID) + selector := fmt.Sprintf("instance-id=%d", instanceID) + + services, err := s.client.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + return nil, fmt.Errorf("failed to list services: %w", err) + } + + if len(services.Items) == 0 { + return nil, fmt.Errorf("service not found for instance %d", instanceID) + } + + return &services.Items[0], nil +} + +// GetServiceInfo gets service information for an instance +func (s *ServiceService) GetServiceInfo(ctx context.Context, userID, instanceID int, targetPort int32) (*ServiceInfo, error) { + service, err := s.GetService(ctx, userID, instanceID) + if err != nil { + return nil, err + } + + return s.extractServiceInfo(service, targetPort), nil +} + +// DeleteService deletes a service +func (s *ServiceService) DeleteService(ctx context.Context, userID, instanceID int) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + service, err := s.GetService(ctx, userID, instanceID) + if err != nil { + // Service doesn't exist, nothing to delete + if isNotFoundError(err) { + return nil + } + return err + } + + err = s.client.Clientset.CoreV1().Services(service.Namespace).Delete(ctx, service.Name, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete service %s: %w", service.Name, err) + } + + return nil +} + +// ServiceExists checks if a service exists +func (s *ServiceService) ServiceExists(ctx context.Context, userID, instanceID int) (bool, error) { + _, err := s.GetService(ctx, userID, instanceID) + if err != nil { + if isNotFoundError(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// GetNodePort gets the NodePort for a service +func (s *ServiceService) GetNodePort(ctx context.Context, userID, instanceID int, targetPort int32) (int32, error) { + service, err := s.GetService(ctx, userID, instanceID) + if err != nil { + return 0, err + } + + nodePort := s.extractNodePort(service, targetPort) + if nodePort == 0 { + return 0, fmt.Errorf("node port not found for target port %d", targetPort) + } + + return nodePort, nil +} + +// extractServiceInfo extracts service information from a Kubernetes service +func (s *ServiceService) extractServiceInfo(service *corev1.Service, targetPort int32) *ServiceInfo { + return &ServiceInfo{ + Name: service.Name, + Namespace: service.Namespace, + ClusterIP: service.Spec.ClusterIP, + NodePort: s.extractNodePort(service, targetPort), + TargetPort: targetPort, + } +} + +// extractNodePort extracts the NodePort for a specific target port +func (s *ServiceService) extractNodePort(service *corev1.Service, targetPort int32) int32 { + for _, port := range service.Spec.Ports { + if port.TargetPort.IntVal == targetPort || port.Port == targetPort { + if port.NodePort != 0 { + return port.NodePort + } + } + } + return 0 +} + +// GetClusterNodes gets all cluster node IPs +func (s *ServiceService) GetClusterNodes(ctx context.Context) ([]string, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + + nodes, err := s.client.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list nodes: %w", err) + } + + var nodeIPs []string + for _, node := range nodes.Items { + // Prefer ExternalIP, fallback to InternalIP + var externalIP, internalIP string + for _, addr := range node.Status.Addresses { + switch addr.Type { + case corev1.NodeExternalIP: + externalIP = addr.Address + case corev1.NodeInternalIP: + internalIP = addr.Address + } + } + + if externalIP != "" { + nodeIPs = append(nodeIPs, externalIP) + } else if internalIP != "" { + nodeIPs = append(nodeIPs, internalIP) + } + } + + return nodeIPs, nil +} + +// GetAccessEndpoint gets the best access endpoint for a service +// Returns nodeIP:nodePort for accessing the service from outside the cluster +func (s *ServiceService) GetAccessEndpoint(ctx context.Context, userID, instanceID int, targetPort int32) (string, error) { + nodePort, err := s.GetNodePort(ctx, userID, instanceID, targetPort) + if err != nil { + return "", err + } + + nodes, err := s.GetClusterNodes(ctx) + if err != nil { + return "", err + } + + if len(nodes) == 0 { + return "", fmt.Errorf("no cluster nodes found") + } + + // Use the first available node + return fmt.Sprintf("%s:%d", nodes[0], nodePort), nil +} diff --git a/backend/internal/services/leader/leader.go b/backend/internal/services/leader/leader.go new file mode 100644 index 0000000..e9050d2 --- /dev/null +++ b/backend/internal/services/leader/leader.go @@ -0,0 +1,124 @@ +// Package leader wraps Kubernetes lease-based leader election so that +// control-plane singleton background loops (the K8s sync loop, Team event +// consumers and the stale-task monitor) run on exactly one clawmanager-app +// replica at a time. The HTTP API and the in-pod nginx desktop data plane run +// on every replica; only these background loops must be gated. +package leader + +import ( + "context" + "log" + "os" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" +) + +// Config holds the resolved leader-election parameters. +type Config struct { + Namespace string + LeaseName string + Identity string + LeaseDuration time.Duration + RenewDeadline time.Duration + RetryPeriod time.Duration +} + +// Callbacks are invoked when this replica acquires or loses leadership. +// +// OnStartedLeading receives a context that is cancelled when leadership is +// lost; callers may use it to scope leader-only work. OnStoppedLeading is +// invoked when leadership is lost (or the parent context is cancelled) and must +// stop any work started in OnStartedLeading. Both callbacks must be safe to +// call repeatedly across leadership transitions. +type Callbacks struct { + OnStartedLeading func(ctx context.Context) + OnStoppedLeading func() +} + +// Run participates in leader election and blocks until ctx is cancelled. It is +// resilient to leadership loss: after losing the lease it re-participates so a +// replica that briefly lost leadership can re-acquire it and restart the +// background loops. Run is intended to be called in its own goroutine. +func Run(ctx context.Context, clientset kubernetes.Interface, cfg Config, cb Callbacks) { + identity := cfg.Identity + if identity == "" { + if host, err := os.Hostname(); err == nil && host != "" { + identity = host + } else { + identity = "clawmanager-app" + } + } + + leaseDuration := cfg.LeaseDuration + if leaseDuration <= 0 { + leaseDuration = 15 * time.Second + } + renewDeadline := cfg.RenewDeadline + if renewDeadline <= 0 || renewDeadline >= leaseDuration { + renewDeadline = leaseDuration * 2 / 3 + } + retryPeriod := cfg.RetryPeriod + if retryPeriod <= 0 { + retryPeriod = 2 * time.Second + } + + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Name: cfg.LeaseName, + Namespace: cfg.Namespace, + }, + Client: clientset.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: identity, + }, + } + + electionCfg := leaderelection.LeaderElectionConfig{ + Lock: lock, + ReleaseOnCancel: true, + LeaseDuration: leaseDuration, + RenewDeadline: renewDeadline, + RetryPeriod: retryPeriod, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(c context.Context) { + log.Printf("[leader] acquired leadership (identity=%s lease=%s/%s)", identity, cfg.Namespace, cfg.LeaseName) + if cb.OnStartedLeading != nil { + cb.OnStartedLeading(c) + } + }, + OnStoppedLeading: func() { + log.Printf("[leader] lost leadership (identity=%s)", identity) + if cb.OnStoppedLeading != nil { + cb.OnStoppedLeading() + } + }, + }, + } + + for { + select { + case <-ctx.Done(): + return + default: + } + + elector, err := leaderelection.NewLeaderElector(electionCfg) + if err != nil { + log.Printf("[leader] failed to create leader elector: %v; retrying in %s", err, retryPeriod) + select { + case <-ctx.Done(): + return + case <-time.After(retryPeriod): + continue + } + } + + // Blocks until leadership is lost or ctx is cancelled. On loss we loop + // and re-participate. + elector.Run(ctx) + } +} diff --git a/backend/internal/services/llm_model_service.go b/backend/internal/services/llm_model_service.go new file mode 100644 index 0000000..2797f9a --- /dev/null +++ b/backend/internal/services/llm_model_service.go @@ -0,0 +1,447 @@ +package services + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// LLMModelService defines admin model catalog operations. +type LLMModelService interface { + ListModels() ([]models.LLMModel, error) + ListActiveModels() ([]models.LLMModel, error) + SaveModel(req SaveLLMModelRequest) (*models.LLMModel, error) + DeleteModel(id int) error + DiscoverProviderModels(req DiscoverLLMModelsRequest) ([]DiscoveredLLMModel, error) +} + +// SaveLLMModelRequest contains editable model catalog fields. +type SaveLLMModelRequest struct { + ID int + DisplayName string + Description *string + ProviderType string + ProtocolType string + BaseURL string + ProviderModelName string + APIKey *string + APIKeySecretRef *string + IsSecure bool + IsActive bool + InputPrice float64 + OutputPrice float64 + Currency string +} + +// DiscoverLLMModelsRequest contains fields needed to query a provider for available models. +type DiscoverLLMModelsRequest struct { + ProviderType string + ProtocolType string + BaseURL string + APIKey *string + APIKeySecretRef *string +} + +// DiscoveredLLMModel is a normalized provider model entry returned by discovery. +type DiscoveredLLMModel struct { + ID string `json:"id"` + DisplayName string `json:"display_name"` +} + +type llmModelService struct { + repo repository.LLMModelRepository + httpClient *http.Client + secretRefService SecretRefService +} + +var versionSegmentPattern = regexp.MustCompile(`(?i)^v\d+(?:[a-z0-9._-]*)?$`) + +// NewLLMModelService creates a new LLM model service. +func NewLLMModelService(repo repository.LLMModelRepository) LLMModelService { + return &llmModelService{ + repo: repo, + secretRefService: NewSecretRefService(), + httpClient: &http.Client{ + Timeout: 20 * time.Second, + }, + } +} + +func (s *llmModelService) ListModels() ([]models.LLMModel, error) { + items, err := s.repo.List() + if err != nil { + return nil, fmt.Errorf("failed to list llm models: %w", err) + } + return items, nil +} + +func (s *llmModelService) ListActiveModels() ([]models.LLMModel, error) { + items, err := s.repo.ListActive() + if err != nil { + return nil, fmt.Errorf("failed to list active llm models: %w", err) + } + return items, nil +} + +func (s *llmModelService) SaveModel(req SaveLLMModelRequest) (*models.LLMModel, error) { + displayName := strings.TrimSpace(req.DisplayName) + if displayName == "" { + return nil, errors.New("display name is required") + } + + providerType := strings.TrimSpace(strings.ToLower(req.ProviderType)) + if providerType == "" { + return nil, errors.New("provider type is required") + } + protocolType, err := models.ResolveLLMProtocolType(providerType, req.ProtocolType) + if err != nil { + return nil, err + } + + baseURL := strings.TrimSpace(req.BaseURL) + if baseURL == "" { + return nil, errors.New("base URL is required") + } + + providerModelName := strings.TrimSpace(req.ProviderModelName) + if providerModelName == "" { + return nil, errors.New("provider model name is required") + } + + if req.InputPrice < 0 { + return nil, errors.New("input price must be non-negative") + } + + if req.OutputPrice < 0 { + return nil, errors.New("output price must be non-negative") + } + + currency := strings.ToUpper(strings.TrimSpace(req.Currency)) + if currency == "" { + currency = "USD" + } + + existingByName, err := s.repo.GetByDisplayName(displayName) + if err != nil { + return nil, fmt.Errorf("failed to validate model display name: %w", err) + } + if existingByName != nil && existingByName.ID != req.ID { + return nil, errors.New("display name already exists") + } + + var current *models.LLMModel + if req.ID != 0 { + current, err = s.repo.GetByID(req.ID) + if err != nil { + return nil, fmt.Errorf("failed to get llm model: %w", err) + } + if current == nil { + return nil, errors.New("model not found") + } + } + + var description *string + if req.Description != nil { + trimmed := strings.TrimSpace(*req.Description) + if trimmed != "" { + description = &trimmed + } + } + + var apiKey *string + if req.APIKey != nil { + trimmed := strings.TrimSpace(*req.APIKey) + if trimmed != "" { + apiKey = &trimmed + } + } + + var apiKeySecretRef *string + if req.APIKeySecretRef != nil { + trimmed := strings.TrimSpace(*req.APIKeySecretRef) + if trimmed != "" { + apiKeySecretRef = &trimmed + } + } + + model := &models.LLMModel{ + ID: req.ID, + DisplayName: displayName, + Description: description, + ProviderType: providerType, + ProtocolType: protocolType, + BaseURL: baseURL, + ProviderModelName: providerModelName, + APIKey: apiKey, + APIKeySecretRef: apiKeySecretRef, + IsSecure: req.IsSecure, + IsActive: req.IsActive, + InputPrice: req.InputPrice, + OutputPrice: req.OutputPrice, + Currency: currency, + } + + if current != nil { + model.CreatedAt = current.CreatedAt + } + + if err := s.repo.Save(model); err != nil { + return nil, fmt.Errorf("failed to save llm model: %w", err) + } + + return model, nil +} + +func (s *llmModelService) DeleteModel(id int) error { + existing, err := s.repo.GetByID(id) + if err != nil { + return fmt.Errorf("failed to get llm model: %w", err) + } + if existing == nil { + return errors.New("model not found") + } + + if err := s.repo.Delete(id); err != nil { + return fmt.Errorf("failed to delete llm model: %w", err) + } + return nil +} + +func (s *llmModelService) DiscoverProviderModels(req DiscoverLLMModelsRequest) ([]DiscoveredLLMModel, error) { + providerType := strings.TrimSpace(strings.ToLower(req.ProviderType)) + if providerType == "" { + return nil, errors.New("provider type is required") + } + protocolType, err := models.ResolveLLMProtocolType(providerType, req.ProtocolType) + if err != nil { + return nil, err + } + + baseURL := strings.TrimSpace(req.BaseURL) + if baseURL == "" { + return nil, errors.New("base URL is required") + } + + resolvedAPIKey, err := s.secretRefService.ResolveString(context.Background(), req.APIKey, req.APIKeySecretRef) + if err != nil { + return nil, err + } + + var apiKey string + if resolvedAPIKey != nil { + apiKey = strings.TrimSpace(*resolvedAPIKey) + } + + switch protocolType { + case models.ProtocolTypeOpenAI, models.ProtocolTypeOpenAICompatible: + return s.discoverOpenAICompatibleModels(baseURL, apiKey) + case models.ProtocolTypeAnthropic: + return s.discoverAnthropicModels(baseURL, apiKey) + case models.ProtocolTypeGoogle: + return s.discoverGoogleModels(baseURL, apiKey) + case models.ProtocolTypeAzureOpenAI: + return nil, errors.New("automatic model discovery for azure-openai is not supported yet") + default: + return nil, errors.New("provider discovery is not supported") + } +} + +func (s *llmModelService) discoverOpenAICompatibleModels(baseURL, apiKey string) ([]DiscoveredLLMModel, error) { + endpoint, err := buildProviderEndpoint(baseURL, "v1", "models") + if err != nil { + return nil, err + } + + request, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to build discovery request: %w", err) + } + + if apiKey != "" { + request.Header.Set("Authorization", "Bearer "+apiKey) + } + + type responsePayload struct { + Data []struct { + ID string `json:"id"` + OwnedBy string `json:"owned_by"` + } `json:"data"` + } + + var payload responsePayload + if err := s.doJSON(request, &payload); err != nil { + return nil, err + } + + models := make([]DiscoveredLLMModel, 0, len(payload.Data)) + for _, item := range payload.Data { + id := strings.TrimSpace(item.ID) + if id == "" { + continue + } + displayName := id + if owner := strings.TrimSpace(item.OwnedBy); owner != "" { + displayName = fmt.Sprintf("%s (%s)", id, owner) + } + models = append(models, DiscoveredLLMModel{ + ID: id, + DisplayName: displayName, + }) + } + return models, nil +} + +func (s *llmModelService) discoverAnthropicModels(baseURL, apiKey string) ([]DiscoveredLLMModel, error) { + endpoint, err := buildProviderEndpoint(baseURL, "v1", "models") + if err != nil { + return nil, err + } + + request, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to build discovery request: %w", err) + } + request.Header.Set("anthropic-version", "2023-06-01") + if apiKey != "" { + request.Header.Set("x-api-key", apiKey) + } + + type responsePayload struct { + Data []struct { + ID string `json:"id"` + DisplayName string `json:"display_name"` + } `json:"data"` + } + + var payload responsePayload + if err := s.doJSON(request, &payload); err != nil { + return nil, err + } + + models := make([]DiscoveredLLMModel, 0, len(payload.Data)) + for _, item := range payload.Data { + id := strings.TrimSpace(item.ID) + if id == "" { + continue + } + displayName := strings.TrimSpace(item.DisplayName) + if displayName == "" { + displayName = id + } + models = append(models, DiscoveredLLMModel{ + ID: id, + DisplayName: displayName, + }) + } + return models, nil +} + +func (s *llmModelService) discoverGoogleModels(baseURL, apiKey string) ([]DiscoveredLLMModel, error) { + endpoint, err := buildProviderEndpoint(baseURL, "v1beta", "models") + if err != nil { + return nil, err + } + + if apiKey != "" { + separator := "?" + if strings.Contains(endpoint, "?") { + separator = "&" + } + endpoint = endpoint + separator + "key=" + url.QueryEscape(apiKey) + } + + request, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to build discovery request: %w", err) + } + + type responsePayload struct { + Models []struct { + Name string `json:"name"` + DisplayName string `json:"displayName"` + } `json:"models"` + } + + var payload responsePayload + if err := s.doJSON(request, &payload); err != nil { + return nil, err + } + + models := make([]DiscoveredLLMModel, 0, len(payload.Models)) + for _, item := range payload.Models { + id := strings.TrimSpace(strings.TrimPrefix(item.Name, "models/")) + if id == "" { + continue + } + displayName := strings.TrimSpace(item.DisplayName) + if displayName == "" { + displayName = id + } + models = append(models, DiscoveredLLMModel{ + ID: id, + DisplayName: displayName, + }) + } + return models, nil +} + +func (s *llmModelService) doJSON(request *http.Request, target any) error { + response, err := s.httpClient.Do(request) + if err != nil { + return fmt.Errorf("failed to call provider discovery endpoint: %w", err) + } + defer response.Body.Close() + + if response.StatusCode < 200 || response.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(response.Body, 2048)) + message := strings.TrimSpace(string(body)) + if message == "" { + message = response.Status + } + return fmt.Errorf("provider discovery failed: %s", message) + } + + if err := json.NewDecoder(response.Body).Decode(target); err != nil { + return fmt.Errorf("failed to decode provider discovery response: %w", err) + } + return nil +} + +func buildProviderEndpoint(baseURL, versionPrefix, resource string) (string, error) { + trimmed := strings.TrimSpace(strings.TrimRight(baseURL, "/")) + if trimmed == "" { + return "", errors.New("base URL is required") + } + + parsed, err := url.Parse(trimmed) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", errors.New("base URL is invalid") + } + + versionPath := "/" + strings.Trim(versionPrefix, "/") + resourcePath := strings.Trim(resource, "/") + if strings.HasSuffix(strings.ToLower(parsed.Path), strings.ToLower(versionPath)) { + return trimmed + "/" + resourcePath, nil + } + + pathSegments := strings.Split(strings.Trim(parsed.Path, "/"), "/") + lastSegment := "" + if len(pathSegments) > 0 { + lastSegment = pathSegments[len(pathSegments)-1] + } + if versionSegmentPattern.MatchString(lastSegment) { + return trimmed + "/" + resourcePath, nil + } + + return trimmed + versionPath + "/" + resourcePath, nil +} diff --git a/backend/internal/services/llm_model_service_test.go b/backend/internal/services/llm_model_service_test.go new file mode 100644 index 0000000..d97df89 --- /dev/null +++ b/backend/internal/services/llm_model_service_test.go @@ -0,0 +1,75 @@ +package services + +import "testing" + +func TestBuildProviderEndpoint(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + baseURL string + versionPrefix string + resource string + want string + wantErr bool + }{ + { + name: "appends default version when base url has no version suffix", + baseURL: "https://api.deepseek.com", + versionPrefix: "v1", + resource: "models", + want: "https://api.deepseek.com/v1/models", + }, + { + name: "keeps matching version suffix", + baseURL: "https://api.openai.com/v1", + versionPrefix: "v1", + resource: "models", + want: "https://api.openai.com/v1/models", + }, + { + name: "respects nonstandard version suffix", + baseURL: "https://open.bigmodel.cn/api/paas/v4", + versionPrefix: "v1", + resource: "models", + want: "https://open.bigmodel.cn/api/paas/v4/models", + }, + { + name: "keeps beta version suffix", + baseURL: "https://generativelanguage.googleapis.com/v1beta", + versionPrefix: "v1beta", + resource: "models", + want: "https://generativelanguage.googleapis.com/v1beta/models", + }, + { + name: "rejects invalid base url", + baseURL: "not-a-url", + versionPrefix: "v1", + resource: "models", + wantErr: true, + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := buildProviderEndpoint(tc.baseURL, tc.versionPrefix, tc.resource) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error, got none and endpoint %q", got) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if got != tc.want { + t.Fatalf("expected %q, got %q", tc.want, got) + } + }) + } +} diff --git a/backend/internal/services/model_invocation_service.go b/backend/internal/services/model_invocation_service.go new file mode 100644 index 0000000..d4ed054 --- /dev/null +++ b/backend/internal/services/model_invocation_service.go @@ -0,0 +1,88 @@ +package services + +import ( + "fmt" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// ModelInvocationService defines application-level operations for governed model calls. +type ModelInvocationService interface { + RecordInvocation(invocation *models.ModelInvocation) error + GetInvocationByID(id int) (*models.ModelInvocation, error) + ListInvocationsByTraceID(traceID string) ([]models.ModelInvocation, error) + ListInvocationsBySessionID(sessionID string, limit int) ([]models.ModelInvocation, error) + ListInvocationsByUserID(userID, limit int) ([]models.ModelInvocation, error) +} + +type modelInvocationService struct { + repo repository.ModelInvocationRepository +} + +// NewModelInvocationService creates a new model invocation service. +func NewModelInvocationService(repo repository.ModelInvocationRepository) ModelInvocationService { + return &modelInvocationService{repo: repo} +} + +func (s *modelInvocationService) RecordInvocation(invocation *models.ModelInvocation) error { + if invocation == nil { + return fmt.Errorf("model invocation is required") + } + if strings.TrimSpace(invocation.TraceID) == "" { + return fmt.Errorf("trace id is required") + } + if strings.TrimSpace(invocation.RequestID) == "" { + return fmt.Errorf("request id is required") + } + if strings.TrimSpace(invocation.ProviderType) == "" { + return fmt.Errorf("provider type is required") + } + if strings.TrimSpace(invocation.RequestedModel) == "" { + return fmt.Errorf("requested model is required") + } + if strings.TrimSpace(invocation.ActualProviderModel) == "" { + return fmt.Errorf("actual provider model is required") + } + if strings.TrimSpace(invocation.TrafficClass) == "" { + invocation.TrafficClass = models.TrafficClassLLM + } + if strings.TrimSpace(invocation.Status) == "" { + invocation.Status = models.ModelInvocationStatusPending + } + + return s.repo.Create(invocation) +} + +func (s *modelInvocationService) GetInvocationByID(id int) (*models.ModelInvocation, error) { + item, err := s.repo.GetByID(id) + if err != nil { + return nil, fmt.Errorf("failed to get model invocation: %w", err) + } + return item, nil +} + +func (s *modelInvocationService) ListInvocationsByTraceID(traceID string) ([]models.ModelInvocation, error) { + items, err := s.repo.ListByTraceID(strings.TrimSpace(traceID)) + if err != nil { + return nil, fmt.Errorf("failed to list model invocations by trace: %w", err) + } + return items, nil +} + +func (s *modelInvocationService) ListInvocationsBySessionID(sessionID string, limit int) ([]models.ModelInvocation, error) { + items, err := s.repo.ListBySessionID(strings.TrimSpace(sessionID), limit) + if err != nil { + return nil, fmt.Errorf("failed to list model invocations by session: %w", err) + } + return items, nil +} + +func (s *modelInvocationService) ListInvocationsByUserID(userID, limit int) ([]models.ModelInvocation, error) { + items, err := s.repo.ListByUserID(userID, limit) + if err != nil { + return nil, fmt.Errorf("failed to list model invocations by user: %w", err) + } + return items, nil +} diff --git a/backend/internal/services/object_storage_service.go b/backend/internal/services/object_storage_service.go new file mode 100644 index 0000000..e29ee0f --- /dev/null +++ b/backend/internal/services/object_storage_service.go @@ -0,0 +1,114 @@ +package services + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "clawreef/internal/config" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type ObjectStorageService interface { + PutObject(ctx context.Context, objectKey string, body []byte, contentType string) error + GetObject(ctx context.Context, objectKey string) ([]byte, error) +} + +type objectStorageService struct { + minioClient *minio.Client + bucket string + basePath string + localPath string +} + +func NewObjectStorageService(cfg config.ObjectStorageConfig) (ObjectStorageService, error) { + service := &objectStorageService{ + bucket: strings.TrimSpace(cfg.Bucket), + basePath: strings.Trim(strings.TrimSpace(cfg.BasePath), "/"), + localPath: strings.TrimSpace(cfg.LocalFallback), + } + if service.localPath == "" { + service.localPath = ".data/object-storage" + } + if strings.TrimSpace(cfg.Endpoint) == "" { + return service, nil + } + + client, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: cfg.UseSSL, + Region: cfg.Region, + BucketLookup: minio.BucketLookupPath, + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize object storage client: %w", err) + } + service.minioClient = client + return service, nil +} + +func (s *objectStorageService) PutObject(ctx context.Context, objectKey string, body []byte, contentType string) error { + if s.minioClient == nil { + target := filepath.Join(s.localPath, filepath.FromSlash(s.resolveObjectKey(objectKey))) + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return fmt.Errorf("failed to prepare local object storage directory: %w", err) + } + if err := os.WriteFile(target, body, 0o644); err != nil { + return fmt.Errorf("failed to write local object storage object: %w", err) + } + return nil + } + + exists, err := s.minioClient.BucketExists(ctx, s.bucket) + if err != nil { + return fmt.Errorf("failed to check object storage bucket: %w", err) + } + if !exists { + if err := s.minioClient.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{}); err != nil { + return fmt.Errorf("failed to create object storage bucket: %w", err) + } + } + _, err = s.minioClient.PutObject(ctx, s.bucket, s.resolveObjectKey(objectKey), bytes.NewReader(body), int64(len(body)), minio.PutObjectOptions{ + ContentType: contentType, + }) + if err != nil { + return fmt.Errorf("failed to upload object: %w", err) + } + return nil +} + +func (s *objectStorageService) GetObject(ctx context.Context, objectKey string) ([]byte, error) { + if s.minioClient == nil { + target := filepath.Join(s.localPath, filepath.FromSlash(s.resolveObjectKey(objectKey))) + content, err := os.ReadFile(target) + if err != nil { + return nil, fmt.Errorf("failed to read local object storage object: %w", err) + } + return content, nil + } + + reader, err := s.minioClient.GetObject(ctx, s.bucket, s.resolveObjectKey(objectKey), minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to open object: %w", err) + } + defer reader.Close() + content, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read object: %w", err) + } + return content, nil +} + +func (s *objectStorageService) resolveObjectKey(objectKey string) string { + objectKey = strings.Trim(strings.TrimSpace(objectKey), "/") + if s.basePath == "" { + return objectKey + } + return s.basePath + "/" + objectKey +} diff --git a/backend/internal/services/openclaw_config_service.go b/backend/internal/services/openclaw_config_service.go new file mode 100644 index 0000000..2f43a1f --- /dev/null +++ b/backend/internal/services/openclaw_config_service.go @@ -0,0 +1,2347 @@ +package services + +import ( + "context" + "encoding/json" + "fmt" + "log" + "regexp" + "sort" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" +) + +const ( + OpenClawConfigResourceTypeChannel = "channel" + OpenClawConfigResourceTypeSkill = "skill" + OpenClawConfigResourceTypeSessionTemplate = "session_template" + OpenClawConfigResourceTypeLogPolicy = "log_policy" + OpenClawConfigResourceTypeAgent = "agent" + OpenClawConfigResourceTypeScheduledTask = "scheduled_task" + + OpenClawConfigPlanModeNone = "none" + OpenClawConfigPlanModeBundle = "bundle" + OpenClawConfigPlanModeManual = "manual" + + OpenClawBootstrapManifestEnv = "CLAWMANAGER_OPENCLAW_BOOTSTRAP_MANIFEST_JSON" + OpenClawChannelsEnv = "CLAWMANAGER_OPENCLAW_CHANNELS_JSON" + OpenClawSkillsEnv = "CLAWMANAGER_OPENCLAW_SKILLS_JSON" + OpenClawSessionTemplatesEnv = "CLAWMANAGER_OPENCLAW_SESSION_TEMPLATES_JSON" + OpenClawLogPoliciesEnv = "CLAWMANAGER_OPENCLAW_LOG_POLICIES_JSON" + OpenClawAgentsEnv = "CLAWMANAGER_OPENCLAW_AGENTS_JSON" + OpenClawScheduledTasksEnv = "CLAWMANAGER_OPENCLAW_SCHEDULED_TASKS_JSON" + HermesBootstrapManifestEnv = "CLAWMANAGER_HERMES_BOOTSTRAP_MANIFEST_JSON" + HermesChannelsEnv = "CLAWMANAGER_HERMES_CHANNELS_JSON" + HermesSkillsEnv = "CLAWMANAGER_HERMES_SKILLS_JSON" + HermesSessionTemplatesEnv = "CLAWMANAGER_HERMES_SESSION_TEMPLATES_JSON" + HermesLogPoliciesEnv = "CLAWMANAGER_HERMES_LOG_POLICIES_JSON" + HermesAgentsEnv = "CLAWMANAGER_HERMES_AGENTS_JSON" + HermesScheduledTasksEnv = "CLAWMANAGER_HERMES_SCHEDULED_TASKS_JSON" + RuntimeBootstrapManifestEnv = "CLAWMANAGER_RUNTIME_BOOTSTRAP_MANIFEST_JSON" + RuntimeChannelsEnv = "CLAWMANAGER_RUNTIME_CHANNELS_JSON" + RuntimeSkillsEnv = "CLAWMANAGER_RUNTIME_SKILLS_JSON" + RuntimeSessionTemplatesEnv = "CLAWMANAGER_RUNTIME_SESSION_TEMPLATES_JSON" + RuntimeLogPoliciesEnv = "CLAWMANAGER_RUNTIME_LOG_POLICIES_JSON" + RuntimeAgentsEnv = "CLAWMANAGER_RUNTIME_AGENTS_JSON" + RuntimeScheduledTasksEnv = "CLAWMANAGER_RUNTIME_SCHEDULED_TASKS_JSON" + openClawBootstrapPayloadMaxBytes = 64 * 1024 + + openClawCompiledSnapshotStatus = "compiled" + openClawActiveSnapshotStatus = "active" + openClawFailedSnapshotStatus = "failed" + defaultSnapshotListLimit = 50 +) + +var ( + openClawAllowedResourceTypes = map[string]struct{}{ + OpenClawConfigResourceTypeChannel: {}, + OpenClawConfigResourceTypeSkill: {}, + OpenClawConfigResourceTypeSessionTemplate: {}, + OpenClawConfigResourceTypeLogPolicy: {}, + OpenClawConfigResourceTypeAgent: {}, + OpenClawConfigResourceTypeScheduledTask: {}, + } + openClawPlanModes = map[string]struct{}{ + OpenClawConfigPlanModeNone: {}, + OpenClawConfigPlanModeBundle: {}, + OpenClawConfigPlanModeManual: {}, + } + openClawResourceTypeOrder = []string{ + OpenClawConfigResourceTypeChannel, + OpenClawConfigResourceTypeSkill, + OpenClawConfigResourceTypeSessionTemplate, + OpenClawConfigResourceTypeLogPolicy, + OpenClawConfigResourceTypeAgent, + OpenClawConfigResourceTypeScheduledTask, + } + openClawEnvByResourceType = map[string]string{ + OpenClawConfigResourceTypeChannel: OpenClawChannelsEnv, + OpenClawConfigResourceTypeSkill: OpenClawSkillsEnv, + OpenClawConfigResourceTypeSessionTemplate: OpenClawSessionTemplatesEnv, + OpenClawConfigResourceTypeLogPolicy: OpenClawLogPoliciesEnv, + OpenClawConfigResourceTypeAgent: OpenClawAgentsEnv, + OpenClawConfigResourceTypeScheduledTask: OpenClawScheduledTasksEnv, + } + hermesBootstrapEnvAliases = map[string]string{ + OpenClawBootstrapManifestEnv: HermesBootstrapManifestEnv, + OpenClawChannelsEnv: HermesChannelsEnv, + OpenClawSkillsEnv: HermesSkillsEnv, + OpenClawSessionTemplatesEnv: HermesSessionTemplatesEnv, + OpenClawLogPoliciesEnv: HermesLogPoliciesEnv, + OpenClawAgentsEnv: HermesAgentsEnv, + OpenClawScheduledTasksEnv: HermesScheduledTasksEnv, + } + runtimeBootstrapEnvAliases = map[string]string{ + OpenClawBootstrapManifestEnv: RuntimeBootstrapManifestEnv, + OpenClawChannelsEnv: RuntimeChannelsEnv, + OpenClawSkillsEnv: RuntimeSkillsEnv, + OpenClawSessionTemplatesEnv: RuntimeSessionTemplatesEnv, + OpenClawLogPoliciesEnv: RuntimeLogPoliciesEnv, + OpenClawAgentsEnv: RuntimeAgentsEnv, + OpenClawScheduledTasksEnv: RuntimeScheduledTasksEnv, + } + openClawResourceKeyPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]{1,99}$`) +) + +type OpenClawConfigDependency struct { + Type string `json:"type"` + Key string `json:"key"` + Required bool `json:"required"` +} + +type OpenClawConfigEnvelope struct { + SchemaVersion int `json:"schemaVersion"` + Kind string `json:"kind"` + Format string `json:"format"` + DependsOn []OpenClawConfigDependency `json:"dependsOn"` + Config json.RawMessage `json:"config"` +} + +type OpenClawConfigPlan struct { + Mode string `json:"mode"` + BundleID *int `json:"bundle_id,omitempty"` + ResourceIDs []int `json:"resource_ids,omitempty"` +} + +type OpenClawConfigResourceSummary struct { + ID int `json:"id"` + ResourceType string `json:"resource_type"` + ResourceKey string `json:"resource_key"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + Version int `json:"version"` +} + +type OpenClawConfigResourcePayload struct { + ID int `json:"id"` + UserID int `json:"user_id"` + ResourceType string `json:"resource_type"` + ResourceKey string `json:"resource_key"` + Name string `json:"name"` + Description *string `json:"description,omitempty"` + Enabled bool `json:"enabled"` + Version int `json:"version"` + Tags []string `json:"tags"` + Content json.RawMessage `json:"content"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type UpsertOpenClawConfigResourceRequest struct { + ResourceType string `json:"resource_type"` + ResourceKey string `json:"resource_key"` + Name string `json:"name"` + Description *string `json:"description,omitempty"` + Enabled bool `json:"enabled"` + Tags []string `json:"tags"` + Content json.RawMessage `json:"content"` +} + +type OpenClawConfigBundleItemPayload struct { + ResourceID int `json:"resource_id"` + SortOrder int `json:"sort_order"` + Required bool `json:"required"` + Resource *OpenClawConfigResourceSummary `json:"resource,omitempty"` +} + +type OpenClawConfigBundleSkillSummary struct { + ID int `json:"id"` + UserID int `json:"user_id"` + SkillKey string `json:"skill_key"` + Name string `json:"name"` + Description *string `json:"description,omitempty"` + Status string `json:"status"` + SourceType string `json:"source_type"` + RiskLevel string `json:"risk_level"` + CurrentVersionID *int `json:"current_version_id,omitempty"` + LastScannedAt *time.Time `json:"last_scanned_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type OpenClawConfigBundleSkillPayload struct { + SkillID int `json:"skill_id"` + SortOrder int `json:"sort_order"` + Required bool `json:"required"` + Skill *OpenClawConfigBundleSkillSummary `json:"skill,omitempty"` +} + +type OpenClawConfigBundlePayload struct { + ID int `json:"id"` + UserID int `json:"user_id"` + Name string `json:"name"` + Description *string `json:"description,omitempty"` + Enabled bool `json:"enabled"` + Version int `json:"version"` + Items []OpenClawConfigBundleItemPayload `json:"items"` + SkillItems []OpenClawConfigBundleSkillPayload `json:"skill_items"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type UpsertOpenClawConfigBundleRequest struct { + Name string `json:"name"` + Description *string `json:"description,omitempty"` + Enabled bool `json:"enabled"` + Items []OpenClawConfigBundleItemPayload `json:"items"` + SkillItems []OpenClawConfigBundleSkillPayload `json:"skill_items"` +} + +type OpenClawConfigCompilePreview struct { + Mode string `json:"mode"` + Bundle *OpenClawConfigBundlePayload `json:"bundle,omitempty"` + SelectedResources []OpenClawConfigResourceSummary `json:"selected_resources"` + ResolvedResources []OpenClawConfigResourceSummary `json:"resolved_resources"` + AutoIncluded []OpenClawConfigResourceSummary `json:"auto_included"` + Warnings []string `json:"warnings"` + EnvNames []string `json:"env_names"` + PayloadSizes map[string]int `json:"payload_sizes"` + TotalPayloadBytes int `json:"total_payload_bytes"` + Manifest json.RawMessage `json:"manifest"` +} + +type OpenClawInjectionSnapshotPayload struct { + ID int `json:"id"` + InstanceID *int `json:"instance_id,omitempty"` + UserID int `json:"user_id"` + Mode string `json:"mode"` + BundleID *int `json:"bundle_id,omitempty"` + SelectedResourceIDs []int `json:"selected_resource_ids"` + ResolvedResources []OpenClawConfigResourceSummary `json:"resolved_resources"` + Manifest json.RawMessage `json:"manifest"` + EnvNames []string `json:"env_names"` + PayloadSizes map[string]int `json:"payload_sizes"` + SecretName *string `json:"secret_name,omitempty"` + Status string `json:"status"` + ErrorMessage *string `json:"error_message,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ActivatedAt *time.Time `json:"activated_at,omitempty"` +} + +type OpenClawConfigService interface { + ListResources(userID int, resourceType string) ([]OpenClawConfigResourcePayload, error) + GetResource(userID, id int) (*OpenClawConfigResourcePayload, error) + CreateResource(userID int, req UpsertOpenClawConfigResourceRequest) (*OpenClawConfigResourcePayload, error) + UpdateResource(userID, id int, req UpsertOpenClawConfigResourceRequest) (*OpenClawConfigResourcePayload, error) + DeleteResource(userID, id int) error + CloneResource(userID, id int) (*OpenClawConfigResourcePayload, error) + ValidateResource(req UpsertOpenClawConfigResourceRequest) error + + ListBundles(userID int) ([]OpenClawConfigBundlePayload, error) + GetBundle(userID, id int) (*OpenClawConfigBundlePayload, error) + CreateBundle(userID int, req UpsertOpenClawConfigBundleRequest) (*OpenClawConfigBundlePayload, error) + UpdateBundle(userID, id int, req UpsertOpenClawConfigBundleRequest) (*OpenClawConfigBundlePayload, error) + DeleteBundle(userID, id int) error + CloneBundle(userID, id int) (*OpenClawConfigBundlePayload, error) + ResolveBundleSkillIDs(userID int, plan *OpenClawConfigPlan) ([]int, error) + + CompilePreview(userID int, plan OpenClawConfigPlan) (*OpenClawConfigCompilePreview, error) + PlanWithoutTeamMemberLeaderOnlyChannels(userID int, plan *OpenClawConfigPlan) (*OpenClawConfigPlan, error) + CreateSnapshotForInstance(userID int, instance *models.Instance, plan *OpenClawConfigPlan) (*models.OpenClawInjectionSnapshot, error) + MarkSnapshotActive(snapshot *models.OpenClawInjectionSnapshot) error + MarkSnapshotFailed(snapshot *models.OpenClawInjectionSnapshot, err error) error + EnsureSnapshotSecret(ctx context.Context, userID int, instance *models.Instance, snapshotID int) (string, error) + ListSnapshots(userID int, limit int) ([]OpenClawInjectionSnapshotPayload, error) + GetSnapshot(userID, id int) (*OpenClawInjectionSnapshotPayload, error) +} + +type openClawConfigService struct { + repo repository.OpenClawConfigRepository + skillRepo repository.SkillRepository + secretService *k8s.SecretService +} + +type compiledOpenClawResource struct { + model models.OpenClawConfigResource + tags []string + envelope OpenClawConfigEnvelope +} + +type compiledOpenClawConfig struct { + plan OpenClawConfigPlan + bundle *models.OpenClawConfigBundle + selected []compiledOpenClawResource + resolved []compiledOpenClawResource + autoIncluded []compiledOpenClawResource + warnings []string + renderedEnv map[string]string + manifest string + payloadSizes map[string]int + totalPayloadSize int +} + +func NewOpenClawConfigService(repo repository.OpenClawConfigRepository, skillRepo repository.SkillRepository) OpenClawConfigService { + return &openClawConfigService{ + repo: repo, + skillRepo: skillRepo, + secretService: k8s.NewSecretService(), + } +} + +func (s *openClawConfigService) ListResources(userID int, resourceType string) ([]OpenClawConfigResourcePayload, error) { + if resourceType != "" && !isValidOpenClawResourceType(resourceType) { + return nil, fmt.Errorf("invalid openclaw resource type") + } + + items, err := s.repo.ListResources(userID, resourceType) + if err != nil { + return nil, err + } + + result := make([]OpenClawConfigResourcePayload, 0, len(items)) + for _, item := range items { + payload, err := resourcePayloadFromModel(item) + if err != nil { + return nil, err + } + result = append(result, payload) + } + return result, nil +} + +func (s *openClawConfigService) GetResource(userID, id int) (*OpenClawConfigResourcePayload, error) { + item, err := s.repo.GetResourceByID(id) + if err != nil { + return nil, err + } + if item == nil || item.UserID != userID { + return nil, fmt.Errorf("openclaw config resource not found") + } + + payload, err := resourcePayloadFromModel(*item) + if err != nil { + return nil, err + } + return &payload, nil +} + +func (s *openClawConfigService) CreateResource(userID int, req UpsertOpenClawConfigResourceRequest) (*OpenClawConfigResourcePayload, error) { + resourceType := normalizeResourceType(req.ResourceType) + resourceKey := normalizeResourceKey(req.ResourceKey) + normalizedContent, err := normalizeOpenClawResourceContent(resourceType, resourceKey, req.Content) + if err != nil { + return nil, err + } + req.Content = normalizedContent + if err := s.ValidateResource(req); err != nil { + return nil, err + } + existing, err := s.repo.GetResourceByUserTypeKey(userID, resourceType, resourceKey) + if err != nil { + return nil, err + } + if existing != nil { + return nil, fmt.Errorf("openclaw config resource key already exists") + } + + now := time.Now() + item := &models.OpenClawConfigResource{ + UserID: userID, + ResourceType: resourceType, + ResourceKey: resourceKey, + Name: strings.TrimSpace(req.Name), + Description: normalizeOptionalString(req.Description), + Enabled: req.Enabled, + Version: 1, + TagsJSON: encodeStringArray(normalizeTags(req.Tags)), + ContentJSON: string(normalizedContent), + CreatedAt: now, + UpdatedAt: now, + } + if err := s.repo.CreateResource(item); err != nil { + return nil, err + } + + payload, err := resourcePayloadFromModel(*item) + if err != nil { + return nil, err + } + return &payload, nil +} + +func (s *openClawConfigService) UpdateResource(userID, id int, req UpsertOpenClawConfigResourceRequest) (*OpenClawConfigResourcePayload, error) { + item, err := s.repo.GetResourceByID(id) + if err != nil { + return nil, err + } + if item == nil || item.UserID != userID { + return nil, fmt.Errorf("openclaw config resource not found") + } + + resourceType := normalizeResourceType(req.ResourceType) + resourceKey := normalizeResourceKey(req.ResourceKey) + normalizedContent, err := normalizeOpenClawResourceContent(resourceType, resourceKey, req.Content) + if err != nil { + return nil, err + } + req.Content = normalizedContent + if err := s.ValidateResource(req); err != nil { + return nil, err + } + existing, err := s.repo.GetResourceByUserTypeKey(userID, resourceType, resourceKey) + if err != nil { + return nil, err + } + if existing != nil && existing.ID != id { + return nil, fmt.Errorf("openclaw config resource key already exists") + } + + item.ResourceType = resourceType + item.ResourceKey = resourceKey + item.Name = strings.TrimSpace(req.Name) + item.Description = normalizeOptionalString(req.Description) + item.Enabled = req.Enabled + item.Version++ + item.TagsJSON = encodeStringArray(normalizeTags(req.Tags)) + item.ContentJSON = string(normalizedContent) + item.UpdatedAt = time.Now() + + if err := s.repo.UpdateResource(item); err != nil { + return nil, err + } + + // Cascade: recompile active snapshots that reference this resource. + // Non-blocking: errors are logged but do not fail the update. + go s.cascadeSnapshotsForResource(userID, id) + + payload, err := resourcePayloadFromModel(*item) + if err != nil { + return nil, err + } + return &payload, nil +} + +func (s *openClawConfigService) DeleteResource(userID, id int) error { + item, err := s.repo.GetResourceByID(id) + if err != nil { + return err + } + if item == nil || item.UserID != userID { + return fmt.Errorf("openclaw config resource not found") + } + return s.repo.DeleteResource(id) +} + +func (s *openClawConfigService) CloneResource(userID, id int) (*OpenClawConfigResourcePayload, error) { + item, err := s.repo.GetResourceByID(id) + if err != nil { + return nil, err + } + if item == nil || item.UserID != userID { + return nil, fmt.Errorf("openclaw config resource not found") + } + + clone := *item + clone.ID = 0 + clone.Name = fmt.Sprintf("%s Copy", item.Name) + clone.ResourceKey = fmt.Sprintf("%s-copy-%d", item.ResourceKey, time.Now().Unix()) + clone.Version = 1 + clone.CreatedAt = time.Now() + clone.UpdatedAt = clone.CreatedAt + if normalizedContent, err := normalizeOpenClawResourceContent(clone.ResourceType, item.ResourceKey, json.RawMessage(clone.ContentJSON)); err == nil { + clone.ContentJSON = string(normalizedContent) + } + + if err := s.repo.CreateResource(&clone); err != nil { + return nil, err + } + + payload, err := resourcePayloadFromModel(clone) + if err != nil { + return nil, err + } + return &payload, nil +} + +func (s *openClawConfigService) ValidateResource(req UpsertOpenClawConfigResourceRequest) error { + resourceType := normalizeResourceType(req.ResourceType) + resourceKey := normalizeResourceKey(req.ResourceKey) + name := strings.TrimSpace(req.Name) + + if !isValidOpenClawResourceType(resourceType) { + return fmt.Errorf("invalid openclaw resource type") + } + if name == "" { + return fmt.Errorf("openclaw config resource name is required") + } + if !openClawResourceKeyPattern.MatchString(resourceKey) { + return fmt.Errorf("openclaw config resource key is invalid") + } + + envelope, err := parseOpenClawEnvelope(resourceType, req.Content) + if err != nil { + return err + } + for _, dep := range envelope.DependsOn { + if !isValidOpenClawResourceType(dep.Type) { + return fmt.Errorf("openclaw config dependency type is invalid") + } + if normalizeResourceKey(dep.Key) == "" { + return fmt.Errorf("openclaw config dependency key is required") + } + } + return nil +} + +func (s *openClawConfigService) ListBundles(userID int) ([]OpenClawConfigBundlePayload, error) { + items, err := s.repo.ListBundles(userID) + if err != nil { + return nil, err + } + + result := make([]OpenClawConfigBundlePayload, 0, len(items)) + for _, item := range items { + payload, err := s.bundlePayloadFromModel(item) + if err != nil { + return nil, err + } + result = append(result, payload) + } + return result, nil +} + +func (s *openClawConfigService) GetBundle(userID, id int) (*OpenClawConfigBundlePayload, error) { + item, err := s.repo.GetBundleByID(id) + if err != nil { + return nil, err + } + if item == nil || item.UserID != userID { + return nil, fmt.Errorf("openclaw config bundle not found") + } + + payload, err := s.bundlePayloadFromModel(*item) + if err != nil { + return nil, err + } + return &payload, nil +} + +func (s *openClawConfigService) CreateBundle(userID int, req UpsertOpenClawConfigBundleRequest) (*OpenClawConfigBundlePayload, error) { + if err := s.validateBundleRequest(userID, req); err != nil { + return nil, err + } + + now := time.Now() + item := &models.OpenClawConfigBundle{ + UserID: userID, + Name: strings.TrimSpace(req.Name), + Description: normalizeOptionalString(req.Description), + Enabled: req.Enabled, + Version: 1, + CreatedAt: now, + UpdatedAt: now, + } + + if err := s.repo.CreateBundle(item); err != nil { + return nil, err + } + if err := s.repo.ReplaceBundleItems(item.ID, normalizeBundleItems(req.Items)); err != nil { + return nil, err + } + if err := s.repo.ReplaceBundleSkills(item.ID, normalizeBundleSkills(req.SkillItems)); err != nil { + return nil, err + } + + payload, err := s.bundlePayloadFromModel(*item) + if err != nil { + return nil, err + } + return &payload, nil +} + +func (s *openClawConfigService) UpdateBundle(userID, id int, req UpsertOpenClawConfigBundleRequest) (*OpenClawConfigBundlePayload, error) { + if err := s.validateBundleRequest(userID, req); err != nil { + return nil, err + } + + item, err := s.repo.GetBundleByID(id) + if err != nil { + return nil, err + } + if item == nil || item.UserID != userID { + return nil, fmt.Errorf("openclaw config bundle not found") + } + + item.Name = strings.TrimSpace(req.Name) + item.Description = normalizeOptionalString(req.Description) + item.Enabled = req.Enabled + item.Version++ + item.UpdatedAt = time.Now() + + if err := s.repo.UpdateBundle(item); err != nil { + return nil, err + } + if err := s.repo.ReplaceBundleItems(item.ID, normalizeBundleItems(req.Items)); err != nil { + return nil, err + } + if err := s.repo.ReplaceBundleSkills(item.ID, normalizeBundleSkills(req.SkillItems)); err != nil { + return nil, err + } + + payload, err := s.bundlePayloadFromModel(*item) + if err != nil { + return nil, err + } + return &payload, nil +} + +func (s *openClawConfigService) DeleteBundle(userID, id int) error { + item, err := s.repo.GetBundleByID(id) + if err != nil { + return err + } + if item == nil || item.UserID != userID { + return fmt.Errorf("openclaw config bundle not found") + } + return s.repo.DeleteBundle(id) +} + +func (s *openClawConfigService) CloneBundle(userID, id int) (*OpenClawConfigBundlePayload, error) { + item, err := s.repo.GetBundleByID(id) + if err != nil { + return nil, err + } + if item == nil || item.UserID != userID { + return nil, fmt.Errorf("openclaw config bundle not found") + } + + items, err := s.repo.ListBundleItems(id) + if err != nil { + return nil, err + } + skills, err := s.repo.ListBundleSkills(id) + if err != nil { + return nil, err + } + + clone := *item + clone.ID = 0 + clone.Name = fmt.Sprintf("%s Copy", item.Name) + clone.Version = 1 + clone.CreatedAt = time.Now() + clone.UpdatedAt = clone.CreatedAt + if err := s.repo.CreateBundle(&clone); err != nil { + return nil, err + } + for idx := range items { + items[idx].ID = 0 + items[idx].BundleID = clone.ID + } + if err := s.repo.ReplaceBundleItems(clone.ID, items); err != nil { + return nil, err + } + for idx := range skills { + skills[idx].ID = 0 + skills[idx].BundleID = clone.ID + } + if err := s.repo.ReplaceBundleSkills(clone.ID, skills); err != nil { + return nil, err + } + + payload, err := s.bundlePayloadFromModel(clone) + if err != nil { + return nil, err + } + return &payload, nil +} + +func (s *openClawConfigService) ResolveBundleSkillIDs(userID int, plan *OpenClawConfigPlan) ([]int, error) { + if plan == nil || plan.Mode != OpenClawConfigPlanModeBundle || plan.BundleID == nil || *plan.BundleID <= 0 { + return nil, nil + } + + bundle, err := s.repo.GetBundleByID(*plan.BundleID) + if err != nil { + return nil, err + } + if bundle == nil || bundle.UserID != userID { + return nil, fmt.Errorf("openclaw config bundle not found") + } + if !bundle.Enabled { + return nil, fmt.Errorf("openclaw config bundle is disabled") + } + + bundleSkills, err := s.repo.ListBundleSkills(bundle.ID) + if err != nil { + return nil, err + } + result := make([]int, 0, len(bundleSkills)) + seen := map[int]struct{}{} + for _, item := range bundleSkills { + if _, exists := seen[item.SkillID]; exists { + continue + } + skill, err := s.getBundleSkill(userID, item.SkillID) + if err != nil { + return nil, err + } + if skill == nil { + return nil, fmt.Errorf("skill not found") + } + seen[item.SkillID] = struct{}{} + result = append(result, item.SkillID) + } + return result, nil +} + +func (s *openClawConfigService) CompilePreview(userID int, plan OpenClawConfigPlan) (*OpenClawConfigCompilePreview, error) { + compiled, err := s.compilePlan(userID, plan) + if err != nil { + return nil, err + } + return s.previewFromCompiled(compiled) +} + +func (s *openClawConfigService) PlanWithoutTeamMemberLeaderOnlyChannels(userID int, plan *OpenClawConfigPlan) (*OpenClawConfigPlan, error) { + if plan == nil || !hasOpenClawConfigSelections(*plan) { + return nil, nil + } + + normalizedPlan := *plan + normalizedPlan.Mode = normalizePlanMode(normalizedPlan.Mode) + if normalizedPlan.Mode == "" { + normalizedPlan.Mode = OpenClawConfigPlanModeNone + } + if normalizedPlan.Mode == OpenClawConfigPlanModeNone { + return nil, nil + } + + selected, _, err := s.loadSelectedResources(userID, normalizedPlan) + if err != nil { + return nil, err + } + + keptResourceIDs := make([]int, 0, len(selected)) + removed := false + for _, resource := range selected { + if isTeamMemberLeaderOnlyOpenClawChannel(resource) { + removed = true + continue + } + keptResourceIDs = append(keptResourceIDs, resource.ID) + } + if !removed { + return cloneOpenClawConfigPlan(plan), nil + } + if len(keptResourceIDs) == 0 { + return nil, nil + } + + return &OpenClawConfigPlan{ + Mode: OpenClawConfigPlanModeManual, + ResourceIDs: keptResourceIDs, + }, nil +} + +func (s *openClawConfigService) CreateSnapshotForInstance(userID int, instance *models.Instance, plan *OpenClawConfigPlan) (*models.OpenClawInjectionSnapshot, error) { + if instance == nil || plan == nil || !hasOpenClawConfigSelections(*plan) { + return nil, nil + } + + compiled, err := s.compilePlan(userID, *plan) + if err != nil { + return nil, err + } + + selectedIDs := make([]int, 0, len(compiled.selected)) + for _, resource := range compiled.selected { + selectedIDs = append(selectedIDs, resource.model.ID) + } + + resolvedSummaries := make([]OpenClawConfigResourceSummary, 0, len(compiled.resolved)) + for _, resource := range compiled.resolved { + resolvedSummaries = append(resolvedSummaries, resourceSummaryFromModel(resource.model)) + } + + selectedJSON, err := marshalJSONString(selectedIDs) + if err != nil { + return nil, err + } + resolvedJSON, err := marshalJSONString(resolvedSummaries) + if err != nil { + return nil, err + } + envJSON, err := marshalJSONString(compiled.renderedEnv) + if err != nil { + return nil, err + } + + now := time.Now() + instanceID := instance.ID + snapshot := &models.OpenClawInjectionSnapshot{ + InstanceID: &instanceID, + UserID: userID, + Mode: compiled.plan.Mode, + BundleID: compiled.plan.BundleID, + SelectedResourceIDsJSON: selectedJSON, + ResolvedResourcesJSON: resolvedJSON, + RenderedManifestJSON: compiled.manifest, + RenderedEnvJSON: envJSON, + Status: openClawCompiledSnapshotStatus, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.repo.CreateSnapshot(snapshot); err != nil { + return nil, err + } + return snapshot, nil +} + +func (s *openClawConfigService) MarkSnapshotActive(snapshot *models.OpenClawInjectionSnapshot) error { + if snapshot == nil { + return nil + } + now := time.Now() + snapshot.Status = openClawActiveSnapshotStatus + snapshot.ActivatedAt = &now + snapshot.UpdatedAt = now + return s.repo.UpdateSnapshot(snapshot) +} + +func (s *openClawConfigService) MarkSnapshotFailed(snapshot *models.OpenClawInjectionSnapshot, failedErr error) error { + if snapshot == nil { + return nil + } + message := strings.TrimSpace(errorString(failedErr)) + if message == "" { + message = "unknown bootstrap failure" + } + now := time.Now() + snapshot.Status = openClawFailedSnapshotStatus + snapshot.ErrorMessage = &message + snapshot.UpdatedAt = now + return s.repo.UpdateSnapshot(snapshot) +} + +func (s *openClawConfigService) EnsureSnapshotSecret(ctx context.Context, userID int, instance *models.Instance, snapshotID int) (string, error) { + if instance == nil || snapshotID <= 0 { + return "", nil + } + + envValues, err := s.RuntimeEnvForSnapshot(userID, instance.Type, snapshotID) + if err != nil { + return "", err + } + + snapshot, err := s.repo.GetSnapshotByID(snapshotID) + if err != nil { + return "", err + } + if snapshot == nil || snapshot.UserID != userID { + return "", fmt.Errorf("openclaw injection snapshot not found") + } + + secretName := snapshot.SecretName + if secretName == nil || strings.TrimSpace(*secretName) == "" { + client := k8s.GetClient() + if client == nil { + return "", fmt.Errorf("k8s client not initialized") + } + name := client.GetOpenClawBootstrapSecretName(instance.ID, instance.Name) + secretName = &name + } + + if err := s.secretService.UpsertSecret(ctx, userID, *secretName, envValues, map[string]string{ + "app": "clawreef", + "instance-id": fmt.Sprintf("%d", instance.ID), + "instance-name": instance.Name, + "user-id": fmt.Sprintf("%d", userID), + "managed-by": "clawreef", + "resource-type": "openclaw-bootstrap", + }); err != nil { + return "", err + } + + if snapshot.SecretName == nil || *snapshot.SecretName != *secretName { + snapshot.SecretName = secretName + snapshot.UpdatedAt = time.Now() + if err := s.repo.UpdateSnapshot(snapshot); err != nil { + return "", err + } + } + + return *secretName, nil +} + +func (s *openClawConfigService) RuntimeEnvForSnapshot(userID int, instanceType string, snapshotID int) (map[string]string, error) { + if s == nil || s.repo == nil || snapshotID <= 0 { + return map[string]string{}, nil + } + + snapshot, err := s.repo.GetSnapshotByID(snapshotID) + if err != nil { + return nil, err + } + if snapshot == nil || snapshot.UserID != userID { + return nil, fmt.Errorf("openclaw injection snapshot not found") + } + + var envValues map[string]string + if err := json.Unmarshal([]byte(snapshot.RenderedEnvJSON), &envValues); err != nil { + return nil, fmt.Errorf("openclaw injection snapshot env payload is invalid") + } + return runtimeBootstrapEnvValues(instanceType, envValues), nil +} +func runtimeBootstrapEnvValues(instanceType string, envValues map[string]string) map[string]string { + result := map[string]string{} + for key, value := range envValues { + result[key] = value + } + + if !strings.EqualFold(instanceType, "hermes") { + return result + } + + addBootstrapEnvAliases(result, hermesBootstrapEnvAliases) + addBootstrapEnvAliases(result, runtimeBootstrapEnvAliases) + return result +} + +func addBootstrapEnvAliases(envValues map[string]string, aliases map[string]string) { + for source, target := range aliases { + value, ok := envValues[source] + if !ok { + continue + } + if source == OpenClawBootstrapManifestEnv { + if aliasedManifest, err := aliasBootstrapManifestEnvNames(value, aliases); err == nil { + value = aliasedManifest + } + } + envValues[target] = value + } +} + +func aliasBootstrapManifestEnvNames(raw string, aliases map[string]string) (string, error) { + var manifest map[string]interface{} + if err := json.Unmarshal([]byte(raw), &manifest); err != nil { + return "", err + } + + payloads, ok := manifest["payloads"].([]interface{}) + if !ok { + return marshalJSONString(manifest) + } + for _, item := range payloads { + payload, ok := item.(map[string]interface{}) + if !ok { + continue + } + envName, ok := payload["env"].(string) + if !ok { + continue + } + if alias, exists := aliases[envName]; exists { + payload["env"] = alias + } + } + return marshalJSONString(manifest) +} + +func (s *openClawConfigService) ListSnapshots(userID int, limit int) ([]OpenClawInjectionSnapshotPayload, error) { + if limit <= 0 { + limit = defaultSnapshotListLimit + } + + items, err := s.repo.ListSnapshotsByUser(userID, limit) + if err != nil { + return nil, err + } + + result := make([]OpenClawInjectionSnapshotPayload, 0, len(items)) + for _, item := range items { + payload, err := snapshotPayloadFromModel(item) + if err != nil { + return nil, err + } + result = append(result, payload) + } + return result, nil +} + +func (s *openClawConfigService) GetSnapshot(userID, id int) (*OpenClawInjectionSnapshotPayload, error) { + item, err := s.repo.GetSnapshotByID(id) + if err != nil { + return nil, err + } + if item == nil || item.UserID != userID { + return nil, fmt.Errorf("openclaw injection snapshot not found") + } + + payload, err := snapshotPayloadFromModel(*item) + if err != nil { + return nil, err + } + return &payload, nil +} + +func (s *openClawConfigService) validateBundleRequest(userID int, req UpsertOpenClawConfigBundleRequest) error { + if strings.TrimSpace(req.Name) == "" { + return fmt.Errorf("openclaw config bundle name is required") + } + if len(req.Items) == 0 && len(req.SkillItems) == 0 { + return fmt.Errorf("openclaw config bundle must include at least one resource or skill") + } + + seen := map[int]struct{}{} + for _, item := range req.Items { + if item.ResourceID <= 0 { + return fmt.Errorf("openclaw config bundle resource id is required") + } + if _, exists := seen[item.ResourceID]; exists { + return fmt.Errorf("openclaw config bundle contains duplicate resources") + } + seen[item.ResourceID] = struct{}{} + + resource, err := s.repo.GetResourceByID(item.ResourceID) + if err != nil { + return err + } + if resource == nil || resource.UserID != userID { + return fmt.Errorf("openclaw config resource not found") + } + } + + seenSkills := map[int]struct{}{} + for _, item := range req.SkillItems { + if item.SkillID <= 0 { + return fmt.Errorf("openclaw config bundle skill id is required") + } + if _, exists := seenSkills[item.SkillID]; exists { + return fmt.Errorf("openclaw config bundle contains duplicate skills") + } + seenSkills[item.SkillID] = struct{}{} + + skill, err := s.getBundleSkill(userID, item.SkillID) + if err != nil { + return err + } + if skill == nil { + return fmt.Errorf("skill not found") + } + } + return nil +} + +func (s *openClawConfigService) getBundleSkill(userID, skillID int) (*models.Skill, error) { + if s.skillRepo == nil { + return nil, fmt.Errorf("skill repository is not initialized") + } + skill, err := s.skillRepo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil || skill.UserID != userID || !isUserManagedSkill(*skill) || !strings.EqualFold(skill.Status, "active") { + return nil, nil + } + return skill, nil +} + +func normalizeBundleItems(items []OpenClawConfigBundleItemPayload) []models.OpenClawConfigBundleItem { + result := make([]models.OpenClawConfigBundleItem, 0, len(items)) + for idx, item := range items { + sortOrder := item.SortOrder + if sortOrder == 0 { + sortOrder = idx + 1 + } + result = append(result, models.OpenClawConfigBundleItem{ + ResourceID: item.ResourceID, + SortOrder: sortOrder, + Required: item.Required, + }) + } + return result +} + +func normalizeBundleSkills(items []OpenClawConfigBundleSkillPayload) []models.OpenClawConfigBundleSkill { + result := make([]models.OpenClawConfigBundleSkill, 0, len(items)) + for idx, item := range items { + sortOrder := item.SortOrder + if sortOrder == 0 { + sortOrder = idx + 1 + } + result = append(result, models.OpenClawConfigBundleSkill{ + SkillID: item.SkillID, + SortOrder: sortOrder, + Required: item.Required, + }) + } + return result +} + +func (s *openClawConfigService) bundlePayloadFromModel(item models.OpenClawConfigBundle) (OpenClawConfigBundlePayload, error) { + bundleItems, err := s.repo.ListBundleItems(item.ID) + if err != nil { + return OpenClawConfigBundlePayload{}, err + } + bundleSkills, err := s.repo.ListBundleSkills(item.ID) + if err != nil { + return OpenClawConfigBundlePayload{}, err + } + + payloadItems := make([]OpenClawConfigBundleItemPayload, 0, len(bundleItems)) + for _, bundleItem := range bundleItems { + var summary *OpenClawConfigResourceSummary + resource, err := s.repo.GetResourceByID(bundleItem.ResourceID) + if err != nil { + return OpenClawConfigBundlePayload{}, err + } + if resource != nil { + resourceSummary := resourceSummaryFromModel(*resource) + summary = &resourceSummary + } + + payloadItems = append(payloadItems, OpenClawConfigBundleItemPayload{ + ResourceID: bundleItem.ResourceID, + SortOrder: bundleItem.SortOrder, + Required: bundleItem.Required, + Resource: summary, + }) + } + + payloadSkills := make([]OpenClawConfigBundleSkillPayload, 0, len(bundleSkills)) + for _, bundleSkill := range bundleSkills { + var summary *OpenClawConfigBundleSkillSummary + skill, err := s.getBundleSkill(item.UserID, bundleSkill.SkillID) + if err != nil { + return OpenClawConfigBundlePayload{}, err + } + if skill != nil { + skillSummary := bundleSkillSummaryFromModel(*skill) + summary = &skillSummary + } + + payloadSkills = append(payloadSkills, OpenClawConfigBundleSkillPayload{ + SkillID: bundleSkill.SkillID, + SortOrder: bundleSkill.SortOrder, + Required: bundleSkill.Required, + Skill: summary, + }) + } + + return OpenClawConfigBundlePayload{ + ID: item.ID, + UserID: item.UserID, + Name: item.Name, + Description: item.Description, + Enabled: item.Enabled, + Version: item.Version, + Items: payloadItems, + SkillItems: payloadSkills, + CreatedAt: item.CreatedAt, + UpdatedAt: item.UpdatedAt, + }, nil +} + +func (s *openClawConfigService) compilePlan(userID int, plan OpenClawConfigPlan) (*compiledOpenClawConfig, error) { + plan.Mode = normalizePlanMode(plan.Mode) + if plan.Mode == "" { + plan.Mode = OpenClawConfigPlanModeNone + } + if _, ok := openClawPlanModes[plan.Mode]; !ok { + return nil, fmt.Errorf("invalid openclaw config plan mode") + } + + if plan.Mode == OpenClawConfigPlanModeNone { + manifestJSON, err := marshalJSONString(map[string]interface{}{ + "schemaVersion": 1, + "mode": OpenClawConfigPlanModeNone, + "resources": []interface{}{}, + "payloads": []interface{}{}, + }) + if err != nil { + return nil, err + } + return &compiledOpenClawConfig{ + plan: plan, + renderedEnv: map[string]string{}, + manifest: manifestJSON, + payloadSizes: map[string]int{}, + }, nil + } + + selected, bundle, err := s.loadSelectedResources(userID, plan) + if err != nil { + return nil, err + } + + selectedCompiled := make([]compiledOpenClawResource, 0, len(selected)) + resolvedOrdered := make([]compiledOpenClawResource, 0, len(selected)) + autoIncluded := make([]compiledOpenClawResource, 0) + resourceByRef := map[string]compiledOpenClawResource{} + warnings := []string{} + + for _, item := range selected { + compiled, err := compiledResourceFromModel(item) + if err != nil { + return nil, err + } + ref := openClawResourceRef(compiled.model.ResourceType, compiled.model.ResourceKey) + resourceByRef[ref] = compiled + selectedCompiled = append(selectedCompiled, compiled) + resolvedOrdered = append(resolvedOrdered, compiled) + } + + queue := append([]compiledOpenClawResource{}, selectedCompiled...) + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + + for _, dependency := range current.envelope.DependsOn { + depType := normalizeResourceType(dependency.Type) + depKey := normalizeResourceKey(dependency.Key) + if depType == "" || depKey == "" { + return nil, fmt.Errorf("openclaw config dependency is invalid") + } + + ref := openClawResourceRef(depType, depKey) + if _, exists := resourceByRef[ref]; exists { + continue + } + + resource, err := s.repo.GetResourceByUserTypeKey(userID, depType, depKey) + if err != nil { + return nil, err + } + if resource == nil { + if dependency.Required { + return nil, fmt.Errorf("missing required openclaw config dependency: %s/%s", depType, depKey) + } + warnings = append(warnings, fmt.Sprintf("Optional dependency %s/%s is not configured", depType, depKey)) + continue + } + if !resource.Enabled { + if dependency.Required { + return nil, fmt.Errorf("required openclaw config dependency is disabled: %s/%s", depType, depKey) + } + warnings = append(warnings, fmt.Sprintf("Optional dependency %s/%s is disabled", depType, depKey)) + continue + } + + compiledDep, err := compiledResourceFromModel(*resource) + if err != nil { + return nil, err + } + resourceByRef[ref] = compiledDep + resolvedOrdered = append(resolvedOrdered, compiledDep) + autoIncluded = append(autoIncluded, compiledDep) + queue = append(queue, compiledDep) + } + } + + sortCompiledResources(resolvedOrdered) + sortCompiledResources(autoIncluded) + + renderedEnv, manifest, payloadSizes, totalPayloadSize, err := renderCompiledOpenClawPayload(plan, bundle, resolvedOrdered) + if err != nil { + return nil, err + } + + return &compiledOpenClawConfig{ + plan: plan, + bundle: bundle, + selected: selectedCompiled, + resolved: resolvedOrdered, + autoIncluded: autoIncluded, + warnings: warnings, + renderedEnv: renderedEnv, + manifest: manifest, + payloadSizes: payloadSizes, + totalPayloadSize: totalPayloadSize, + }, nil +} + +func (s *openClawConfigService) loadSelectedResources(userID int, plan OpenClawConfigPlan) ([]models.OpenClawConfigResource, *models.OpenClawConfigBundle, error) { + switch plan.Mode { + case OpenClawConfigPlanModeBundle: + if plan.BundleID == nil || *plan.BundleID <= 0 { + return nil, nil, fmt.Errorf("openclaw config bundle is required") + } + + bundle, err := s.repo.GetBundleByID(*plan.BundleID) + if err != nil { + return nil, nil, err + } + if bundle == nil || bundle.UserID != userID { + return nil, nil, fmt.Errorf("openclaw config bundle not found") + } + if !bundle.Enabled { + return nil, nil, fmt.Errorf("openclaw config bundle is disabled") + } + + items, err := s.repo.ListBundleItems(bundle.ID) + if err != nil { + return nil, nil, err + } + skills, err := s.repo.ListBundleSkills(bundle.ID) + if err != nil { + return nil, nil, err + } + if len(items) == 0 && len(skills) == 0 { + return nil, nil, fmt.Errorf("openclaw config bundle is empty") + } + + result := make([]models.OpenClawConfigResource, 0, len(items)) + for _, item := range items { + resource, err := s.repo.GetResourceByID(item.ResourceID) + if err != nil { + return nil, nil, err + } + if resource == nil || resource.UserID != userID { + return nil, nil, fmt.Errorf("openclaw config resource not found") + } + if !resource.Enabled { + return nil, nil, fmt.Errorf("openclaw config bundle contains a disabled resource") + } + result = append(result, *resource) + } + return result, bundle, nil + case OpenClawConfigPlanModeManual: + if len(plan.ResourceIDs) == 0 { + return nil, nil, fmt.Errorf("at least one openclaw config resource must be selected") + } + + result := make([]models.OpenClawConfigResource, 0, len(plan.ResourceIDs)) + seen := map[int]struct{}{} + for _, id := range plan.ResourceIDs { + if id <= 0 { + return nil, nil, fmt.Errorf("openclaw config resource id is invalid") + } + if _, exists := seen[id]; exists { + continue + } + seen[id] = struct{}{} + + resource, err := s.repo.GetResourceByID(id) + if err != nil { + return nil, nil, err + } + if resource == nil || resource.UserID != userID { + return nil, nil, fmt.Errorf("openclaw config resource not found") + } + if !resource.Enabled { + return nil, nil, fmt.Errorf("openclaw config resource is disabled") + } + result = append(result, *resource) + } + return result, nil, nil + default: + return nil, nil, fmt.Errorf("invalid openclaw config plan mode") + } +} + +func (s *openClawConfigService) previewFromCompiled(compiled *compiledOpenClawConfig) (*OpenClawConfigCompilePreview, error) { + var bundlePayload *OpenClawConfigBundlePayload + if compiled.bundle != nil { + payload, err := s.bundlePayloadFromModel(*compiled.bundle) + if err != nil { + return nil, err + } + bundlePayload = &payload + } + + return &OpenClawConfigCompilePreview{ + Mode: compiled.plan.Mode, + Bundle: bundlePayload, + SelectedResources: summarizeCompiledResources(compiled.selected), + ResolvedResources: summarizeCompiledResources(compiled.resolved), + AutoIncluded: summarizeCompiledResources(compiled.autoIncluded), + Warnings: compiled.warnings, + EnvNames: sortedEnvNames(compiled.renderedEnv), + PayloadSizes: compiled.payloadSizes, + TotalPayloadBytes: compiled.totalPayloadSize, + Manifest: json.RawMessage(compiled.manifest), + }, nil +} + +func resourcePayloadFromModel(item models.OpenClawConfigResource) (OpenClawConfigResourcePayload, error) { + tags, err := decodeStringArray(item.TagsJSON) + if err != nil { + return OpenClawConfigResourcePayload{}, fmt.Errorf("failed to parse openclaw config resource tags") + } + + content := json.RawMessage(item.ContentJSON) + if len(content) == 0 { + content = json.RawMessage("{}") + } + if normalizedContent, err := normalizeOpenClawResourceContent(item.ResourceType, item.ResourceKey, content); err == nil { + content = normalizedContent + } + + return OpenClawConfigResourcePayload{ + ID: item.ID, + UserID: item.UserID, + ResourceType: item.ResourceType, + ResourceKey: item.ResourceKey, + Name: item.Name, + Description: item.Description, + Enabled: item.Enabled, + Version: item.Version, + Tags: tags, + Content: content, + CreatedAt: item.CreatedAt, + UpdatedAt: item.UpdatedAt, + }, nil +} + +func resourceSummaryFromModel(item models.OpenClawConfigResource) OpenClawConfigResourceSummary { + return OpenClawConfigResourceSummary{ + ID: item.ID, + ResourceType: item.ResourceType, + ResourceKey: item.ResourceKey, + Name: item.Name, + Enabled: item.Enabled, + Version: item.Version, + } +} + +func bundleSkillSummaryFromModel(item models.Skill) OpenClawConfigBundleSkillSummary { + return OpenClawConfigBundleSkillSummary{ + ID: item.ID, + UserID: item.UserID, + SkillKey: item.SkillKey, + Name: item.Name, + Description: item.Description, + Status: item.Status, + SourceType: item.SourceType, + RiskLevel: item.RiskLevel, + CurrentVersionID: item.CurrentVersionID, + LastScannedAt: item.LastScannedAt, + CreatedAt: item.CreatedAt, + UpdatedAt: item.UpdatedAt, + } +} + +func snapshotPayloadFromModel(item models.OpenClawInjectionSnapshot) (OpenClawInjectionSnapshotPayload, error) { + selectedIDs := []int{} + if strings.TrimSpace(item.SelectedResourceIDsJSON) != "" { + if err := json.Unmarshal([]byte(item.SelectedResourceIDsJSON), &selectedIDs); err != nil { + return OpenClawInjectionSnapshotPayload{}, fmt.Errorf("failed to parse openclaw snapshot selected ids") + } + } + + resolved := []OpenClawConfigResourceSummary{} + if strings.TrimSpace(item.ResolvedResourcesJSON) != "" { + if err := json.Unmarshal([]byte(item.ResolvedResourcesJSON), &resolved); err != nil { + return OpenClawInjectionSnapshotPayload{}, fmt.Errorf("failed to parse openclaw snapshot resources") + } + } + + envValues := map[string]string{} + if strings.TrimSpace(item.RenderedEnvJSON) != "" { + if err := json.Unmarshal([]byte(item.RenderedEnvJSON), &envValues); err != nil { + return OpenClawInjectionSnapshotPayload{}, fmt.Errorf("failed to parse openclaw snapshot env payloads") + } + } + + payloadSizes := map[string]int{} + for key, value := range envValues { + payloadSizes[key] = len(value) + } + + return OpenClawInjectionSnapshotPayload{ + ID: item.ID, + InstanceID: item.InstanceID, + UserID: item.UserID, + Mode: item.Mode, + BundleID: item.BundleID, + SelectedResourceIDs: selectedIDs, + ResolvedResources: resolved, + Manifest: json.RawMessage(item.RenderedManifestJSON), + EnvNames: sortedEnvNames(envValues), + PayloadSizes: payloadSizes, + SecretName: item.SecretName, + Status: item.Status, + ErrorMessage: item.ErrorMessage, + CreatedAt: item.CreatedAt, + UpdatedAt: item.UpdatedAt, + ActivatedAt: item.ActivatedAt, + }, nil +} + +func renderCompiledOpenClawPayload(plan OpenClawConfigPlan, bundle *models.OpenClawConfigBundle, resources []compiledOpenClawResource) (map[string]string, string, map[string]int, int, error) { + payloads := map[string]interface{}{} + payloadCounts := map[string]int{} + for _, resourceType := range openClawResourceTypeOrder { + payloads[resourceType] = defaultOpenClawEnvPayload(resourceType) + payloadCounts[resourceType] = 0 + } + + resolvedSummaries := make([]map[string]interface{}, 0, len(resources)) + for _, resource := range resources { + nextPayload, err := appendCompiledOpenClawEnvPayload(payloads[resource.model.ResourceType], resource) + if err != nil { + return nil, "", nil, 0, err + } + payloads[resource.model.ResourceType] = nextPayload + payloadCounts[resource.model.ResourceType]++ + resolvedSummaries = append(resolvedSummaries, map[string]interface{}{ + "id": resource.model.ID, + "type": resource.model.ResourceType, + "key": resource.model.ResourceKey, + "name": resource.model.Name, + "version": resource.model.Version, + }) + } + + renderedEnv := map[string]string{} + payloadSizes := map[string]int{} + totalPayloadBytes := 0 + + for _, resourceType := range openClawResourceTypeOrder { + envName := openClawEnvByResourceType[resourceType] + value, err := marshalJSONString(payloads[resourceType]) + if err != nil { + return nil, "", nil, 0, err + } + renderedEnv[envName] = value + payloadSizes[envName] = len(value) + totalPayloadBytes += len(value) + } + + manifest := map[string]interface{}{ + "schemaVersion": 1, + "mode": plan.Mode, + "resources": resolvedSummaries, + "payloads": []map[string]interface{}{}, + } + if bundle != nil { + manifest["bundle"] = map[string]interface{}{ + "id": bundle.ID, + "name": bundle.Name, + "version": bundle.Version, + } + } + + for _, resourceType := range openClawResourceTypeOrder { + envName := openClawEnvByResourceType[resourceType] + manifest["payloads"] = append(manifest["payloads"].([]map[string]interface{}), map[string]interface{}{ + "env": envName, + "count": payloadCounts[resourceType], + }) + } + + manifestJSON, err := marshalJSONString(manifest) + if err != nil { + return nil, "", nil, 0, err + } + renderedEnv[OpenClawBootstrapManifestEnv] = manifestJSON + payloadSizes[OpenClawBootstrapManifestEnv] = len(manifestJSON) + totalPayloadBytes += len(manifestJSON) + + if totalPayloadBytes > openClawBootstrapPayloadMaxBytes { + return nil, "", nil, 0, fmt.Errorf("openclaw bootstrap payload is too large") + } + + return renderedEnv, manifestJSON, payloadSizes, totalPayloadBytes, nil +} + +func defaultOpenClawEnvPayload(resourceType string) interface{} { + if resourceType == OpenClawConfigResourceTypeChannel { + return map[string]interface{}{} + } + + return map[string]interface{}{ + "schemaVersion": 1, + "items": []map[string]interface{}{}, + } +} + +func appendCompiledOpenClawEnvPayload(payload interface{}, resource compiledOpenClawResource) (interface{}, error) { + if resource.model.ResourceType == OpenClawConfigResourceTypeChannel { + channelPayload, ok := payload.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("failed to render openclaw channels payload") + } + + var configPayload interface{} + if len(resource.envelope.Config) == 0 { + configPayload = map[string]interface{}{} + } else if err := json.Unmarshal(resource.envelope.Config, &configPayload); err != nil { + return nil, fmt.Errorf("failed to parse openclaw channel config") + } + + channelKey := openClawChannelEnvKey(resource.model.ResourceKey, configPayload) + nextConfig := normalizeOpenClawChannelConfigForEnv(resource.model.ResourceKey, configPayload) + if existingConfig, ok := channelPayload[channelKey]; ok { + channelPayload[channelKey] = mergeOpenClawChannelEnvConfig(channelKey, resource.model.ResourceKey, existingConfig, nextConfig) + } else { + channelPayload[channelKey] = nextConfig + } + return channelPayload, nil + } + + group, ok := payload.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("failed to render openclaw payload for resource type %s", resource.model.ResourceType) + } + + payloadItem := map[string]interface{}{ + "id": resource.model.ID, + "type": resource.model.ResourceType, + "key": resource.model.ResourceKey, + "name": resource.model.Name, + "version": resource.model.Version, + "tags": resource.tags, + "content": json.RawMessage(resource.model.ContentJSON), + } + group["items"] = append(group["items"].([]map[string]interface{}), payloadItem) + return group, nil +} + +func normalizeOpenClawChannelConfigForEnv(resourceKey string, configPayload interface{}) interface{} { + switch detectOpenClawChannelProvider(resourceKey, configPayload) { + case "dingtalk-connector": + return normalizeDingTalkChannelConfigForEnv(configPayload) + case "feishu": + return normalizeFeishuChannelConfigForEnv(configPayload) + case "slack": + return normalizeSlackChannelConfigForEnv(configPayload) + case "telegram": + return normalizeTelegramChannelConfigForEnv(configPayload) + case "wecom": + return normalizeWeComChannelConfigForEnv(configPayload) + } + + return configPayload +} + +func openClawChannelEnvKey(resourceKey string, configPayload interface{}) string { + provider := detectOpenClawChannelProvider(resourceKey, configPayload) + switch provider { + case "dingtalk-connector", "feishu", "slack", "telegram", "wecom": + return provider + } + return normalizeResourceKey(resourceKey) +} + +func mergeOpenClawChannelEnvConfig(channelKey, resourceKey string, existing, next interface{}) interface{} { + if channelKey == "feishu" { + return mergeFeishuChannelEnvConfig(resourceKey, existing, next) + } + return next +} + +func mergeFeishuChannelEnvConfig(resourceKey string, existing, next interface{}) interface{} { + existingMap, existingOk := existing.(map[string]interface{}) + nextMap, nextOk := next.(map[string]interface{}) + if !existingOk || !nextOk { + return next + } + + merged := make(map[string]interface{}, len(existingMap)+len(nextMap)) + for k, v := range existingMap { + merged[k] = v + } + for k, v := range nextMap { + if k != "accounts" && k != "defaultAccount" { + merged[k] = v + } + } + if _, ok := merged["defaultAccount"]; !ok { + if defaultAccount, ok := nextMap["defaultAccount"]; ok { + merged["defaultAccount"] = defaultAccount + } + } + + mergedAccounts := map[string]interface{}{} + if existingAccounts, ok := existingMap["accounts"].(map[string]interface{}); ok { + for k, v := range existingAccounts { + mergedAccounts[k] = v + } + } + if nextAccounts, ok := nextMap["accounts"].(map[string]interface{}); ok { + for accountKey, account := range nextAccounts { + mergedAccountKey := accountKey + if accountKey == "main" { + key := normalizeResourceKey(resourceKey) + if _, exists := mergedAccounts["main"]; exists && key != "" && key != "feishu" { + mergedAccountKey = key + } + } + mergedAccounts[mergedAccountKey] = account + } + } + merged["accounts"] = mergedAccounts + return merged +} + +func detectOpenClawChannelProvider(resourceKey string, configPayload interface{}) string { + key := strings.ToLower(strings.TrimSpace(resourceKey)) + switch key { + case "dingtalk-connector", "feishu", "slack", "telegram", "wecom": + return key + } + + config, ok := configPayload.(map[string]interface{}) + if !ok { + return key + } + domain, _ := config["domain"].(string) + switch strings.ToLower(strings.TrimSpace(domain)) { + case "dingtalk", "dingding", "dingtalk-connector": + return "dingtalk-connector" + case "feishu", "lark": + return "feishu" + case "wecom", "wechat-work", "work-wechat", "enterprise-wechat", "qywx": + return "wecom" + } + if accounts, ok := config["accounts"].(map[string]interface{}); ok && len(accounts) > 0 { + return "feishu" + } + if _, hasClientID := config["clientId"].(string); hasClientID { + if _, hasClientSecret := config["clientSecret"].(string); hasClientSecret { + return "dingtalk-connector" + } + } + if _, hasBotID := config["botId"].(string); hasBotID { + if _, hasSecret := config["secret"].(string); hasSecret { + return "wecom" + } + } + + return key +} + +func isTeamMemberLeaderOnlyOpenClawChannel(resource models.OpenClawConfigResource) bool { + return normalizeResourceType(resource.ResourceType) == OpenClawConfigResourceTypeChannel +} + +func cloneOpenClawConfigPlan(plan *OpenClawConfigPlan) *OpenClawConfigPlan { + if plan == nil { + return nil + } + clone := *plan + if plan.ResourceIDs != nil { + clone.ResourceIDs = append([]int(nil), plan.ResourceIDs...) + } + return &clone +} + +// mergeOpenClawChannelConfigForStorage combines the allowlist-normalized config +// with the original parsed payload so storage retains keys the env-render path +// does not surface (e.g. webhook, custom capabilities, additional feishu +// accounts). Normalized keys always override the original; unknown keys pass +// through untouched. For feishu the accounts map is merged member-wise so +// accounts other than main, and sibling fields inside main, both survive. +func mergeOpenClawChannelConfigForStorage(resourceKey string, original, normalized interface{}) interface{} { + normalizedMap, normalizedOk := normalized.(map[string]interface{}) + originalMap, originalOk := original.(map[string]interface{}) + if !normalizedOk { + return normalized + } + if !originalOk { + return normalizedMap + } + + merged := make(map[string]interface{}, len(originalMap)+len(normalizedMap)) + for k, v := range originalMap { + merged[k] = v + } + for k, v := range normalizedMap { + merged[k] = v + } + + if detectOpenClawChannelProvider(resourceKey, original) == "feishu" { + originalAccounts, _ := originalMap["accounts"].(map[string]interface{}) + normalizedAccounts, _ := normalizedMap["accounts"].(map[string]interface{}) + if originalAccounts != nil || normalizedAccounts != nil { + mergedAccounts := make(map[string]interface{}) + for k, v := range originalAccounts { + mergedAccounts[k] = v + } + if normalizedMain, ok := normalizedAccounts["main"].(map[string]interface{}); ok { + mergedMain := make(map[string]interface{}) + if existingMain, ok := originalAccounts["main"].(map[string]interface{}); ok { + for k, v := range existingMain { + mergedMain[k] = v + } + } + for k, v := range normalizedMain { + mergedMain[k] = v + } + mergedAccounts["main"] = mergedMain + } + merged["accounts"] = mergedAccounts + } + } + + return merged +} + +func normalizeFeishuChannelConfigForEnv(configPayload interface{}) map[string]interface{} { + config, ok := configPayload.(map[string]interface{}) + if !ok { + config = map[string]interface{}{} + } + + accounts, _ := config["accounts"].(map[string]interface{}) + normalizedAccounts := make(map[string]interface{}, len(accounts)+1) + for accountKey, rawAccount := range accounts { + accountMap, ok := rawAccount.(map[string]interface{}) + if !ok { + normalizedAccounts[accountKey] = rawAccount + continue + } + normalizedAccount := make(map[string]interface{}, len(accountMap)) + for k, v := range accountMap { + normalizedAccount[k] = v + } + if _, ok := normalizedAccount["appId"].(string); !ok { + normalizedAccount["appId"] = "" + } + if _, ok := normalizedAccount["appSecret"].(string); !ok { + normalizedAccount["appSecret"] = "" + } + normalizedAccounts[accountKey] = normalizedAccount + } + + var account map[string]interface{} + if mainAccount, ok := accounts["main"].(map[string]interface{}); ok { + account = mainAccount + } else if defaultAccount, ok := accounts["default"].(map[string]interface{}); ok { + account = defaultAccount + } else { + account = map[string]interface{}{} + } + + appID, _ := account["appId"].(string) + if appID == "" { + appID, _ = config["appId"].(string) + } + + appSecret, _ := account["appSecret"].(string) + if appSecret == "" { + appSecret, _ = config["appSecret"].(string) + } + if _, ok := normalizedAccounts["main"]; !ok { + normalizedAccounts["main"] = map[string]interface{}{ + "appId": appID, + "appSecret": appSecret, + } + } + + defaultAccount, _ := config["defaultAccount"].(string) + if strings.TrimSpace(defaultAccount) == "" { + defaultAccount = "main" + } + + result := map[string]interface{}{ + "enabled": true, + "domain": "feishu", + "defaultAccount": defaultAccount, + "accounts": normalizedAccounts, + } + if requireMention, ok := config["requireMention"].(bool); ok { + result["requireMention"] = requireMention + } + + return result +} + +func normalizeSlackChannelConfigForEnv(configPayload interface{}) map[string]interface{} { + config, ok := configPayload.(map[string]interface{}) + if !ok { + config = map[string]interface{}{} + } + + botToken, _ := config["botToken"].(string) + appToken, _ := config["appToken"].(string) + groupPolicy, _ := config["groupPolicy"].(string) + if strings.TrimSpace(groupPolicy) == "" { + groupPolicy = "allowlist" + } + + channels, _ := config["channels"].(map[string]interface{}) + if channels == nil { + channels = map[string]interface{}{ + "#general": map[string]interface{}{ + "allow": true, + }, + } + } + + capabilities, _ := config["capabilities"].(map[string]interface{}) + if capabilities == nil { + capabilities = map[string]interface{}{ + "interactiveReplies": true, + } + } + + return map[string]interface{}{ + "enabled": true, + "botToken": botToken, + "appToken": appToken, + "groupPolicy": groupPolicy, + "channels": channels, + "capabilities": capabilities, + } +} + +func normalizeTelegramChannelConfigForEnv(configPayload interface{}) map[string]interface{} { + config, ok := configPayload.(map[string]interface{}) + if !ok { + config = map[string]interface{}{} + } + + botToken, _ := config["botToken"].(string) + dmPolicy, _ := config["dmPolicy"].(string) + if strings.TrimSpace(dmPolicy) == "" { + dmPolicy = "open" + } + + allowFrom := normalizeStringArrayForEnv(config["allowFrom"]) + if len(allowFrom) == 0 { + allowFrom = []string{"*"} + } + + return map[string]interface{}{ + "enabled": true, + "botToken": botToken, + "dmPolicy": dmPolicy, + "allowFrom": allowFrom, + } +} + +func normalizeDingTalkChannelConfigForEnv(configPayload interface{}) map[string]interface{} { + config, ok := configPayload.(map[string]interface{}) + if !ok { + config = map[string]interface{}{} + } + + clientID, _ := config["clientId"].(string) + clientSecret, _ := config["clientSecret"].(string) + + allowFrom := normalizeStringArrayForEnv(config["allowFrom"]) + if len(allowFrom) == 0 { + allowFrom = []string{"*"} + } + + return map[string]interface{}{ + "enabled": true, + "clientId": clientID, + "clientSecret": clientSecret, + "allowFrom": allowFrom, + } +} + +func normalizeWeComChannelConfigForEnv(configPayload interface{}) map[string]interface{} { + config, ok := configPayload.(map[string]interface{}) + if !ok { + config = map[string]interface{}{} + } + + botID, _ := config["botId"].(string) + secret, _ := config["secret"].(string) + dmPolicy, _ := config["dmPolicy"].(string) + if strings.TrimSpace(dmPolicy) == "" { + dmPolicy = "pairing" + } + + allowFrom := normalizeStringArrayForEnv(config["allowFrom"]) + if len(allowFrom) == 0 { + allowFrom = []string{"*"} + } + + return map[string]interface{}{ + "botId": botID, + "secret": secret, + "dmPolicy": dmPolicy, + "allowFrom": allowFrom, + } +} + +func normalizeStringArrayForEnv(value interface{}) []string { + items, ok := value.([]interface{}) + if !ok { + return nil + } + + result := make([]string, 0, len(items)) + for _, item := range items { + text, ok := item.(string) + if !ok { + continue + } + result = append(result, text) + } + return result +} + +func normalizeOpenClawResourceContent(resourceType, resourceKey string, raw json.RawMessage) (json.RawMessage, error) { + if normalizeResourceType(resourceType) != OpenClawConfigResourceTypeChannel { + return raw, nil + } + + envelope, err := parseOpenClawEnvelope(resourceType, raw) + if err != nil { + return nil, err + } + + var configPayload interface{} + if err := json.Unmarshal(envelope.Config, &configPayload); err != nil { + return nil, fmt.Errorf("failed to parse openclaw channel config") + } + + normalizedConfig := normalizeOpenClawChannelConfigForEnv(resourceKey, configPayload) + // Preserve unknown fields at storage time: the *ForEnv helpers rebuild the + // config from a known-field allowlist, which is correct for rendering runtime + // env but would silently drop tenant-authored keys (e.g. webhook, custom + // capabilities, additional feishu accounts) if applied verbatim to stored + // content. Merge the allowlist output back over the original payload so + // storage is a superset of the normalized render. + mergedConfig := mergeOpenClawChannelConfigForStorage(resourceKey, configPayload, normalizedConfig) + envelope.Format = openClawChannelFormat(resourceKey) + envelope.Config, err = json.Marshal(mergedConfig) + if err != nil { + return nil, fmt.Errorf("failed to marshal normalized openclaw channel config") + } + + normalizedEnvelope, err := json.Marshal(envelope) + if err != nil { + return nil, fmt.Errorf("failed to marshal normalized openclaw content") + } + + return normalizedEnvelope, nil +} + +func compiledResourceFromModel(item models.OpenClawConfigResource) (compiledOpenClawResource, error) { + tags, err := decodeStringArray(item.TagsJSON) + if err != nil { + return compiledOpenClawResource{}, fmt.Errorf("failed to parse openclaw config tags") + } + + envelope, err := parseOpenClawEnvelope(item.ResourceType, json.RawMessage(item.ContentJSON)) + if err != nil { + return compiledOpenClawResource{}, err + } + + return compiledOpenClawResource{ + model: item, + tags: tags, + envelope: envelope, + }, nil +} + +func parseOpenClawEnvelope(resourceType string, raw json.RawMessage) (OpenClawConfigEnvelope, error) { + if len(raw) == 0 { + return OpenClawConfigEnvelope{}, fmt.Errorf("openclaw config content is required") + } + + var envelope OpenClawConfigEnvelope + if err := json.Unmarshal(raw, &envelope); err != nil { + return OpenClawConfigEnvelope{}, fmt.Errorf("openclaw config content must be valid JSON") + } + if envelope.SchemaVersion <= 0 { + return OpenClawConfigEnvelope{}, fmt.Errorf("openclaw config schemaVersion is required") + } + if canonicalizeOpenClawKind(envelope.Kind) != normalizeResourceType(resourceType) { + return OpenClawConfigEnvelope{}, fmt.Errorf("openclaw config kind does not match resource type") + } + if strings.TrimSpace(envelope.Format) == "" { + return OpenClawConfigEnvelope{}, fmt.Errorf("openclaw config format is required") + } + if len(envelope.Config) == 0 { + return OpenClawConfigEnvelope{}, fmt.Errorf("openclaw config config payload is required") + } + return envelope, nil +} + +func hasOpenClawConfigSelections(plan OpenClawConfigPlan) bool { + switch normalizePlanMode(plan.Mode) { + case OpenClawConfigPlanModeBundle: + return plan.BundleID != nil && *plan.BundleID > 0 + case OpenClawConfigPlanModeManual: + return len(plan.ResourceIDs) > 0 + default: + return false + } +} + +func normalizePlanMode(mode string) string { + return strings.ToLower(strings.TrimSpace(mode)) +} + +func normalizeResourceType(resourceType string) string { + value := strings.TrimSpace(strings.ToLower(resourceType)) + value = strings.ReplaceAll(value, "-", "_") + value = strings.ReplaceAll(value, " ", "_") + return value +} + +func normalizeResourceKey(resourceKey string) string { + return strings.TrimSpace(resourceKey) +} + +func openClawChannelFormat(resourceKey string) string { + key := normalizeResourceKey(resourceKey) + if key == "" { + return "channel/custom@v1" + } + return fmt.Sprintf("channel/%s@v1", key) +} + +func canonicalizeOpenClawKind(kind string) string { + value := normalizeResourceType(kind) + switch value { + case "sessiontemplate": + return OpenClawConfigResourceTypeSessionTemplate + case "logpolicy": + return OpenClawConfigResourceTypeLogPolicy + case "scheduledtask": + return OpenClawConfigResourceTypeScheduledTask + default: + return value + } +} + +func isValidOpenClawResourceType(resourceType string) bool { + _, ok := openClawAllowedResourceTypes[normalizeResourceType(resourceType)] + return ok +} + +func normalizeTags(tags []string) []string { + result := make([]string, 0, len(tags)) + seen := map[string]struct{}{} + for _, tag := range tags { + trimmed := strings.TrimSpace(tag) + if trimmed == "" { + continue + } + if _, exists := seen[trimmed]; exists { + continue + } + seen[trimmed] = struct{}{} + result = append(result, trimmed) + } + sort.Strings(result) + return result +} + +func encodeStringArray(values []string) string { + if len(values) == 0 { + return "[]" + } + raw, _ := json.Marshal(values) + return string(raw) +} + +func decodeStringArray(raw string) ([]string, error) { + if strings.TrimSpace(raw) == "" { + return []string{}, nil + } + var values []string + if err := json.Unmarshal([]byte(raw), &values); err != nil { + return nil, err + } + return values, nil +} + +func marshalJSONString(value interface{}) (string, error) { + raw, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("failed to marshal openclaw payload") + } + return string(raw), nil +} + +func openClawResourceRef(resourceType, resourceKey string) string { + return fmt.Sprintf("%s:%s", normalizeResourceType(resourceType), normalizeResourceKey(resourceKey)) +} + +func summarizeCompiledResources(resources []compiledOpenClawResource) []OpenClawConfigResourceSummary { + result := make([]OpenClawConfigResourceSummary, 0, len(resources)) + for _, resource := range resources { + result = append(result, resourceSummaryFromModel(resource.model)) + } + return result +} + +func sortCompiledResources(items []compiledOpenClawResource) { + priority := map[string]int{} + for idx, resourceType := range openClawResourceTypeOrder { + priority[resourceType] = idx + } + + sort.SliceStable(items, func(i, j int) bool { + leftType := normalizeResourceType(items[i].model.ResourceType) + rightType := normalizeResourceType(items[j].model.ResourceType) + if priority[leftType] != priority[rightType] { + return priority[leftType] < priority[rightType] + } + if items[i].model.ResourceKey != items[j].model.ResourceKey { + return items[i].model.ResourceKey < items[j].model.ResourceKey + } + return items[i].model.ID < items[j].model.ID + }) +} + +func sortedEnvNames(values map[string]string) []string { + names := make([]string, 0, len(values)) + for key := range values { + names = append(names, key) + } + sort.Strings(names) + return names +} + +func normalizeOptionalString(value *string) *string { + if value == nil { + return nil + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return nil + } + return &trimmed +} + +func errorString(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +// cascadeSnapshotsForResource recompiles all active snapshots that reference +// the given resource ID, so that their rendered_env_json reflects the latest +// resource content. It does NOT restart instances; the new env takes effect +// on the next instance restart. +func (s *openClawConfigService) cascadeSnapshotsForResource(userID, resourceID int) { + snapshots, err := s.repo.ListActiveSnapshots(userID) + if err != nil { + log.Printf("[cascade] failed to list active snapshots for user %d: %v", userID, err) + return + } + + for _, snap := range snapshots { + if !snapshotReferencesResource(snap, resourceID) { + continue + } + + // Record the version stamp at read time for CAS. + readVersion := snap.UpdatedAt + + plan, err := planFromSnapshot(snap) + if err != nil { + log.Printf("[cascade] snapshot %d: failed to reconstruct plan: %v", snap.ID, err) + continue + } + + compiled, err := s.compilePlan(userID, plan) + if err != nil { + log.Printf("[cascade] snapshot %d: recompile failed: %v", snap.ID, err) + continue + } + + envJSON, err := marshalJSONString(compiled.renderedEnv) + if err != nil { + log.Printf("[cascade] snapshot %d: marshal env failed: %v", snap.ID, err) + continue + } + + resolvedSummaries := make([]OpenClawConfigResourceSummary, 0, len(compiled.resolved)) + for _, r := range compiled.resolved { + resolvedSummaries = append(resolvedSummaries, resourceSummaryFromModel(r.model)) + } + resolvedJSON, err := marshalJSONString(resolvedSummaries) + if err != nil { + log.Printf("[cascade] snapshot %d: marshal resolved failed: %v", snap.ID, err) + continue + } + + snap.RenderedEnvJSON = envJSON + snap.ResolvedResourcesJSON = resolvedJSON + snap.RenderedManifestJSON = compiled.manifest + snap.UpdatedAt = time.Now() + + // CAS write: only succeeds if updated_at has not changed since our read. + ok, err := s.repo.UpdateSnapshotIfUnchanged(&snap, readVersion) + if err != nil { + log.Printf("[cascade] snapshot %d: update failed: %v", snap.ID, err) + } else if !ok { + log.Printf("[cascade] snapshot %d: skipped; already updated by a newer cascade", snap.ID) + } else { + log.Printf("[cascade] snapshot %d: refreshed successfully", snap.ID) + } + } +} + +// snapshotReferencesResource checks whether a snapshot references the given +// resource ID. It checks ResolvedResourcesJSON first (which contains the full +// set of selected + dependsOn resources), falling back to SelectedResourceIDsJSON +// for backward compatibility with older snapshots. +func snapshotReferencesResource(snap models.OpenClawInjectionSnapshot, resourceID int) bool { + // Primary: check ResolvedResourcesJSON (contains selected + dependsOn full set). + if strings.TrimSpace(snap.ResolvedResourcesJSON) != "" { + var summaries []struct { + ID int `json:"id"` + } + if err := json.Unmarshal([]byte(snap.ResolvedResourcesJSON), &summaries); err == nil { + for _, s := range summaries { + if s.ID == resourceID { + return true + } + } + return false + } + // If ResolvedResourcesJSON is present but malformed, fall through to fallback. + } + + // Fallback: check SelectedResourceIDsJSON for older snapshots. + if strings.TrimSpace(snap.SelectedResourceIDsJSON) == "" { + return false + } + var ids []int + if err := json.Unmarshal([]byte(snap.SelectedResourceIDsJSON), &ids); err != nil { + return false + } + for _, id := range ids { + if id == resourceID { + return true + } + } + return false +} + +// planFromSnapshot reconstructs an OpenClawConfigPlan from a snapshot's stored +// mode, bundle_id, and selected_resource_ids_json. +func planFromSnapshot(snap models.OpenClawInjectionSnapshot) (OpenClawConfigPlan, error) { + plan := OpenClawConfigPlan{ + Mode: snap.Mode, + BundleID: snap.BundleID, + } + if snap.Mode == OpenClawConfigPlanModeManual { + var ids []int + if err := json.Unmarshal([]byte(snap.SelectedResourceIDsJSON), &ids); err != nil { + return plan, fmt.Errorf("failed to parse selected resource ids: %w", err) + } + plan.ResourceIDs = ids + } + return plan, nil +} diff --git a/backend/internal/services/openclaw_config_service_test.go b/backend/internal/services/openclaw_config_service_test.go new file mode 100644 index 0000000..f0aabb9 --- /dev/null +++ b/backend/internal/services/openclaw_config_service_test.go @@ -0,0 +1,785 @@ +package services + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "clawreef/internal/models" +) + +func TestRenderCompiledOpenClawPayloadRendersChannelsAsKeyedConfigMap(t *testing.T) { + t.Parallel() + + resources := []compiledOpenClawResource{ + { + model: models.OpenClawConfigResource{ + ID: 1, + ResourceType: OpenClawConfigResourceTypeChannel, + ResourceKey: "dingtalk-connector", + Name: "DingTalk", + Version: 1, + ContentJSON: `{"schemaVersion":1,"kind":"channel","format":"channel/dingtalk-connector@v1","dependsOn":[],"config":{"enabled":false,"clientId":"ding-xxxxxxxxxxxxxx","clientSecret":"xxxxxxxxxxxxxxxxxxxxxxx","allowFrom":["*"],"legacyField":"drop-me"}}`, + }, + envelope: OpenClawConfigEnvelope{ + SchemaVersion: 1, + Kind: "channel", + Format: "channel/dingtalk-connector@v1", + Config: json.RawMessage(`{"enabled":false,"clientId":"ding-xxxxxxxxxxxxxx","clientSecret":"xxxxxxxxxxxxxxxxxxxxxxx","allowFrom":["*"],"legacyField":"drop-me"}`), + }, + }, + { + model: models.OpenClawConfigResource{ + ID: 2, + ResourceType: OpenClawConfigResourceTypeChannel, + ResourceKey: "feishu", + Name: "Feishu", + Version: 1, + ContentJSON: `{"schemaVersion":1,"kind":"channel","format":"channel/feishu@v1","dependsOn":[],"config":{"enabled":false,"domain":"feishu","appId":"cli_top","appSecret":"top_secret","defaultAccount":"default","accounts":{"default":{"appId":"cli_xxx","appSecret":"xxx","botName":"old-bot","enabled":true}},"requireMention":true}}`, + }, + envelope: OpenClawConfigEnvelope{ + SchemaVersion: 1, + Kind: "channel", + Format: "channel/feishu@v1", + Config: json.RawMessage(`{"enabled":false,"domain":"feishu","appId":"cli_top","appSecret":"top_secret","defaultAccount":"default","accounts":{"default":{"appId":"cli_xxx","appSecret":"xxx","botName":"old-bot","enabled":true}},"requireMention":true}`), + }, + }, + { + model: models.OpenClawConfigResource{ + ID: 3, + ResourceType: OpenClawConfigResourceTypeChannel, + ResourceKey: "feishu-ops", + Name: "Feishu Ops", + Version: 1, + ContentJSON: `{"schemaVersion":1,"kind":"channel","format":"channel/feishu-ops@v1","dependsOn":[],"config":{"enabled":true,"domain":"feishu","defaultAccount":"main","accounts":{"main":{"appId":"cli_ops","appSecret":"ops_secret"}}}}`, + }, + envelope: OpenClawConfigEnvelope{ + SchemaVersion: 1, + Kind: "channel", + Format: "channel/feishu-ops@v1", + Config: json.RawMessage(`{"enabled":true,"domain":"feishu","defaultAccount":"main","accounts":{"main":{"appId":"cli_ops","appSecret":"ops_secret"}}}`), + }, + }, + { + model: models.OpenClawConfigResource{ + ID: 4, + ResourceType: OpenClawConfigResourceTypeChannel, + ResourceKey: "slack", + Name: "Slack", + Version: 1, + ContentJSON: `{"schemaVersion":1,"kind":"channel","format":"channel/slack@v1","dependsOn":[],"config":{"enabled":false,"botToken":"xoxb-xxx","appToken":"xapp-xxx","groupPolicy":"allowlist","channels":{"#general":{"allow":true}},"capabilities":{"interactiveReplies":true}}}`, + }, + envelope: OpenClawConfigEnvelope{ + SchemaVersion: 1, + Kind: "channel", + Format: "channel/slack@v1", + Config: json.RawMessage(`{"enabled":false,"botToken":"xoxb-xxx","appToken":"xapp-xxx","groupPolicy":"allowlist","channels":{"#general":{"allow":true}},"capabilities":{"interactiveReplies":true}}`), + }, + }, + { + model: models.OpenClawConfigResource{ + ID: 5, + ResourceType: OpenClawConfigResourceTypeChannel, + ResourceKey: "telegram", + Name: "Telegram", + Version: 1, + ContentJSON: `{"schemaVersion":1,"kind":"channel","format":"channel/telegram@v1","dependsOn":[],"config":{"enabled":false,"botToken":"123456:xxx","dmPolicy":"open","allowFrom":["*"],"network":{"autoSelectFamily":false}}}`, + }, + envelope: OpenClawConfigEnvelope{ + SchemaVersion: 1, + Kind: "channel", + Format: "channel/telegram@v1", + Config: json.RawMessage(`{"enabled":false,"botToken":"123456:xxx","dmPolicy":"open","allowFrom":["*"],"network":{"autoSelectFamily":false}}`), + }, + }, + { + model: models.OpenClawConfigResource{ + ID: 6, + ResourceType: OpenClawConfigResourceTypeChannel, + ResourceKey: "wecom", + Name: "WeCom", + Version: 1, + ContentJSON: `{"schemaVersion":1,"kind":"channel","format":"channel/wecom@v1","dependsOn":[],"config":{"enabled":false,"botId":"ww-bot","secret":"wecom-secret","dmPolicy":"pairing","allowFrom":["*"],"legacyField":"drop-me"}}`, + }, + envelope: OpenClawConfigEnvelope{ + SchemaVersion: 1, + Kind: "channel", + Format: "channel/wecom@v1", + Config: json.RawMessage(`{"enabled":false,"botId":"ww-bot","secret":"wecom-secret","dmPolicy":"pairing","allowFrom":["*"],"legacyField":"drop-me"}`), + }, + }, + { + model: models.OpenClawConfigResource{ + ID: 7, + ResourceType: OpenClawConfigResourceTypeSkill, + ResourceKey: "support-bot", + Name: "Support Bot", + Version: 1, + ContentJSON: `{"schemaVersion":1,"kind":"skill","format":"skill/custom@v1","dependsOn":[],"config":{"prompt":"help"}}`, + }, + tags: []string{"skill"}, + envelope: OpenClawConfigEnvelope{ + SchemaVersion: 1, + Kind: "skill", + Format: "skill/custom@v1", + Config: json.RawMessage(`{"prompt":"help"}`), + }, + }, + } + + renderedEnv, _, _, _, err := renderCompiledOpenClawPayload(OpenClawConfigPlan{Mode: OpenClawConfigPlanModeManual}, nil, resources) + if err != nil { + t.Fatalf("renderCompiledOpenClawPayload returned error: %v", err) + } + + gotChannels := renderedEnv[OpenClawChannelsEnv] + wantChannels := `{"dingtalk-connector":{"allowFrom":["*"],"clientId":"ding-xxxxxxxxxxxxxx","clientSecret":"xxxxxxxxxxxxxxxxxxxxxxx","enabled":true},"feishu":{"accounts":{"default":{"appId":"cli_xxx","appSecret":"xxx","botName":"old-bot","enabled":true},"feishu-ops":{"appId":"cli_ops","appSecret":"ops_secret"},"main":{"appId":"cli_xxx","appSecret":"xxx"}},"defaultAccount":"default","domain":"feishu","enabled":true,"requireMention":true},"slack":{"appToken":"xapp-xxx","botToken":"xoxb-xxx","capabilities":{"interactiveReplies":true},"channels":{"#general":{"allow":true}},"enabled":true,"groupPolicy":"allowlist"},"telegram":{"allowFrom":["*"],"botToken":"123456:xxx","dmPolicy":"open","enabled":true},"wecom":{"allowFrom":["*"],"botId":"ww-bot","dmPolicy":"pairing","secret":"wecom-secret"}}` + if gotChannels != wantChannels { + t.Fatalf("unexpected channel payload:\nwant: %s\ngot: %s", wantChannels, gotChannels) + } + + gotSkills := renderedEnv[OpenClawSkillsEnv] + wantSkills := `{"items":[{"content":{"schemaVersion":1,"kind":"skill","format":"skill/custom@v1","dependsOn":[],"config":{"prompt":"help"}},"id":7,"key":"support-bot","name":"Support Bot","tags":["skill"],"type":"skill","version":1}],"schemaVersion":1}` + if gotSkills != wantSkills { + t.Fatalf("unexpected skill payload:\nwant: %s\ngot: %s", wantSkills, gotSkills) + } +} + +func TestRuntimeBootstrapEnvValuesAddsHermesAndRuntimeAliases(t *testing.T) { + env := map[string]string{ + OpenClawChannelsEnv: `{"slack":{"enabled":true}}`, + OpenClawSkillsEnv: `{"schemaVersion":1,"items":[]}`, + OpenClawBootstrapManifestEnv: `{"schemaVersion":1,"payloads":[{"env":"CLAWMANAGER_OPENCLAW_CHANNELS_JSON","count":1},{"env":"CLAWMANAGER_OPENCLAW_SKILLS_JSON","count":0}]}`, + } + + got := runtimeBootstrapEnvValues("hermes", env) + + if got[HermesChannelsEnv] != env[OpenClawChannelsEnv] { + t.Fatalf("expected Hermes channels alias to mirror OpenClaw channels") + } + if got[RuntimeSkillsEnv] != env[OpenClawSkillsEnv] { + t.Fatalf("expected runtime skills alias to mirror OpenClaw skills") + } + if got[OpenClawChannelsEnv] != env[OpenClawChannelsEnv] { + t.Fatalf("expected original OpenClaw channels env to be preserved") + } + if !strings.Contains(got[HermesBootstrapManifestEnv], HermesChannelsEnv) { + t.Fatalf("expected Hermes manifest alias to reference Hermes env names, got %s", got[HermesBootstrapManifestEnv]) + } + if !strings.Contains(got[RuntimeBootstrapManifestEnv], RuntimeChannelsEnv) { + t.Fatalf("expected runtime manifest alias to reference runtime env names, got %s", got[RuntimeBootstrapManifestEnv]) + } +} + +func TestOpenClawChannelEnvKeyUsesProviderKeyForResourceAliases(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + resourceKey string + config map[string]interface{} + want string + }{ + { + name: "feishu resource alias", + resourceKey: "feishu-2", + config: map[string]interface{}{ + "domain": "feishu", + "accounts": map[string]interface{}{ + "main": map[string]interface{}{ + "appId": "cli_xxx", + "appSecret": "secret", + }, + }, + }, + want: "feishu", + }, + { + name: "dingtalk resource alias", + resourceKey: "dingtalk-2", + config: map[string]interface{}{ + "clientId": "ding_xxx", + "clientSecret": "secret", + }, + want: "dingtalk-connector", + }, + { + name: "wecom resource alias", + resourceKey: "wecom-2", + config: map[string]interface{}{ + "botId": "ww-bot", + "secret": "secret", + }, + want: "wecom", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := openClawChannelEnvKey(tt.resourceKey, tt.config); got != tt.want { + t.Fatalf("unexpected env key: want %s, got %s", tt.want, got) + } + }) + } +} + +func TestResourcePayloadFromModelNormalizesStoredChannelJSON(t *testing.T) { + t.Parallel() + + item := models.OpenClawConfigResource{ + ID: 10, + UserID: 1, + ResourceType: OpenClawConfigResourceTypeChannel, + ResourceKey: "slack", + Name: "Slack", + Enabled: true, + Version: 1, + TagsJSON: `["channel","builtin","slack"]`, + ContentJSON: `{"schemaVersion":1,"kind":"channel","format":"channel/slack@v1","dependsOn":[],"config":{"enabled":false,"botToken":"xoxb-xxxxxxxxx","appToken":"xapp-xxxxxxxxxxxxxx","groupPolicy":"allowlist","channels":{"#general":{"allow":true}},"capabilities":{"interactiveReplies":true},"legacyField":"drop-me"}}`, + } + + payload, err := resourcePayloadFromModel(item) + if err != nil { + t.Fatalf("resourcePayloadFromModel returned error: %v", err) + } + + got := string(payload.Content) + want := `{"schemaVersion":1,"kind":"channel","format":"channel/slack@v1","dependsOn":[],"config":{"enabled":true,"botToken":"xoxb-xxxxxxxxx","appToken":"xapp-xxxxxxxxxxxxxx","groupPolicy":"allowlist","channels":{"#general":{"allow":true}},"capabilities":{"interactiveReplies":true},"legacyField":"drop-me"}}` + + var gotJSON interface{} + if err := json.Unmarshal([]byte(got), &gotJSON); err != nil { + t.Fatalf("failed to unmarshal normalized resource content: %v", err) + } + + var wantJSON interface{} + if err := json.Unmarshal([]byte(want), &wantJSON); err != nil { + t.Fatalf("failed to unmarshal expected normalized resource content: %v", err) + } + + if !reflect.DeepEqual(gotJSON, wantJSON) { + t.Fatalf("unexpected normalized resource content:\nwant: %s\ngot: %s", want, got) + } +} + +func TestResourcePayloadFromModelNormalizesStoredDingTalkChannelJSON(t *testing.T) { + t.Parallel() + + item := models.OpenClawConfigResource{ + ID: 11, + UserID: 1, + ResourceType: OpenClawConfigResourceTypeChannel, + ResourceKey: "dingtalk-connector", + Name: "DingTalk", + Enabled: true, + Version: 1, + TagsJSON: `["channel","builtin","dingtalk-connector"]`, + ContentJSON: `{"schemaVersion":1,"kind":"channel","format":"channel/dingtalk-connector@v1","dependsOn":[],"config":{"enabled":false,"clientId":"ding-xxxxxxxxxxxxxx","clientSecret":"xxxxxxxxxxxxxxxxxxxxxxx","allowFrom":[],"legacyField":"drop-me"}}`, + } + + payload, err := resourcePayloadFromModel(item) + if err != nil { + t.Fatalf("resourcePayloadFromModel returned error: %v", err) + } + + got := string(payload.Content) + want := `{"schemaVersion":1,"kind":"channel","format":"channel/dingtalk-connector@v1","dependsOn":[],"config":{"enabled":true,"clientId":"ding-xxxxxxxxxxxxxx","clientSecret":"xxxxxxxxxxxxxxxxxxxxxxx","allowFrom":["*"],"legacyField":"drop-me"}}` + + var gotJSON interface{} + if err := json.Unmarshal([]byte(got), &gotJSON); err != nil { + t.Fatalf("failed to unmarshal normalized resource content: %v", err) + } + + var wantJSON interface{} + if err := json.Unmarshal([]byte(want), &wantJSON); err != nil { + t.Fatalf("failed to unmarshal expected normalized resource content: %v", err) + } + + if !reflect.DeepEqual(gotJSON, wantJSON) { + t.Fatalf("unexpected normalized resource content:\nwant: %s\ngot: %s", want, got) + } +} + +func TestResourcePayloadFromModelNormalizesStoredWeComChannelJSON(t *testing.T) { + t.Parallel() + + item := models.OpenClawConfigResource{ + ID: 12, + UserID: 1, + ResourceType: OpenClawConfigResourceTypeChannel, + ResourceKey: "wecom", + Name: "WeCom", + Enabled: true, + Version: 1, + TagsJSON: `["channel","builtin","wecom"]`, + ContentJSON: `{"schemaVersion":1,"kind":"channel","format":"channel/wecom@v1","dependsOn":[],"config":{"enabled":false,"botId":"ww-bot","secret":"wecom-secret","allowFrom":[],"legacyField":"keep-me"}}`, + } + + payload, err := resourcePayloadFromModel(item) + if err != nil { + t.Fatalf("resourcePayloadFromModel returned error: %v", err) + } + + got := string(payload.Content) + want := `{"schemaVersion":1,"kind":"channel","format":"channel/wecom@v1","dependsOn":[],"config":{"enabled":false,"botId":"ww-bot","secret":"wecom-secret","allowFrom":["*"],"legacyField":"keep-me","dmPolicy":"pairing"}}` + + var gotJSON interface{} + if err := json.Unmarshal([]byte(got), &gotJSON); err != nil { + t.Fatalf("failed to unmarshal normalized resource content: %v", err) + } + + var wantJSON interface{} + if err := json.Unmarshal([]byte(want), &wantJSON); err != nil { + t.Fatalf("failed to unmarshal expected normalized resource content: %v", err) + } + + if !reflect.DeepEqual(gotJSON, wantJSON) { + t.Fatalf("unexpected normalized resource content:\nwant: %s\ngot: %s", want, got) + } +} + +func intPtr(v int) *int { return &v } + +func TestSnapshotReferencesResource(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + snap models.OpenClawInjectionSnapshot + resourceID int + want bool + }{ + { + name: "matching ID in SelectedResourceIDsJSON", + snap: models.OpenClawInjectionSnapshot{SelectedResourceIDsJSON: `[8, 15, 22]`}, + resourceID: 15, + want: true, + }, + { + name: "single matching ID", + snap: models.OpenClawInjectionSnapshot{SelectedResourceIDsJSON: `[8]`}, + resourceID: 8, + want: true, + }, + { + name: "no match", + snap: models.OpenClawInjectionSnapshot{SelectedResourceIDsJSON: `[8, 15, 22]`}, + resourceID: 99, + want: false, + }, + { + name: "empty JSON", + snap: models.OpenClawInjectionSnapshot{SelectedResourceIDsJSON: ""}, + resourceID: 1, + want: false, + }, + { + name: "whitespace only", + snap: models.OpenClawInjectionSnapshot{SelectedResourceIDsJSON: " "}, + resourceID: 1, + want: false, + }, + { + name: "invalid JSON", + snap: models.OpenClawInjectionSnapshot{SelectedResourceIDsJSON: `broken`}, + resourceID: 1, + want: false, + }, + // v2: ResolvedResourcesJSON cases + { + name: "matching ID only in ResolvedResourcesJSON (indirect dependency)", + snap: models.OpenClawInjectionSnapshot{ + SelectedResourceIDsJSON: `[10]`, + ResolvedResourcesJSON: `[{"id":10,"type":"agent","key":"bot","name":"Bot","version":1},{"id":20,"type":"skill","key":"helper","name":"Helper","version":1}]`, + }, + resourceID: 20, + want: true, + }, + { + name: "no match in ResolvedResourcesJSON", + snap: models.OpenClawInjectionSnapshot{ + SelectedResourceIDsJSON: `[10]`, + ResolvedResourcesJSON: `[{"id":10,"type":"agent","key":"bot","name":"Bot","version":1}]`, + }, + resourceID: 99, + want: false, + }, + { + name: "ResolvedResourcesJSON malformed — fallback to SelectedResourceIDsJSON succeeds", + snap: models.OpenClawInjectionSnapshot{ + SelectedResourceIDsJSON: `[5, 10]`, + ResolvedResourcesJSON: `not-valid-json`, + }, + resourceID: 10, + want: true, + }, + { + name: "ResolvedResourcesJSON malformed — fallback to SelectedResourceIDsJSON no match", + snap: models.OpenClawInjectionSnapshot{ + SelectedResourceIDsJSON: `[5, 10]`, + ResolvedResourcesJSON: `not-valid-json`, + }, + resourceID: 99, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := snapshotReferencesResource(tt.snap, tt.resourceID) + if got != tt.want { + t.Errorf("snapshotReferencesResource() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPlanFromSnapshot(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + snap models.OpenClawInjectionSnapshot + want OpenClawConfigPlan + wantErr bool + }{ + { + name: "bundle mode", + snap: models.OpenClawInjectionSnapshot{ + Mode: "bundle", + BundleID: intPtr(5), + SelectedResourceIDsJSON: `[8, 15]`, + }, + want: OpenClawConfigPlan{ + Mode: "bundle", + BundleID: intPtr(5), + }, + }, + { + name: "manual mode", + snap: models.OpenClawInjectionSnapshot{ + Mode: "manual", + SelectedResourceIDsJSON: `[8, 15, 22]`, + }, + want: OpenClawConfigPlan{ + Mode: "manual", + ResourceIDs: []int{8, 15, 22}, + }, + }, + { + name: "manual mode with invalid JSON", + snap: models.OpenClawInjectionSnapshot{ + Mode: "manual", + SelectedResourceIDsJSON: `broken`, + }, + wantErr: true, + }, + { + name: "none mode", + snap: models.OpenClawInjectionSnapshot{ + Mode: "none", + }, + want: OpenClawConfigPlan{ + Mode: "none", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := planFromSnapshot(tt.snap) + if (err != nil) != tt.wantErr { + t.Fatalf("planFromSnapshot() error = %v, wantErr %v", err, tt.wantErr) + } + if err != nil { + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("planFromSnapshot() = %+v, want %+v", got, tt.want) + } + }) + } + +} + +// TestNormalizeOpenClawResourceContentPreservesUnknownChannelFields asserts +// that write-time normalization keeps tenant-authored keys that the runtime +// env-render allowlist does not surface, for every supported channel editor. +// Regression guard for the "save drops unknown fields" data-loss bug. +func TestNormalizeOpenClawResourceContentPreservesUnknownChannelFields(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + resourceKey string + content string + wantKeep map[string]interface{} + }{ + { + name: "telegram preserves webhook and custom capabilities", + resourceKey: "telegram", + content: `{"schemaVersion":1,"kind":"channel","format":"channel/telegram@v1","dependsOn":[],"config":{"enabled":true,"botToken":"123:abc","dmPolicy":"open","allowFrom":["*"],"webhook":"https://example.com/hook","capabilities":{"interactiveReplies":true}}}`, + wantKeep: map[string]interface{}{ + "webhook": "https://example.com/hook", + "capabilities": map[string]interface{}{"interactiveReplies": true}, + }, + }, + { + name: "dingtalk-connector preserves dmPolicy and webhook", + resourceKey: "dingtalk-connector", + content: `{"schemaVersion":1,"kind":"channel","format":"channel/dingtalk-connector@v1","dependsOn":[],"config":{"enabled":true,"clientId":"ding-x","clientSecret":"sec","allowFrom":["*"],"dmPolicy":"closed","webhook":"https://example.com/hook"}}`, + wantKeep: map[string]interface{}{ + "dmPolicy": "closed", + "webhook": "https://example.com/hook", + }, + }, + { + name: "slack preserves allowFrom and dmPolicy", + resourceKey: "slack", + content: `{"schemaVersion":1,"kind":"channel","format":"channel/slack@v1","dependsOn":[],"config":{"enabled":true,"botToken":"xoxb-x","appToken":"xapp-x","groupPolicy":"allowlist","channels":{"#general":{"allow":true}},"capabilities":{"interactiveReplies":true},"allowFrom":["C123","C456"],"dmPolicy":"closed"}}`, + wantKeep: map[string]interface{}{ + "allowFrom": []interface{}{"C123", "C456"}, + "dmPolicy": "closed", + }, + }, + { + name: "wecom preserves webhook and custom capabilities", + resourceKey: "wecom", + content: `{"schemaVersion":1,"kind":"channel","format":"channel/wecom@v1","dependsOn":[],"config":{"enabled":true,"botId":"ww-bot","secret":"sec","dmPolicy":"pairing","allowFrom":[],"webhook":"https://example.com/wecom","capabilities":{"interactiveReplies":true}}}`, + wantKeep: map[string]interface{}{ + "webhook": "https://example.com/wecom", + "capabilities": map[string]interface{}{"interactiveReplies": true}, + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + normalized, err := normalizeOpenClawResourceContent(OpenClawConfigResourceTypeChannel, tc.resourceKey, json.RawMessage(tc.content)) + if err != nil { + t.Fatalf("normalizeOpenClawResourceContent returned error: %v", err) + } + var env OpenClawConfigEnvelope + if err := json.Unmarshal(normalized, &env); err != nil { + t.Fatalf("failed to unmarshal normalized envelope: %v", err) + } + var config map[string]interface{} + if err := json.Unmarshal(env.Config, &config); err != nil { + t.Fatalf("failed to unmarshal normalized config: %v", err) + } + for k, want := range tc.wantKeep { + got, ok := config[k] + if !ok { + t.Fatalf("expected key %q to survive normalization, got dropped; full config: %s", k, env.Config) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("key %q mismatched after normalization:\nwant: %v\ngot: %v", k, want, got) + } + } + }) + } +} + +// TestNormalizeOpenClawResourceContentPreservesFeishuSiblingAccounts asserts +// the deep-merge behavior of the feishu accounts map: accounts other than +// "main" survive, and sibling fields inside accounts.main survive too. The +// main account's allowlisted fields (appId, appSecret) are still normalized. +func TestNormalizeOpenClawResourceContentPreservesFeishuSiblingAccounts(t *testing.T) { + t.Parallel() + + input := `{"schemaVersion":1,"kind":"channel","format":"channel/feishu@v1","dependsOn":[],"config":{"enabled":true,"accounts":{"main":{"appId":"cli_main","appSecret":"main_sec","verificationToken":"vt_keep"},"default":{"appId":"cli_default","appSecret":"default_sec"},"acme":{"appId":"cli_acme","appSecret":"acme_sec","botName":"acme-bot"}},"requireMention":true}}` + + normalized, err := normalizeOpenClawResourceContent(OpenClawConfigResourceTypeChannel, "feishu", json.RawMessage(input)) + if err != nil { + t.Fatalf("normalizeOpenClawResourceContent returned error: %v", err) + } + + var env OpenClawConfigEnvelope + if err := json.Unmarshal(normalized, &env); err != nil { + t.Fatalf("failed to unmarshal envelope: %v", err) + } + var config map[string]interface{} + if err := json.Unmarshal(env.Config, &config); err != nil { + t.Fatalf("failed to unmarshal config: %v", err) + } + + if got := config["requireMention"]; got != true { + t.Fatalf("expected top-level requireMention=true to survive, got %v", got) + } + + accounts, ok := config["accounts"].(map[string]interface{}) + if !ok { + t.Fatalf("expected accounts map, got %T", config["accounts"]) + } + + wantDefault := map[string]interface{}{"appId": "cli_default", "appSecret": "default_sec"} + if !reflect.DeepEqual(accounts["default"], wantDefault) { + t.Fatalf("default account not preserved:\nwant: %v\ngot: %v", wantDefault, accounts["default"]) + } + wantAcme := map[string]interface{}{"appId": "cli_acme", "appSecret": "acme_sec", "botName": "acme-bot"} + if !reflect.DeepEqual(accounts["acme"], wantAcme) { + t.Fatalf("acme account not preserved:\nwant: %v\ngot: %v", wantAcme, accounts["acme"]) + } + + main, ok := accounts["main"].(map[string]interface{}) + if !ok { + t.Fatalf("expected accounts.main map, got %T", accounts["main"]) + } + if main["verificationToken"] != "vt_keep" { + t.Fatalf("expected accounts.main.verificationToken to survive, got %v", main["verificationToken"]) + } + if main["appId"] != "cli_main" { + t.Fatalf("expected accounts.main.appId normalized to cli_main, got %v", main["appId"]) + } + if main["appSecret"] != "main_sec" { + t.Fatalf("expected accounts.main.appSecret normalized to main_sec, got %v", main["appSecret"]) + } +} + +func TestPlanWithoutTeamMemberLeaderOnlyChannelsFiltersAllChannels(t *testing.T) { + service := &openClawConfigService{ + repo: &openClawConfigRepositoryStub{ + resourcesByID: map[int]*models.OpenClawConfigResource{ + 1: openClawConfigTestResource(1, 9, OpenClawConfigResourceTypeChannel, "dingtalk-connector"), + 2: openClawConfigTestResource(2, 9, OpenClawConfigResourceTypeChannel, "wecom"), + 3: openClawConfigTestResource(3, 9, OpenClawConfigResourceTypeChannel, "feishu"), + 4: openClawConfigTestResource(4, 9, OpenClawConfigResourceTypeSkill, "review-skill"), + }, + }, + } + + plan := &OpenClawConfigPlan{ + Mode: OpenClawConfigPlanModeManual, + ResourceIDs: []int{1, 2, 3, 4}, + } + filtered, err := service.PlanWithoutTeamMemberLeaderOnlyChannels(9, plan) + if err != nil { + t.Fatalf("PlanWithoutTeamMemberLeaderOnlyChannels returned error: %v", err) + } + if filtered == nil { + t.Fatalf("expected non-channel resources to remain") + } + if filtered.Mode != OpenClawConfigPlanModeManual || !reflect.DeepEqual(filtered.ResourceIDs, []int{4}) { + t.Fatalf("unexpected filtered plan: %#v", filtered) + } + if !reflect.DeepEqual(plan.ResourceIDs, []int{1, 2, 3, 4}) { + t.Fatalf("expected original plan to remain unchanged, got %#v", plan.ResourceIDs) + } + + channelsOnly, err := service.PlanWithoutTeamMemberLeaderOnlyChannels(9, &OpenClawConfigPlan{ + Mode: OpenClawConfigPlanModeManual, + ResourceIDs: []int{1, 2, 3}, + }) + if err != nil { + t.Fatalf("channels-only filtering returned error: %v", err) + } + if channelsOnly != nil { + t.Fatalf("expected channels-only worker plan to be nil, got %#v", channelsOnly) + } +} + +func openClawConfigTestResource(id, userID int, resourceType, resourceKey string) *models.OpenClawConfigResource { + return &models.OpenClawConfigResource{ + ID: id, + UserID: userID, + ResourceType: resourceType, + ResourceKey: resourceKey, + Name: resourceKey, + Enabled: true, + Version: 1, + ContentJSON: `{"schemaVersion":1,"kind":"` + resourceType + `","format":"test@v1","dependsOn":[],"config":{}}`, + } +} + +type openClawConfigRepositoryStub struct { + resourcesByID map[int]*models.OpenClawConfigResource +} + +func (s *openClawConfigRepositoryStub) ListResources(userID int, resourceType string) ([]models.OpenClawConfigResource, error) { + return nil, nil +} + +func (s *openClawConfigRepositoryStub) GetResourceByID(id int) (*models.OpenClawConfigResource, error) { + if s.resourcesByID == nil { + return nil, nil + } + return s.resourcesByID[id], nil +} + +func (s *openClawConfigRepositoryStub) GetResourceByUserTypeKey(userID int, resourceType, resourceKey string) (*models.OpenClawConfigResource, error) { + return nil, nil +} + +func (s *openClawConfigRepositoryStub) CreateResource(resource *models.OpenClawConfigResource) error { + return nil +} + +func (s *openClawConfigRepositoryStub) UpdateResource(resource *models.OpenClawConfigResource) error { + return nil +} + +func (s *openClawConfigRepositoryStub) DeleteResource(id int) error { return nil } + +func (s *openClawConfigRepositoryStub) ListBundles(userID int) ([]models.OpenClawConfigBundle, error) { + return nil, nil +} + +func (s *openClawConfigRepositoryStub) GetBundleByID(id int) (*models.OpenClawConfigBundle, error) { + return nil, nil +} + +func (s *openClawConfigRepositoryStub) CreateBundle(bundle *models.OpenClawConfigBundle) error { + return nil +} + +func (s *openClawConfigRepositoryStub) UpdateBundle(bundle *models.OpenClawConfigBundle) error { + return nil +} + +func (s *openClawConfigRepositoryStub) DeleteBundle(id int) error { return nil } + +func (s *openClawConfigRepositoryStub) ListBundleItems(bundleID int) ([]models.OpenClawConfigBundleItem, error) { + return nil, nil +} + +func (s *openClawConfigRepositoryStub) ReplaceBundleItems(bundleID int, items []models.OpenClawConfigBundleItem) error { + return nil +} + +func (s *openClawConfigRepositoryStub) ListBundleSkills(bundleID int) ([]models.OpenClawConfigBundleSkill, error) { + return nil, nil +} + +func (s *openClawConfigRepositoryStub) ReplaceBundleSkills(bundleID int, items []models.OpenClawConfigBundleSkill) error { + return nil +} + +func (s *openClawConfigRepositoryStub) CreateSnapshot(snapshot *models.OpenClawInjectionSnapshot) error { + return nil +} + +func (s *openClawConfigRepositoryStub) UpdateSnapshot(snapshot *models.OpenClawInjectionSnapshot) error { + return nil +} + +func (s *openClawConfigRepositoryStub) GetSnapshotByID(id int) (*models.OpenClawInjectionSnapshot, error) { + return nil, nil +} + +func (s *openClawConfigRepositoryStub) ListSnapshotsByUser(userID int, limit int) ([]models.OpenClawInjectionSnapshot, error) { + return nil, nil +} + +func (s *openClawConfigRepositoryStub) ListActiveSnapshots(userID int) ([]models.OpenClawInjectionSnapshot, error) { + return nil, nil +} + +func (s *openClawConfigRepositoryStub) UpdateSnapshotIfUnchanged(snapshot *models.OpenClawInjectionSnapshot, expectedUpdatedAt time.Time) (bool, error) { + return true, nil +} diff --git a/backend/internal/services/openclaw_transfer_integration_test.go b/backend/internal/services/openclaw_transfer_integration_test.go new file mode 100644 index 0000000..9bc7e51 --- /dev/null +++ b/backend/internal/services/openclaw_transfer_integration_test.go @@ -0,0 +1,200 @@ +//go:build integration + +// Integration tests for OpenClawTransferService against a real Kubernetes +// pod. Skipped unless OPENCLAW_TEST_POD is set. +// +// Usage: +// +// export KUBECONFIG=/path/to/kubeconfig +// export OPENCLAW_TEST_POD=/ +// go test -tags integration ./internal/services -run TestOpenClawTransfer_ -v +// +// The test pod must be a running OpenClaw/webtop instance (container name +// "desktop") with /config mounted as the persistent workspace. + +package services + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" +) + +type testExecRunner struct { + cfg *rest.Config + client *kubernetes.Clientset + ns string + pod string +} + +func mustTestRunner(t *testing.T) *testExecRunner { + t.Helper() + target := os.Getenv("OPENCLAW_TEST_POD") + if target == "" { + t.Skip("OPENCLAW_TEST_POD not set (format: /); skipping integration test") + } + parts := strings.SplitN(target, "/", 2) + if len(parts) != 2 { + t.Fatalf("OPENCLAW_TEST_POD must be /, got %q", target) + } + + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + kubeconfig = os.Getenv("HOME") + "/.kube/config" + } + cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + t.Fatalf("load kubeconfig: %v", err) + } + cs, err := kubernetes.NewForConfig(cfg) + if err != nil { + t.Fatalf("k8s client: %v", err) + } + return &testExecRunner{cfg: cfg, client: cs, ns: parts[0], pod: parts[1]} +} + +func (r *testExecRunner) exec(ctx context.Context, command []string, stdin io.Reader, stdout, stderr io.Writer) error { + req := r.client.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(r.pod). + Namespace(r.ns). + SubResource("exec") + req.VersionedParams(&corev1.PodExecOptions{ + Container: "desktop", + Command: command, + Stdin: stdin != nil, + Stdout: stdout != nil, + Stderr: stderr != nil, + }, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(r.cfg, "POST", req.URL()) + if err != nil { + return err + } + return exec.StreamWithContext(ctx, remotecommand.StreamOptions{Stdin: stdin, Stdout: stdout, Stderr: stderr}) +} + +func TestOpenClawTransfer_Export_RoundTrip(t *testing.T) { + r := mustTestRunner(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // Seed .openclaw with a marker file under /config. + markerName := fmt.Sprintf("marker-%d.txt", time.Now().UnixNano()) + seed := []string{"sh", "-lc", fmt.Sprintf(`mkdir -p /config/.openclaw && echo hello > /config/.openclaw/%s`, markerName)} + if err := r.exec(ctx, seed, nil, io.Discard, io.Discard); err != nil { + t.Fatalf("seed marker: %v", err) + } + t.Cleanup(func() { + _ = r.exec(context.Background(), []string{"sh", "-lc", fmt.Sprintf(`rm -f /config/.openclaw/%s`, markerName)}, nil, io.Discard, io.Discard) + }) + + // Run the real Export command. + var stdout, stderr bytes.Buffer + if err := r.exec(ctx, buildExportCommand(), nil, &stdout, &stderr); err != nil { + t.Fatalf("export exec: %v (stderr=%s)", err, stderr.String()) + } + if stdout.Len() < 100 { + t.Fatalf("export archive too small (%d bytes), stderr=%s", stdout.Len(), stderr.String()) + } + + // Verify marker entry exists in the archive. + gz, err := gzip.NewReader(&stdout) + if err != nil { + t.Fatalf("gzip reader: %v", err) + } + defer gz.Close() + tr := tar.NewReader(gz) + wantSuffix := ".openclaw/" + markerName + found := false + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("tar next: %v", err) + } + if strings.HasSuffix(hdr.Name, wantSuffix) { + found = true + break + } + } + if !found { + t.Fatalf("marker %q not found in archive", wantSuffix) + } +} + +func TestOpenClawTransfer_Export_MissingWorkspace(t *testing.T) { + r := mustTestRunner(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Temporarily move .openclaw out of the way. + backup := fmt.Sprintf(".openclaw.bak-%d", time.Now().UnixNano()) + if err := r.exec(ctx, []string{"sh", "-lc", fmt.Sprintf(`if [ -d /config/.openclaw ]; then mv /config/.openclaw /config/%s; fi`, backup)}, nil, io.Discard, io.Discard); err != nil { + t.Fatalf("hide workspace: %v", err) + } + t.Cleanup(func() { + _ = r.exec(context.Background(), []string{"sh", "-lc", fmt.Sprintf(`if [ -d /config/%s ]; then rm -rf /config/.openclaw && mv /config/%s /config/.openclaw; fi`, backup, backup)}, nil, io.Discard, io.Discard) + }) + + var stdout, stderr bytes.Buffer + err := r.exec(ctx, buildExportCommand(), nil, &stdout, &stderr) + if !isExportEmptyWorkspaceError(err) { + t.Fatalf("expected exit-code-42 empty-workspace error, got err=%v stderr=%q", err, stderr.String()) + } +} + +func TestOpenClawTransfer_Import_OwnershipIsAbc(t *testing.T) { + r := mustTestRunner(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // Build a tiny tar.gz containing .openclaw/ownership-probe. + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + content := []byte("probe\n") + hdr := &tar.Header{Name: ".openclaw/ownership-probe", Mode: 0o644, Size: int64(len(content)), ModTime: time.Now()} + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("tar header: %v", err) + } + if _, err := tw.Write(content); err != nil { + t.Fatalf("tar write: %v", err) + } + tw.Close() + gz.Close() + + var stderr bytes.Buffer + if err := r.exec(ctx, buildImportCommand(), &buf, io.Discard, &stderr); err != nil { + t.Fatalf("import exec: %v (stderr=%s)", err, stderr.String()) + } + t.Cleanup(func() { + _ = r.exec(context.Background(), []string{"sh", "-lc", `rm -f /config/.openclaw/ownership-probe`}, nil, io.Discard, io.Discard) + }) + + // stat the restored file; expect uid:gid 1000:1000. + var out bytes.Buffer + if err := r.exec(ctx, []string{"sh", "-lc", `stat -c '%u:%g' /config/.openclaw/ownership-probe`}, nil, &out, io.Discard); err != nil { + t.Fatalf("stat probe: %v", err) + } + got := strings.TrimSpace(out.String()) + if got != "1000:1000" { + t.Fatalf("ownership = %q, want 1000:1000", got) + } +} diff --git a/backend/internal/services/openclaw_transfer_service.go b/backend/internal/services/openclaw_transfer_service.go new file mode 100644 index 0000000..22b8814 --- /dev/null +++ b/backend/internal/services/openclaw_transfer_service.go @@ -0,0 +1,395 @@ +package services + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "strings" + "sync" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" + k8sexec "k8s.io/client-go/util/exec" +) + +const ( + openclawConfigDirName = ".openclaw" + hermesConfigDirName = ".hermes" + openclawBaseDir = "/config" + openclawExportEmptyExitCode = 42 +) + +// ErrOpenClawWorkspaceMissing is returned by Export when the .openclaw +// workspace does not exist inside the desktop container. Handlers should +// map this to an HTTP 404 rather than returning an empty 200 body. +var ErrOpenClawWorkspaceMissing = errors.New("openclaw workspace is empty or missing") + +// ErrHermesWorkspaceMissing is returned by ExportHermes when the .hermes +// workspace does not exist inside the runtime container. +var ErrHermesWorkspaceMissing = errors.New("hermes workspace is empty or missing") + +type OpenClawTransferService interface { + Export(ctx context.Context, userID, instanceID int) ([]byte, error) + Import(ctx context.Context, userID, instanceID int, archive io.Reader) error + ExportHermes(ctx context.Context, userID, instanceID int) ([]byte, error) + ImportHermes(ctx context.Context, userID, instanceID int, archive io.Reader) error +} + +type openClawTransferService struct { + podService *k8s.PodService + instanceRepo repository.InstanceRepository + bindingRepo repository.InstanceRuntimeBindingRepository + runtimePodRepo repository.RuntimePodRepository +} + +type openClawTransferRuntimeRepositories struct { + instanceRepo repository.InstanceRepository + bindingRepo repository.InstanceRuntimeBindingRepository + runtimePodRepo repository.RuntimePodRepository +} + +type transferExecTarget struct { + namespace string + podName string + container string +} + +var ( + openClawTransferReposMu sync.RWMutex + openClawTransferRepos openClawTransferRuntimeRepositories +) + +type workspaceTransferSpec struct { + dirName string + baseDirExpr string + missingErr error + actionLabel string + preserveTargetDir bool +} + +func NewOpenClawTransferService() OpenClawTransferService { + openClawTransferReposMu.RLock() + repos := openClawTransferRepos + openClawTransferReposMu.RUnlock() + return &openClawTransferService{ + podService: k8s.NewPodService(), + instanceRepo: repos.instanceRepo, + bindingRepo: repos.bindingRepo, + runtimePodRepo: repos.runtimePodRepo, + } +} + +func SetOpenClawTransferRuntimeRepositories(instanceRepo repository.InstanceRepository, bindingRepo repository.InstanceRuntimeBindingRepository, runtimePodRepo repository.RuntimePodRepository) { + openClawTransferReposMu.Lock() + openClawTransferRepos = openClawTransferRuntimeRepositories{ + instanceRepo: instanceRepo, + bindingRepo: bindingRepo, + runtimePodRepo: runtimePodRepo, + } + openClawTransferReposMu.Unlock() +} + +// buildBaseDirExpr returns a POSIX shell expression that resolves the +// OpenClaw persistent directory inside the desktop container. It honors the +// CLAWMANAGER_AGENT_PERSISTENT_DIR env var (injected by ClawManager at pod +// creation) and falls back to the hardcoded PVC mount path. +// +// $HOME is intentionally NOT used: `kubectl exec` spawns a fresh process as +// root with HOME=/root, which does not match the linuxserver entrypoint's +// runtime user `abc` (HOME=/config). +func buildBaseDirExpr() string { + return fmt.Sprintf("${CLAWMANAGER_AGENT_PERSISTENT_DIR:-%s}", openclawBaseDir) +} + +func openClawWorkspaceSpec() workspaceTransferSpec { + return workspaceTransferSpec{ + dirName: openclawConfigDirName, + baseDirExpr: buildBaseDirExpr(), + missingErr: ErrOpenClawWorkspaceMissing, + actionLabel: ".openclaw", + } +} + +func hermesWorkspaceSpec() workspaceTransferSpec { + return workspaceTransferSpec{ + dirName: hermesConfigDirName, + baseDirExpr: openclawBaseDir, + missingErr: ErrHermesWorkspaceMissing, + actionLabel: ".hermes", + preserveTargetDir: true, + } +} + +// buildExportCommand returns the sh -lc command used to stream a gzipped +// tarball of the .openclaw workspace from the desktop container over stdout. +// When the workspace does not exist, the command exits with +// openclawExportEmptyExitCode so the service layer can map it to +// ErrOpenClawWorkspaceMissing instead of returning an empty archive. +func buildExportCommand() []string { + return buildWorkspaceExportCommand(openClawWorkspaceSpec()) +} + +func buildHermesExportCommand() []string { + return buildWorkspaceExportCommand(hermesWorkspaceSpec()) +} + +func buildWorkspaceExportCommand(spec workspaceTransferSpec) []string { + script := fmt.Sprintf( + `base_dir="%s"; target_dir="$base_dir/%s"; `+ + `if [ ! -d "$target_dir" ]; then exit %d; fi; `+ + `tar czf - -C "$base_dir" %s`, + spec.baseDirExpr, + spec.dirName, + openclawExportEmptyExitCode, + shellQuote(spec.dirName), + ) + return []string{"sh", "-lc", script} +} + +// buildImportCommand returns the sh -lc command used to restore a gzipped +// tarball of the .openclaw workspace into the desktop container from stdin. +// The extract is re-exec'd as user `abc` (uid 1000) via `su` so restored +// files are owned by the runtime user, matching how the linuxserver +// entrypoint writes /config. +func buildImportCommand() []string { + return buildWorkspaceImportCommand(openClawWorkspaceSpec()) +} + +func buildHermesImportCommand() []string { + return buildWorkspaceImportCommand(hermesWorkspaceSpec()) +} + +func buildWorkspaceImportCommand(spec workspaceTransferSpec) []string { + clearTarget := `rm -rf "$target_dir" && mkdir -p "$base_dir"` + if spec.preserveTargetDir { + script := fmt.Sprintf( + `base_dir="%s"; target_dir="$base_dir/%s"; `+ + `mkdir -p "$target_dir" && find "$target_dir" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && `+ + `tar xzf - -C "$base_dir" && chown -R abc:abc "$target_dir"`, + spec.baseDirExpr, + spec.dirName, + ) + return []string{"sh", "-lc", script} + } + inner := fmt.Sprintf( + `base_dir="%s"; target_dir="$base_dir/%s"; `+ + `%s && tar xzf - -C "$base_dir"`, + spec.baseDirExpr, + spec.dirName, + clearTarget, + ) + outer := fmt.Sprintf(`exec su abc -s /bin/sh -c %s`, shellQuote(inner)) + return []string{"sh", "-lc", outer} +} + +func (s *openClawTransferService) Export(ctx context.Context, userID, instanceID int) ([]byte, error) { + return s.exportWorkspace(ctx, userID, instanceID, openClawWorkspaceSpec()) +} + +func (s *openClawTransferService) ExportHermes(ctx context.Context, userID, instanceID int) ([]byte, error) { + return s.exportWorkspace(ctx, userID, instanceID, hermesWorkspaceSpec()) +} + +func (s *openClawTransferService) exportWorkspace(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec) ([]byte, error) { + resolvedSpec, err := s.workspaceSpecForInstance(ctx, userID, instanceID, spec) + if err != nil { + return nil, err + } + command := buildWorkspaceExportCommand(resolvedSpec) + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := s.exec(ctx, userID, instanceID, command, nil, &stdout, &stderr); err != nil { + if isExportEmptyWorkspaceError(err) { + return nil, resolvedSpec.missingErr + } + return nil, formatExecError("export "+resolvedSpec.actionLabel, err, stderr.String()) + } + + return stdout.Bytes(), nil +} + +// isExportEmptyWorkspaceError reports whether err indicates the export +// command exited with openclawExportEmptyExitCode (signalling that the +// .openclaw workspace does not exist). +func isExportEmptyWorkspaceError(err error) bool { + if err == nil { + return false + } + var codeErr k8sexec.CodeExitError + if errors.As(err, &codeErr) { + return codeErr.Code == openclawExportEmptyExitCode + } + // Fallback: remotecommand sometimes wraps the exit code in a plain + // error whose message contains "exit code N". + return strings.Contains(err.Error(), fmt.Sprintf("exit code %d", openclawExportEmptyExitCode)) +} + +func (s *openClawTransferService) Import(ctx context.Context, userID, instanceID int, archive io.Reader) error { + return s.importWorkspace(ctx, userID, instanceID, archive, openClawWorkspaceSpec()) +} + +func (s *openClawTransferService) ImportHermes(ctx context.Context, userID, instanceID int, archive io.Reader) error { + return s.importWorkspace(ctx, userID, instanceID, archive, hermesWorkspaceSpec()) +} + +func (s *openClawTransferService) importWorkspace(ctx context.Context, userID, instanceID int, archive io.Reader, spec workspaceTransferSpec) error { + resolvedSpec, err := s.workspaceSpecForInstance(ctx, userID, instanceID, spec) + if err != nil { + return err + } + command := buildWorkspaceImportCommand(resolvedSpec) + var stderr bytes.Buffer + if err := s.exec(ctx, userID, instanceID, command, archive, nil, &stderr); err != nil { + return formatExecError("import "+resolvedSpec.actionLabel, err, stderr.String()) + } + + return nil +} + +func (s *openClawTransferService) workspaceSpecForInstance(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec) (workspaceTransferSpec, error) { + instance, err := s.runtimeManagedInstance(ctx, userID, instanceID) + if err != nil || instance == nil { + return spec, err + } + baseDir := "" + if instance.WorkspacePath != nil { + baseDir = strings.TrimSpace(*instance.WorkspacePath) + } + if baseDir == "" { + runtimeType, ok := NormalizeV2RuntimeType(instance.Type) + if !ok { + return spec, nil + } + baseDir = RuntimeWorkspacePath(runtimeType, instance.UserID, instance.ID) + } + spec.baseDirExpr = baseDir + return spec, nil +} + +func (s *openClawTransferService) exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error { + if s.podService == nil || s.podService.GetClient() == nil || s.podService.GetClient().Clientset == nil { + return fmt.Errorf("k8s client not initialized") + } + + target, err := s.runtimeExecTarget(ctx, userID, instanceID) + if err != nil { + return err + } + if target == nil { + pod, podErr := s.podService.GetPod(ctx, userID, instanceID) + if podErr != nil { + return fmt.Errorf("failed to get pod: %w", podErr) + } + target = &transferExecTarget{ + namespace: pod.Namespace, + podName: pod.Name, + container: "desktop", + } + } + + req := s.podService.GetClient().Clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(target.podName). + Namespace(target.namespace). + SubResource("exec") + + req.VersionedParams(&corev1.PodExecOptions{ + Container: target.container, + Command: command, + Stdin: stdin != nil, + Stdout: stdout != nil, + Stderr: stderr != nil, + TTY: false, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(s.podService.GetClient().Config, "POST", req.URL()) + if err != nil { + return fmt.Errorf("failed to initialize exec stream: %w", err) + } + + return exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + Tty: false, + }) +} + +func (s *openClawTransferService) runtimeManagedInstance(ctx context.Context, userID, instanceID int) (*models.Instance, error) { + if s == nil || s.instanceRepo == nil { + return nil, nil + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, fmt.Errorf("failed to get instance: %w", err) + } + if instance == nil { + return nil, nil + } + if instance.UserID != userID { + return nil, fmt.Errorf("instance does not belong to user") + } + if !isLiteRuntimeInstance(instance) { + return nil, nil + } + return instance, nil +} + +func (s *openClawTransferService) runtimeExecTarget(ctx context.Context, userID, instanceID int) (*transferExecTarget, error) { + instance, err := s.runtimeManagedInstance(ctx, userID, instanceID) + if err != nil || instance == nil { + return nil, err + } + if s.bindingRepo == nil || s.runtimePodRepo == nil { + return nil, fmt.Errorf("runtime repositories are not configured for lite workspace transfer") + } + binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID) + if err != nil { + return nil, fmt.Errorf("failed to get runtime binding: %w", err) + } + if binding == nil { + return nil, fmt.Errorf("runtime binding not found for instance %d", instanceID) + } + if binding.Generation != instance.RuntimeGeneration { + return nil, fmt.Errorf("runtime binding generation does not match instance") + } + runtimePod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if err != nil { + return nil, fmt.Errorf("failed to get runtime pod: %w", err) + } + if runtimePod == nil { + return nil, fmt.Errorf("runtime pod not found for instance %d", instanceID) + } + return &transferExecTarget{ + namespace: runtimePod.Namespace, + podName: runtimePod.PodName, + container: "runtime", + }, nil +} + +func isLiteRuntimeInstance(instance *models.Instance) bool { + if instance == nil { + return false + } + if strings.EqualFold(strings.TrimSpace(instance.InstanceMode), InstanceModeLite) { + return true + } + return strings.EqualFold(strings.TrimSpace(instance.RuntimeType), RuntimeBackendGateway) +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} + +func formatExecError(action string, execErr error, stderr string) error { + if stderr != "" { + return fmt.Errorf("failed to %s: %s", action, stderr) + } + return fmt.Errorf("failed to %s: %w", action, execErr) +} diff --git a/backend/internal/services/openclaw_transfer_service_test.go b/backend/internal/services/openclaw_transfer_service_test.go new file mode 100644 index 0000000..8afe957 --- /dev/null +++ b/backend/internal/services/openclaw_transfer_service_test.go @@ -0,0 +1,172 @@ +package services + +import ( + "context" + "strings" + "testing" + + "clawreef/internal/models" +) + +func TestBuildBaseDirExpr_UsesEnvVarWithFallback(t *testing.T) { + got := buildBaseDirExpr() + want := "${CLAWMANAGER_AGENT_PERSISTENT_DIR:-/config}" + if got != want { + t.Fatalf("buildBaseDirExpr() = %q, want %q", got, want) + } +} + +func TestBuildExportCommand_UsesBaseDirNotHome(t *testing.T) { + cmd := buildExportCommand() + if len(cmd) != 3 || cmd[0] != "sh" || cmd[1] != "-lc" { + t.Fatalf("unexpected command shape: %#v", cmd) + } + script := cmd[2] + + if !strings.Contains(script, "CLAWMANAGER_AGENT_PERSISTENT_DIR") { + t.Errorf("expected CLAWMANAGER_AGENT_PERSISTENT_DIR in script, got: %s", script) + } + if strings.Contains(script, "HOME") || strings.Contains(script, "/home/user") { + t.Errorf("export script must not depend on $HOME or /home/user, got: %s", script) + } + if !strings.Contains(script, "exit 42") { + t.Errorf("expected `exit 42` for empty-workspace branch, got: %s", script) + } + if !strings.Contains(script, "tar czf -") { + t.Errorf("expected `tar czf -` streaming, got: %s", script) + } +} + +func TestBuildExportCommand_QuotesConfigDirName(t *testing.T) { + cmd := buildExportCommand() + quoted := shellQuote(openclawConfigDirName) + if !strings.Contains(cmd[2], quoted) { + t.Errorf("expected quoted %q in script, got: %s", quoted, cmd[2]) + } +} + +func TestBuildHermesExportCommand_ExportsHermesDirectoryFromConfig(t *testing.T) { + cmd := buildHermesExportCommand() + if len(cmd) != 3 || cmd[0] != "sh" || cmd[1] != "-lc" { + t.Fatalf("unexpected command shape: %#v", cmd) + } + script := cmd[2] + + if !strings.Contains(script, `base_dir="/config"`) { + t.Errorf("expected Hermes export to use /config as archive base, got: %s", script) + } + if !strings.Contains(script, `target_dir="$base_dir/.hermes"`) { + t.Errorf("expected Hermes export target to be /config/.hermes, got: %s", script) + } + if !strings.Contains(script, shellQuote(hermesConfigDirName)) { + t.Errorf("expected quoted .hermes in script, got: %s", script) + } +} + +func TestBuildImportCommand_UsesSuAbc(t *testing.T) { + cmd := buildImportCommand() + if len(cmd) != 3 || cmd[0] != "sh" || cmd[1] != "-lc" { + t.Fatalf("unexpected command shape: %#v", cmd) + } + script := cmd[2] + + if !strings.Contains(script, "su abc -s /bin/sh -c") { + t.Errorf("expected `su abc -s /bin/sh -c` wrap to land files as uid 1000, got: %s", script) + } + if !strings.Contains(script, "tar xzf -") { + t.Errorf("expected `tar xzf -` in extract, got: %s", script) + } +} + +func TestBuildImportCommand_NoHomeReference(t *testing.T) { + cmd := buildImportCommand() + script := cmd[2] + if strings.Contains(script, "HOME") || strings.Contains(script, "/home/user") { + t.Errorf("import script must not depend on $HOME or /home/user, got: %s", script) + } + if !strings.Contains(script, "CLAWMANAGER_AGENT_PERSISTENT_DIR") { + t.Errorf("expected CLAWMANAGER_AGENT_PERSISTENT_DIR in import script, got: %s", script) + } +} + +func TestBuildHermesImportCommand_PreservesMountedHermesDirectory(t *testing.T) { + cmd := buildHermesImportCommand() + if len(cmd) != 3 || cmd[0] != "sh" || cmd[1] != "-lc" { + t.Fatalf("unexpected command shape: %#v", cmd) + } + script := cmd[2] + + if !strings.Contains(script, `base_dir="/config"`) { + t.Errorf("expected Hermes import to use /config as archive base, got: %s", script) + } + if strings.Contains(script, `rm -rf "$target_dir"`) { + t.Errorf("Hermes import must not remove the /config/.hermes mount point, got: %s", script) + } + if !strings.Contains(script, `find "$target_dir" -mindepth 1 -maxdepth 1`) { + t.Errorf("expected Hermes import to clear contents below mount point, got: %s", script) + } + if !strings.Contains(script, "tar xzf -") { + t.Errorf("expected `tar xzf -` in extract, got: %s", script) + } + if !strings.Contains(script, `chown -R abc:abc "$target_dir"`) { + t.Errorf("expected Hermes import to restore runtime user ownership, got: %s", script) + } +} + +func TestWorkspaceSpecForLiteUsesRuntimeWorkspacePath(t *testing.T) { + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[123] = &models.Instance{ + ID: 123, + UserID: 45, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + RuntimeGeneration: 2, + } + service := &openClawTransferService{instanceRepo: instanceRepo} + + spec, err := service.workspaceSpecForInstance(context.Background(), 45, 123, openClawWorkspaceSpec()) + if err != nil { + t.Fatalf("workspaceSpecForInstance returned error: %v", err) + } + if got, want := spec.baseDirExpr, RuntimeWorkspacePath(RuntimeTypeOpenClaw, 45, 123); got != want { + t.Fatalf("base dir = %q, want %q", got, want) + } +} + +func TestWorkspaceSpecForProKeepsDesktopConfigPath(t *testing.T) { + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[124] = &models.Instance{ + ID: 124, + UserID: 45, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendDesktop, + InstanceMode: InstanceModePro, + } + service := &openClawTransferService{instanceRepo: instanceRepo} + + spec, err := service.workspaceSpecForInstance(context.Background(), 45, 124, openClawWorkspaceSpec()) + if err != nil { + t.Fatalf("workspaceSpecForInstance returned error: %v", err) + } + if got, want := spec.baseDirExpr, buildBaseDirExpr(); got != want { + t.Fatalf("base dir = %q, want %q", got, want) + } +} + +func TestShellQuote_HandlesSingleQuotes(t *testing.T) { + cases := []struct { + in, want string + }{ + {"", "''"}, + {"abc", "'abc'"}, + {"a'b", `'a'"'"'b'`}, + {".openclaw", "'.openclaw'"}, + } + for _, tc := range cases { + got := shellQuote(tc.in) + if got != tc.want { + t.Errorf("shellQuote(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/backend/internal/services/quota_service.go b/backend/internal/services/quota_service.go new file mode 100644 index 0000000..8f26ffd --- /dev/null +++ b/backend/internal/services/quota_service.go @@ -0,0 +1,100 @@ +package services + +import ( + "errors" + "fmt" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// QuotaService defines the interface for quota operations +type QuotaService interface { + GetUserQuota(userID int) (*models.UserQuota, error) + UpdateUserQuota(userID int, quota *models.UserQuota) error + CreateDefaultQuota(userID int) (*models.UserQuota, error) + CheckUserQuota(userID int, requiredCPU float64, requiredMemory, requiredStorage int) error +} + +// quotaService implements QuotaService +type quotaService struct { + quotaRepo repository.QuotaRepository +} + +// NewQuotaService creates a new quota service +func NewQuotaService(quotaRepo repository.QuotaRepository) QuotaService { + return "aService{ + quotaRepo: quotaRepo, + } +} + +// GetUserQuota gets quota for a user +func (s *quotaService) GetUserQuota(userID int) (*models.UserQuota, error) { + quota, err := s.quotaRepo.GetByUserID(userID) + if err != nil { + return nil, fmt.Errorf("failed to get quota: %w", err) + } + + if quota == nil { + // Create default quota if not exists + quota, err = s.quotaRepo.CreateDefaultQuota(userID) + if err != nil { + return nil, fmt.Errorf("failed to create default quota: %w", err) + } + } + + return quota, nil +} + +// UpdateUserQuota updates quota for a user +func (s *quotaService) UpdateUserQuota(userID int, quota *models.UserQuota) error { + existingQuota, err := s.quotaRepo.GetByUserID(userID) + if err != nil { + return fmt.Errorf("failed to get existing quota: %w", err) + } + + if existingQuota == nil { + // Create new quota + quota.UserID = userID + return s.quotaRepo.Create(quota) + } + + // Update existing quota + existingQuota.MaxInstances = quota.MaxInstances + existingQuota.MaxCPUCores = quota.MaxCPUCores + existingQuota.MaxMemoryGB = quota.MaxMemoryGB + existingQuota.MaxStorageGB = quota.MaxStorageGB + existingQuota.MaxGPUCount = quota.MaxGPUCount + + return s.quotaRepo.Update(existingQuota) +} + +// CreateDefaultQuota creates default quota for a user +func (s *quotaService) CreateDefaultQuota(userID int) (*models.UserQuota, error) { + return s.quotaRepo.CreateDefaultQuota(userID) +} + +// CheckUserQuota checks if user has enough quota for new instance +func (s *quotaService) CheckUserQuota(userID int, requiredCPU float64, requiredMemory, requiredStorage int) error { + quota, err := s.GetUserQuota(userID) + if err != nil { + return fmt.Errorf("failed to get user quota: %w", err) + } + + // Check instance count (will be implemented when instance repo is ready) + // For now, just check resource limits + + if requiredCPU > quota.MaxCPUCores { + return errors.New("insufficient CPU quota") + } + + if requiredMemory > quota.MaxMemoryGB { + return errors.New("insufficient memory quota") + } + + if requiredStorage > quota.MaxStorageGB { + return errors.New("insufficient storage quota") + } + + return nil +} diff --git a/backend/internal/services/redis_client.go b/backend/internal/services/redis_client.go new file mode 100644 index 0000000..b10baae --- /dev/null +++ b/backend/internal/services/redis_client.go @@ -0,0 +1,17 @@ +package services + +import ( + "context" + "time" +) + +type PlatformRedisClient interface { + SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) + Del(ctx context.Context, key string) error + XAdd(ctx context.Context, key string, fields map[string]string) (string, error) + XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) +} + +func NewPlatformRedisClient(rawURL string) (PlatformRedisClient, error) { + return newRedisBus(rawURL) +} diff --git a/backend/internal/services/risk_detection_service.go b/backend/internal/services/risk_detection_service.go new file mode 100644 index 0000000..6f7c473 --- /dev/null +++ b/backend/internal/services/risk_detection_service.go @@ -0,0 +1,157 @@ +package services + +import ( + "fmt" + "regexp" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// RiskMatch is a normalized detector match. +type RiskMatch struct { + RuleID string `json:"rule_id"` + RuleName string `json:"rule_name"` + Severity string `json:"severity"` + Action string `json:"action"` + MatchSummary string `json:"match_summary"` +} + +// RiskAnalysis is the aggregate result of scanning request content. +type RiskAnalysis struct { + IsSensitive bool `json:"is_sensitive"` + HighestSeverity string `json:"highest_severity"` + HighestAction string `json:"highest_action"` + Hits []RiskMatch `json:"hits"` +} + +// RiskDetectionService defines message scanning operations. +type RiskDetectionService interface { + AnalyzeText(text string) RiskAnalysis +} + +type compiledRiskRule struct { + rule models.RiskRule + pattern *regexp.Regexp +} + +type riskDetectionService struct { + repo repository.RiskRuleRepository +} + +// NewRiskDetectionService creates a risk detector backed by configurable rules. +func NewRiskDetectionService(repo repository.RiskRuleRepository) RiskDetectionService { + return &riskDetectionService{repo: repo} +} + +func (s *riskDetectionService) AnalyzeText(text string) RiskAnalysis { + normalized := strings.TrimSpace(text) + if normalized == "" { + return RiskAnalysis{ + HighestSeverity: models.RiskSeverityLow, + HighestAction: models.RiskActionAllow, + Hits: []RiskMatch{}, + } + } + + rules, err := s.repo.ListEnabled() + if err != nil || len(rules) == 0 { + return RiskAnalysis{ + HighestSeverity: models.RiskSeverityLow, + HighestAction: models.RiskActionAllow, + Hits: []RiskMatch{}, + } + } + + compiled := compileRiskRules(rules) + return analyzeWithCompiledRules(normalized, compiled) +} + +func analyzeWithCompiledRules(normalized string, compiled []compiledRiskRule) RiskAnalysis { + hits := []RiskMatch{} + for _, item := range compiled { + if item.pattern == nil || !item.pattern.MatchString(normalized) { + continue + } + summary := strings.TrimSpace(derefString(item.rule.Description)) + if summary == "" { + summary = fmt.Sprintf("Matched configured rule %s.", item.rule.DisplayName) + } + hits = append(hits, RiskMatch{ + RuleID: item.rule.RuleID, + RuleName: item.rule.DisplayName, + Severity: item.rule.Severity, + Action: item.rule.Action, + MatchSummary: summary, + }) + } + + highestSeverity := models.RiskSeverityLow + highestAction := models.RiskActionAllow + for _, hit := range hits { + if compareRiskSeverity(hit.Severity, highestSeverity) > 0 { + highestSeverity = hit.Severity + } + if compareRiskAction(hit.Action, highestAction) > 0 { + highestAction = hit.Action + } + } + + return RiskAnalysis{ + IsSensitive: len(hits) > 0, + HighestSeverity: highestSeverity, + HighestAction: highestAction, + Hits: hits, + } +} + +func compileRiskRules(rules []models.RiskRule) []compiledRiskRule { + compiled := make([]compiledRiskRule, 0, len(rules)) + for _, rule := range rules { + pattern := strings.TrimSpace(rule.Pattern) + if pattern == "" { + continue + } + re, err := regexp.Compile(pattern) + if err != nil { + continue + } + compiled = append(compiled, compiledRiskRule{ + rule: rule, + pattern: re, + }) + } + return compiled +} + +func compareRiskSeverity(left, right string) int { + rank := map[string]int{ + models.RiskSeverityLow: 1, + models.RiskSeverityMedium: 2, + models.RiskSeverityHigh: 3, + } + return rank[left] - rank[right] +} + +func compareRiskAction(left, right string) int { + if left == "require_approval" { + left = models.RiskActionBlock + } + if right == "require_approval" { + right = models.RiskActionBlock + } + rank := map[string]int{ + models.RiskActionAllow: 1, + models.RiskActionRouteSecureModel: 2, + models.RiskActionBlock: 3, + } + return rank[left] - rank[right] +} + +func derefString(value *string) string { + if value == nil { + return "" + } + return *value +} diff --git a/backend/internal/services/risk_hit_service.go b/backend/internal/services/risk_hit_service.go new file mode 100644 index 0000000..28df2d7 --- /dev/null +++ b/backend/internal/services/risk_hit_service.go @@ -0,0 +1,83 @@ +package services + +import ( + "fmt" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// RiskHitService defines operations for persisted risk hits. +type RiskHitService interface { + RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, attribution *RiskHitAttribution, action string, hits []RiskMatch) error + ListHitsByTraceID(traceID string) ([]models.RiskHit, error) +} + +type RiskHitAttribution struct { + InstanceMode *string + RuntimeType *string + GatewayID *string + RuntimePodID *int64 +} + +type riskHitService struct { + repo repository.RiskHitRepository +} + +// NewRiskHitService creates a new risk hit service. +func NewRiskHitService(repo repository.RiskHitRepository) RiskHitService { + return &riskHitService{repo: repo} +} + +func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, attribution *RiskHitAttribution, action string, hits []RiskMatch) error { + traceID = strings.TrimSpace(traceID) + if traceID == "" || len(hits) == 0 { + return nil + } + if strings.TrimSpace(action) == "" { + action = models.RiskActionAllow + } + + for _, hit := range hits { + record := &models.RiskHit{ + TraceID: traceID, + SessionID: sessionID, + RequestID: requestID, + UserID: userID, + InstanceID: instanceID, + InstanceMode: nil, + RuntimeType: nil, + GatewayID: nil, + RuntimePodID: nil, + InvocationID: invocationID, + RuleID: strings.TrimSpace(hit.RuleID), + RuleName: strings.TrimSpace(hit.RuleName), + Severity: strings.TrimSpace(hit.Severity), + Action: action, + MatchSummary: strings.TrimSpace(hit.MatchSummary), + } + if attribution != nil { + record.InstanceMode = attribution.InstanceMode + record.RuntimeType = attribution.RuntimeType + record.GatewayID = attribution.GatewayID + record.RuntimePodID = attribution.RuntimePodID + } + if record.RuleID == "" || record.RuleName == "" || record.Severity == "" || record.MatchSummary == "" { + return fmt.Errorf("risk hit record is incomplete") + } + if err := s.repo.Create(record); err != nil { + return fmt.Errorf("failed to record risk hit: %w", err) + } + } + + return nil +} + +func (s *riskHitService) ListHitsByTraceID(traceID string) ([]models.RiskHit, error) { + items, err := s.repo.ListByTraceID(strings.TrimSpace(traceID)) + if err != nil { + return nil, fmt.Errorf("failed to list risk hits by trace: %w", err) + } + return items, nil +} diff --git a/backend/internal/services/risk_rule_service.go b/backend/internal/services/risk_rule_service.go new file mode 100644 index 0000000..956f6f0 --- /dev/null +++ b/backend/internal/services/risk_rule_service.go @@ -0,0 +1,194 @@ +package services + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +// RiskRuleService defines admin risk rule management operations. +type RiskRuleService interface { + ListRules() ([]models.RiskRule, error) + SaveRule(req SaveRiskRuleRequest) (*models.RiskRule, error) + DeleteRule(ruleID string) error + BulkSetEnabled(ruleIDs []string, enabled bool) error + TestRules(req TestRiskRulesRequest) (RiskAnalysis, error) +} + +// SaveRiskRuleRequest contains editable fields for risk rules. +type SaveRiskRuleRequest struct { + RuleID string + DisplayName string + Description *string + Pattern string + Severity string + Action string + IsEnabled bool + SortOrder int +} + +// TestRiskRulesRequest contains data for rule preview and testing. +type TestRiskRulesRequest struct { + Text string + Rule *SaveRiskRuleRequest +} + +type riskRuleService struct { + repo repository.RiskRuleRepository +} + +// NewRiskRuleService creates a new risk rule service. +func NewRiskRuleService(repo repository.RiskRuleRepository) RiskRuleService { + return &riskRuleService{repo: repo} +} + +func (s *riskRuleService) ListRules() ([]models.RiskRule, error) { + items, err := s.repo.List() + if err != nil { + return nil, fmt.Errorf("failed to list risk rules: %w", err) + } + return items, nil +} + +func (s *riskRuleService) SaveRule(req SaveRiskRuleRequest) (*models.RiskRule, error) { + rule, err := s.buildRuleModel(req) + if err != nil { + return nil, err + } + + if err := s.repo.Upsert(rule); err != nil { + return nil, fmt.Errorf("failed to save risk rule: %w", err) + } + + return rule, nil +} + +func (s *riskRuleService) DeleteRule(ruleID string) error { + ruleID = strings.TrimSpace(ruleID) + if ruleID == "" { + return errors.New("rule id is required") + } + + existing, err := s.repo.GetByRuleID(ruleID) + if err != nil { + return fmt.Errorf("failed to get risk rule: %w", err) + } + if existing == nil { + return errors.New("risk rule not found") + } + + existing.IsEnabled = false + if err := s.repo.Upsert(existing); err != nil { + return fmt.Errorf("failed to disable risk rule: %w", err) + } + return nil +} + +func (s *riskRuleService) BulkSetEnabled(ruleIDs []string, enabled bool) error { + if len(ruleIDs) == 0 { + return errors.New("at least one rule id is required") + } + + for _, ruleID := range ruleIDs { + trimmed := strings.TrimSpace(ruleID) + if trimmed == "" { + return errors.New("rule id is required") + } + + existing, err := s.repo.GetByRuleID(trimmed) + if err != nil { + return fmt.Errorf("failed to get risk rule: %w", err) + } + if existing == nil { + return errors.New("risk rule not found") + } + + existing.IsEnabled = enabled + if err := s.repo.Upsert(existing); err != nil { + return fmt.Errorf("failed to update risk rule status: %w", err) + } + } + + return nil +} + +func (s *riskRuleService) TestRules(req TestRiskRulesRequest) (RiskAnalysis, error) { + text := strings.TrimSpace(req.Text) + if text == "" { + return RiskAnalysis{}, errors.New("sample text is required") + } + + var rules []models.RiskRule + if req.Rule != nil { + rule, err := s.buildRuleModel(*req.Rule) + if err != nil { + return RiskAnalysis{}, err + } + rules = []models.RiskRule{*rule} + } else { + items, err := s.repo.ListEnabled() + if err != nil { + return RiskAnalysis{}, fmt.Errorf("failed to list enabled risk rules: %w", err) + } + rules = items + } + + compiled := compileRiskRules(rules) + return analyzeWithCompiledRules(text, compiled), nil +} + +func (s *riskRuleService) buildRuleModel(req SaveRiskRuleRequest) (*models.RiskRule, error) { + ruleID := strings.TrimSpace(req.RuleID) + if ruleID == "" { + return nil, errors.New("rule id is required") + } + + displayName := strings.TrimSpace(req.DisplayName) + if displayName == "" { + return nil, errors.New("rule display name is required") + } + + pattern := strings.TrimSpace(req.Pattern) + if pattern == "" { + return nil, errors.New("rule pattern is required") + } + if _, err := regexp.Compile(pattern); err != nil { + return nil, errors.New("rule pattern is invalid") + } + + severity := strings.TrimSpace(req.Severity) + if severity != models.RiskSeverityLow && severity != models.RiskSeverityMedium && severity != models.RiskSeverityHigh { + return nil, errors.New("risk severity is invalid") + } + + action := strings.TrimSpace(req.Action) + if action == "require_approval" { + action = models.RiskActionBlock + } + if action != models.RiskActionAllow && action != models.RiskActionRouteSecureModel && action != models.RiskActionBlock { + return nil, errors.New("risk action is invalid") + } + + var description *string + if req.Description != nil { + trimmed := strings.TrimSpace(*req.Description) + if trimmed != "" { + description = &trimmed + } + } + + return &models.RiskRule{ + RuleID: ruleID, + DisplayName: displayName, + Description: description, + Pattern: pattern, + Severity: severity, + Action: action, + IsEnabled: req.IsEnabled, + SortOrder: req.SortOrder, + }, nil +} diff --git a/backend/internal/services/runtime_agent_client.go b/backend/internal/services/runtime_agent_client.go new file mode 100644 index 0000000..fcf5b0d --- /dev/null +++ b/backend/internal/services/runtime_agent_client.go @@ -0,0 +1,130 @@ +package services + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +type RuntimeAgentClient interface { + Health(ctx context.Context, endpoint string) error + CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) + DeleteGateway(ctx context.Context, endpoint, gatewayID string) error + Drain(ctx context.Context, endpoint string) error +} + +type RuntimeAgentPortRange struct { + Start int `json:"start"` + End int `json:"end"` +} + +type RuntimeAgentCreateGatewayRequest struct { + InstanceID int `json:"instance_id"` + UserID int `json:"user_id"` + AgentType string `json:"agent_type"` + WorkspacePath string `json:"workspace_path"` + PortRange RuntimeAgentPortRange `json:"port_range"` + UID int `json:"uid"` + GID int `json:"gid"` + CPUCores float64 `json:"cpu_cores"` + MemoryMB int `json:"memory_mb"` + DiskQuotaMB int `json:"disk_quota_mb"` + Generation int `json:"generation"` + Environment map[string]string `json:"environment,omitempty"` +} + +type RuntimeAgentCreateGatewayResponse struct { + GatewayID string `json:"gateway_id"` + Port int `json:"port"` + PID *int `json:"pid,omitempty"` + Status string `json:"status"` +} + +type runtimeAgentHTTPClient struct { + controlToken string + httpClient *http.Client +} + +func NewRuntimeAgentClient(controlToken string) RuntimeAgentClient { + return NewRuntimeAgentClientWithHTTPClient(controlToken, nil) +} + +func NewRuntimeAgentClientWithHTTPClient(controlToken string, httpClient *http.Client) RuntimeAgentClient { + if httpClient == nil { + httpClient = &http.Client{ + Timeout: 30 * time.Second, + } + } + clientCopy := *httpClient + clientCopy.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + return &runtimeAgentHTTPClient{ + controlToken: controlToken, + httpClient: &clientCopy, + } +} + +func (c *runtimeAgentHTTPClient) Health(ctx context.Context, endpoint string) error { + return c.do(ctx, http.MethodGet, endpoint, "/v1/health", nil, nil) +} + +func (c *runtimeAgentHTTPClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) { + var resp RuntimeAgentCreateGatewayResponse + if err := c.do(ctx, http.MethodPost, endpoint, "/v1/gateways", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *runtimeAgentHTTPClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error { + return c.do(ctx, http.MethodDelete, endpoint, "/v1/gateways/"+url.PathEscape(gatewayID), nil, nil) +} + +func (c *runtimeAgentHTTPClient) Drain(ctx context.Context, endpoint string) error { + return c.do(ctx, http.MethodPost, endpoint, "/v1/drain", map[string]bool{"draining": true}, nil) +} + +func (c *runtimeAgentHTTPClient) do(ctx context.Context, method, endpoint, path string, body any, out any) error { + endpoint = strings.TrimRight(endpoint, "/") + var reader io.Reader + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return err + } + reader = bytes.NewReader(payload) + } + + req, err := http.NewRequestWithContext(ctx, method, endpoint+path, reader) + if err != nil { + return err + } + req.Header.Set("X-ClawManager-Control-Token", c.controlToken) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode == http.StatusConflict { + return fmt.Errorf("runtime agent conflict: %s", string(msg)) + } + return fmt.Errorf("runtime agent status %d: %s", resp.StatusCode, string(msg)) + } + if out == nil { + return nil + } + return json.NewDecoder(resp.Body).Decode(out) +} diff --git a/backend/internal/services/runtime_agent_client_test.go b/backend/internal/services/runtime_agent_client_test.go new file mode 100644 index 0000000..fc461bf --- /dev/null +++ b/backend/internal/services/runtime_agent_client_test.go @@ -0,0 +1,223 @@ +package services + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestRuntimeAgentClientCreateGateway(t *testing.T) { + var gotToken string + var gotReq RuntimeAgentCreateGatewayRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/gateways" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + gotToken = r.Header.Get("X-ClawManager-Control-Token") + if err := json.NewDecoder(r.Body).Decode(&gotReq); err != nil { + t.Fatalf("decode request: %v", err) + } + _ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-7-3", + Port: 20017, + PID: runtimeAgentIntPtr(8842), + Status: "running", + }) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + resp, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{ + InstanceID: 7, + UserID: 8, + AgentType: "openclaw", + WorkspacePath: "/workspaces/openclaw/user-8/instance-7", + PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099}, + UID: 200007, + GID: 200007, + CPUCores: 2, + MemoryMB: 4096, + DiskQuotaMB: 20480, + Generation: 3, + }) + if err != nil { + t.Fatalf("CreateGateway returned error: %v", err) + } + if gotToken != "secret" { + t.Fatalf("unexpected token %q", gotToken) + } + if gotReq.InstanceID != 7 || gotReq.PortRange.Start != 20000 { + t.Fatalf("unexpected request %#v", gotReq) + } + if resp.GatewayID != "gw-7-3" || resp.Port != 20017 { + t.Fatalf("unexpected response %#v", resp) + } +} + +func TestRuntimeAgentClientCreateGatewayTrimsEndpointSlash(t *testing.T) { + var calls int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.URL.Path != "/v1/gateways" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + _ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-7-3", + Port: 20017, + Status: "running", + }) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + if _, err := client.CreateGateway(context.Background(), server.URL+"/", RuntimeAgentCreateGatewayRequest{}); err != nil { + t.Fatalf("CreateGateway returned error: %v", err) + } + if calls != 1 { + t.Fatalf("expected one gateway create call, got %d", calls) + } +} + +func TestRuntimeAgentClientDefaultTimeoutAllowsGatewayStartup(t *testing.T) { + client := NewRuntimeAgentClient("secret") + agentClient, ok := client.(*runtimeAgentHTTPClient) + if !ok { + t.Fatalf("unexpected client type %T", client) + } + if agentClient.httpClient.Timeout != 30*time.Second { + t.Fatalf("default timeout = %v, want 30s", agentClient.httpClient.Timeout) + } +} + +func TestRuntimeAgentClientDeleteGatewayEscapesGatewayID(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Fatalf("unexpected method %s", r.Method) + } + if r.URL.EscapedPath() != "/v1/gateways/gw-7%2F3" { + t.Fatalf("unexpected escaped path %s", r.URL.EscapedPath()) + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + if err := client.DeleteGateway(context.Background(), server.URL, "gw-7/3"); err != nil { + t.Fatalf("DeleteGateway returned error: %v", err) + } +} + +func TestRuntimeAgentClientHealthHandlesSuccessAndRedirect(t *testing.T) { + successServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/health" { + t.Fatalf("unexpected success path %s", r.URL.Path) + } + w.WriteHeader(http.StatusNoContent) + })) + defer successServer.Close() + redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/health": + w.Header().Set("Location", "/ok") + w.WriteHeader(http.StatusFound) + _, _ = w.Write([]byte("redirect not allowed")) + case "/ok": + w.WriteHeader(http.StatusOK) + default: + t.Fatalf("unexpected redirect path %s", r.URL.Path) + } + })) + defer redirectServer.Close() + + client := NewRuntimeAgentClient("secret") + if err := client.Health(context.Background(), successServer.URL); err != nil { + t.Fatalf("Health returned error for 204: %v", err) + } + err := client.Health(context.Background(), redirectServer.URL) + if err == nil || !strings.Contains(err.Error(), "runtime agent status 302") || !strings.Contains(err.Error(), "redirect not allowed") { + t.Fatalf("unexpected redirect error %v", err) + } +} + +func TestRuntimeAgentClientWithHTTPClientDoesNotMutateRedirectPolicy(t *testing.T) { + supplied := &http.Client{Timeout: 10 * time.Second} + client := NewRuntimeAgentClientWithHTTPClient("secret", supplied) + + if supplied.CheckRedirect != nil { + t.Fatalf("constructor mutated supplied CheckRedirect") + } + + agentClient, ok := client.(*runtimeAgentHTTPClient) + if !ok { + t.Fatalf("unexpected client type %T", client) + } + if agentClient.httpClient == supplied { + t.Fatalf("constructor reused supplied client pointer") + } + if agentClient.httpClient.Timeout != supplied.Timeout { + t.Fatalf("timeout was not preserved: got %v want %v", agentClient.httpClient.Timeout, supplied.Timeout) + } + if agentClient.httpClient.CheckRedirect == nil { + t.Fatalf("copied client has no redirect policy") + } +} + +func TestRuntimeAgentClientDrainSendsJSONBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/drain" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Fatalf("unexpected content type %q", r.Header.Get("Content-Type")) + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + if strings.TrimSpace(string(body)) != `{"draining":true}` { + t.Fatalf("unexpected body %s", body) + } + w.WriteHeader(http.StatusAccepted) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + if err := client.Drain(context.Background(), server.URL); err != nil { + t.Fatalf("Drain returned error: %v", err) + } +} + +func TestRuntimeAgentClientConflict(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "no free port", http.StatusConflict) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + _, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{}) + if err == nil || !strings.Contains(err.Error(), "runtime agent conflict") || !strings.Contains(err.Error(), "no free port") { + t.Fatalf("unexpected error %v", err) + } +} + +func TestRuntimeAgentClientNonConflictErrorIncludesStatusAndBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "agent exploded", http.StatusInternalServerError) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + _, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{}) + if err == nil || !strings.Contains(err.Error(), "runtime agent status 500") || !strings.Contains(err.Error(), "agent exploded") { + t.Fatalf("unexpected error %v", err) + } +} + +func runtimeAgentIntPtr(v int) *int { + return &v +} diff --git a/backend/internal/services/runtime_capacity.go b/backend/internal/services/runtime_capacity.go new file mode 100644 index 0000000..e5a59b7 --- /dev/null +++ b/backend/internal/services/runtime_capacity.go @@ -0,0 +1,80 @@ +package services + +import ( + "fmt" + "path" + "strings" +) + +const ( + RuntimeTypeOpenClaw = "openclaw" + RuntimeTypeHermes = "hermes" + + InstanceModeLite = "lite" + InstanceModePro = "pro" + + RuntimeBackendGateway = "gateway" + RuntimeBackendDesktop = "desktop" + RuntimeBackendShell = "shell" + + RuntimeGatewayPortStart = 20000 + RuntimeGatewayPortEnd = 20099 + RuntimePodCapacity = 100 + RuntimeLinuxIDBase = 200000 +) + +func NormalizeV2RuntimeType(instanceType string) (string, bool) { + switch strings.ToLower(strings.TrimSpace(instanceType)) { + case RuntimeTypeOpenClaw: + return RuntimeTypeOpenClaw, true + case RuntimeTypeHermes: + return RuntimeTypeHermes, true + default: + return "", false + } +} + +func NormalizeInstanceMode(mode string) (string, bool) { + switch strings.ToLower(strings.TrimSpace(mode)) { + case InstanceModeLite: + return InstanceModeLite, true + case InstanceModePro: + return InstanceModePro, true + default: + return "", false + } +} + +func RuntimeTypeForInstanceMode(mode string) (string, bool) { + normalized, ok := NormalizeInstanceMode(mode) + if !ok { + return "", false + } + if normalized == InstanceModeLite { + return RuntimeBackendGateway, true + } + return RuntimeBackendDesktop, true +} + +func InstanceModeForRuntimeType(runtimeType string) string { + if strings.EqualFold(strings.TrimSpace(runtimeType), RuntimeBackendGateway) { + return InstanceModeLite + } + return InstanceModePro +} + +func RuntimeWorkspacePath(runtimeType string, userID int, instanceID int) string { + return RuntimeWorkspacePathWithRoot("/workspaces", runtimeType, userID, instanceID) +} + +func RuntimeWorkspacePathWithRoot(root, runtimeType string, userID int, instanceID int) string { + root = strings.TrimSpace(root) + if root == "" { + root = "/workspaces" + } + return path.Join(root, fmt.Sprintf("%s/user-%d/instance-%d", runtimeType, userID, instanceID)) +} + +func RuntimeLinuxID(instanceID int) int { + return RuntimeLinuxIDBase + instanceID +} diff --git a/backend/internal/services/runtime_capacity_test.go b/backend/internal/services/runtime_capacity_test.go new file mode 100644 index 0000000..3435615 --- /dev/null +++ b/backend/internal/services/runtime_capacity_test.go @@ -0,0 +1,52 @@ +package services + +import "testing" + +func TestNormalizeV2RuntimeTypeAcceptsManagedRuntimeTypes(t *testing.T) { + for _, input := range []string{"openclaw", " OpenClaw ", "hermes", "Hermes"} { + got, ok := NormalizeV2RuntimeType(input) + if !ok { + t.Fatalf("expected %q to be accepted", input) + } + if got != RuntimeTypeOpenClaw && got != RuntimeTypeHermes { + t.Fatalf("expected normalized managed runtime type, got %q", got) + } + } +} + +func TestNormalizeV2RuntimeTypeRejectsLegacyRuntimeTypes(t *testing.T) { + for _, input := range []string{"webtop", "ubuntu", "", "desktop", "shell"} { + if got, ok := NormalizeV2RuntimeType(input); ok { + t.Fatalf("expected %q to be rejected, got %q", input, got) + } + } +} + +func TestRuntimeWorkspacePath(t *testing.T) { + got := RuntimeWorkspacePath("openclaw", 45, 123) + want := "/workspaces/openclaw/user-45/instance-123" + if got != want { + t.Fatalf("expected workspace path %q, got %q", want, got) + } +} + +func TestRuntimeLinuxID(t *testing.T) { + if got, want := RuntimeLinuxID(123), 200123; got != want { + t.Fatalf("expected linux id %d, got %d", want, got) + } +} + +func TestInstanceModeRuntimeTypeMapping(t *testing.T) { + if got, ok := RuntimeTypeForInstanceMode(" lite "); !ok || got != RuntimeBackendGateway { + t.Fatalf("lite runtime type = %q/%v, want gateway/true", got, ok) + } + if got, ok := RuntimeTypeForInstanceMode("Pro"); !ok || got != RuntimeBackendDesktop { + t.Fatalf("pro runtime type = %q/%v, want desktop/true", got, ok) + } + if got := InstanceModeForRuntimeType(RuntimeBackendGateway); got != InstanceModeLite { + t.Fatalf("gateway mode = %q, want lite", got) + } + if got := InstanceModeForRuntimeType(RuntimeBackendDesktop); got != InstanceModePro { + t.Fatalf("desktop mode = %q, want pro", got) + } +} diff --git a/backend/internal/services/runtime_events.go b/backend/internal/services/runtime_events.go new file mode 100644 index 0000000..d0f1f5b --- /dev/null +++ b/backend/internal/services/runtime_events.go @@ -0,0 +1,68 @@ +package services + +import ( + "context" + "encoding/json" + "time" +) + +const runtimeEventStreamKey = "clawmanager:runtime-events" + +type RuntimeEvent struct { + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` + CreatedAt time.Time `json:"created_at"` +} + +type RuntimeEventService interface { + Publish(ctx context.Context, eventType string, payload any) error + Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) +} + +func NewRuntimeEventService(redis PlatformRedisClient) RuntimeEventService { + if redis == nil { + return noopRuntimeEventService{} + } + return &redisRuntimeEventService{redis: redis} +} + +type redisRuntimeEventService struct { + redis PlatformRedisClient +} + +func (s *redisRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error { + payloadJSON, err := json.Marshal(payload) + if err != nil { + return err + } + event := RuntimeEvent{ + Type: eventType, + Payload: payloadJSON, + CreatedAt: time.Now().UTC(), + } + eventJSON, err := json.Marshal(event) + if err != nil { + return err + } + _, err = s.redis.XAdd(ctx, runtimeEventStreamKey, map[string]string{ + "type": event.Type, + "payload": string(event.Payload), + "created_at": event.CreatedAt.Format(time.RFC3339Nano), + "event": string(eventJSON), + }) + return err +} + +func (s *redisRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) { + return s.redis.XRead(ctx, runtimeEventStreamKey, lastID, block) +} + +type noopRuntimeEventService struct{} + +func (noopRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error { + return nil +} + +func (noopRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) { + return nil, nil +} diff --git a/backend/internal/services/runtime_events_test.go b/backend/internal/services/runtime_events_test.go new file mode 100644 index 0000000..51b1dd4 --- /dev/null +++ b/backend/internal/services/runtime_events_test.go @@ -0,0 +1,75 @@ +package services + +import ( + "context" + "encoding/json" + "testing" + "time" +) + +func TestRuntimeEventServiceNoopWhenRedisNil(t *testing.T) { + service := NewRuntimeEventService(nil) + + if err := service.Publish(context.Background(), "runtime.test", map[string]string{"ok": "true"}); err != nil { + t.Fatalf("Publish returned error: %v", err) + } + messages, err := service.Read(context.Background(), "0", time.Millisecond) + if err != nil { + t.Fatalf("Read returned error: %v", err) + } + if messages != nil { + t.Fatalf("messages = %#v, want nil", messages) + } +} + +func TestRuntimeEventServicePublishesJSONFields(t *testing.T) { + redis := &fakePlatformRedisClient{} + service := NewRuntimeEventService(redis) + + if err := service.Publish(context.Background(), "runtime.instance.running", map[string]int{"instance_id": 17}); err != nil { + t.Fatalf("Publish returned error: %v", err) + } + if redis.xaddKey != runtimeEventStreamKey { + t.Fatalf("XAdd key = %q", redis.xaddKey) + } + if redis.xaddFields["type"] != "runtime.instance.running" { + t.Fatalf("type field = %q", redis.xaddFields["type"]) + } + var payload map[string]int + if err := json.Unmarshal([]byte(redis.xaddFields["payload"]), &payload); err != nil { + t.Fatalf("payload is not json: %v", err) + } + if payload["instance_id"] != 17 { + t.Fatalf("payload instance_id = %d", payload["instance_id"]) + } + var event RuntimeEvent + if err := json.Unmarshal([]byte(redis.xaddFields["event"]), &event); err != nil { + t.Fatalf("event is not json: %v", err) + } + if event.Type != "runtime.instance.running" { + t.Fatalf("event type = %q", event.Type) + } +} + +type fakePlatformRedisClient struct { + xaddKey string + xaddFields map[string]string +} + +func (c *fakePlatformRedisClient) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) { + return true, nil +} + +func (c *fakePlatformRedisClient) Del(ctx context.Context, key string) error { + return nil +} + +func (c *fakePlatformRedisClient) XAdd(ctx context.Context, key string, fields map[string]string) (string, error) { + c.xaddKey = key + c.xaddFields = fields + return "1-0", nil +} + +func (c *fakePlatformRedisClient) XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) { + return []redisStreamMessage{{ID: "1-0", Fields: map[string]string{"type": "runtime.test"}}}, nil +} diff --git a/backend/internal/services/runtime_leader.go b/backend/internal/services/runtime_leader.go new file mode 100644 index 0000000..26d994e --- /dev/null +++ b/backend/internal/services/runtime_leader.go @@ -0,0 +1,99 @@ +package services + +import ( + "context" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + runtimeLeaderLeaseName = "clawmanager-runtime-scheduler" + runtimeLeaderLeaseTTL = 15 * time.Second +) + +type RuntimeLeaderService interface { + IsLeader(ctx context.Context) bool +} + +type runtimeLeaderService struct { + client kubernetes.Interface + namespace string + holder string +} + +func NewRuntimeLeaderService(client kubernetes.Interface, namespace, holder string) RuntimeLeaderService { + return &runtimeLeaderService{ + client: client, + namespace: namespace, + holder: holder, + } +} + +func (s *runtimeLeaderService) IsLeader(ctx context.Context) bool { + if s == nil || s.client == nil || s.namespace == "" || s.holder == "" { + return false + } + + leases := s.client.CoordinationV1().Leases(s.namespace) + now := metav1.NewMicroTime(time.Now()) + lease, err := leases.Get(ctx, runtimeLeaderLeaseName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + _, createErr := leases.Create(ctx, s.newLease(now), metav1.CreateOptions{}) + if createErr == nil { + return true + } + if !errors.IsAlreadyExists(createErr) { + return false + } + lease, err = leases.Get(ctx, runtimeLeaderLeaseName, metav1.GetOptions{}) + } + if err != nil { + return false + } + + if !s.canAcquire(lease, now.Time) { + return false + } + + lease.Spec.HolderIdentity = &s.holder + lease.Spec.RenewTime = &now + ttlSeconds := int32(runtimeLeaderLeaseTTL / time.Second) + lease.Spec.LeaseDurationSeconds = &ttlSeconds + if _, err := leases.Update(ctx, lease, metav1.UpdateOptions{}); err != nil { + return false + } + return true +} + +func (s *runtimeLeaderService) newLease(now metav1.MicroTime) *coordinationv1.Lease { + ttlSeconds := int32(runtimeLeaderLeaseTTL / time.Second) + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: runtimeLeaderLeaseName, + Namespace: s.namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &s.holder, + AcquireTime: &now, + RenewTime: &now, + LeaseDurationSeconds: &ttlSeconds, + }, + } +} + +func (s *runtimeLeaderService) canAcquire(lease *coordinationv1.Lease, now time.Time) bool { + if lease == nil { + return true + } + if lease.Spec.HolderIdentity != nil && *lease.Spec.HolderIdentity == s.holder { + return true + } + if lease.Spec.RenewTime == nil { + return true + } + return lease.Spec.RenewTime.Add(runtimeLeaderLeaseTTL).Before(now) +} diff --git a/backend/internal/services/runtime_leader_test.go b/backend/internal/services/runtime_leader_test.go new file mode 100644 index 0000000..fe8e1e4 --- /dev/null +++ b/backend/internal/services/runtime_leader_test.go @@ -0,0 +1,67 @@ +package services + +import ( + "context" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestRuntimeLeaderServiceFirstCallerBecomesLeader(t *testing.T) { + service := NewRuntimeLeaderService(fake.NewSimpleClientset(), "runtime-system", "backend-a") + + if !service.IsLeader(context.Background()) { + t.Fatalf("expected first caller to become leader") + } +} + +func TestRuntimeLeaderServiceDifferentCallerCannotStealUnexpiredLease(t *testing.T) { + now := metav1.NewMicroTime(time.Now()) + holder := "backend-a" + client := fake.NewSimpleClientset(&coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: "clawmanager-runtime-scheduler", Namespace: "runtime-system"}, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &holder, + RenewTime: &now, + LeaseDurationSeconds: int32Ptr(15), + }, + }) + service := NewRuntimeLeaderService(client, "runtime-system", "backend-b") + + if service.IsLeader(context.Background()) { + t.Fatalf("expected different caller not to steal unexpired lease") + } +} + +func TestRuntimeLeaderServiceDifferentCallerCanAcquireExpiredLease(t *testing.T) { + renewTime := metav1.NewMicroTime(time.Now().Add(-30 * time.Second)) + holder := "backend-a" + client := fake.NewSimpleClientset(&coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: "clawmanager-runtime-scheduler", Namespace: "runtime-system"}, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &holder, + RenewTime: &renewTime, + LeaseDurationSeconds: int32Ptr(15), + }, + }) + service := NewRuntimeLeaderService(client, "runtime-system", "backend-b") + + if !service.IsLeader(context.Background()) { + t.Fatalf("expected different caller to acquire expired lease") + } + + lease, err := client.CoordinationV1().Leases("runtime-system").Get(context.Background(), "clawmanager-runtime-scheduler", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to load lease: %v", err) + } + if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity != "backend-b" { + t.Fatalf("expected lease holder backend-b, got %#v", lease.Spec.HolderIdentity) + } +} + +func int32Ptr(value int32) *int32 { + return &value +} diff --git a/backend/internal/services/runtime_scheduler.go b/backend/internal/services/runtime_scheduler.go new file mode 100644 index 0000000..60ea429 --- /dev/null +++ b/backend/internal/services/runtime_scheduler.go @@ -0,0 +1,948 @@ +package services + +import ( + "context" + "errors" + "fmt" + "log" + "strconv" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" +) + +type RuntimeScheduler struct { + instanceRepo repository.InstanceRepository + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + rolloutRepo repository.RuntimeRolloutRepository + agentClient RuntimeAgentClient + events RuntimeEventService + leader RuntimeLeaderService + deployments k8s.RuntimeDeploymentService + envBuilder RuntimeGatewayEnvBuilder + tick time.Duration + + workspaceRoot string + runtimeNamespace string + gatewayPortStart int + gatewayPortEnd int + heartbeatTimeout time.Duration + maxGatewaysPerPod int +} + +var errRuntimeScaleOutPending = errors.New("runtime scale-out pending") + +const runtimeRolloutStaleWindowMultiplier = 3 + +type RuntimeGatewayEnvBuilder func(*models.Instance) (map[string]string, error) + +type RuntimeSchedulerOption func(*RuntimeScheduler) + +func WithRuntimeSchedulerWorkspaceRoot(root string) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + if strings.TrimSpace(root) != "" { + s.workspaceRoot = strings.TrimSpace(root) + } + } +} + +func WithRuntimeSchedulerGatewayPortRange(start, end int) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + if start > 0 { + s.gatewayPortStart = start + } + if end > 0 { + s.gatewayPortEnd = end + } + if s.gatewayPortEnd < s.gatewayPortStart { + s.gatewayPortEnd = s.gatewayPortStart + } + } +} + +func WithRuntimeSchedulerHeartbeatTimeout(timeout time.Duration) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + s.heartbeatTimeout = timeout + } +} + +func WithRuntimeSchedulerNamespace(namespace string) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + if strings.TrimSpace(namespace) != "" { + s.runtimeNamespace = strings.TrimSpace(namespace) + } + } +} + +func WithRuntimeSchedulerMaxGatewaysPerPod(capacity int) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + if capacity > 0 { + s.maxGatewaysPerPod = capacity + } + } +} + +func WithRuntimeSchedulerGatewayEnvBuilder(builder RuntimeGatewayEnvBuilder) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + s.envBuilder = builder + } +} + +func NewRuntimeScheduler( + instanceRepo repository.InstanceRepository, + podRepo repository.RuntimePodRepository, + bindingRepo repository.InstanceRuntimeBindingRepository, + rolloutRepo repository.RuntimeRolloutRepository, + agentClient RuntimeAgentClient, + events RuntimeEventService, + leader RuntimeLeaderService, + deployments k8s.RuntimeDeploymentService, + tick time.Duration, + opts ...RuntimeSchedulerOption, +) *RuntimeScheduler { + if tick <= 0 { + tick = 2 * time.Second + } + s := &RuntimeScheduler{ + instanceRepo: instanceRepo, + podRepo: podRepo, + bindingRepo: bindingRepo, + rolloutRepo: rolloutRepo, + agentClient: agentClient, + events: events, + leader: leader, + deployments: deployments, + tick: tick, + workspaceRoot: "/workspaces", + runtimeNamespace: "clawmanager-system", + gatewayPortStart: RuntimeGatewayPortStart, + gatewayPortEnd: RuntimeGatewayPortEnd, + heartbeatTimeout: 10 * time.Second, + maxGatewaysPerPod: RuntimePodCapacity, + } + for _, opt := range opts { + opt(s) + } + return s +} + +func (s *RuntimeScheduler) Start(ctx context.Context) { + if s == nil { + return + } + go func() { + ticker := time.NewTicker(s.tick) + defer ticker.Stop() + + s.reconcileIfLeader(ctx) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.reconcileIfLeader(ctx) + } + } + }() +} + +func (s *RuntimeScheduler) HeartbeatTimeout() time.Duration { + if s == nil { + return 0 + } + return s.heartbeatTimeout +} + +func (s *RuntimeScheduler) DrainPod(ctx context.Context, podID int64) error { + if s == nil || s.podRepo == nil { + return fmt.Errorf("runtime scheduler pod repository is not configured") + } + pod, err := s.podRepo.GetByID(ctx, podID) + if err != nil { + return err + } + if pod == nil { + return fmt.Errorf("runtime pod %d not found", podID) + } + if pod.AgentEndpoint != nil && strings.TrimSpace(*pod.AgentEndpoint) != "" && s.agentClient != nil { + if err := s.agentClient.Drain(ctx, strings.TrimSpace(*pod.AgentEndpoint)); err != nil { + return err + } + } + return s.podRepo.MarkState(ctx, podID, "draining", true) +} + +func (s *RuntimeScheduler) FailoverPod(ctx context.Context, podID int64, reason string) error { + if s == nil || s.podRepo == nil || s.bindingRepo == nil || s.instanceRepo == nil { + return fmt.Errorf("runtime scheduler dependencies are not configured") + } + var errs []error + if err := s.podRepo.MarkState(ctx, podID, "unhealthy", false); err != nil { + errs = append(errs, err) + } + + bindings, err := s.bindingRepo.ListByRuntimePodID(ctx, podID) + if err != nil { + errs = append(errs, err) + return errors.Join(errs...) + } + for _, binding := range bindings { + nextGeneration := binding.Generation + 1 + instance, err := s.instanceRepo.GetByID(binding.InstanceID) + if err != nil { + errs = append(errs, err) + continue + } + if instance != nil { + nextGeneration = instance.RuntimeGeneration + 1 + } + message := reason + if err := s.instanceRepo.UpdateRuntimeState(ctx, binding.InstanceID, "creating", nextGeneration, &message); err != nil { + errs = append(errs, err) + continue + } + if err := s.bindingRepo.DeleteByInstanceID(ctx, binding.InstanceID); err != nil { + errs = append(errs, err) + continue + } + if err := s.podRepo.ReleaseSlot(ctx, podID); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) StartRollout(ctx context.Context, rolloutID int64) error { + if s == nil || s.rolloutRepo == nil || s.podRepo == nil { + return fmt.Errorf("runtime scheduler rollout dependencies are not configured") + } + rollout, err := s.rolloutRepo.GetByID(ctx, rolloutID) + if err != nil { + return err + } + if rollout == nil { + return fmt.Errorf("runtime rollout %d not found", rolloutID) + } + + startedAt := time.Now().UTC() + if err := s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "running", &startedAt, nil, nil); err != nil { + return err + } + + allPods, err := s.podRepo.List(ctx, rollout.RuntimeType) + if err != nil { + message := err.Error() + _ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message) + return err + } + currentPods := s.currentRuntimePods(allPods, time.Now().UTC()) + if runtimePodsAlreadyAtImage(currentPods, rollout.RuntimeType, rollout.TargetImageRef) { + finishedAt := time.Now().UTC() + return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", &startedAt, &finishedAt, nil) + } + batchSize := rollout.BatchSize + if batchSize <= 0 { + batchSize = 1 + } + maxUnavailable := rollout.MaxUnavailable + if maxUnavailable <= 0 { + maxUnavailable = 1 + } + if err := s.rolloutRuntimeDeployments(ctx, rollout, allPods, maxUnavailable, batchSize); err != nil { + message := err.Error() + _ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message) + return err + } + unavailable := 0 + readyCandidates := 0 + for _, pod := range currentPods { + if pod.Draining || pod.State != "ready" { + unavailable++ + continue + } + readyCandidates++ + } + remaining := maxUnavailable - unavailable + if remaining <= 0 { + return nil + } + drainLimit := minInt(batchSize, remaining) + var errs []error + drained := 0 + for _, pod := range currentPods { + if drained >= drainLimit { + break + } + if pod.State != "ready" || pod.Draining { + continue + } + if err := s.DrainPod(ctx, pod.ID); err != nil { + errs = append(errs, err) + continue + } + drained++ + } + if len(errs) > 0 { + joined := errors.Join(errs...) + message := joined.Error() + _ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message) + return joined + } + if drained == 0 && readyCandidates == 0 && unavailable == 0 && len(currentPods) > 0 { + finishedAt := time.Now().UTC() + return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", &startedAt, &finishedAt, nil) + } + return nil +} + +func (s *RuntimeScheduler) reconcileRollouts(ctx context.Context) error { + if s == nil || s.rolloutRepo == nil { + return nil + } + rollouts, err := s.rolloutRepo.ListActive(ctx, "") + if err != nil { + return fmt.Errorf("list active runtime rollouts: %w", err) + } + var errs []error + for _, rollout := range rollouts { + switch rollout.Status { + case "pending": + if err := s.StartRollout(ctx, rollout.ID); err != nil { + errs = append(errs, fmt.Errorf("start runtime rollout %d: %w", rollout.ID, err)) + } + case "running": + if err := s.finishRolloutIfReady(ctx, rollout); err != nil { + errs = append(errs, fmt.Errorf("finish runtime rollout %d: %w", rollout.ID, err)) + } + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) rolloutRuntimeDeployments(ctx context.Context, rollout *models.RuntimeRollout, pods []models.RuntimePod, maxUnavailable, maxSurge int) error { + if s == nil || s.deployments == nil || rollout == nil { + return nil + } + targetImage := strings.TrimSpace(rollout.TargetImageRef) + if targetImage == "" { + return nil + } + type deploymentRef struct { + namespace string + name string + } + refs := map[deploymentRef]struct{}{} + for _, pod := range pods { + if pod.RuntimeType != rollout.RuntimeType { + continue + } + namespace := strings.TrimSpace(pod.Namespace) + name := strings.TrimSpace(pod.DeploymentName) + if namespace == "" || name == "" { + continue + } + refs[deploymentRef{namespace: namespace, name: name}] = struct{}{} + } + if len(refs) == 0 { + name := defaultRuntimeDeploymentName(rollout.RuntimeType) + if name == "" { + return fmt.Errorf("no runtime deployment found for %s rollout", rollout.RuntimeType) + } + refs[deploymentRef{namespace: s.runtimeNamespace, name: name}] = struct{}{} + } + var errs []error + for ref := range refs { + if err := s.deployments.RolloutImage(ctx, ref.namespace, ref.name, targetImage, maxUnavailable, maxSurge); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) RuntimeDeploymentPods(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + if s == nil || s.deployments == nil { + return nil, nil + } + runtimeType = strings.TrimSpace(runtimeType) + pods, err := s.deployments.ListPods(ctx, s.runtimeNamespace, runtimeType) + if err != nil { + return nil, err + } + result := make([]models.RuntimePod, 0, len(pods)) + for index, pod := range pods { + result = append(result, models.RuntimePod{ + ID: -int64(index + 1), + RuntimeType: pod.RuntimeType, + Namespace: pod.Namespace, + PodName: pod.PodName, + PodIP: pod.PodIP, + NodeName: pod.NodeName, + DeploymentName: pod.DeploymentName, + ImageRef: pod.ImageRef, + State: runtimeDeploymentFallbackState(pod.State), + Capacity: s.maxGatewaysPerPod, + UsedSlots: 0, + Draining: false, + }) + } + return result, nil +} + +func defaultRuntimeDeploymentName(runtimeType string) string { + switch strings.ToLower(strings.TrimSpace(runtimeType)) { + case RuntimeTypeOpenClaw: + return "openclaw-runtime" + case RuntimeTypeHermes: + return "hermes-runtime" + default: + return "" + } +} + +func runtimeDeploymentFallbackState(k8sState string) string { + switch strings.TrimSpace(k8sState) { + case "pending", "deleted": + return strings.TrimSpace(k8sState) + default: + return "unhealthy" + } +} + +func (s *RuntimeScheduler) finishRolloutIfReady(ctx context.Context, rollout models.RuntimeRollout) error { + if s == nil || s.podRepo == nil || s.rolloutRepo == nil { + return nil + } + targetImage := strings.TrimSpace(rollout.TargetImageRef) + if targetImage == "" { + return nil + } + pods, err := s.podRepo.List(ctx, rollout.RuntimeType) + if err != nil { + return err + } + pods = s.currentRuntimePods(pods, time.Now().UTC()) + if len(pods) == 0 { + return nil + } + for _, pod := range pods { + if pod.RuntimeType != rollout.RuntimeType { + continue + } + if pod.State != "ready" || pod.Draining || strings.TrimSpace(pod.ImageRef) != targetImage { + return nil + } + } + finishedAt := time.Now().UTC() + return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", rollout.StartedAt, &finishedAt, nil) +} + +func (s *RuntimeScheduler) currentRuntimePods(pods []models.RuntimePod, now time.Time) []models.RuntimePod { + if s == nil || s.heartbeatTimeout <= 0 { + return pods + } + cutoff := now.UTC().Add(-runtimeRolloutStaleWindowMultiplier * s.heartbeatTimeout) + current := pods[:0] + for _, pod := range pods { + if pod.LastSeenAt != nil && pod.LastSeenAt.UTC().Before(cutoff) { + continue + } + current = append(current, pod) + } + return current +} + +func runtimePodsAlreadyAtImage(pods []models.RuntimePod, runtimeType, targetImage string) bool { + targetImage = strings.TrimSpace(targetImage) + if targetImage == "" { + return false + } + matched := 0 + for _, pod := range pods { + if pod.RuntimeType != runtimeType { + continue + } + matched++ + if pod.State != "ready" || pod.Draining || strings.TrimSpace(pod.ImageRef) != targetImage { + return false + } + } + return matched > 0 +} + +func (s *RuntimeScheduler) reconcileIfLeader(ctx context.Context) { + if s.leader != nil && !s.leader.IsLeader(ctx) { + return + } + if err := s.reconcile(ctx); err != nil { + log.Printf("runtime scheduler reconcile failed: %v", err) + } +} + +func (s *RuntimeScheduler) reconcile(ctx context.Context) error { + if s == nil || s.instanceRepo == nil || s.bindingRepo == nil || s.podRepo == nil { + return fmt.Errorf("runtime scheduler dependencies are not configured") + } + var errs []error + if err := s.failoverStalePods(ctx); err != nil { + errs = append(errs, err) + } + if err := s.reconcileRollouts(ctx); err != nil { + errs = append(errs, err) + } + creating, err := s.instanceRepo.GetV2Creating(ctx, 100) + if err != nil { + errs = append(errs, err) + } else { + for _, instance := range creating { + if !isSchedulerManagedV2Instance(instance) { + continue + } + binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID) + if err != nil { + errs = append(errs, fmt.Errorf("get binding for creating instance %d: %w", instance.ID, err)) + continue + } + if binding != nil { + if err := s.syncInstanceStateFromBinding(ctx, instance, binding); err != nil { + errs = append(errs, fmt.Errorf("sync creating instance %d from binding: %w", instance.ID, err)) + } + continue + } + if err := s.assignInstance(ctx, instance); err != nil { + if errors.Is(err, errRuntimeScaleOutPending) { + continue + } + errs = append(errs, fmt.Errorf("assign creating instance %d: %w", instance.ID, err)) + s.markInstanceError(ctx, instance, err, &errs) + } + } + } + + desired, err := s.instanceRepo.GetV2DesiredRunning(ctx, 100) + if err != nil { + errs = append(errs, err) + } else { + for _, instance := range desired { + if !isSchedulerManagedV2Instance(instance) { + continue + } + if isRuntimeErrorInstance(instance) && !isRecoverableRuntimeSchedulingError(instance) { + continue + } + binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID) + if err != nil { + errs = append(errs, fmt.Errorf("get running binding for instance %d: %w", instance.ID, err)) + continue + } + if binding != nil { + if err := s.syncInstanceStateFromBinding(ctx, instance, binding); err != nil { + errs = append(errs, fmt.Errorf("sync desired instance %d from binding: %w", instance.ID, err)) + } + continue + } + binding, err = s.bindingRepo.GetByInstanceID(ctx, instance.ID) + if err != nil { + errs = append(errs, fmt.Errorf("get binding for desired instance %d: %w", instance.ID, err)) + continue + } + if binding != nil { + if err := s.syncInstanceStateFromBinding(ctx, instance, binding); err != nil { + errs = append(errs, fmt.Errorf("sync desired instance %d from binding: %w", instance.ID, err)) + } + continue + } + if assignErr := s.assignInstance(ctx, instance); assignErr != nil { + if errors.Is(assignErr, errRuntimeScaleOutPending) { + continue + } + errs = append(errs, fmt.Errorf("assign desired instance %d: %w", instance.ID, assignErr)) + s.markInstanceError(ctx, instance, assignErr, &errs) + } + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) syncInstanceStateFromBinding(ctx context.Context, instance models.Instance, binding *models.InstanceRuntimeBinding) error { + if s == nil || s.instanceRepo == nil || binding == nil { + return nil + } + state := strings.ToLower(strings.TrimSpace(binding.State)) + switch state { + case "running", "ready", "healthy": + if strings.EqualFold(strings.TrimSpace(instance.Status), "running") && instance.RuntimeErrorMessage == nil { + return nil + } + return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "running", maxInt(instance.RuntimeGeneration, binding.Generation), nil) + case "error", "failed": + message := "runtime gateway failed" + if binding.ErrorMessage != nil && strings.TrimSpace(*binding.ErrorMessage) != "" { + message = strings.TrimSpace(*binding.ErrorMessage) + } + if strings.EqualFold(strings.TrimSpace(instance.Status), "error") && + instance.RuntimeErrorMessage != nil && + strings.TrimSpace(*instance.RuntimeErrorMessage) == message { + return nil + } + return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", maxInt(instance.RuntimeGeneration, binding.Generation), &message) + default: + return nil + } +} + +func (s *RuntimeScheduler) failoverStalePods(ctx context.Context) error { + if s.heartbeatTimeout <= 0 { + return nil + } + pods, err := s.podRepo.List(ctx, "") + if err != nil { + return fmt.Errorf("list runtime pods for heartbeat failover: %w", err) + } + cutoff := time.Now().UTC().Add(-s.heartbeatTimeout) + var errs []error + for _, pod := range pods { + if pod.State == "unhealthy" || pod.State == "pending" { + continue + } + if pod.LastSeenAt == nil || !pod.LastSeenAt.Before(cutoff) { + continue + } + reason := fmt.Sprintf("runtime pod heartbeat lost since %s", pod.LastSeenAt.UTC().Format(time.RFC3339)) + if err := s.FailoverPod(ctx, pod.ID, reason); err != nil { + errs = append(errs, fmt.Errorf("failover stale runtime pod %d: %w", pod.ID, err)) + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.Instance) error { + if s == nil || s.podRepo == nil { + return fmt.Errorf("runtime scheduler pod repository is not configured") + } + if !isSchedulerManagedV2Instance(instance) { + return fmt.Errorf("instance %d is not scheduler-managed v2", instance.ID) + } + if s.bindingRepo != nil { + binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID) + if err != nil { + return fmt.Errorf("failed to check existing binding: %w", err) + } + if binding != nil { + return nil + } + } + runtimeType, ok := schedulerRuntimeType(instance) + if !ok { + return fmt.Errorf("unsupported runtime type %q", instance.Type) + } + pods, err := s.podRepo.ListSchedulable(ctx, runtimeType) + if err != nil { + return err + } + if len(pods) == 0 { + scaled, err := s.scaleOutIfAtCapacity(ctx, runtimeType) + if err != nil { + return fmt.Errorf("scale out %s runtime deployment: %w", runtimeType, err) + } + if scaled { + return errRuntimeScaleOutPending + } + } + var lastErr error + for _, pod := range pods { + if pod.AgentEndpoint == nil || strings.TrimSpace(*pod.AgentEndpoint) == "" { + continue + } + claimed, err := s.podRepo.TryClaimSlot(ctx, pod.ID) + if err != nil { + lastErr = err + continue + } + if !claimed { + continue + } + if err := s.createGatewayOnPod(ctx, instance, runtimeType, pod); err != nil { + if releaseErr := s.podRepo.ReleaseSlot(ctx, pod.ID); releaseErr != nil { + return errors.Join(err, releaseErr) + } + lastErr = err + continue + } + return nil + } + if lastErr != nil { + return fmt.Errorf("no schedulable %s runtime pod: %w", runtimeType, lastErr) + } + return fmt.Errorf("no schedulable %s runtime pod", runtimeType) +} + +type runtimeDeploymentCapacity struct { + namespace string + name string + active int + full int +} + +func (s *RuntimeScheduler) scaleOutIfAtCapacity(ctx context.Context, runtimeType string) (bool, error) { + if s == nil || s.deployments == nil || s.podRepo == nil { + return false, nil + } + pods, err := s.podRepo.List(ctx, runtimeType) + if err != nil { + return false, fmt.Errorf("list %s runtime pods for scale-out: %w", runtimeType, err) + } + groups := map[string]*runtimeDeploymentCapacity{} + for _, pod := range pods { + if pod.RuntimeType != runtimeType || pod.State != "ready" || pod.Draining || pod.Capacity <= 0 { + continue + } + namespace := strings.TrimSpace(pod.Namespace) + name := strings.TrimSpace(pod.DeploymentName) + if namespace == "" || name == "" { + continue + } + key := namespace + "/" + name + group := groups[key] + if group == nil { + group = &runtimeDeploymentCapacity{namespace: namespace, name: name} + groups[key] = group + } + group.active++ + if pod.UsedSlots >= pod.Capacity { + group.full++ + } + } + var target *runtimeDeploymentCapacity + for _, group := range groups { + if group.active == 0 || group.active != group.full { + continue + } + if target == nil || group.active > target.active { + target = group + } + } + if target == nil { + return false, nil + } + replicas := int32(target.active + 1) + if err := s.deployments.Scale(ctx, target.namespace, target.name, replicas); err != nil { + return false, err + } + if s.events != nil { + if err := s.events.Publish(ctx, "runtime.pool.scaleout", map[string]any{ + "runtime_type": runtimeType, + "namespace": target.namespace, + "deployment": target.name, + "replicas": replicas, + }); err != nil { + log.Printf("runtime scheduler publish scale-out event failed: %v", err) + } + } + return true, nil +} + +func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance models.Instance, runtimeType string, pod models.RuntimePod) error { + if s.agentClient == nil || s.bindingRepo == nil || s.instanceRepo == nil { + return fmt.Errorf("runtime scheduler gateway dependencies are not configured") + } + endpoint := strings.TrimSpace(*pod.AgentEndpoint) + workspacePath := RuntimeWorkspacePathWithRoot(s.workspaceRoot, runtimeType, instance.UserID, instance.ID) + environment, err := s.gatewayEnvironment(&instance) + if err != nil { + return fmt.Errorf("build runtime gateway environment: %w", err) + } + uid, gid := runtimeGatewayLinuxIDs(instance.ID, environment) + resp, err := s.agentClient.CreateGateway(ctx, endpoint, RuntimeAgentCreateGatewayRequest{ + InstanceID: instance.ID, + UserID: instance.UserID, + AgentType: runtimeType, + WorkspacePath: workspacePath, + PortRange: RuntimeAgentPortRange{ + Start: s.gatewayPortStart, + End: s.gatewayPortEnd, + }, + UID: uid, + GID: gid, + CPUCores: instance.CPUCores, + MemoryMB: instance.MemoryGB * 1024, + DiskQuotaMB: instance.DiskGB * 1024, + Generation: instance.RuntimeGeneration, + Environment: environment, + }) + if err != nil { + return err + } + if resp == nil { + return fmt.Errorf("runtime agent returned empty gateway response") + } + + now := time.Now().UTC() + gatewayState := normalizeRuntimeGatewayCreateState(resp.Status) + binding := &models.InstanceRuntimeBinding{ + InstanceID: instance.ID, + RuntimePodID: pod.ID, + RuntimeType: runtimeType, + GatewayID: resp.GatewayID, + GatewayPort: resp.Port, + GatewayPID: resp.PID, + WorkspacePath: workspacePath, + State: gatewayState, + Generation: instance.RuntimeGeneration, + } + if gatewayState == "running" { + binding.LastHealthAt = &now + } + if err := s.bindingRepo.Create(ctx, binding); err != nil { + return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, false, err) + } + if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil { + return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err) + } + instanceState := "creating" + var stateMessage *string + if gatewayState == "running" { + instanceState = "running" + } else { + message := "runtime gateway starting" + stateMessage = &message + } + if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, instanceState, instance.RuntimeGeneration, stateMessage); err != nil { + return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err) + } + if s.events != nil { + eventType := "runtime.instance.starting" + if gatewayState == "running" { + eventType = "runtime.instance.running" + } + if err := s.events.Publish(ctx, eventType, map[string]any{ + "instance_id": instance.ID, + "runtime_type": runtimeType, + "runtime_pod_id": pod.ID, + "gateway_id": resp.GatewayID, + "gateway_port": resp.Port, + "gateway_state": gatewayState, + "workspace_path": workspacePath, + "generation": instance.RuntimeGeneration, + }); err != nil { + log.Printf("runtime scheduler publish event failed: %v", err) + } + } + return nil +} + +func normalizeRuntimeGatewayCreateState(status string) string { + switch strings.ToLower(strings.TrimSpace(status)) { + case "running", "ready", "healthy": + return "running" + default: + return "starting" + } +} + +func runtimeGatewayLinuxIDs(instanceID int, environment map[string]string) (int, int) { + linuxID := RuntimeLinuxID(instanceID) + if !strings.EqualFold(strings.TrimSpace(environment["CLAWMANAGER_TEAM_ENABLED"]), "true") { + return linuxID, linuxID + } + sharedGID, err := strconv.Atoi(strings.TrimSpace(environment["CLAWMANAGER_TEAM_SHARED_GID"])) + if err != nil || sharedGID <= 0 { + return linuxID, linuxID + } + return linuxID, sharedGID +} + +func (s *RuntimeScheduler) gatewayEnvironment(instance *models.Instance) (map[string]string, error) { + if s == nil || s.envBuilder == nil { + return nil, nil + } + env, err := s.envBuilder(instance) + if err != nil { + return nil, err + } + if len(env) == 0 { + return nil, nil + } + copied := make(map[string]string, len(env)) + for key, value := range env { + if strings.TrimSpace(key) != "" { + copied[key] = value + } + } + if len(copied) == 0 { + return nil, nil + } + return copied, nil +} + +func (s *RuntimeScheduler) cleanupGatewayAfterAssignFailure(ctx context.Context, endpoint string, instanceID int, gatewayID string, bindingCreated bool, cause error) error { + errs := []error{cause} + if gatewayID != "" && s.agentClient != nil { + if err := s.agentClient.DeleteGateway(ctx, endpoint, gatewayID); err != nil { + errs = append(errs, fmt.Errorf("delete gateway %s: %w", gatewayID, err)) + } + } + if bindingCreated && s.bindingRepo != nil { + if err := s.bindingRepo.DeleteByInstanceID(ctx, instanceID); err != nil { + errs = append(errs, fmt.Errorf("delete binding for instance %d: %w", instanceID, err)) + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) markInstanceError(ctx context.Context, instance models.Instance, cause error, errs *[]error) { + message := cause.Error() + if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &message); err != nil { + *errs = append(*errs, fmt.Errorf("mark instance %d error: %w", instance.ID, err)) + } +} + +func schedulerRuntimeType(instance models.Instance) (string, bool) { + return NormalizeV2RuntimeType(instance.Type) +} + +func isSchedulerManagedV2Instance(instance models.Instance) bool { + if _, ok := schedulerRuntimeType(instance); !ok { + return false + } + if normalizeInstanceRuntimeType(instance.RuntimeType) != RuntimeBackendGateway { + return false + } + if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok && mode != InstanceModeLite { + return false + } + return instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != "" +} + +func isRuntimeErrorInstance(instance models.Instance) bool { + return strings.EqualFold(strings.TrimSpace(instance.Status), "error") +} + +func isRecoverableRuntimeSchedulingError(instance models.Instance) bool { + if !isRuntimeErrorInstance(instance) || instance.RuntimeErrorMessage == nil { + return false + } + runtimeType, ok := schedulerRuntimeType(instance) + if !ok { + return false + } + message := strings.TrimSpace(*instance.RuntimeErrorMessage) + return message == fmt.Sprintf("no schedulable %s runtime pod", runtimeType) +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/backend/internal/services/runtime_scheduler_test.go b/backend/internal/services/runtime_scheduler_test.go new file mode 100644 index 0000000..8ae017a --- /dev/null +++ b/backend/internal/services/runtime_scheduler_test.go @@ -0,0 +1,1890 @@ +package services + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" +) + +func TestRuntimeSchedulerAssignsCreatingInstanceToReadyPod(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-45/instance-17" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 17, + UserID: 45, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + CPUCores: 1.5, + MemoryGB: 2, + DiskGB: 8, + WorkspacePath: &workspacePath, + RuntimeGeneration: 3, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: { + ID: 9, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 1, + }, + }, + schedulable: []models.RuntimePod{{ + ID: 9, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 1, + }}, + } + bindingRepo := newFakeRuntimeBindingRepo() + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-17", + Port: 20017, + Status: "running", + }, + } + events := &fakeRuntimeEventService{} + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + events, + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + + if got := len(agent.createRequests); got != 1 { + t.Fatalf("CreateGateway calls = %d, want 1", got) + } + req := agent.createRequests[0] + if req.endpoint != endpoint { + t.Fatalf("CreateGateway endpoint = %q, want %q", req.endpoint, endpoint) + } + if req.req.WorkspacePath != "/workspaces/openclaw/user-45/instance-17" { + t.Fatalf("workspace path = %q", req.req.WorkspacePath) + } + if req.req.PortRange.Start != 20000 || req.req.PortRange.End != 20099 { + t.Fatalf("port range = %+v", req.req.PortRange) + } + if req.req.UID != RuntimeLinuxID(17) || req.req.GID != RuntimeLinuxID(17) { + t.Fatalf("linux IDs = %d/%d", req.req.UID, req.req.GID) + } + if req.req.MemoryMB != 2048 || req.req.DiskQuotaMB != 8192 { + t.Fatalf("resource MB = %d/%d", req.req.MemoryMB, req.req.DiskQuotaMB) + } + + binding := bindingRepo.bindings[17] + if binding == nil { + t.Fatal("binding was not created") + } + if binding.RuntimePodID != 9 || binding.GatewayPort != 20017 { + t.Fatalf("binding pod/port = %d/%d", binding.RuntimePodID, binding.GatewayPort) + } + if instanceRepo.workspacePaths[17] != "/workspaces/openclaw/user-45/instance-17" { + t.Fatalf("instance workspace path = %q", instanceRepo.workspacePaths[17]) + } + state := instanceRepo.runtimeStates[17] + if state.status != "running" || state.generation != 3 { + t.Fatalf("runtime state = %+v", state) + } + if podRepo.claims[9] != 1 { + t.Fatalf("pod claims = %d, want 1", podRepo.claims[9]) + } + if got := len(events.published); got != 1 { + t.Fatalf("published events = %d, want 1", got) + } + if events.published[0].eventType != "runtime.instance.running" { + t.Fatalf("published event = %q, want runtime.instance.running", events.published[0].eventType) + } +} + +func TestRuntimeSchedulerPassesGatewayEnvironmentToRuntimeAgent(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + token := "igt_instance_68" + workspacePath := "/workspaces/openclaw/user-1/instance-68" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 68, + UserID: 1, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + AccessToken: &token, + WorkspacePath: &workspacePath, + RuntimeGeneration: 4, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 10: { + ID: 10, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 10, + }, + }, + schedulable: []models.RuntimePod{{ + ID: 10, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 10, + }}, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-68-4", + Port: 20068, + Status: "starting", + }, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + WithRuntimeSchedulerGatewayEnvBuilder(func(instance *models.Instance) (map[string]string, error) { + if instance == nil || instance.ID != 68 { + t.Fatalf("gateway env builder received instance %#v, want 68", instance) + } + return map[string]string{ + "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm", + "CLAWMANAGER_LLM_API_KEY": token, + "CLAWMANAGER_LLM_MODEL": `["auto","gpt-5.5"]`, + "OPENAI_API_KEY": token, + }, nil + }), + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + + if got := len(agent.createRequests); got != 1 { + t.Fatalf("CreateGateway calls = %d, want 1", got) + } + env := agent.createRequests[0].req.Environment + if env["CLAWMANAGER_LLM_API_KEY"] != token || env["OPENAI_API_KEY"] != token { + t.Fatalf("gateway environment missing instance token aliases: %#v", env) + } + if env["CLAWMANAGER_LLM_BASE_URL"] != "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm" { + t.Fatalf("CLAWMANAGER_LLM_BASE_URL = %q", env["CLAWMANAGER_LLM_BASE_URL"]) + } + if env["CLAWMANAGER_LLM_MODEL"] != `["auto","gpt-5.5"]` { + t.Fatalf("CLAWMANAGER_LLM_MODEL = %q", env["CLAWMANAGER_LLM_MODEL"]) + } + state := instanceRepo.runtimeStates[68] + if state.status != "creating" || state.generation != 4 || state.message == nil || !strings.Contains(*state.message, "gateway starting") { + t.Fatalf("runtime state after starting gateway = %+v, want creating with startup message", state) + } +} + +func TestRuntimeSchedulerUsesTeamSharedGIDForTeamLiteGateway(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-1/instance-130" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 130, + UserID: 1, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + WorkspacePath: &workspacePath, + RuntimeGeneration: 1, + MemoryGB: 1, + DiskGB: 1, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 11: { + ID: 11, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 10, + }, + }, + schedulable: []models.RuntimePod{{ + ID: 11, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 10, + }}, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-130-1", Port: 20030, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + WithRuntimeSchedulerGatewayEnvBuilder(func(instance *models.Instance) (map[string]string, error) { + return map[string]string{ + "CLAWMANAGER_TEAM_ENABLED": "true", + "CLAWMANAGER_TEAM_SHARED_GID": "1000", + }, nil + }), + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 1 { + t.Fatalf("CreateGateway calls = %d, want 1", got) + } + req := agent.createRequests[0].req + if req.UID != RuntimeLinuxID(130) { + t.Fatalf("UID = %d, want isolated instance UID %d", req.UID, RuntimeLinuxID(130)) + } + if req.GID != 1000 { + t.Fatalf("GID = %d, want Team shared GID 1000", req.GID) + } +} + +func TestRuntimeSchedulerNoSchedulablePodReturnsErrorAndReleasesNothing(t *testing.T) { + ctx := context.Background() + instanceRepo := newFakeRuntimeInstanceRepo() + podRepo := &fakeRuntimePodRepo{} + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + err := scheduler.assignInstance(ctx, models.Instance{ + ID: 1, + UserID: 2, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: ptrString("/workspaces/openclaw/user-2/instance-1"), + }) + if err == nil { + t.Fatal("assignInstance returned nil error") + } + if podRepo.releaseCount != 0 { + t.Fatalf("release count = %d, want 0", podRepo.releaseCount) + } +} + +func TestRuntimeSchedulerScalesOutWhenAllReadyPodsAtCapacity(t *testing.T) { + ctx := context.Background() + endpoint := "http://pod-1.runtime" + workspacePath := "/workspaces/openclaw/user-12/instance-27" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 27, + UserID: 12, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + WorkspacePath: &workspacePath, + RuntimeGeneration: 2, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 5: { + ID: 5, + RuntimeType: RuntimeTypeOpenClaw, + Namespace: "clawmanager-system", + PodName: "openclaw-runtime-abc", + DeploymentName: "openclaw-runtime", + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 100, + UsedSlots: 100, + }, + }, + } + deployments := &fakeRuntimeDeploymentService{} + agent := &fakeRuntimeAgentClient{} + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(deployments.scales); got != 1 { + t.Fatalf("deployment Scale calls = %d, want 1", got) + } + scale := deployments.scales[0] + if scale.namespace != "clawmanager-system" || scale.name != "openclaw-runtime" || scale.replicas != 2 { + t.Fatalf("deployment Scale call = %+v, want clawmanager-system/openclaw-runtime replicas 2", scale) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0 while waiting for scale-out pod", got) + } + if state, ok := instanceRepo.runtimeStates[27]; ok { + t.Fatalf("instance runtime state = %+v, want unchanged while waiting for scale-out pod", state) + } +} + +func TestRuntimeSchedulerDoesNotScaleOutWhenGatewayCreateFailsWithFreeSlot(t *testing.T) { + ctx := context.Background() + endpoint := "http://pod-1.runtime" + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 5: { + ID: 5, + RuntimeType: RuntimeTypeOpenClaw, + Namespace: "clawmanager-system", + DeploymentName: "openclaw-runtime", + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 100, + UsedSlots: 1, + }, + }, + schedulable: []models.RuntimePod{{ + ID: 5, + RuntimeType: RuntimeTypeOpenClaw, + Namespace: "clawmanager-system", + DeploymentName: "openclaw-runtime", + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 100, + UsedSlots: 1, + }}, + } + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{createErr: errors.New("agent timeout")}, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + + err := scheduler.assignInstance(ctx, models.Instance{ + ID: 28, + UserID: 12, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: ptrString("/workspaces/openclaw/user-12/instance-28"), + RuntimeGeneration: 1, + }) + if err == nil { + t.Fatal("assignInstance returned nil error") + } + if got := len(deployments.scales); got != 0 { + t.Fatalf("deployment Scale calls = %d, want 0", got) + } +} + +func TestRuntimeSchedulerReconcileAssignsDesiredRunningInstanceWithMissingBinding(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-46/instance-18" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.desiredRunning = []models.Instance{{ + ID: 18, + UserID: 46, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "running", + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 4, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-18", Port: 20018, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 1 { + t.Fatalf("CreateGateway calls = %d, want 1", got) + } +} + +func TestRuntimeSchedulerReconcileRetriesRecoverableNoSchedulableError(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-46/instance-68" + errorMessage := "no schedulable openclaw runtime pod" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.desiredRunning = []models.Instance{{ + ID: 68, + UserID: 46, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "error", + RuntimeErrorMessage: &errorMessage, + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 29, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 50: {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0}, + }, + schedulable: []models.RuntimePod{ + {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-68", Port: 20068, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 1 { + t.Fatalf("CreateGateway calls = %d, want 1", got) + } + state := instanceRepo.runtimeStates[68] + if state.status != "running" || state.generation != 29 || state.message != nil { + t.Fatalf("runtime state = %+v, want running generation 29 without error message", state) + } +} + +func TestRuntimeSchedulerReconcileSkipsNonRecoverableErrorInstance(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-46/instance-69" + errorMessage := "create gateway failed: invalid runtime response" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.desiredRunning = []models.Instance{{ + ID: 69, + UserID: 46, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "error", + RuntimeErrorMessage: &errorMessage, + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 24, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 50: {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0}, + }, + schedulable: []models.RuntimePod{ + {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-69", Port: 20069, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0 for non-recoverable error", got) + } + if _, ok := instanceRepo.runtimeStates[69]; ok { + t.Fatalf("runtime state was changed for non-recoverable error: %+v", instanceRepo.runtimeStates[69]) + } +} + +func TestRuntimeSchedulerReconcileSkipsAssignmentWhenBindingLookupErrors(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + lookupErr := errors.New("binding db unavailable") + workspacePath := "/workspaces/openclaw/user-47/instance-19" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.desiredRunning = []models.Instance{{ + ID: 19, + UserID: 47, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "running", + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.runningErr = lookupErr + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-19", Port: 20019, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + err := scheduler.reconcile(ctx) + if err == nil { + t.Fatal("reconcile returned nil error") + } + if !errors.Is(err, lookupErr) { + t.Fatalf("reconcile error = %v, want lookup error", err) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0", got) + } + if bindingRepo.bindings[19] != nil { + t.Fatal("binding was created despite lookup error") + } + if podRepo.claims[9] != 0 { + t.Fatalf("pod claims = %d, want 0", podRepo.claims[9]) + } +} + +func TestRuntimeSchedulerReconcileSkipsLegacyInstanceWithoutWorkspacePath(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 20, + UserID: 48, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + MemoryGB: 1, + DiskGB: 1, + RuntimeGeneration: 1, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-20", Port: 20020, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0", got) + } + if _, ok := instanceRepo.runtimeStates[20]; ok { + t.Fatalf("legacy instance was marked with runtime state: %+v", instanceRepo.runtimeStates[20]) + } +} + +func TestRuntimeSchedulerSkipsCreatingInstanceWithExistingBinding(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-49/instance-21" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 21, + UserID: 49, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 1, + }} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[21] = &models.InstanceRuntimeBinding{InstanceID: 21, RuntimePodID: 9, State: "creating", Generation: 1} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-21", Port: 20021, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0", got) + } + if podRepo.claims[9] != 0 { + t.Fatalf("pod claims = %d, want 0", podRepo.claims[9]) + } +} + +func TestRuntimeSchedulerSyncsCreatingInstanceFromRunningBinding(t *testing.T) { + ctx := context.Background() + workspacePath := "/workspaces/openclaw/user-1/instance-25" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 25, + UserID: 1, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + WorkspacePath: &workspacePath, + RuntimeGeneration: 2, + }} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[25] = &models.InstanceRuntimeBinding{ + InstanceID: 25, + RuntimeType: RuntimeTypeOpenClaw, + State: "running", + Generation: 2, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + &fakeRuntimePodRepo{}, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + + state := instanceRepo.runtimeStates[25] + if state.status != "running" || state.generation != 2 || state.message != nil { + t.Fatalf("expected instance to sync to running from binding, got %+v", state) + } +} + +func TestRuntimeSchedulerCleansUpGatewayAndReleasesSlotWhenBindingCreateFails(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + createErr := errors.New("binding insert failed") + instanceRepo := newFakeRuntimeInstanceRepo() + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.createErr = createErr + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-22", Port: 20022, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + err := scheduler.assignInstance(ctx, models.Instance{ + ID: 22, + UserID: 50, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: ptrString("/workspaces/openclaw/user-50/instance-22"), + MemoryGB: 1, + DiskGB: 1, + RuntimeGeneration: 1, + }) + if err == nil { + t.Fatal("assignInstance returned nil error") + } + if !errors.Is(err, createErr) { + t.Fatalf("assignInstance error = %v, want binding create error", err) + } + if got := len(agent.deleteRequests); got != 1 { + t.Fatalf("DeleteGateway calls = %d, want 1", got) + } + if agent.deleteRequests[0].gatewayID != "gw-22" { + t.Fatalf("deleted gateway = %q, want gw-22", agent.deleteRequests[0].gatewayID) + } + if podRepo.releases[9] != 1 { + t.Fatalf("pod releases = %d, want 1", podRepo.releases[9]) + } + if bindingRepo.bindings[22] != nil { + t.Fatal("binding remains after failed create") + } +} + +func TestRuntimeSchedulerCleansUpGatewayBindingAndSlotWhenWorkspacePathUpdateFails(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspaceErr := errors.New("workspace path update failed") + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.setWorkspacePathErr = workspaceErr + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-23", Port: 20023, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + err := scheduler.assignInstance(ctx, models.Instance{ + ID: 23, + UserID: 51, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: ptrString("/workspaces/openclaw/user-51/instance-23"), + MemoryGB: 1, + DiskGB: 1, + RuntimeGeneration: 1, + }) + if err == nil { + t.Fatal("assignInstance returned nil error") + } + if !errors.Is(err, workspaceErr) { + t.Fatalf("assignInstance error = %v, want workspace update error", err) + } + if got := len(agent.deleteRequests); got != 1 { + t.Fatalf("DeleteGateway calls = %d, want 1", got) + } + if agent.deleteRequests[0].gatewayID != "gw-23" { + t.Fatalf("deleted gateway = %q, want gw-23", agent.deleteRequests[0].gatewayID) + } + if bindingRepo.deleteCalls[23] != 1 { + t.Fatalf("binding delete calls = %d, want 1", bindingRepo.deleteCalls[23]) + } + if bindingRepo.bindings[23] != nil { + t.Fatal("binding remains after workspace update failure") + } + if podRepo.releases[9] != 1 { + t.Fatalf("pod releases = %d, want 1", podRepo.releases[9]) + } +} + +func TestRuntimeSchedulerDesiredRunningSkipsNonRunningExistingBinding(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-52/instance-24" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.desiredRunning = []models.Instance{{ + ID: 24, + UserID: 52, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "running", + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 1, + }} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[24] = &models.InstanceRuntimeBinding{ + InstanceID: 24, + RuntimePodID: 9, + State: "creating", + Generation: 1, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-24", Port: 20024, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0", got) + } + if podRepo.claims[9] != 0 { + t.Fatalf("pod claims = %d, want 0", podRepo.claims[9]) + } + if bindingRepo.bindings[24].GatewayID != "" { + t.Fatalf("binding was overwritten: %+v", bindingRepo.bindings[24]) + } +} + +func TestRuntimeSchedulerFailoverPodMarksUnhealthyDeletesBindingsReleasesSlotsAndRecreatesInstances(t *testing.T) { + ctx := context.Background() + reason := "agent heartbeat lost" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{ + InstanceID: 17, + RuntimePodID: 9, + Generation: 3, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready"}, + }, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.FailoverPod(ctx, 9, reason); err != nil { + t.Fatalf("FailoverPod returned error: %v", err) + } + + if got := podRepo.marked[9]; got.state != "unhealthy" || got.draining { + t.Fatalf("marked pod state = %+v", got) + } + if bindingRepo.bindings[17] != nil { + t.Fatal("binding was not deleted") + } + if podRepo.releases[9] != 1 { + t.Fatalf("pod releases = %d, want 1", podRepo.releases[9]) + } + state := instanceRepo.runtimeStates[17] + if state.status != "creating" || state.generation != 4 { + t.Fatalf("runtime state = %+v, want creating generation 4", state) + } + if state.message == nil || *state.message != reason { + t.Fatalf("runtime state message = %v, want %q", state.message, reason) + } +} + +func TestRuntimeSchedulerFailoverLeavesBindingAndSlotWhenInstanceUpdateFails(t *testing.T) { + ctx := context.Background() + reason := "agent heartbeat lost" + updateErr := errors.New("instance update failed") + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3} + instanceRepo.updateRuntimeStateErr = updateErr + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{ + InstanceID: 17, + RuntimePodID: 9, + Generation: 3, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready"}, + }, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + err := scheduler.FailoverPod(ctx, 9, reason) + if err == nil { + t.Fatal("FailoverPod returned nil error") + } + if !errors.Is(err, updateErr) { + t.Fatalf("FailoverPod error = %v, want update error", err) + } + if bindingRepo.bindings[17] == nil { + t.Fatal("binding was deleted despite instance update failure") + } + if podRepo.releases[9] != 0 { + t.Fatalf("pod releases = %d, want 0", podRepo.releases[9]) + } +} + +func TestRuntimeSchedulerReconcileFailoversStalePod(t *testing.T) { + ctx := context.Background() + staleSeen := time.Now().UTC().Add(-30 * time.Second) + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{ + InstanceID: 17, + RuntimePodID: 9, + Generation: 3, + State: "running", + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready", LastSeenAt: &staleSeen}, + }, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + WithRuntimeSchedulerHeartbeatTimeout(10*time.Second), + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + + if got := podRepo.marked[9]; got.state != "unhealthy" || got.draining { + t.Fatalf("marked pod state = %+v", got) + } + if bindingRepo.bindings[17] != nil { + t.Fatal("binding was not deleted") + } + if podRepo.releases[9] != 1 { + t.Fatalf("pod releases = %d, want 1", podRepo.releases[9]) + } + state := instanceRepo.runtimeStates[17] + if state.status != "creating" || state.generation != 4 { + t.Fatalf("runtime state = %+v, want creating generation 4", state) + } + if state.message == nil || !strings.Contains(*state.message, "runtime pod heartbeat lost") { + t.Fatalf("runtime state message = %v, want heartbeat lost reason", state.message) + } +} + +func TestRuntimeSchedulerReconcileKeepsRecentPodBinding(t *testing.T) { + ctx := context.Background() + recentSeen := time.Now().UTC() + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{ + InstanceID: 17, + RuntimePodID: 9, + Generation: 3, + State: "running", + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready", LastSeenAt: &recentSeen}, + }, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + WithRuntimeSchedulerHeartbeatTimeout(10*time.Second), + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + + if got := podRepo.marked[9]; got.state != "" || got.draining { + t.Fatalf("pod was unexpectedly marked: %+v", got) + } + if bindingRepo.bindings[17] == nil { + t.Fatal("binding was deleted") + } + if podRepo.releases[9] != 0 { + t.Fatalf("pod releases = %d, want 0", podRepo.releases[9]) + } +} + +func TestRuntimeSchedulerRolloutExistingDrainingPodsPreventAdditionalDrains(t *testing.T) { + ctx := context.Background() + readyEndpoint := "http://ready.runtime" + drainingEndpoint := "http://draining.runtime" + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 7: {ID: 7, RuntimeType: RuntimeTypeOpenClaw, Status: "pending", BatchSize: 3, MaxUnavailable: 1}, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 1: {ID: 1, RuntimeType: RuntimeTypeOpenClaw, State: "ready", Draining: true, AgentEndpoint: &drainingEndpoint}, + 2: {ID: 2, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &readyEndpoint}, + }, + } + agent := &fakeRuntimeAgentClient{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.StartRollout(ctx, 7); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(agent.drainEndpoints); got != 0 { + t.Fatalf("Drain calls = %d, want 0", got) + } + if len(rolloutRepo.statuses) != 1 || rolloutRepo.statuses[0].status != "running" { + t.Fatalf("rollout statuses = %+v, want only running", rolloutRepo.statuses) + } +} + +func TestRuntimeSchedulerRolloutBatchLimitedByMaxUnavailable(t *testing.T) { + ctx := context.Background() + endpoint1 := "http://pod-1.runtime" + endpoint2 := "http://pod-2.runtime" + endpoint3 := "http://pod-3.runtime" + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 8: {ID: 8, RuntimeType: RuntimeTypeOpenClaw, Status: "pending", BatchSize: 3, MaxUnavailable: 2}, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 1: {ID: 1, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &endpoint1}, + 2: {ID: 2, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &endpoint2}, + 3: {ID: 3, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &endpoint3}, + }, + } + agent := &fakeRuntimeAgentClient{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.StartRollout(ctx, 8); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(agent.drainEndpoints); got != 2 { + t.Fatalf("Drain calls = %d, want 2", got) + } +} + +func TestRuntimeSchedulerRolloutUpdatesDeployments(t *testing.T) { + ctx := context.Background() + endpoint := "http://pod-1.runtime" + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, TargetImageRef: "new-image", Status: "pending", BatchSize: 1, MaxUnavailable: 1}, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 1: { + ID: 1, + RuntimeType: RuntimeTypeOpenClaw, + State: "ready", + Namespace: "runtime-system", + DeploymentName: "openclaw-runtime", + AgentEndpoint: &endpoint, + }, + }, + } + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + + if err := scheduler.StartRollout(ctx, 9); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(deployments.rolloutImageCalls); got != 1 { + t.Fatalf("deployment RolloutImage calls = %d, want 1", got) + } + call := deployments.rolloutImageCalls[0] + if call.namespace != "runtime-system" || call.name != "openclaw-runtime" || call.image != "new-image" { + t.Fatalf("RolloutImage call = %+v, want runtime-system/openclaw-runtime new-image", call) + } + if call.maxUnavailable != 1 || call.maxSurge != 1 { + t.Fatalf("RolloutImage strategy = %+v, want maxUnavailable=1 maxSurge=1", call) + } +} + +func TestRuntimeSchedulerRolloutUsesStalePodDeploymentRefWhenNoCurrentPods(t *testing.T) { + ctx := context.Background() + staleSeen := time.Now().UTC().Add(-5 * time.Minute) + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 12: {ID: 12, RuntimeType: RuntimeTypeOpenClaw, TargetImageRef: "registry/openclaw:v2", Status: "pending", BatchSize: 1, MaxUnavailable: 1}, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 41: { + ID: 41, + RuntimeType: RuntimeTypeOpenClaw, + State: "unhealthy", + Namespace: "runtime-system", + DeploymentName: "openclaw-runtime", + ImageRef: "registry/openclaw:v1", + LastSeenAt: &staleSeen, + }, + }, + } + agent := &fakeRuntimeAgentClient{} + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + agent, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + WithRuntimeSchedulerHeartbeatTimeout(10*time.Second), + ) + + if err := scheduler.StartRollout(ctx, 12); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(deployments.rolloutImageCalls); got != 1 { + t.Fatalf("deployment RolloutImage calls = %d, want 1", got) + } + call := deployments.rolloutImageCalls[0] + if call.namespace != "runtime-system" || call.name != "openclaw-runtime" || call.image != "registry/openclaw:v2" { + t.Fatalf("RolloutImage call = %+v, want runtime-system/openclaw-runtime registry/openclaw:v2", call) + } + if got := len(agent.drainEndpoints); got != 0 { + t.Fatalf("Drain calls = %d, want 0 with no current runtime pods", got) + } + if len(rolloutRepo.statuses) != 1 || rolloutRepo.statuses[0].status != "running" { + t.Fatalf("rollout statuses = %+v, want only running", rolloutRepo.statuses) + } +} + +func TestRuntimeSchedulerRolloutFallsBackToDefaultDeploymentWhenNoPodRows(t *testing.T) { + ctx := context.Background() + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 13: {ID: 13, RuntimeType: RuntimeTypeOpenClaw, TargetImageRef: "registry/openclaw:v3", Status: "pending", BatchSize: 2, MaxUnavailable: 1}, + }, + } + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{}}, + newFakeRuntimeBindingRepo(), + rolloutRepo, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + WithRuntimeSchedulerNamespace("runtime-system"), + ) + + if err := scheduler.StartRollout(ctx, 13); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(deployments.rolloutImageCalls); got != 1 { + t.Fatalf("deployment RolloutImage calls = %d, want 1", got) + } + call := deployments.rolloutImageCalls[0] + if call.namespace != "runtime-system" || call.name != "openclaw-runtime" || call.image != "registry/openclaw:v3" { + t.Fatalf("RolloutImage call = %+v, want runtime-system/openclaw-runtime registry/openclaw:v3", call) + } +} + +func TestRuntimeSchedulerRolloutSameImageFinishesWithoutDrain(t *testing.T) { + ctx := context.Background() + recentSeen := time.Now().UTC() + endpoint := "http://pod-1.runtime" + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 11: {ID: 11, RuntimeType: RuntimeTypeHermes, TargetImageRef: "registry/hermes:v2", Status: "pending", BatchSize: 1, MaxUnavailable: 1}, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 4: { + ID: 4, + RuntimeType: RuntimeTypeHermes, + State: "ready", + Namespace: "runtime-system", + DeploymentName: "hermes-runtime", + ImageRef: "registry/hermes:v2", + AgentEndpoint: &endpoint, + LastSeenAt: &recentSeen, + }, + }, + } + agent := &fakeRuntimeAgentClient{} + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + agent, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + + if err := scheduler.StartRollout(ctx, 11); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(agent.drainEndpoints); got != 0 { + t.Fatalf("Drain calls = %d, want 0", got) + } + if got := len(deployments.rolloutImageCalls); got != 0 { + t.Fatalf("RolloutImage calls = %d, want 0", got) + } + if len(rolloutRepo.statuses) != 2 || rolloutRepo.statuses[0].status != "running" || rolloutRepo.statuses[1].status != "finished" { + t.Fatalf("rollout statuses = %+v, want running then finished", rolloutRepo.statuses) + } +} + +func TestRuntimeSchedulerReconcileStartsPendingRollouts(t *testing.T) { + ctx := context.Background() + recentSeen := time.Now().UTC() + endpoint := "http://pod-1.runtime" + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 10: { + ID: 10, + RuntimeType: RuntimeTypeHermes, + TargetImageRef: "registry/hermes:v2", + Status: "pending", + BatchSize: 2, + MaxUnavailable: 1, + }, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 3: { + ID: 3, + RuntimeType: RuntimeTypeHermes, + State: "ready", + Namespace: "runtime-system", + DeploymentName: "hermes-runtime", + AgentEndpoint: &endpoint, + LastSeenAt: &recentSeen, + }, + }, + } + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(deployments.rolloutImageCalls); got != 1 { + t.Fatalf("deployment RolloutImage calls = %d, want 1", got) + } + call := deployments.rolloutImageCalls[0] + if call.namespace != "runtime-system" || call.name != "hermes-runtime" || call.image != "registry/hermes:v2" { + t.Fatalf("RolloutImage call = %+v, want runtime-system/hermes-runtime registry/hermes:v2", call) + } + if len(rolloutRepo.statuses) == 0 || rolloutRepo.statuses[0].status != "running" { + t.Fatalf("rollout statuses = %+v, want running", rolloutRepo.statuses) + } +} + +type fakeRuntimeInstanceRepo struct { + creating []models.Instance + desiredRunning []models.Instance + byID map[int]*models.Instance + workspacePaths map[int]string + runtimeStates map[int]fakeRuntimeState + updateRuntimeStateErr error + setWorkspacePathErr error +} + +type fakeRuntimeState struct { + status string + generation int + message *string +} + +func newFakeRuntimeInstanceRepo() *fakeRuntimeInstanceRepo { + return &fakeRuntimeInstanceRepo{ + byID: map[int]*models.Instance{}, + workspacePaths: map[int]string{}, + runtimeStates: map[int]fakeRuntimeState{}, + } +} + +func (r *fakeRuntimeInstanceRepo) Create(instance *models.Instance) error { return nil } +func (r *fakeRuntimeInstanceRepo) GetByID(id int) (*models.Instance, error) { + return r.byID[id], nil +} +func (r *fakeRuntimeInstanceRepo) GetByAccessToken(accessToken string) (*models.Instance, error) { + return nil, nil +} +func (r *fakeRuntimeInstanceRepo) GetByAgentBootstrapToken(bootstrapToken string) (*models.Instance, error) { + return nil, nil +} +func (r *fakeRuntimeInstanceRepo) GetAll(offset, limit int) ([]models.Instance, error) { + return nil, nil +} +func (r *fakeRuntimeInstanceRepo) CountAll() (int, error) { return 0, nil } +func (r *fakeRuntimeInstanceRepo) GetByUserID(userID int, offset, limit int) ([]models.Instance, error) { + return nil, nil +} +func (r *fakeRuntimeInstanceRepo) CountByUserID(userID int) (int, error) { return 0, nil } +func (r *fakeRuntimeInstanceRepo) CountActiveByMode(ctx context.Context, mode string) (int, error) { + return 0, nil +} +func (r *fakeRuntimeInstanceRepo) ExistsByUserIDAndName(userID int, name string) (bool, error) { + return false, nil +} +func (r *fakeRuntimeInstanceRepo) GetAllRunning() ([]models.Instance, error) { return nil, nil } +func (r *fakeRuntimeInstanceRepo) GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) { + return r.desiredRunning, nil +} +func (r *fakeRuntimeInstanceRepo) GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) { + return r.creating, nil +} +func (r *fakeRuntimeInstanceRepo) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error { + if r.updateRuntimeStateErr != nil { + return r.updateRuntimeStateErr + } + r.runtimeStates[id] = fakeRuntimeState{status: status, generation: generation, message: message} + return nil +} +func (r *fakeRuntimeInstanceRepo) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error { + if r.setWorkspacePathErr != nil { + return r.setWorkspacePathErr + } + r.workspacePaths[id] = workspacePath + return nil +} +func (r *fakeRuntimeInstanceRepo) UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error { + return nil +} +func (r *fakeRuntimeInstanceRepo) Update(instance *models.Instance) error { return nil } +func (r *fakeRuntimeInstanceRepo) Delete(id int) error { return nil } + +type fakeRuntimePodRepo struct { + pods map[int64]*models.RuntimePod + schedulable []models.RuntimePod + claims map[int64]int + releases map[int64]int + marked map[int64]fakePodMark + releaseCount int + getErr error +} + +type fakePodMark struct { + state string + draining bool +} + +func (r *fakeRuntimePodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error { + return nil +} +func (r *fakeRuntimePodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) { + if r.getErr != nil { + return nil, r.getErr + } + return r.pods[id], nil +} +func (r *fakeRuntimePodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) { + return nil, nil +} +func (r *fakeRuntimePodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + var pods []models.RuntimePod + for _, pod := range r.pods { + if runtimeType == "" || pod.RuntimeType == runtimeType { + pods = append(pods, *pod) + } + } + return pods, nil +} +func (r *fakeRuntimePodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + return r.schedulable, nil +} +func (r *fakeRuntimePodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) { + if r.claims == nil { + r.claims = map[int64]int{} + } + r.claims[podID]++ + return true, nil +} +func (r *fakeRuntimePodRepo) ReleaseSlot(ctx context.Context, podID int64) error { + if r.releases == nil { + r.releases = map[int64]int{} + } + r.releases[podID]++ + r.releaseCount++ + return nil +} +func (r *fakeRuntimePodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error { + if r.marked == nil { + r.marked = map[int64]fakePodMark{} + } + r.marked[podID] = fakePodMark{state: state, draining: draining} + return nil +} +func (r *fakeRuntimePodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error { + if r.pods != nil { + if pod := r.pods[podID]; pod != nil { + pod.State = state + pod.UsedSlots = usedSlots + pod.Capacity = capacity + pod.Draining = draining + pod.LastSeenAt = &lastSeenAt + } + } + return nil +} +func (r *fakeRuntimePodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error { + return nil +} +func (r *fakeRuntimePodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error { + return nil +} + +type fakeRuntimeBindingRepo struct { + bindings map[int]*models.InstanceRuntimeBinding + deleteCalls map[int]int + deleteAndReleaseCalls map[int]int + runningErr error + getErr error + createErr error +} + +func newFakeRuntimeBindingRepo() *fakeRuntimeBindingRepo { + return &fakeRuntimeBindingRepo{ + bindings: map[int]*models.InstanceRuntimeBinding{}, + deleteCalls: map[int]int{}, + deleteAndReleaseCalls: map[int]int{}, + } +} + +func (r *fakeRuntimeBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error { + if r.createErr != nil { + return r.createErr + } + copy := *binding + r.bindings[binding.InstanceID] = © + return nil +} +func (r *fakeRuntimeBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + if r.getErr != nil { + return nil, r.getErr + } + return r.bindings[instanceID], nil +} +func (r *fakeRuntimeBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + if r.runningErr != nil { + return nil, r.runningErr + } + binding := r.bindings[instanceID] + if binding == nil || binding.State != "running" { + return nil, nil + } + return binding, nil +} +func (r *fakeRuntimeBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) { + var bindings []models.InstanceRuntimeBinding + for _, binding := range r.bindings { + if binding.RuntimePodID == runtimePodID { + bindings = append(bindings, *binding) + } + } + return bindings, nil +} +func (r *fakeRuntimeBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) { + return nil, nil +} +func (r *fakeRuntimeBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error { + return nil +} +func (r *fakeRuntimeBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error { + return nil +} +func (r *fakeRuntimeBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error { + r.deleteCalls[instanceID]++ + delete(r.bindings, instanceID) + return nil +} +func (r *fakeRuntimeBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error { + r.deleteAndReleaseCalls[instanceID]++ + delete(r.bindings, instanceID) + return nil +} + +type fakeRuntimeRolloutRepo struct { + rollouts map[int64]*models.RuntimeRollout + statuses []fakeRolloutStatus +} + +type fakeRolloutStatus struct { + id int64 + status string + startedAt *time.Time + finishedAt *time.Time + message *string +} + +func (r *fakeRuntimeRolloutRepo) Create(ctx context.Context, rollout *models.RuntimeRollout) error { + return nil +} +func (r *fakeRuntimeRolloutRepo) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) { + if r.rollouts == nil { + return nil, nil + } + return r.rollouts[id], nil +} +func (r *fakeRuntimeRolloutRepo) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) { + var active []models.RuntimeRollout + for _, rollout := range r.rollouts { + if rollout == nil { + continue + } + if runtimeType != "" && rollout.RuntimeType != runtimeType { + continue + } + if rollout.Status == "pending" || rollout.Status == "running" { + active = append(active, *rollout) + } + } + return active, nil +} +func (r *fakeRuntimeRolloutRepo) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error { + r.statuses = append(r.statuses, fakeRolloutStatus{id: id, status: status, startedAt: startedAt, finishedAt: finishedAt, message: message}) + return nil +} + +type fakeRuntimeAgentClient struct { + createResponse *RuntimeAgentCreateGatewayResponse + createErr error + deleteErr error + createRequests []fakeCreateGatewayRequest + deleteRequests []fakeDeleteGatewayRequest + drainEndpoints []string +} + +type fakeCreateGatewayRequest struct { + endpoint string + req RuntimeAgentCreateGatewayRequest +} + +type fakeDeleteGatewayRequest struct { + endpoint string + gatewayID string +} + +func (c *fakeRuntimeAgentClient) Health(ctx context.Context, endpoint string) error { return nil } +func (c *fakeRuntimeAgentClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) { + c.createRequests = append(c.createRequests, fakeCreateGatewayRequest{endpoint: endpoint, req: req}) + if c.createErr != nil { + return nil, c.createErr + } + if c.createResponse == nil { + return nil, errors.New("missing create response") + } + return c.createResponse, nil +} +func (c *fakeRuntimeAgentClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error { + c.deleteRequests = append(c.deleteRequests, fakeDeleteGatewayRequest{endpoint: endpoint, gatewayID: gatewayID}) + return c.deleteErr +} +func (c *fakeRuntimeAgentClient) Drain(ctx context.Context, endpoint string) error { + c.drainEndpoints = append(c.drainEndpoints, endpoint) + return nil +} + +type fakeRuntimeEventService struct { + published []fakeRuntimeEvent +} + +type fakeRuntimeEvent struct { + eventType string + payload any +} + +func (s *fakeRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error { + s.published = append(s.published, fakeRuntimeEvent{eventType: eventType, payload: payload}) + return nil +} +func (s *fakeRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) { + return nil, nil +} + +type fakeRuntimeDeploymentService struct { + ensureCalls int + scaleCalls int + scales []fakeScaleCall + rolloutImageCalls []fakeRolloutImageCall + pods []k8s.RuntimeDeploymentPod +} + +type fakeScaleCall struct { + namespace string + name string + replicas int32 +} + +type fakeRolloutImageCall struct { + namespace string + name string + image string + maxUnavailable int + maxSurge int +} + +func (s *fakeRuntimeDeploymentService) Ensure(ctx context.Context, spec k8s.RuntimeDeploymentSpec) error { + s.ensureCalls++ + return nil +} +func (s *fakeRuntimeDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error { + s.scaleCalls++ + s.scales = append(s.scales, fakeScaleCall{namespace: namespace, name: name, replicas: replicas}) + return nil +} +func (s *fakeRuntimeDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error { + s.rolloutImageCalls = append(s.rolloutImageCalls, fakeRolloutImageCall{ + namespace: namespace, + name: name, + image: image, + maxUnavailable: maxUnavailable, + maxSurge: maxSurge, + }) + return nil +} +func (s *fakeRuntimeDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]k8s.RuntimeDeploymentPod, error) { + var pods []k8s.RuntimeDeploymentPod + for _, pod := range s.pods { + if namespace != "" && pod.Namespace != namespace { + continue + } + if runtimeType != "" && pod.RuntimeType != runtimeType { + continue + } + pods = append(pods, pod) + } + return pods, nil +} + +func TestIsSchedulerManagedV2InstanceRejectsProDesktop(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-7/instance-42" + instance := models.Instance{ + ID: 42, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendDesktop, + InstanceMode: InstanceModePro, + WorkspacePath: &workspacePath, + } + if isSchedulerManagedV2Instance(instance) { + t.Fatal("pro desktop instance must not be scheduler-managed") + } + + instance.RuntimeType = RuntimeBackendGateway + instance.InstanceMode = InstanceModeLite + if !isSchedulerManagedV2Instance(instance) { + t.Fatal("lite gateway instance should be scheduler-managed") + } +} + +func ptrString(value string) *string { + return &value +} diff --git a/backend/internal/services/runtime_workspace_file_service.go b/backend/internal/services/runtime_workspace_file_service.go new file mode 100644 index 0000000..7af41fe --- /dev/null +++ b/backend/internal/services/runtime_workspace_file_service.go @@ -0,0 +1,660 @@ +package services + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" +) + +const ( + runtimeWorkspaceErrPrefix = "CLAW_WORKSPACE_ERR:" + runtimeWorkspaceBaseDir = "/config" +) + +type runtimeWorkspaceExecutor interface { + exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error +} + +type runtimeWorkspaceFileService struct { + auditRepo repository.WorkspaceFileAuditRepository + executor runtimeWorkspaceExecutor +} + +type k8sRuntimeWorkspaceExecutor struct { + deploymentService *k8s.InstanceDeploymentService +} + +type runtimeWorkspaceEntryPayload struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` + ModifiedAt time.Time `json:"modified_at"` +} + +func NewRuntimeWorkspaceFileService(auditRepo repository.WorkspaceFileAuditRepository) WorkspaceFileService { + return &runtimeWorkspaceFileService{ + auditRepo: auditRepo, + executor: &k8sRuntimeWorkspaceExecutor{ + deploymentService: k8s.NewInstanceDeploymentService(), + }, + } +} + +func (s *runtimeWorkspaceFileService) List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return nil, err + } + var response struct { + Entries []runtimeWorkspaceEntryPayload `json:"entries"` + } + if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceListScript, runtimeWorkspaceBase(scope), relative}, nil, &response); err != nil { + return nil, err + } + entries := make([]WorkspaceEntry, 0, len(response.Entries)) + for _, payload := range response.Entries { + entry := runtimeWorkspaceEntryFromPayload(payload) + entries = append(entries, entry) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].IsDir != entries[j].IsDir { + return entries[i].IsDir + } + left := strings.ToLower(entries[i].Name) + right := strings.ToLower(entries[j].Name) + if left == right { + return entries[i].Name < entries[j].Name + } + return left < right + }) + return entries, nil +} + +func (s *runtimeWorkspaceFileService) Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) { + relative, entry, err := s.statFile(ctx, scope, relativePath) + if err != nil { + return nil, err + } + kind, contentType, previewable, maxBytes := workspacePreviewKind(entry.Name, entry.Size) + downloadURL := workspaceDownloadURL(scope.InstanceID, relative) + if !previewable { + return &WorkspacePreview{ + Kind: "binary", + ContentType: "application/octet-stream", + DownloadURL: downloadURL, + }, nil + } + if maxBytes > 0 && entry.Size > maxBytes { + return nil, ErrWorkspacePreviewTooLarge + } + if kind == "text" { + file, _, _, err := s.openRemoteFile(ctx, scope, relative, false) + if err != nil { + return nil, err + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, textPreviewMaxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > textPreviewMaxBytes { + return nil, ErrWorkspacePreviewTooLarge + } + return &WorkspacePreview{ + Kind: "text", + ContentType: contentType, + Text: string(data), + DownloadURL: downloadURL, + }, nil + } + + return &WorkspacePreview{ + Kind: kind, + ContentType: contentType, + PreviewURL: workspacePreviewURL(scope.InstanceID, relative), + DownloadURL: downloadURL, + }, nil +} + +func (s *runtimeWorkspaceFileService) OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + _, entry, err := s.statFile(ctx, scope, relativePath) + if err != nil { + return nil, "", 0, err + } + _, contentType, previewable, maxBytes := workspacePreviewKind(entry.Name, entry.Size) + if !previewable { + return nil, "", 0, ErrWorkspaceFileExpected + } + if maxBytes > 0 && entry.Size > maxBytes { + return nil, "", 0, ErrWorkspacePreviewTooLarge + } + file, _, size, err := s.openRemoteFile(ctx, scope, relativePath, false) + if err != nil { + return nil, "", 0, err + } + return file, contentType, size, nil +} + +func (s *runtimeWorkspaceFileService) OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + relative, entry, err := s.statFile(ctx, scope, relativePath) + if err != nil { + return nil, "", 0, err + } + file, filename, size, err := s.openRemoteFile(ctx, scope, relative, true) + if err != nil { + return nil, "", 0, err + } + if filename == "" { + filename = entry.Name + } + return file, filename, size, nil +} + +func (s *runtimeWorkspaceFileService) Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + maxBytes := WorkspaceUploadMaxBytes() + if size > maxBytes { + return nil, ErrWorkspaceUploadTooLarge + } + cleanName, err := sanitizeWorkspaceFilename(filename) + if err != nil { + return nil, err + } + dir, err := cleanWorkspaceRelativePath(relativeDir) + if err != nil { + return nil, err + } + targetRelative := joinWorkspaceRelative(dir, cleanName) + limited := io.LimitReader(reader, maxBytes+1) + var payload runtimeWorkspaceEntryPayload + if err := s.runJSON(ctx, scope, []string{ + "python3", + "-c", + runtimeWorkspaceUploadScript, + runtimeWorkspaceBase(scope), + dir, + cleanName, + fmt.Sprintf("%d", maxBytes), + }, limited, &payload); err != nil { + return nil, err + } + if err := s.recordAudit(ctx, scope, "upload", targetRelative, payload.Size); err != nil { + return nil, err + } + entry := runtimeWorkspaceEntryFromPayload(payload) + return &entry, nil +} + +func (s *runtimeWorkspaceFileService) Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error) { + if isWorkspaceRootRelative(relativePath) { + return nil, ErrWorkspaceRootOperation + } + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return nil, err + } + var payload runtimeWorkspaceEntryPayload + if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceMkdirScript, runtimeWorkspaceBase(scope), relative}, nil, &payload); err != nil { + return nil, err + } + if err := s.recordAudit(ctx, scope, "mkdir", relative, 0); err != nil { + return nil, err + } + entry := runtimeWorkspaceEntryFromPayload(payload) + return &entry, nil +} + +func (s *runtimeWorkspaceFileService) Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error) { + if isWorkspaceRootRelative(oldPath) || isWorkspaceRootRelative(newPath) { + return nil, ErrWorkspaceRootOperation + } + oldRelative, err := cleanWorkspaceRelativePath(oldPath) + if err != nil { + return nil, err + } + newRelative, err := cleanWorkspaceRelativePath(newPath) + if err != nil { + return nil, err + } + var payload runtimeWorkspaceEntryPayload + if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceRenameScript, runtimeWorkspaceBase(scope), oldRelative, newRelative}, nil, &payload); err != nil { + return nil, err + } + if err := s.recordAudit(ctx, scope, "rename", oldRelative+" -> "+newRelative, 0); err != nil { + return nil, err + } + entry := runtimeWorkspaceEntryFromPayload(payload) + return &entry, nil +} + +func (s *runtimeWorkspaceFileService) Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error { + if isWorkspaceRootRelative(relativePath) { + return ErrWorkspaceRootOperation + } + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return err + } + if err := s.run(ctx, scope, []string{"python3", "-c", runtimeWorkspaceDeleteScript, runtimeWorkspaceBase(scope), relative}, nil, io.Discard); err != nil { + return err + } + return s.recordAudit(ctx, scope, "delete", relative, 0) +} + +func (s *runtimeWorkspaceFileService) statFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (string, WorkspaceEntry, error) { + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return "", WorkspaceEntry{}, err + } + var payload runtimeWorkspaceEntryPayload + if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceStatFileScript, runtimeWorkspaceBase(scope), relative}, nil, &payload); err != nil { + return "", WorkspaceEntry{}, err + } + entry := runtimeWorkspaceEntryFromPayload(payload) + return relative, entry, nil +} + +func (s *runtimeWorkspaceFileService) openRemoteFile(ctx context.Context, scope WorkspaceFileScope, relativePath string, audit bool) (*os.File, string, int64, error) { + relative, entry, err := s.statFile(ctx, scope, relativePath) + if err != nil { + return nil, "", 0, err + } + file, err := os.CreateTemp("", "clawmanager-runtime-workspace-*") + if err != nil { + return nil, "", 0, err + } + ok := false + defer func() { + if !ok { + name := file.Name() + _ = file.Close() + _ = os.Remove(name) + } + }() + if err := s.run(ctx, scope, []string{"python3", "-c", runtimeWorkspaceStreamFileScript, runtimeWorkspaceBase(scope), relative}, nil, file); err != nil { + return nil, "", 0, err + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, "", 0, err + } + if audit { + if err := s.recordAudit(ctx, scope, "download", relative, entry.Size); err != nil { + return nil, "", 0, err + } + } + ok = true + return file, entry.Name, entry.Size, nil +} + +func (s *runtimeWorkspaceFileService) runJSON(ctx context.Context, scope WorkspaceFileScope, command []string, stdin io.Reader, target any) error { + var stdout bytes.Buffer + if err := s.run(ctx, scope, command, stdin, &stdout); err != nil { + return err + } + if err := json.Unmarshal(stdout.Bytes(), target); err != nil { + return fmt.Errorf("failed to parse runtime workspace response: %w", err) + } + return nil +} + +func (s *runtimeWorkspaceFileService) run(ctx context.Context, scope WorkspaceFileScope, command []string, stdin io.Reader, stdout io.Writer) error { + if s == nil || s.executor == nil { + return fmt.Errorf("runtime workspace executor is not configured") + } + var stderr bytes.Buffer + if err := s.executor.exec(ctx, scope.UserID, scope.InstanceID, command, stdin, stdout, &stderr); err != nil { + return mapRuntimeWorkspaceError(err, stderr.String()) + } + return nil +} + +func (s *runtimeWorkspaceFileService) recordAudit(ctx context.Context, scope WorkspaceFileScope, action, relativePath string, bytes int64) error { + if s.auditRepo == nil { + return nil + } + if err := ctx.Err(); err != nil { + return err + } + return s.auditRepo.Create(ctx, &models.WorkspaceFileAudit{ + InstanceID: scope.InstanceID, + UserID: scope.UserID, + Action: action, + RelativePath: relativePath, + Bytes: bytes, + CreatedAt: time.Now().UTC(), + }) +} + +func (e *k8sRuntimeWorkspaceExecutor) exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error { + client := k8s.GetClient() + if client == nil || client.Clientset == nil || client.Config == nil { + return fmt.Errorf("k8s client not initialized") + } + deploymentService := e.deploymentService + if deploymentService == nil { + deploymentService = k8s.NewInstanceDeploymentService() + } + pod, err := deploymentService.GetActivePod(ctx, userID, instanceID) + if err != nil { + return fmt.Errorf("failed to get active instance pod: %w", err) + } + req := client.Clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("exec") + req.VersionedParams(&corev1.PodExecOptions{ + Container: "desktop", + Command: command, + Stdin: stdin != nil, + Stdout: stdout != nil, + Stderr: stderr != nil, + TTY: false, + }, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(client.Config, "POST", req.URL()) + if err != nil { + return fmt.Errorf("failed to initialize runtime workspace exec stream: %w", err) + } + return exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + Tty: false, + }) +} + +func runtimeWorkspaceBase(scope WorkspaceFileScope) string { + if base := strings.TrimSpace(scope.WorkspacePath); base != "" { + return base + } + return runtimeWorkspaceBaseDir +} + +func runtimeWorkspaceEntryFromPayload(payload runtimeWorkspaceEntryPayload) WorkspaceEntry { + isDir := payload.IsDir + return WorkspaceEntry{ + Name: payload.Name, + Path: filepath.ToSlash(payload.Path), + IsDir: isDir, + Size: payload.Size, + ModifiedAt: payload.ModifiedAt.UTC(), + Previewable: workspaceEntryPreviewable(payload.Name, payload.Size, isDir), + Downloadable: !isDir, + } +} + +func mapRuntimeWorkspaceError(execErr error, stderr string) error { + code := runtimeWorkspaceErrorCode(stderr) + switch code { + case "not_found": + return ErrWorkspacePathNotFound + case "escape": + return ErrWorkspacePathEscape + case "dir_required": + return ErrWorkspaceDirectoryExpected + case "file_required": + return ErrWorkspaceFileExpected + case "exists": + return ErrWorkspaceEntryExists + case "upload_too_large": + return ErrWorkspaceUploadTooLarge + case "filename_invalid", "invalid": + return ErrWorkspacePathInvalid + } + if strings.TrimSpace(stderr) != "" { + return fmt.Errorf("runtime workspace operation failed: %s", strings.TrimSpace(stderr)) + } + if execErr != nil { + return execErr + } + return errors.New("runtime workspace operation failed") +} + +func runtimeWorkspaceErrorCode(stderr string) string { + for _, line := range strings.Split(stderr, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, runtimeWorkspaceErrPrefix) { + remainder := strings.TrimPrefix(line, runtimeWorkspaceErrPrefix) + if index := strings.Index(remainder, ":"); index >= 0 { + return remainder[:index] + } + return remainder + } + } + return "" +} + +const runtimeWorkspacePythonPrelude = ` +import datetime +import json +import os +import shutil +import stat +import sys +import tempfile + +ERR_PREFIX = "CLAW_WORKSPACE_ERR:" + +def fail(code, message=""): + print(ERR_PREFIX + code + ":" + str(message), file=sys.stderr) + sys.exit(1) + +def clean_rel(value): + raw = (value or "").strip().replace("\\", "/") + if raw in ("", "."): + return "" + if "\x00" in raw: + fail("invalid", "path contains null byte") + if raw.startswith("/"): + fail("escape", "absolute paths are not allowed") + parts = [] + for part in raw.split("/"): + if part in ("", "."): + continue + if part == "..": + fail("escape", "traversal is not allowed") + parts.append(part) + return "/".join(parts) + +def safe_base(value): + base = os.path.realpath(value or "/config") + if not os.path.isdir(base): + fail("not_found", "workspace root") + return base + +def is_subpath(base, target): + try: + return os.path.commonpath([base, target]) == base + except ValueError: + return False + +def joined_path(base, rel): + target = os.path.abspath(os.path.join(base, rel)) if rel else base + if not is_subpath(base, target): + fail("escape", rel) + return target + +def target_for(base, rel, allow_missing=False): + rel = clean_rel(rel) + target = joined_path(base, rel) + if os.path.lexists(target): + real = os.path.realpath(target) + if not is_subpath(base, real): + fail("escape", rel) + return rel, target + if allow_missing: + parent = os.path.realpath(os.path.dirname(target)) + if not os.path.isdir(parent): + fail("invalid", "parent is not a directory") + if not is_subpath(base, parent): + fail("escape", rel) + return rel, target + fail("not_found", rel) + +def entry_payload(base, rel, target): + try: + st = os.lstat(target) + except FileNotFoundError: + fail("not_found", rel) + name = os.path.basename(target) if rel else os.path.basename(base) + modified = datetime.datetime.fromtimestamp(st.st_mtime, datetime.timezone.utc).isoformat().replace("+00:00", "Z") + return { + "name": name, + "path": rel, + "is_dir": stat.S_ISDIR(st.st_mode), + "size": int(st.st_size), + "modified_at": modified, + } + +def json_out(value): + print(json.dumps(value, separators=(",", ":"))) + +def chown_abc(path): + try: + import grp + import pwd + os.chown(path, pwd.getpwnam("abc").pw_uid, grp.getgrnam("abc").gr_gid) + except Exception: + pass +` + +var runtimeWorkspaceListScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +if not os.path.isdir(target): + fail("dir_required", rel) +entries = [] +for entry in os.scandir(target): + child_rel = (rel + "/" + entry.name) if rel else entry.name + child_path = os.path.join(target, entry.name) + try: + real = os.path.realpath(child_path) + if not is_subpath(base, real): + continue + except Exception: + continue + entries.append(entry_payload(base, child_rel, child_path)) +json_out({"entries": entries}) +` + +var runtimeWorkspaceStatFileScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +if not os.path.isfile(target): + fail("file_required", rel) +json_out(entry_payload(base, rel, target)) +` + +var runtimeWorkspaceStreamFileScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +if not os.path.isfile(target): + fail("file_required", rel) +with open(target, "rb") as source: + shutil.copyfileobj(source, sys.stdout.buffer) +` + +var runtimeWorkspaceUploadScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +dir_rel, dir_path = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +name = sys.argv[3] if len(sys.argv) > 3 else "" +max_bytes = int(sys.argv[4]) if len(sys.argv) > 4 else 524288000 +if not name or name in (".", "..") or "/" in name or "\\" in name or "\x00" in name: + fail("filename_invalid", name) +if not os.path.isdir(dir_path): + fail("dir_required", dir_rel) +target_rel = (dir_rel + "/" + name) if dir_rel else name +target_rel, target = target_for(base, target_rel, allow_missing=True) +if os.path.isdir(target): + fail("file_required", target_rel) +tmp_fd, tmp_name = tempfile.mkstemp(prefix="." + name + ".tmp-", dir=dir_path) +written = 0 +try: + with os.fdopen(tmp_fd, "wb") as out: + while True: + chunk = sys.stdin.buffer.read(1024 * 1024) + if not chunk: + break + written += len(chunk) + if written > max_bytes: + fail("upload_too_large", target_rel) + out.write(chunk) + chown_abc(tmp_name) + os.replace(tmp_name, target) + chown_abc(target) + json_out(entry_payload(base, target_rel, target)) +finally: + try: + if os.path.exists(tmp_name): + os.unlink(tmp_name) + except Exception: + pass +` + +var runtimeWorkspaceMkdirScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +rel = clean_rel(sys.argv[2] if len(sys.argv) > 2 else "") +if not rel: + fail("invalid", "workspace root cannot be modified") +rel, target = target_for(base, rel, allow_missing=True) +if os.path.lexists(target): + fail("exists", rel) +os.makedirs(target, mode=0o750, exist_ok=False) +chown_abc(target) +json_out(entry_payload(base, rel, target)) +` + +var runtimeWorkspaceRenameScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +old_rel, old_target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +new_rel, new_target = target_for(base, sys.argv[3] if len(sys.argv) > 3 else "", allow_missing=True) +if not old_rel or not new_rel: + fail("invalid", "workspace root cannot be modified") +if os.path.lexists(new_target): + fail("exists", new_rel) +os.rename(old_target, new_target) +json_out(entry_payload(base, new_rel, new_target)) +` + +var runtimeWorkspaceDeleteScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +if not rel: + fail("invalid", "workspace root cannot be modified") +if os.path.islink(target) or os.path.isfile(target): + os.unlink(target) +elif os.path.isdir(target): + shutil.rmtree(target) +else: + fail("not_found", rel) +` + +func runtimeWorkspaceContentType(name string) string { + contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(name))) + if contentType == "" { + return "application/octet-stream" + } + return contentType +} diff --git a/backend/internal/services/secret_ref_service.go b/backend/internal/services/secret_ref_service.go new file mode 100644 index 0000000..c42e40a --- /dev/null +++ b/backend/internal/services/secret_ref_service.go @@ -0,0 +1,129 @@ +package services + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "clawreef/internal/services/k8s" +) + +const inClusterNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + +// SecretReference describes a parsed Kubernetes Secret reference. +type SecretReference struct { + Namespace string + Name string + Key string +} + +// SecretRefService resolves model secret references against Kubernetes Secrets. +type SecretRefService interface { + ResolveString(ctx context.Context, value *string, secretRef *string) (*string, error) +} + +type secretRefService struct { + secretService *k8s.SecretService +} + +// NewSecretRefService creates a new secret ref resolver. +func NewSecretRefService() SecretRefService { + return &secretRefService{ + secretService: k8s.NewSecretService(), + } +} + +func (s *secretRefService) ResolveString(ctx context.Context, value *string, secretRef *string) (*string, error) { + if value != nil { + trimmed := strings.TrimSpace(*value) + if trimmed != "" { + return &trimmed, nil + } + } + + if secretRef == nil || strings.TrimSpace(*secretRef) == "" { + return nil, nil + } + + ref, err := ParseSecretReference(*secretRef) + if err != nil { + return nil, err + } + if ref.Namespace == "" { + ref.Namespace = defaultSecretNamespace() + if ref.Namespace == "" { + return nil, errors.New("secret namespace is required in secret ref") + } + } + + secretValue, err := s.secretService.GetSecretValue(ctx, ref.Namespace, ref.Name, ref.Key) + if err != nil { + return nil, err + } + secretValue = strings.TrimSpace(secretValue) + if secretValue == "" { + return nil, fmt.Errorf("secret value is empty for %s/%s:%s", ref.Namespace, ref.Name, ref.Key) + } + + return &secretValue, nil +} + +// ParseSecretReference supports: +// - name:key +// - namespace/name:key +// - k8s-secret/name:key +// - k8s-secret/namespace/name:key +func ParseSecretReference(value string) (*SecretReference, error) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return nil, errors.New("secret ref is required") + } + + trimmed = strings.TrimPrefix(trimmed, "k8s-secret/") + parts := strings.SplitN(trimmed, ":", 2) + if len(parts) != 2 { + return nil, errors.New("secret ref format is invalid") + } + + pathPart := strings.TrimSpace(parts[0]) + keyPart := strings.TrimSpace(parts[1]) + if pathPart == "" || keyPart == "" { + return nil, errors.New("secret ref format is invalid") + } + + pathSegments := strings.Split(pathPart, "/") + switch len(pathSegments) { + case 1: + return &SecretReference{ + Name: pathSegments[0], + Key: keyPart, + }, nil + case 2: + return &SecretReference{ + Namespace: pathSegments[0], + Name: pathSegments[1], + Key: keyPart, + }, nil + default: + return nil, errors.New("secret ref format is invalid") + } +} + +func defaultSecretNamespace() string { + if value := strings.TrimSpace(os.Getenv("CLAWMANAGER_SECRET_NAMESPACE")); value != "" { + return value + } + if value := strings.TrimSpace(os.Getenv("MODEL_SECRET_NAMESPACE")); value != "" { + return value + } + + if data, err := os.ReadFile(inClusterNamespacePath); err == nil { + if value := strings.TrimSpace(string(data)); value != "" { + return value + } + } + + return "" +} diff --git a/backend/internal/services/security_scan_service.go b/backend/internal/services/security_scan_service.go new file mode 100644 index 0000000..a610318 --- /dev/null +++ b/backend/internal/services/security_scan_service.go @@ -0,0 +1,995 @@ +package services + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +type SecurityScanConfigPayload struct { + ActiveMode string `json:"active_mode"` + DefaultMode string `json:"default_mode"` + QuickAnalyzers []string `json:"quick_analyzers"` + DeepAnalyzers []string `json:"deep_analyzers"` + QuickTimeoutSeconds int `json:"quick_timeout_seconds"` + DeepTimeoutSeconds int `json:"deep_timeout_seconds"` + AllowFallback bool `json:"allow_fallback"` + ScannerStatus SecurityScannerStatusPayload `json:"scanner_status"` + SkillScannerConfig SkillScannerRuntimeConfigPayload `json:"skill_scanner_config"` +} + +type SecurityScannerStatusPayload struct { + Connected bool `json:"connected"` + LLMEnabled bool `json:"llm_enabled"` + StatusLabel string `json:"status_label"` + AvailableCapabilities []string `json:"available_capabilities"` +} + +type SkillScannerRuntimeConfigPayload struct { + Namespace string `json:"namespace"` + DeploymentName string `json:"deployment_name"` + LLMAPIKey string `json:"llm_api_key"` + LLMModel string `json:"llm_model"` + LLMBaseURL string `json:"llm_base_url"` + MetaLLMAPIKey string `json:"meta_llm_api_key"` + MetaLLMModel string `json:"meta_llm_model"` + MetaLLMBaseURL string `json:"meta_llm_base_url"` +} + +type StartSecurityScanRequest struct { + AssetType string `json:"asset_type"` + ScanMode string `json:"scan_mode"` + ScanScope string `json:"scan_scope"` + AssetID *int `json:"asset_id,omitempty"` +} + +type securityScanScope struct { + ScanScope string `json:"scan_scope"` + AssetID *int `json:"asset_id,omitempty"` +} + +type SecurityScanJobItemPayload struct { + ID int `json:"id"` + AssetType string `json:"asset_type"` + AssetID int `json:"asset_id"` + AssetName string `json:"asset_name"` + Status string `json:"status"` + ProgressPct int `json:"progress_pct"` + RiskLevel *string `json:"risk_level,omitempty"` + Summary *string `json:"summary,omitempty"` + ScanResultID *int `json:"scan_result_id,omitempty"` + CachedResult bool `json:"cached_result"` + TriggeredAnalyzers []string `json:"triggered_analyzers,omitempty"` + Findings []SkillFindingPayload `json:"findings,omitempty"` + ErrorMessage *string `json:"error_message,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` +} + +type SecurityScanReportPayload struct { + JobID int `json:"job_id"` + AssetType string `json:"asset_type"` + ScanMode string `json:"scan_mode"` + ScanScope string `json:"scan_scope"` + Status string `json:"status"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + TotalItems int `json:"total_items"` + CompletedItems int `json:"completed_items"` + FailedItems int `json:"failed_items"` + RiskCounts map[string]int `json:"risk_counts"` + FindingsSummary []map[string]string `json:"findings_summary"` + ConfiguredAnalyzers []string `json:"configured_analyzers"` + AvailableAnalyzers []string `json:"available_analyzers"` + TriggeredAnalyzers []string `json:"triggered_analyzers"` + Items []SecurityScanJobItemPayload `json:"items"` + Config SecurityScanConfigPayload `json:"config"` +} + +type SecurityScanJobPayload struct { + ID int `json:"id"` + AssetType string `json:"asset_type"` + ScanMode string `json:"scan_mode"` + ScanScope string `json:"scan_scope"` + Status string `json:"status"` + RequestedBy *int `json:"requested_by,omitempty"` + TotalItems int `json:"total_items"` + CompletedItems int `json:"completed_items"` + FailedItems int `json:"failed_items"` + CurrentItemName *string `json:"current_item_name,omitempty"` + ProgressPct int `json:"progress_pct"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Items []SecurityScanJobItemPayload `json:"items,omitempty"` + Report *SecurityScanReportPayload `json:"report,omitempty"` +} + +type SecurityScanService interface { + GetConfig() (*SecurityScanConfigPayload, error) + SaveConfig(updatedBy int, req SecurityScanConfigPayload) (*SecurityScanConfigPayload, error) + StartScan(requestedBy int, req StartSecurityScanRequest) (*SecurityScanJobPayload, error) + RescanSkill(requestedBy, skillID int, scanMode string) (*SecurityScanJobPayload, error) + ListJobs(limit int) ([]SecurityScanJobPayload, error) + GetJob(jobID int) (*SecurityScanJobPayload, error) +} + +type securityScanService struct { + repo repository.SecurityScanRepository + skillRepo repository.SkillRepository + storage ObjectStorageService + scanner SkillScannerClient + running sync.Map +} + +func NewSecurityScanService(repo repository.SecurityScanRepository, skillRepo repository.SkillRepository, storage ObjectStorageService, scanner SkillScannerClient) SecurityScanService { + return &securityScanService{repo: repo, skillRepo: skillRepo, storage: storage, scanner: scanner} +} + +func (s *securityScanService) GetConfig() (*SecurityScanConfigPayload, error) { + item, err := s.ensureConfig() + if err != nil { + return nil, err + } + payload, err := s.toConfigPayload(*item) + if err != nil { + return nil, err + } + return payload, nil +} + +func (s *securityScanService) SaveConfig(updatedBy int, req SecurityScanConfigPayload) (*SecurityScanConfigPayload, error) { + item, err := s.ensureConfig() + if err != nil { + return nil, err + } + activeMode := strings.TrimSpace(req.ActiveMode) + if activeMode == "" { + activeMode = strings.TrimSpace(req.DefaultMode) + } + if activeMode == "" { + activeMode = "quick" + } + quickJSON, _ := json.Marshal(normalizeAnalyzerList(req.QuickAnalyzers, defaultQuickAnalyzers())) + deepJSON, _ := json.Marshal(normalizeAnalyzerList(req.DeepAnalyzers, defaultDeepAnalyzers())) + item.DefaultMode = normalizeScanMode(activeMode) + item.QuickAnalyzersJSON = string(quickJSON) + item.DeepAnalyzersJSON = string(deepJSON) + item.QuickTimeoutSeconds = normalizeTimeout(req.QuickTimeoutSeconds, 30) + item.DeepTimeoutSeconds = normalizeTimeout(req.DeepTimeoutSeconds, 120) + item.AllowFallback = false + item.UpdatedBy = &updatedBy + if err := s.repo.UpsertConfig(item); err != nil { + return nil, err + } + if err := s.applySkillScannerConfig(req.SkillScannerConfig); err != nil { + return nil, err + } + return s.toConfigPayload(*item) +} + +func (s *securityScanService) StartScan(requestedBy int, req StartSecurityScanRequest) (*SecurityScanJobPayload, error) { + assetType := strings.TrimSpace(req.AssetType) + if assetType == "" { + assetType = "skill" + } + if assetType != "skill" { + return nil, fmt.Errorf("unsupported asset type") + } + config, err := s.ensureConfig() + if err != nil { + return nil, err + } + mode := normalizeScanMode(config.DefaultMode) + scope := normalizeScanScope(req.ScanScope) + skills, err := s.resolveScanSkills(req.AssetID) + if err != nil { + return nil, err + } + scopePayload := securityScanScope{ + ScanScope: scope, + AssetID: req.AssetID, + } + scopeJSONBytes, _ := json.Marshal(scopePayload) + scopeJSON := string(scopeJSONBytes) + job := &models.SecurityScanJob{ + AssetType: assetType, + ScanMode: mode, + ScopeJSON: &scopeJSON, + Status: "queued", + RequestedBy: &requestedBy, + TotalItems: len(skills), + CompletedItems: 0, + FailedItems: 0, + } + if err := s.repo.CreateJob(job); err != nil { + return nil, err + } + for _, skill := range skills { + item := &models.SecurityScanJobItem{ + JobID: job.ID, + AssetType: assetType, + AssetID: skill.ID, + AssetName: skill.Name, + Status: "pending", + ProgressPct: 0, + } + if err := s.repo.CreateJobItem(item); err != nil { + return nil, err + } + } + go s.runJob(job.ID) + return s.GetJob(job.ID) +} + +func (s *securityScanService) RescanSkill(requestedBy, skillID int, scanMode string) (*SecurityScanJobPayload, error) { + return s.StartScan(requestedBy, StartSecurityScanRequest{ + AssetType: "skill", + ScanMode: scanMode, + ScanScope: "full", + AssetID: &skillID, + }) +} + +func (s *securityScanService) resolveScanSkills(assetID *int) ([]models.Skill, error) { + if assetID != nil && *assetID > 0 { + skill, err := s.skillRepo.GetSkillByID(*assetID) + if err != nil { + return nil, err + } + if skill == nil { + return nil, fmt.Errorf("skill not found") + } + return []models.Skill{*skill}, nil + } + return s.skillRepo.ListAllSkills() +} + +func (s *securityScanService) ListJobs(limit int) ([]SecurityScanJobPayload, error) { + items, err := s.repo.ListJobs(limit) + if err != nil { + return nil, err + } + result := make([]SecurityScanJobPayload, 0, len(items)) + for _, item := range items { + payload, err := s.toJobPayload(item, false) + if err != nil { + return nil, err + } + result = append(result, *payload) + } + return result, nil +} + +func (s *securityScanService) GetJob(jobID int) (*SecurityScanJobPayload, error) { + item, err := s.repo.GetJobByID(jobID) + if err != nil { + return nil, err + } + if item == nil { + return nil, fmt.Errorf("security scan job not found") + } + return s.toJobPayload(*item, true) +} + +func (s *securityScanService) runJob(jobID int) { + if _, loaded := s.running.LoadOrStore(jobID, struct{}{}); loaded { + return + } + defer s.running.Delete(jobID) + + job, err := s.repo.GetJobByID(jobID) + if err != nil || job == nil { + return + } + config, err := s.ensureConfig() + if err != nil { + return + } + configPayload, err := s.toConfigPayload(*config) + if err != nil { + return + } + scope := parseSecurityScanScope(job.ScopeJSON) + now := time.Now().UTC() + job.Status = "running" + job.StartedAt = &now + job.UpdatedAt = now + _ = s.repo.UpdateJob(job) + + items, err := s.repo.ListJobItems(jobID) + if err != nil { + return + } + + for _, item := range items { + currentName := item.AssetName + startedAt := time.Now().UTC() + item.Status = "running" + item.ProgressPct = 10 + item.StartedAt = &startedAt + item.UpdatedAt = startedAt + job.CurrentItemName = ¤tName + job.UpdatedAt = startedAt + _ = s.repo.UpdateJobItem(&item) + _ = s.repo.UpdateJob(job) + + skill, err := s.skillRepo.GetSkillByID(item.AssetID) + if err != nil || skill == nil { + s.markJobItemFailed(job, &item, "skill not found") + continue + } + if skill.CurrentVersionID == nil { + s.markJobItemFailed(job, &item, "skill has no active version") + continue + } + version, err := s.skillRepo.GetVersionByID(*skill.CurrentVersionID) + if err != nil || version == nil { + s.markJobItemFailed(job, &item, "skill version not found") + continue + } + blob, err := s.skillRepo.GetBlobByID(version.BlobID) + if err != nil || blob == nil { + s.markJobItemFailed(job, &item, "skill blob not found") + continue + } + + result, cached, err := s.scanBlob(skill, blob, job.ScanMode, scope.ScanScope, *configPayload) + if err != nil { + s.markJobItemFailed(job, &item, err.Error()) + continue + } + + finishedAt := time.Now().UTC() + item.Status = "completed" + item.ProgressPct = 100 + item.CachedResult = cached + item.RiskLevel = &result.RiskLevel + item.Summary = result.Summary + item.ScanResultID = &result.ID + item.FinishedAt = &finishedAt + item.UpdatedAt = finishedAt + job.CompletedItems++ + job.CurrentItemName = ¤tName + job.UpdatedAt = finishedAt + _ = s.repo.UpdateJobItem(&item) + _ = s.repo.UpdateJob(job) + } + + doneAt := time.Now().UTC() + job.Status = "completed" + job.CurrentItemName = nil + job.FinishedAt = &doneAt + job.UpdatedAt = doneAt + _ = s.repo.UpdateJob(job) + _ = s.generateReport(job.ID, *config) +} + +func (s *securityScanService) markJobItemFailed(job *models.SecurityScanJob, item *models.SecurityScanJobItem, message string) { + finishedAt := time.Now().UTC() + item.Status = "failed" + item.ProgressPct = 100 + item.ErrorMessage = optionalString(message) + item.FinishedAt = &finishedAt + item.UpdatedAt = finishedAt + job.CompletedItems++ + job.FailedItems++ + job.UpdatedAt = finishedAt + _ = s.repo.UpdateJobItem(item) + _ = s.repo.UpdateJob(job) +} + +func (s *securityScanService) scanBlob(skill *models.Skill, blob *models.SkillBlob, mode, scope string, config SecurityScanConfigPayload) (*models.SkillScanResult, bool, error) { + latest, err := s.skillRepo.GetLatestScanResultByBlobID(blob.ID) + if err != nil { + return nil, false, err + } + if scope == "incremental" && latest != nil && latest.Status == "completed" && strings.EqualFold(strings.TrimSpace(latest.Engine), "skill-scanner") { + return latest, true, nil + } + if strings.TrimSpace(blob.ObjectKey) == "" || blob.SizeBytes <= 0 { + now := time.Now().UTC() + blob.ScanStatus = "pending" + blob.UpdatedAt = now + _ = s.skillRepo.UpdateBlob(blob) + return nil, false, fmt.Errorf("skill package has not been collected from agent yet") + } + + content, err := s.storage.GetObject(context.Background(), blob.ObjectKey) + if err != nil { + now := time.Now().UTC() + blob.ScanStatus = "pending" + blob.UpdatedAt = now + _ = s.skillRepo.UpdateBlob(blob) + if strings.Contains(strings.ToLower(err.Error()), "specified key does not exist") { + return nil, false, fmt.Errorf("skill package has not been collected from agent yet") + } + return nil, false, err + } + timeout := config.QuickTimeoutSeconds + if mode == "deep" { + timeout = config.DeepTimeoutSeconds + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + defer cancel() + + options := map[string]string{ + "scan_mode": mode, + } + analyzers := config.QuickAnalyzers + if mode == "deep" { + analyzers = config.DeepAnalyzers + } + if len(analyzers) > 0 { + options["analyzers"] = strings.Join(analyzers, ",") + } + + if s.scanner == nil { + return nil, false, fmt.Errorf("skill scanner is not configured") + } + riskLevel, findings, summary, err := s.scanner.ScanArchive(ctx, blob.FileName, content, options) + if err != nil { + now := time.Now().UTC() + blob.ScanStatus = "failed" + blob.UpdatedAt = now + _ = s.skillRepo.UpdateBlob(blob) + return nil, false, fmt.Errorf("skill scanner failed: %w", err) + } + if strings.TrimSpace(summary) == "" { + summary = "Skill scanned by external skill-scanner service" + } + + scannedAt := time.Now().UTC() + findingsJSON, _ := json.Marshal(findings) + result := &models.SkillScanResult{ + BlobID: blob.ID, Engine: "skill-scanner", RiskLevel: riskLevel, Status: "completed", + Summary: &summary, FindingsJSON: optionalString(string(findingsJSON)), ScannedAt: &scannedAt, + } + if err := s.skillRepo.CreateScanResult(result); err != nil { + return nil, false, err + } + blob.ScanStatus = "completed" + blob.RiskLevel = riskLevel + blob.LastScannedAt = &scannedAt + blob.LastScanResultID = &result.ID + if err := s.skillRepo.UpdateBlob(blob); err != nil { + return nil, false, err + } + if skill != nil { + skill.RiskLevel = riskLevel + skill.LastScannedAt = &scannedAt + skill.LastScanResultID = &result.ID + skill.UpdatedAt = scannedAt + _ = s.skillRepo.UpdateSkill(skill) + } + return result, false, nil +} + +func (s *securityScanService) generateReport(jobID int, config models.SecurityScanConfig) error { + job, err := s.repo.GetJobByID(jobID) + if err != nil || job == nil { + return err + } + jobItems, err := s.repo.ListJobItems(jobID) + if err != nil { + return err + } + cfgPayload, err := s.toConfigPayload(config) + if err != nil { + return err + } + riskCounts := map[string]int{} + findingsSummary := make([]map[string]string, 0, len(jobItems)) + itemPayloads := make([]SecurityScanJobItemPayload, 0, len(jobItems)) + triggeredAnalyzers := map[string]struct{}{} + for _, item := range jobItems { + if item.RiskLevel != nil { + riskCounts[*item.RiskLevel]++ + } + itemPayload := toSecurityScanJobItemPayload(item) + if item.ScanResultID != nil { + result, err := s.skillRepo.GetScanResultByID(*item.ScanResultID) + if err != nil { + return err + } + findings := parseSkillFindings(result) + itemPayload.TriggeredAnalyzers = extractTriggeredAnalyzers(result) + itemPayload.Findings = topRiskFindings(findings, 5) + if summary := summarizeScanJobItem(item, findings); summary != nil { + itemPayload.Summary = summary + findingsSummary = append(findingsSummary, map[string]string{ + "asset_name": item.AssetName, + "summary": *summary, + }) + } + for _, analyzer := range itemPayload.TriggeredAnalyzers { + triggeredAnalyzers[analyzer] = struct{}{} + } + } else if summary := summarizeScanJobItem(item, nil); summary != nil { + itemPayload.Summary = summary + findingsSummary = append(findingsSummary, map[string]string{ + "asset_name": item.AssetName, + "summary": *summary, + }) + } + itemPayloads = append(itemPayloads, itemPayload) + } + configuredAnalyzers := cfgPayload.QuickAnalyzers + if job.ScanMode == "deep" { + configuredAnalyzers = cfgPayload.DeepAnalyzers + } + availableAnalyzers := []string{} + if s.scanner != nil { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if analyzers, err := s.scanner.AvailableAnalyzers(ctx); err == nil { + availableAnalyzers = normalizeAnalyzerList(analyzers, []string{}) + } + } + reportPayload := SecurityScanReportPayload{ + JobID: job.ID, + AssetType: job.AssetType, + ScanMode: job.ScanMode, + ScanScope: parseSecurityScanScope(job.ScopeJSON).ScanScope, + Status: job.Status, + StartedAt: job.StartedAt, + FinishedAt: job.FinishedAt, + TotalItems: job.TotalItems, + CompletedItems: job.CompletedItems, + FailedItems: job.FailedItems, + RiskCounts: riskCounts, + FindingsSummary: findingsSummary, + ConfiguredAnalyzers: append([]string{}, configuredAnalyzers...), + AvailableAnalyzers: availableAnalyzers, + TriggeredAnalyzers: sortedAnalyzerKeys(triggeredAnalyzers), + Items: itemPayloads, + Config: *cfgPayload, + } + summaryJSON, _ := json.Marshal(reportPayload) + return s.repo.UpsertReport(&models.SecurityScanReport{ + JobID: job.ID, + SummaryJSON: string(summaryJSON), + }) +} + +func (s *securityScanService) ensureConfig() (*models.SecurityScanConfig, error) { + item, err := s.repo.GetConfig() + if err != nil { + return nil, err + } + if item != nil { + return item, nil + } + quickJSON, _ := json.Marshal(defaultQuickAnalyzers()) + deepJSON, _ := json.Marshal(defaultDeepAnalyzers()) + item = &models.SecurityScanConfig{ + ID: 1, + DefaultMode: "quick", + QuickAnalyzersJSON: string(quickJSON), + DeepAnalyzersJSON: string(deepJSON), + QuickTimeoutSeconds: 30, + DeepTimeoutSeconds: 120, + AllowFallback: false, + } + if err := s.repo.UpsertConfig(item); err != nil { + return nil, err + } + return item, nil +} + +func summarizeScanJobItem(item models.SecurityScanJobItem, findings []SkillFindingPayload) *string { + if summary := summarizeRiskReason(topRiskFindings(findings, 2)); summary != nil { + return summary + } + if item.ErrorMessage != nil && strings.TrimSpace(*item.ErrorMessage) != "" { + message := strings.TrimSpace(*item.ErrorMessage) + return &message + } + if item.RiskLevel != nil { + switch strings.ToLower(strings.TrimSpace(*item.RiskLevel)) { + case "none": + text := "未发现风险项" + return &text + case "low": + text := "发现低风险提示,建议查看逐项结果" + return &text + case "medium": + text := "发现中风险问题,建议尽快整改" + return &text + case "high": + text := "发现高风险问题,建议立即处置" + return &text + } + } + if item.Status == "completed" { + text := "扫描完成,未返回详细发现" + return &text + } + if item.Summary != nil && strings.TrimSpace(*item.Summary) != "" { + text := strings.TrimSpace(*item.Summary) + return &text + } + return nil +} + +func (s *securityScanService) toConfigPayload(item models.SecurityScanConfig) (*SecurityScanConfigPayload, error) { + var quickAnalyzers []string + var deepAnalyzers []string + if strings.TrimSpace(item.QuickAnalyzersJSON) != "" { + if err := json.Unmarshal([]byte(item.QuickAnalyzersJSON), &quickAnalyzers); err != nil { + return nil, fmt.Errorf("failed to decode quick analyzers: %w", err) + } + } + if strings.TrimSpace(item.DeepAnalyzersJSON) != "" { + if err := json.Unmarshal([]byte(item.DeepAnalyzersJSON), &deepAnalyzers); err != nil { + return nil, fmt.Errorf("failed to decode deep analyzers: %w", err) + } + } + return &SecurityScanConfigPayload{ + ActiveMode: item.DefaultMode, + DefaultMode: item.DefaultMode, + QuickAnalyzers: normalizeAnalyzerList(quickAnalyzers, defaultQuickAnalyzers()), + DeepAnalyzers: normalizeAnalyzerList(deepAnalyzers, defaultDeepAnalyzers()), + QuickTimeoutSeconds: item.QuickTimeoutSeconds, + DeepTimeoutSeconds: item.DeepTimeoutSeconds, + AllowFallback: item.AllowFallback, + ScannerStatus: resolveSecurityScannerStatus(), + SkillScannerConfig: resolveSkillScannerRuntimeConfig(), + }, nil +} + +func resolveSecurityScannerStatus() SecurityScannerStatusPayload { + runtime := resolveSkillScannerRuntimeConfig() + enabled := strings.EqualFold(strings.TrimSpace(os.Getenv("SKILL_SCANNER_ENABLED")), "true") && + strings.TrimSpace(os.Getenv("SKILL_SCANNER_BASE_URL")) != "" + llmConfigured := strings.TrimSpace(runtime.LLMModel) != "" && strings.TrimSpace(runtime.LLMBaseURL) != "" && strings.TrimSpace(runtime.LLMAPIKey) != "" || + (strings.TrimSpace(runtime.MetaLLMModel) != "" && strings.TrimSpace(runtime.MetaLLMBaseURL) != "" && strings.TrimSpace(runtime.MetaLLMAPIKey) != "") + + status := SecurityScannerStatusPayload{ + Connected: enabled, + LLMEnabled: llmConfigured, + StatusLabel: "未启用", + AvailableCapabilities: []string{}, + } + if !enabled { + return status + } + status.StatusLabel = "静态扫描可用" + status.AvailableCapabilities = []string{"静态扫描"} + if llmConfigured { + status.StatusLabel = "静态 + LLM 扫描可用" + status.AvailableCapabilities = []string{"静态扫描", "LLM 扫描"} + } + return status +} + +func resolveSkillScannerRuntimeConfig() SkillScannerRuntimeConfigPayload { + cfg := SkillScannerRuntimeConfigPayload{ + Namespace: skillScannerNamespace(), + DeploymentName: skillScannerDeploymentName(), + } + if envs, err := loadSkillScannerDeploymentEnv(); err == nil { + cfg.LLMAPIKey = envs["SKILL_SCANNER_LLM_API_KEY"] + cfg.LLMModel = envs["SKILL_SCANNER_LLM_MODEL"] + cfg.LLMBaseURL = envs["SKILL_SCANNER_LLM_BASE_URL"] + cfg.MetaLLMAPIKey = envs["SKILL_SCANNER_META_LLM_API_KEY"] + cfg.MetaLLMModel = envs["SKILL_SCANNER_META_LLM_MODEL"] + cfg.MetaLLMBaseURL = envs["SKILL_SCANNER_META_LLM_BASE_URL"] + return cfg + } + cfg.LLMAPIKey = os.Getenv("SKILL_SCANNER_LLM_API_KEY") + cfg.LLMModel = os.Getenv("SKILL_SCANNER_LLM_MODEL") + cfg.LLMBaseURL = os.Getenv("SKILL_SCANNER_LLM_BASE_URL") + cfg.MetaLLMAPIKey = os.Getenv("SKILL_SCANNER_META_LLM_API_KEY") + cfg.MetaLLMModel = os.Getenv("SKILL_SCANNER_META_LLM_MODEL") + cfg.MetaLLMBaseURL = os.Getenv("SKILL_SCANNER_META_LLM_BASE_URL") + return cfg +} + +func (s *securityScanService) applySkillScannerConfig(req SkillScannerRuntimeConfigPayload) error { + client := k8s.GetClient() + if client == nil || client.Clientset == nil { + return fmt.Errorf("kubernetes client is unavailable, cannot update skill-scanner config") + } + namespace := skillScannerNamespace() + deploymentName := skillScannerDeploymentName() + deployment, err := client.Clientset.AppsV1().Deployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to load skill-scanner deployment: %w", err) + } + if len(deployment.Spec.Template.Spec.Containers) == 0 { + return fmt.Errorf("skill-scanner deployment has no containers") + } + container := &deployment.Spec.Template.Spec.Containers[0] + upsertEnvVar(container, "SKILL_SCANNER_LLM_API_KEY", strings.TrimSpace(req.LLMAPIKey)) + upsertEnvVar(container, "SKILL_SCANNER_LLM_MODEL", strings.TrimSpace(req.LLMModel)) + upsertEnvVar(container, "SKILL_SCANNER_LLM_BASE_URL", strings.TrimSpace(req.LLMBaseURL)) + upsertEnvVar(container, "SKILL_SCANNER_META_LLM_API_KEY", strings.TrimSpace(req.MetaLLMAPIKey)) + upsertEnvVar(container, "SKILL_SCANNER_META_LLM_MODEL", strings.TrimSpace(req.MetaLLMModel)) + upsertEnvVar(container, "SKILL_SCANNER_META_LLM_BASE_URL", strings.TrimSpace(req.MetaLLMBaseURL)) + + updated, err := client.Clientset.AppsV1().Deployments(namespace).Update(context.Background(), deployment, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update skill-scanner deployment: %w", err) + } + return waitForSkillScannerRollout(context.Background(), client.Clientset, namespace, deploymentName, updated.Generation, 90*time.Second) +} + +func loadSkillScannerDeploymentEnv() (map[string]string, error) { + client := k8s.GetClient() + if client == nil || client.Clientset == nil { + return nil, fmt.Errorf("kubernetes client is unavailable") + } + deployment, err := client.Clientset.AppsV1().Deployments(skillScannerNamespace()).Get(context.Background(), skillScannerDeploymentName(), metav1.GetOptions{}) + if err != nil { + return nil, err + } + if len(deployment.Spec.Template.Spec.Containers) == 0 { + return nil, fmt.Errorf("skill-scanner deployment has no containers") + } + envs := map[string]string{} + for _, env := range deployment.Spec.Template.Spec.Containers[0].Env { + envs[env.Name] = env.Value + } + return envs, nil +} + +func upsertEnvVar(container *corev1.Container, name, value string) { + for i := range container.Env { + if container.Env[i].Name == name { + container.Env[i].Value = value + container.Env[i].ValueFrom = nil + return + } + } + container.Env = append(container.Env, corev1.EnvVar{Name: name, Value: value}) +} + +func waitForSkillScannerRollout(ctx context.Context, clientset kubernetes.Interface, namespace, deploymentName string, generation int64, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + deployment, err := clientset.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to observe skill-scanner rollout: %w", err) + } + replicas := int32(1) + if deployment.Spec.Replicas != nil { + replicas = *deployment.Spec.Replicas + } + if deployment.Status.ObservedGeneration >= generation && + deployment.Status.UpdatedReplicas == replicas && + deployment.Status.AvailableReplicas == replicas && + deployment.Status.UnavailableReplicas == 0 { + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("skill-scanner rollout timed out") +} + +func skillScannerNamespace() string { + if value := strings.TrimSpace(os.Getenv("SKILL_SCANNER_NAMESPACE")); value != "" { + return value + } + if value := strings.TrimSpace(readInClusterNamespace()); value != "" { + return value + } + return "clawmanager-system" +} + +func skillScannerDeploymentName() string { + if value := strings.TrimSpace(os.Getenv("SKILL_SCANNER_DEPLOYMENT")); value != "" { + return value + } + return "skill-scanner" +} + +func readInClusterNamespace() string { + data, err := os.ReadFile(filepath.Clean("/var/run/secrets/kubernetes.io/serviceaccount/namespace")) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +func (s *securityScanService) toJobPayload(item models.SecurityScanJob, includeDetails bool) (*SecurityScanJobPayload, error) { + payload := &SecurityScanJobPayload{ + ID: item.ID, + AssetType: item.AssetType, + ScanMode: item.ScanMode, + ScanScope: parseSecurityScanScope(item.ScopeJSON).ScanScope, + Status: item.Status, + RequestedBy: item.RequestedBy, + TotalItems: item.TotalItems, + CompletedItems: item.CompletedItems, + FailedItems: item.FailedItems, + CurrentItemName: item.CurrentItemName, + ProgressPct: computeProgress(item.TotalItems, item.CompletedItems), + StartedAt: item.StartedAt, + FinishedAt: item.FinishedAt, + CreatedAt: item.CreatedAt, + UpdatedAt: item.UpdatedAt, + } + if !includeDetails { + return payload, nil + } + items, err := s.repo.ListJobItems(item.ID) + if err != nil { + return nil, err + } + payload.Items = make([]SecurityScanJobItemPayload, 0, len(items)) + for _, entry := range items { + payload.Items = append(payload.Items, toSecurityScanJobItemPayload(entry)) + } + report, err := s.repo.GetReportByJobID(item.ID) + if err != nil { + return nil, err + } + if report != nil && strings.TrimSpace(report.SummaryJSON) != "" { + var reportPayload SecurityScanReportPayload + if err := json.Unmarshal([]byte(report.SummaryJSON), &reportPayload); err == nil { + payload.Report = &reportPayload + } + } + return payload, nil +} + +func toSecurityScanJobItemPayload(item models.SecurityScanJobItem) SecurityScanJobItemPayload { + return SecurityScanJobItemPayload{ + ID: item.ID, + AssetType: item.AssetType, + AssetID: item.AssetID, + AssetName: item.AssetName, + Status: item.Status, + ProgressPct: item.ProgressPct, + RiskLevel: item.RiskLevel, + Summary: item.Summary, + ScanResultID: item.ScanResultID, + CachedResult: item.CachedResult, + TriggeredAnalyzers: []string{}, + Findings: []SkillFindingPayload{}, + ErrorMessage: item.ErrorMessage, + StartedAt: item.StartedAt, + FinishedAt: item.FinishedAt, + } +} + +func extractTriggeredAnalyzers(result *models.SkillScanResult) []string { + if result == nil || result.FindingsJSON == nil || strings.TrimSpace(*result.FindingsJSON) == "" { + return []string{} + } + var raw struct { + Findings []struct { + Analyzer string `json:"analyzer"` + } `json:"findings"` + } + if err := json.Unmarshal([]byte(*result.FindingsJSON), &raw); err != nil { + return []string{} + } + seen := map[string]struct{}{} + resultList := make([]string, 0, len(raw.Findings)) + for _, item := range raw.Findings { + analyzer := strings.TrimSpace(strings.ToLower(item.Analyzer)) + if analyzer == "" { + continue + } + if _, ok := seen[analyzer]; ok { + continue + } + seen[analyzer] = struct{}{} + resultList = append(resultList, analyzer) + } + return resultList +} + +func sortedAnalyzerKeys(values map[string]struct{}) []string { + if len(values) == 0 { + return []string{} + } + result := make([]string, 0, len(values)) + for value := range values { + result = append(result, value) + } + slices.Sort(result) + return result +} + +func defaultQuickAnalyzers() []string { + return []string{"static", "behavioral", "trigger"} +} + +func defaultDeepAnalyzers() []string { + return []string{"static", "bytecode", "pipeline", "behavioral", "trigger", "llm", "meta"} +} + +func normalizeAnalyzerList(values []string, fallback []string) []string { + if len(values) == 0 { + return append([]string{}, fallback...) + } + allowed := map[string]struct{}{ + "static": {}, + "bytecode": {}, + "pipeline": {}, + "behavioral": {}, + "trigger": {}, + "llm": {}, + "meta": {}, + } + seen := map[string]struct{}{} + result := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(strings.ToLower(value)) + if value == "" { + continue + } + if _, ok := allowed[value]; !ok { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + if len(result) == 0 { + return append([]string{}, fallback...) + } + return result +} + +func normalizeScanMode(value string) string { + if strings.EqualFold(strings.TrimSpace(value), "deep") { + return "deep" + } + return "quick" +} + +func normalizeScanScope(value string) string { + if strings.EqualFold(strings.TrimSpace(value), "full") { + return "full" + } + return "incremental" +} + +func parseSecurityScanScope(raw *string) securityScanScope { + scope := securityScanScope{ScanScope: "incremental"} + if raw == nil || strings.TrimSpace(*raw) == "" { + return scope + } + if err := json.Unmarshal([]byte(*raw), &scope); err != nil { + return securityScanScope{ScanScope: "incremental"} + } + scope.ScanScope = normalizeScanScope(scope.ScanScope) + return scope +} + +func normalizeTimeout(value, fallback int) int { + if value <= 0 { + return fallback + } + return value +} + +func computeProgress(total, completed int) int { + if total <= 0 { + return 0 + } + if completed >= total { + return 100 + } + return int(float64(completed) / float64(total) * 100) +} diff --git a/backend/internal/services/skill_scanner_client.go b/backend/internal/services/skill_scanner_client.go new file mode 100644 index 0000000..0b3c0c3 --- /dev/null +++ b/backend/internal/services/skill_scanner_client.go @@ -0,0 +1,240 @@ +package services + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "mime/multipart" + "net/http" + "net/url" + "strings" + "time" + + "clawreef/internal/config" +) + +type SkillScannerClient interface { + ScanArchive(ctx context.Context, fileName string, content []byte, options map[string]string) (string, map[string]interface{}, string, error) + AvailableAnalyzers(ctx context.Context) ([]string, error) +} + +type noopSkillScannerClient struct{} + +func (n *noopSkillScannerClient) ScanArchive(ctx context.Context, fileName string, content []byte, options map[string]string) (string, map[string]interface{}, string, error) { + return "", nil, "", fmt.Errorf("skill scanner is disabled") +} + +func (n *noopSkillScannerClient) AvailableAnalyzers(ctx context.Context) ([]string, error) { + return nil, fmt.Errorf("skill scanner is disabled") +} + +type httpSkillScannerClient struct { + baseURL string + apiKey string + client *http.Client +} + +func NewSkillScannerClient(cfg config.SkillScannerConfig) SkillScannerClient { + if !cfg.Enabled || strings.TrimSpace(cfg.BaseURL) == "" { + return &noopSkillScannerClient{} + } + timeout := time.Duration(cfg.TimeoutSeconds) * time.Second + if timeout <= 0 { + timeout = 30 * time.Second + } + return &httpSkillScannerClient{ + baseURL: strings.TrimRight(strings.TrimSpace(cfg.BaseURL), "/"), + apiKey: strings.TrimSpace(cfg.APIKey), + client: &http.Client{Timeout: timeout}, + } +} + +func (c *httpSkillScannerClient) ScanArchive(ctx context.Context, fileName string, content []byte, options map[string]string) (string, map[string]interface{}, string, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, err := writer.CreateFormFile("file", fileName) + if err != nil { + return "", nil, "", fmt.Errorf("failed to create skill scanner upload: %w", err) + } + if _, err := part.Write(content); err != nil { + return "", nil, "", fmt.Errorf("failed to write skill scanner upload: %w", err) + } + _ = writer.WriteField("format", "json") + if err := writer.Close(); err != nil { + return "", nil, "", fmt.Errorf("failed to finalize skill scanner upload: %w", err) + } + + endpoint, err := url.Parse(c.baseURL + "/scan-upload") + if err != nil { + return "", nil, "", fmt.Errorf("failed to build skill scanner url: %w", err) + } + query := endpoint.Query() + applyScanUploadOptions(query, options) + endpoint.RawQuery = query.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint.String(), &body) + if err != nil { + return "", nil, "", fmt.Errorf("failed to create skill scanner request: %w", err) + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Accept", "application/json") + if c.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+c.apiKey) + } + + resp, err := c.client.Do(req) + if err != nil { + return "", nil, "", fmt.Errorf("skill scanner request failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", nil, "", fmt.Errorf("skill scanner returned status %d", resp.StatusCode) + } + + var raw map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + return "", nil, "", fmt.Errorf("failed to decode skill scanner response: %w", err) + } + riskLevel := normalizeScannerRiskLevel(raw) + summary := extractScannerSummary(raw) + return riskLevel, raw, summary, nil +} + +func (c *httpSkillScannerClient) AvailableAnalyzers(ctx context.Context) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health", nil) + if err != nil { + return nil, fmt.Errorf("failed to create skill scanner health request: %w", err) + } + req.Header.Set("Accept", "application/json") + if c.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+c.apiKey) + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("skill scanner health request failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("skill scanner health returned status %d", resp.StatusCode) + } + + var raw struct { + Analyzers []string `json:"analyzers_available"` + } + if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + return nil, fmt.Errorf("failed to decode skill scanner health response: %w", err) + } + result := make([]string, 0, len(raw.Analyzers)) + for _, item := range raw.Analyzers { + item = strings.TrimSpace(item) + if item == "" { + continue + } + item = strings.TrimSuffix(item, "_analyzer") + result = append(result, item) + } + return result, nil +} + +func applyScanUploadOptions(query url.Values, options map[string]string) { + analyzers := splitAnalyzerOption(options["analyzers"]) + if hasAnalyzer(analyzers, "behavioral") { + query.Set("use_behavioral", "true") + } + if hasAnalyzer(analyzers, "llm") || hasAnalyzer(analyzers, "meta") { + query.Set("use_llm", "true") + query.Set("llm_provider", "openai") + } +} + +func splitAnalyzerOption(value string) []string { + if strings.TrimSpace(value) == "" { + return nil + } + items := strings.Split(value, ",") + result := make([]string, 0, len(items)) + for _, item := range items { + item = strings.ToLower(strings.TrimSpace(item)) + if item == "" { + continue + } + result = append(result, item) + } + return result +} + +func hasAnalyzer(items []string, target string) bool { + target = strings.ToLower(strings.TrimSpace(target)) + for _, item := range items { + if item == target { + return true + } + } + return false +} + +func normalizeScannerRiskLevel(raw map[string]interface{}) string { + if safe, ok := readBool(raw["is_safe"]); ok && safe { + return skillRiskNone + } + candidates := []string{ + readString(raw["risk_level"]), + readString(raw["severity"]), + readString(raw["verdict"]), + readString(raw["max_severity"]), + } + if result, ok := raw["result"].(map[string]interface{}); ok { + if safe, ok := readBool(result["is_safe"]); ok && safe { + return skillRiskNone + } + candidates = append(candidates, + readString(result["risk_level"]), + readString(result["severity"]), + readString(result["verdict"]), + readString(result["max_severity"]), + ) + } + for _, value := range candidates { + switch strings.ToLower(strings.TrimSpace(value)) { + case "critical", "high": + return skillRiskHigh + case "medium", "moderate": + return skillRiskMedium + case "low", "warning": + return skillRiskLow + case "none", "clean", "safe", "pass", "info", "informational": + return skillRiskNone + } + } + return skillRiskUnknown +} + +func extractScannerSummary(raw map[string]interface{}) string { + candidates := []string{ + readString(raw["summary"]), + readString(raw["message"]), + } + if result, ok := raw["result"].(map[string]interface{}); ok { + candidates = append(candidates, readString(result["summary"]), readString(result["message"])) + } + for _, candidate := range candidates { + if strings.TrimSpace(candidate) != "" { + return candidate + } + } + return "Skill scanned by external skill-scanner service" +} + +func readString(value interface{}) string { + if text, ok := value.(string); ok { + return text + } + return "" +} + +func readBool(value interface{}) (bool, bool) { + boolean, ok := value.(bool) + return boolean, ok +} diff --git a/backend/internal/services/skill_service.go b/backend/internal/services/skill_service.go new file mode 100644 index 0000000..addbca8 --- /dev/null +++ b/backend/internal/services/skill_service.go @@ -0,0 +1,1934 @@ +package services + +import ( + "archive/zip" + "bytes" + "context" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +var ( + chownRuntimePathOwner = os.Chown + currentEffectiveUID = os.Geteuid +) + +const ( + skillRiskUnknown = "unknown" + skillRiskNone = "none" + skillRiskLow = "low" + skillRiskMedium = "medium" + skillRiskHigh = "high" + + skillSourceUploaded = "uploaded" + skillSourceDiscovered = "discovered" +) + +type SkillPayload struct { + ID int `json:"id"` + ExternalSkillID string `json:"external_skill_id"` + UserID int `json:"user_id"` + SkillKey string `json:"skill_key"` + Name string `json:"name"` + Description *string `json:"description,omitempty"` + Status string `json:"status"` + SourceType string `json:"source_type"` + RiskLevel string `json:"risk_level"` + ScanStatus string `json:"scan_status"` + LastScannedAt *time.Time `json:"last_scanned_at,omitempty"` + CurrentVersionID *int `json:"current_version_id,omitempty"` + CurrentVersionNo *int `json:"current_version_no,omitempty"` + ContentHash *string `json:"content_hash,omitempty"` + ContentMD5 *string `json:"content_md5,omitempty"` + ArchiveHash *string `json:"archive_hash,omitempty"` + RiskReason *string `json:"risk_reason,omitempty"` + TopFindings []SkillFindingPayload `json:"top_findings,omitempty"` + InstanceCount int `json:"instance_count"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type SkillFindingPayload struct { + Analyzer string `json:"analyzer"` + Severity string `json:"severity"` + Category string `json:"category"` + RuleID string `json:"rule_id"` + Title string `json:"title"` + Description string `json:"description"` + FilePath *string `json:"file_path,omitempty"` + LineNumber *int `json:"line_number,omitempty"` + Remediation string `json:"remediation"` + Snippet *string `json:"snippet,omitempty"` +} + +type SkillVersionPayload struct { + ID int `json:"id"` + ExternalVersionID string `json:"external_version_id"` + SkillID int `json:"skill_id"` + BlobID int `json:"blob_id"` + VersionNo int `json:"version_no"` + SourceType string `json:"source_type"` + ContentHash string `json:"content_hash"` + ContentMD5 string `json:"content_md5"` + ArchiveHash string `json:"archive_hash"` + ObjectKey string `json:"object_key"` + FileName string `json:"file_name"` + RiskLevel string `json:"risk_level"` + CreatedAt time.Time `json:"created_at"` +} + +type InstanceSkillPayload struct { + ID int `json:"id"` + InstanceID int `json:"instance_id"` + SkillID int `json:"skill_id"` + SkillVersionID *int `json:"skill_version_id,omitempty"` + SourceType string `json:"source_type"` + InstallPath *string `json:"install_path,omitempty"` + ObservedHash *string `json:"observed_hash,omitempty"` + ContentMD5 *string `json:"content_md5,omitempty"` + Status string `json:"status"` + LastSeenAt *time.Time `json:"last_seen_at,omitempty"` + RemovedAt *time.Time `json:"removed_at,omitempty"` + Skill *SkillPayload `json:"skill,omitempty"` +} + +type SkillScanResultPayload struct { + ID int `json:"id"` + BlobID int `json:"blob_id"` + Engine string `json:"engine"` + RiskLevel string `json:"risk_level"` + Status string `json:"status"` + Summary *string `json:"summary,omitempty"` + Findings map[string]interface{} `json:"findings,omitempty"` + ParsedFindings []SkillFindingPayload `json:"parsed_findings,omitempty"` + ScannedAt *time.Time `json:"scanned_at,omitempty"` +} + +type UpdateSkillRequest struct { + Name string `json:"name"` + Description *string `json:"description"` + Status string `json:"status"` +} + +type AttachSkillToInstanceRequest struct { + SkillID int `json:"skill_id" binding:"required,min=1"` +} + +type AgentSkillRecord struct { + SkillID string `json:"skill_id"` + SkillVersion string `json:"skill_version"` + Identifier string `json:"identifier" binding:"required"` + InstallPath string `json:"install_path"` + ContentMD5 string `json:"content_md5" binding:"required"` + Source string `json:"source"` + Type string `json:"type"` + SizeBytes int64 `json:"size_bytes"` + FileCount int `json:"file_count"` + CollectedAt *time.Time `json:"collected_at,omitempty"` + Metadata map[string]interface{} `json:"metadata"` +} + +type AgentSkillInventoryReportRequest struct { + AgentID string `json:"agent_id" binding:"required"` + ReportedAt *time.Time `json:"reported_at,omitempty"` + Mode string `json:"mode"` + Trigger string `json:"trigger"` + Skills []AgentSkillRecord `json:"skills" binding:"required"` +} + +type AgentSkillPackageUploadRequest struct { + AgentID string `json:"agent_id"` + SkillID string `json:"skill_id"` + SkillVersion string `json:"skill_version"` + Identifier string `json:"identifier"` + ContentMD5 string `json:"content_md5"` + Source string `json:"source"` +} + +type SkillService interface { + ImportArchive(ctx context.Context, userID int, fileHeader *multipart.FileHeader) ([]SkillPayload, error) + ListSkills(userID int) ([]SkillPayload, error) + ListAllSkills() ([]SkillPayload, error) + ListAvailableSkillsForInstance(instanceID int, userID int, userRole string) ([]SkillPayload, error) + GetSkill(userID, skillID int) (*SkillPayload, error) + UpdateSkill(userID, skillID int, req UpdateSkillRequest) (*SkillPayload, error) + DeleteSkill(userID, skillID int) error + DownloadSkill(userID, skillID int) ([]byte, string, error) + DownloadSkillVersionByExternalID(externalVersionID string) ([]byte, string, error) + ListVersions(userID, skillID int) ([]SkillVersionPayload, error) + ListInstanceSkills(instanceID int) ([]InstanceSkillPayload, error) + AttachSkillToInstance(instanceID int, skillID int) (*InstanceSkillPayload, error) + AttachSkillToInstanceForActor(instanceID int, skillID int, userID int, userRole string) (*InstanceSkillPayload, error) + RemoveSkillFromInstance(instanceID int, skillID int) error + SyncAgentSkills(instanceID int, req AgentSkillInventoryReportRequest) error + UploadAgentSkillPackage(ctx context.Context, instanceID int, req AgentSkillPackageUploadRequest, fileHeader *multipart.FileHeader) (*SkillPayload, error) + ListScanResults(userID, skillID int) ([]SkillScanResultPayload, error) +} + +type skillService struct { + repo repository.SkillRepository + instanceRepo repository.InstanceRepository + commandService InstanceCommandService + storage ObjectStorageService + scanner SkillScannerClient +} + +func NewSkillService(repo repository.SkillRepository, instanceRepo repository.InstanceRepository, commandService InstanceCommandService, storage ObjectStorageService, scanner SkillScannerClient) SkillService { + return &skillService{repo: repo, instanceRepo: instanceRepo, commandService: commandService, storage: storage, scanner: scanner} +} + +func (s *skillService) ImportArchive(ctx context.Context, userID int, fileHeader *multipart.FileHeader) ([]SkillPayload, error) { + if !strings.HasSuffix(strings.ToLower(strings.TrimSpace(fileHeader.Filename)), ".zip") { + return nil, fmt.Errorf("only .zip skill archives are supported") + } + file, err := fileHeader.Open() + if err != nil { + return nil, fmt.Errorf("failed to open uploaded archive: %w", err) + } + defer file.Close() + + raw, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("failed to read uploaded archive: %w", err) + } + directories, err := extractSkillDirectories(fileHeader.Filename, raw) + if err != nil { + return nil, err + } + if len(directories) == 0 { + return nil, fmt.Errorf("no skill directories found in archive") + } + + results := make([]SkillPayload, 0, len(directories)) + for _, dir := range directories { + payload, err := s.importDirectory(ctx, userID, dir, fileHeader.Filename) + if err != nil { + return nil, err + } + results = append(results, *payload) + } + return results, nil +} + +func (s *skillService) ListSkills(userID int) ([]SkillPayload, error) { + items, err := s.repo.ListSkillsByUser(userID) + if err != nil { + return nil, err + } + filtered := make([]models.Skill, 0, len(items)) + for _, item := range items { + if isUserManagedSkill(item) { + filtered = append(filtered, item) + } + } + return s.toSkillPayloads(filtered) +} + +func (s *skillService) ListAllSkills() ([]SkillPayload, error) { + items, err := s.repo.ListAllSkills() + if err != nil { + return nil, err + } + return s.toSkillPayloads(items) +} + +func (s *skillService) ListAvailableSkillsForInstance(instanceID int, userID int, userRole string) ([]SkillPayload, error) { + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, err + } + if instance == nil { + return nil, fmt.Errorf("instance not found") + } + if !strings.EqualFold(userRole, "admin") && instance.UserID != userID { + return nil, fmt.Errorf("access denied") + } + + items, err := s.repo.ListAllSkills() + if err != nil { + return nil, err + } + + filtered := make([]models.Skill, 0, len(items)) + for _, item := range items { + if !isUserManagedSkill(item) || !strings.EqualFold(item.Status, "active") { + continue + } + if !canActorAttachSkillToInstance(instance, item, userID, userRole) { + continue + } + filtered = append(filtered, item) + } + return s.toSkillPayloads(filtered) +} +func (s *skillService) GetSkill(userID, skillID int) (*SkillPayload, error) { + item, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if item == nil || item.UserID != userID { + return nil, fmt.Errorf("skill not found") + } + if !isUserManagedSkill(*item) { + return nil, fmt.Errorf("skill not found") + } + return s.toSkillPayload(*item) +} + +func (s *skillService) UpdateSkill(userID, skillID int, req UpdateSkillRequest) (*SkillPayload, error) { + item, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if item == nil || item.UserID != userID { + return nil, fmt.Errorf("skill not found") + } + if !isUserManagedSkill(*item) { + return nil, fmt.Errorf("skill not found") + } + item.Name = strings.TrimSpace(req.Name) + item.Description = req.Description + if status := strings.TrimSpace(req.Status); status != "" { + item.Status = status + } + item.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateSkill(item); err != nil { + return nil, err + } + return s.toSkillPayload(*item) +} + +func (s *skillService) DeleteSkill(userID, skillID int) error { + item, err := s.repo.GetSkillByID(skillID) + if err != nil { + return err + } + if item == nil || item.UserID != userID { + return fmt.Errorf("skill not found") + } + if !isUserManagedSkill(*item) { + return fmt.Errorf("skill not found") + } + return s.repo.DeleteSkill(skillID) +} + +func (s *skillService) DownloadSkill(userID, skillID int) ([]byte, string, error) { + item, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, "", err + } + if item == nil || item.UserID != userID { + return nil, "", fmt.Errorf("skill not found") + } + if !isUserManagedSkill(*item) { + return nil, "", fmt.Errorf("skill not found") + } + if item.CurrentVersionID == nil { + return nil, "", fmt.Errorf("skill has no version") + } + version, err := s.repo.GetVersionByID(*item.CurrentVersionID) + if err != nil { + return nil, "", err + } + blob, err := s.repo.GetBlobByID(version.BlobID) + if err != nil { + return nil, "", err + } + content, err := s.storage.GetObject(context.Background(), blob.ObjectKey) + if err != nil { + return nil, "", err + } + return content, blob.FileName, nil +} + +func (s *skillService) DownloadSkillVersionByExternalID(externalVersionID string) ([]byte, string, error) { + versionID, err := parseExternalVersionID(externalVersionID) + if err != nil { + return nil, "", err + } + version, err := s.repo.GetVersionByID(versionID) + if err != nil { + return nil, "", err + } + if version == nil { + return nil, "", fmt.Errorf("skill version not found") + } + blob, err := s.repo.GetBlobByID(version.BlobID) + if err != nil { + return nil, "", err + } + if blob == nil { + return nil, "", fmt.Errorf("skill blob not found") + } + content, err := s.storage.GetObject(context.Background(), blob.ObjectKey) + if err != nil { + return nil, "", err + } + return content, blob.FileName, nil +} + +func (s *skillService) ListVersions(userID, skillID int) ([]SkillVersionPayload, error) { + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil || skill.UserID != userID { + return nil, fmt.Errorf("skill not found") + } + if !isUserManagedSkill(*skill) { + return nil, fmt.Errorf("skill not found") + } + items, err := s.repo.ListVersionsBySkillID(skillID) + if err != nil { + return nil, err + } + result := make([]SkillVersionPayload, 0, len(items)) + for _, item := range items { + blob, err := s.repo.GetBlobByID(item.BlobID) + if err != nil { + return nil, err + } + result = append(result, SkillVersionPayload{ + ID: item.ID, ExternalVersionID: formatExternalVersionID(item.ID), SkillID: item.SkillID, BlobID: item.BlobID, VersionNo: item.VersionNo, + SourceType: item.SourceType, ContentHash: blob.ContentHash, ContentMD5: s.resolveContentMD5(blob), ArchiveHash: blob.ArchiveHash, + ObjectKey: blob.ObjectKey, FileName: blob.FileName, RiskLevel: blob.RiskLevel, CreatedAt: item.CreatedAt, + }) + } + return result, nil +} + +func (s *skillService) ListInstanceSkills(instanceID int) ([]InstanceSkillPayload, error) { + if err := s.reconcileRemovedInstanceSkillsFromCommands(instanceID); err != nil { + return nil, err + } + items, err := s.repo.ListInstanceSkills(instanceID) + if err != nil { + return nil, err + } + result := make([]InstanceSkillPayload, 0, len(items)) + for _, item := range items { + if isRemovedInstanceSkill(&item) { + continue + } + payload := InstanceSkillPayload{ + ID: item.ID, InstanceID: item.InstanceID, SkillID: item.SkillID, SkillVersionID: item.SkillVersionID, + SourceType: item.SourceType, InstallPath: item.InstallPath, ObservedHash: item.ObservedHash, + Status: item.Status, LastSeenAt: item.LastSeenAt, RemovedAt: item.RemovedAt, + } + skill, err := s.repo.GetSkillByID(item.SkillID) + if err != nil { + return nil, err + } + if skill != nil { + skillPayload, err := s.toSkillPayload(*skill) + if err != nil { + return nil, err + } + payload.Skill = skillPayload + } + result = append(result, payload) + } + return result, nil +} + +func (s *skillService) reconcileRemovedInstanceSkillsFromCommands(instanceID int) error { + if s.commandService == nil { + return nil + } + commands, err := s.commandService.ListByInstanceID(instanceID, 500) + if err != nil { + return err + } + for _, command := range commands { + if command.CommandType != InstanceCommandTypeUninstallSkill || command.Status != instanceCommandStatusSucceeded { + continue + } + if skillID, ok := intPayloadValue(command.Payload["skill_id"]); ok { + if err := s.repo.MarkInstanceSkillRemoved(instanceID, skillID, commandFinishedAt(command)); err != nil { + return err + } + } + if skillKey, ok := stringPayloadValue(command.Payload["target_name"]); ok { + if err := s.repo.MarkInstanceSkillRemovedBySkillKey(instanceID, skillKey, commandFinishedAt(command)); err != nil { + return err + } + } + } + return nil +} + +func commandFinishedAt(command InstanceCommandPayload) time.Time { + if command.FinishedAt != nil && !command.FinishedAt.IsZero() { + return *command.FinishedAt + } + if !command.IssuedAt.IsZero() { + return command.IssuedAt + } + return time.Time{} +} + +func (s *skillService) AttachSkillToInstanceForActor(instanceID int, skillID int, userID int, userRole string) (*InstanceSkillPayload, error) { + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, err + } + if instance == nil { + return nil, fmt.Errorf("instance not found") + } + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil || !canActorAttachSkillToInstance(instance, *skill, userID, userRole) { + return nil, fmt.Errorf("skill not found") + } + return s.attachSkillModelToInstance(instanceID, skill) +} + +func (s *skillService) AttachSkillToInstance(instanceID int, skillID int) (*InstanceSkillPayload, error) { + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil { + return nil, fmt.Errorf("skill not found") + } + return s.attachSkillModelToInstance(instanceID, skill) +} + +func (s *skillService) attachSkillModelToInstance(instanceID int, skill *models.Skill) (*InstanceSkillPayload, error) { + if !isUserManagedSkill(*skill) { + return nil, fmt.Errorf("skill not found") + } + if skill.Status != "active" { + return nil, fmt.Errorf("skill is not active") + } + if isBlockedSkillRisk(skill.RiskLevel) { + return nil, fmt.Errorf("skill is blocked by risk policy") + } + + skillID := skill.ID + versionID := skill.CurrentVersionID + var blob *models.SkillBlob + if versionID != nil { + version, err := s.repo.GetVersionByID(*versionID) + if err != nil { + return nil, err + } + blob, err = s.repo.GetBlobByID(version.BlobID) + if err != nil { + return nil, err + } + if err := s.materializeLiteInstanceSkill(context.Background(), instanceID, skill, blob); err != nil { + return nil, err + } + } + + now := time.Now().UTC() + item := &models.InstanceSkill{ + InstanceID: instanceID, SkillID: skillID, SkillVersionID: versionID, + SourceType: "injected_by_clawmanager", Status: "active", LastSeenAt: &now, UpdatedAt: now, + } + if err := s.repo.UpsertInstanceSkill(item); err != nil { + return nil, err + } + if versionID != nil && blob != nil { + if _, err := s.commandService.Create(instanceID, nil, CreateInstanceCommandRequest{ + CommandType: InstanceCommandTypeInstallSkill, + Payload: map[string]interface{}{ + "skill_id": formatExternalSkillID(skillID), + "skill_version": formatExternalVersionID(*versionID), + "target_name": skill.SkillKey, + "content_md5": s.resolveContentMD5(blob), + }, + IdempotencyKey: fmt.Sprintf("install-skill-%d-%d-%d", instanceID, skillID, now.UnixNano()), + TimeoutSeconds: 300, + }); err != nil { + return nil, fmt.Errorf("failed to queue install skill command: %w", err) + } + } + items, err := s.ListInstanceSkills(instanceID) + if err != nil { + return nil, err + } + for _, candidate := range items { + if candidate.SkillID == skillID { + return &candidate, nil + } + } + return nil, fmt.Errorf("instance skill not found after attach") +} + +func (s *skillService) materializeLiteInstanceSkill(ctx context.Context, instanceID int, skill *models.Skill, blob *models.SkillBlob) error { + if s == nil || s.instanceRepo == nil || s.storage == nil || skill == nil || blob == nil { + return nil + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return err + } + if !isLiteRuntimeInstance(instance) || instance.WorkspacePath == nil || strings.TrimSpace(*instance.WorkspacePath) == "" { + return nil + } + + content, err := s.storage.GetObject(ctx, blob.ObjectKey) + if err != nil { + return fmt.Errorf("failed to load skill archive for lite materialization: %w", err) + } + dirs, err := extractSkillDirectories(blob.FileName, content) + if err != nil { + return fmt.Errorf("failed to read skill archive for lite materialization: %w", err) + } + if len(dirs) != 1 { + return fmt.Errorf("lite skill materialization requires exactly one skill directory") + } + + targetName := sanitizeSkillKey(skill.SkillKey) + if targetName == "" { + targetName = sanitizeSkillKey(dirs[0].Name) + } + if targetName == "" { + return fmt.Errorf("lite skill materialization target is invalid") + } + + targetRoot := liteSkillInstallRoot(instance) + if targetRoot == "" { + return nil + } + if err := writeSkillDirectoryAtomically(targetRoot, targetName, dirs[0].Files); err != nil { + return err + } + return ensureLiteRuntimePersistentOwnership(instance) +} + +func liteSkillInstallRoot(instance *models.Instance) string { + if instance == nil || instance.WorkspacePath == nil || strings.TrimSpace(*instance.WorkspacePath) == "" { + return "" + } + workspacePath := filepath.Clean(strings.TrimSpace(*instance.WorkspacePath)) + if strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeHermes) { + return filepath.Join(workspacePath, "home", ".hermes", "skills") + } + return filepath.Join(workspacePath, "home", ".openclaw", "workspace", "skills") +} + +func liteRuntimePersistentRoot(instance *models.Instance) string { + if instance == nil || instance.WorkspacePath == nil || strings.TrimSpace(*instance.WorkspacePath) == "" { + return "" + } + workspacePath := filepath.Clean(strings.TrimSpace(*instance.WorkspacePath)) + if strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeHermes) { + return filepath.Join(workspacePath, "home", ".hermes") + } + return filepath.Join(workspacePath, "home", ".openclaw") +} + +func ensureLiteRuntimePersistentOwnership(instance *models.Instance) error { + if os.PathSeparator != '/' || instance == nil || instance.ID <= 0 || instance.WorkspacePath == nil { + return nil + } + workspacePath := filepath.Clean(strings.TrimSpace(*instance.WorkspacePath)) + persistentRoot := liteRuntimePersistentRoot(instance) + if workspacePath == "" || persistentRoot == "" || !isPathWithin(workspacePath, persistentRoot) { + return nil + } + + uid := RuntimeLinuxID(instance.ID) + gid := uid + for _, dir := range liteRuntimePersistentAncestors(workspacePath, persistentRoot) { + if err := chownRuntimePath(dir, uid, gid, 0750); err != nil { + return err + } + } + return filepath.WalkDir(persistentRoot, func(current string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !isPathWithin(workspacePath, current) { + return fmt.Errorf("lite runtime path escapes workspace: %s", current) + } + info, err := entry.Info() + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + return nil + } + mode := os.FileMode(0640) + if entry.IsDir() { + mode = 0750 + } + return chownRuntimePath(current, uid, gid, mode) + }) +} + +func liteRuntimePersistentAncestors(workspacePath, persistentRoot string) []string { + workspacePath = filepath.Clean(workspacePath) + persistentRoot = filepath.Clean(persistentRoot) + result := []string{workspacePath} + rel, err := filepath.Rel(workspacePath, persistentRoot) + if err != nil || rel == "." || rel == "" || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return result + } + current := workspacePath + for _, part := range strings.Split(rel, string(filepath.Separator)) { + if part == "" || part == "." { + continue + } + current = filepath.Join(current, part) + result = append(result, current) + } + return result +} + +func chownRuntimePath(targetPath string, uid, gid int, mode os.FileMode) error { + if err := chownRuntimePathOwner(targetPath, uid, gid); err != nil { + if currentEffectiveUID() != 0 && errors.Is(err, os.ErrPermission) { + if chmodErr := os.Chmod(targetPath, mode); chmodErr != nil { + return fmt.Errorf("failed to set lite runtime permissions on %s: %w", targetPath, chmodErr) + } + return nil + } + return fmt.Errorf("failed to set lite runtime owner on %s: %w", targetPath, err) + } + if err := os.Chmod(targetPath, mode); err != nil { + return fmt.Errorf("failed to set lite runtime permissions on %s: %w", targetPath, err) + } + return nil +} + +func writeSkillDirectoryAtomically(targetRoot, targetName string, files map[string][]byte) error { + targetRoot = filepath.Clean(strings.TrimSpace(targetRoot)) + targetName = strings.TrimSpace(targetName) + if targetRoot == "." || targetRoot == "" || targetName == "" || strings.ContainsAny(targetName, `/\\`) { + return fmt.Errorf("invalid lite skill target") + } + if err := os.MkdirAll(targetRoot, 0750); err != nil { + return fmt.Errorf("failed to prepare lite skill root: %w", err) + } + tmpRoot := filepath.Join(targetRoot, ".tmp") + if err := os.MkdirAll(tmpRoot, 0750); err != nil { + return fmt.Errorf("failed to prepare lite skill temp root: %w", err) + } + + tmpDir, err := os.MkdirTemp(tmpRoot, ".tmp-skill-"+targetName+"-") + if err != nil { + return fmt.Errorf("failed to create lite skill temp dir: %w", err) + } + cleanupTmp := true + defer func() { + if cleanupTmp { + _ = os.RemoveAll(tmpDir) + } + }() + + for relPath, body := range files { + clean := normalizeSkillRelPath(relPath) + if clean == "" || hasHiddenPathSegment(clean) { + continue + } + parts := strings.Split(clean, "/") + if len(parts) == 0 { + continue + } + entryPath := filepath.Join(append([]string{tmpDir}, parts...)...) + if !isPathWithin(tmpDir, entryPath) { + return fmt.Errorf("skill archive entry escapes target: %s", relPath) + } + if err := os.MkdirAll(filepath.Dir(entryPath), 0750); err != nil { + return fmt.Errorf("failed to prepare lite skill directory: %w", err) + } + if err := os.WriteFile(entryPath, body, 0640); err != nil { + return fmt.Errorf("failed to write lite skill file: %w", err) + } + } + + targetPath := filepath.Join(targetRoot, targetName) + if !isPathWithin(targetRoot, targetPath) { + return fmt.Errorf("lite skill target escapes root") + } + backupPath := targetPath + ".old" + _ = os.RemoveAll(backupPath) + if _, err := os.Stat(targetPath); err == nil { + if err := os.Rename(targetPath, backupPath); err != nil { + return fmt.Errorf("failed to stage existing lite skill directory: %w", err) + } + } else if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to inspect existing lite skill directory: %w", err) + } + if err := os.Rename(tmpDir, targetPath); err != nil { + if _, statErr := os.Stat(backupPath); statErr == nil { + _ = os.Rename(backupPath, targetPath) + } + return fmt.Errorf("failed to install lite skill directory: %w", err) + } + cleanupTmp = false + _ = os.RemoveAll(backupPath) + return nil +} + +func isPathWithin(root, target string) bool { + root = filepath.Clean(root) + target = filepath.Clean(target) + rel, err := filepath.Rel(root, target) + return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel) +} + +func (s *skillService) RemoveSkillFromInstance(instanceID int, skillID int) error { + item, err := s.repo.GetInstanceSkill(instanceID, skillID) + if err != nil { + return err + } + if item == nil { + return nil + } + if err := s.removeLiteInstanceSkillDirectory(instanceID, item); err != nil { + return err + } + now := time.Now().UTC() + item.Status = "removed" + item.RemovedAt = &now + item.UpdatedAt = now + if err := s.repo.UpsertInstanceSkill(item); err != nil { + return err + } + if _, err := s.commandService.Create(instanceID, nil, CreateInstanceCommandRequest{ + CommandType: InstanceCommandTypeUninstallSkill, + Payload: map[string]interface{}{"skill_id": skillID, "target_name": skillKeyForRemoval(item)}, + IdempotencyKey: fmt.Sprintf("remove-skill-%d-%d-%d", instanceID, skillID, now.UnixNano()), + TimeoutSeconds: 300, + }); err != nil { + return fmt.Errorf("failed to queue uninstall skill command: %w", err) + } + return nil +} + +func (s *skillService) removeLiteInstanceSkillDirectory(instanceID int, item *models.InstanceSkill) error { + if s == nil || s.instanceRepo == nil || item == nil { + return nil + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return err + } + if !isLiteRuntimeInstance(instance) { + return nil + } + skillKey := strings.TrimSpace(skillKeyForRemoval(item)) + if skillKey == "" || strings.HasPrefix(skillKey, "skill-") { + skill, err := s.repo.GetSkillByID(item.SkillID) + if err != nil { + return err + } + if skill != nil { + skillKey = strings.TrimSpace(skill.SkillKey) + } + } + targetName := sanitizeSkillKey(skillKey) + if targetName == "" { + return nil + } + targetRoot := liteSkillInstallRoot(instance) + if targetRoot == "" { + return nil + } + targetPath := filepath.Join(targetRoot, targetName) + if !isPathWithin(targetRoot, targetPath) { + return fmt.Errorf("lite skill removal target escapes root") + } + if err := os.RemoveAll(targetPath); err != nil { + return fmt.Errorf("failed to remove lite skill directory: %w", err) + } + return ensureLiteRuntimePersistentOwnership(instance) +} +func isBlockedSkillRisk(value string) bool { + value = strings.TrimSpace(value) + return strings.EqualFold(value, skillRiskMedium) || strings.EqualFold(value, skillRiskHigh) +} + +func isRemovedInstanceSkill(item *models.InstanceSkill) bool { + return item != nil && (strings.EqualFold(strings.TrimSpace(item.Status), "removed") || item.RemovedAt != nil) +} + +func (s *skillService) SyncAgentSkills(instanceID int, req AgentSkillInventoryReportRequest) error { + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return err + } + if instance == nil { + return fmt.Errorf("instance not found") + } + ownerUserID := instance.UserID + + reportedAt := time.Now().UTC() + if req.ReportedAt != nil && !req.ReportedAt.IsZero() { + reportedAt = req.ReportedAt.UTC() + } + active := make([]int, 0, len(req.Skills)) + for _, record := range req.Skills { + hash := strings.TrimSpace(record.ContentMD5) + if hash == "" { + continue + } + normalizedSource := normalizeSkillSource(record.Source) + var skill *models.Skill + var version *models.SkillVersion + + if normalizedSource == "injected_by_clawmanager" { + if skillID, err := parseExternalSkillID(record.SkillID); err == nil { + item, err := s.repo.GetSkillByID(skillID) + if err != nil { + return err + } + if item != nil && item.UserID == ownerUserID { + skill = item + } + } + } + + skillKey := sanitizeSkillKey(record.Identifier) + if skillKey == "" { + skillKey = hash[:skillMin(16, len(hash))] + } + if skill == nil { + item, err := s.repo.GetSkillByUserKey(ownerUserID, skillKey) + if err != nil { + return err + } + if item != nil && (normalizedSource != "discovered_in_instance" || strings.EqualFold(item.SourceType, skillSourceDiscovered)) { + skill = item + } + } + + blob, err := s.repo.GetBlobByContentHash(hash) + if err != nil { + return err + } + if blob == nil && skill != nil { + version, blob, err = s.findVersionByContentMD5(skill.ID, hash) + if err != nil { + return err + } + } + if blob == nil { + blob = &models.SkillBlob{ + ContentHash: hash, + ArchiveHash: hash, + ObjectKey: "", + FileName: sanitizeSkillKey(record.Identifier) + ".zip", + MediaType: "application/zip", + SizeBytes: 0, + ScanStatus: "pending", + RiskLevel: skillRiskUnknown, + } + if err := s.repo.CreateBlob(blob); err != nil { + return err + } + } + if strings.TrimSpace(blob.ObjectKey) == "" { + _, _ = s.commandService.Create(instanceID, nil, CreateInstanceCommandRequest{ + CommandType: InstanceCommandTypeCollectSkillPackage, + Payload: map[string]interface{}{ + "skill_id": record.SkillID, + "skill_version": record.SkillVersion, + "identifier": record.Identifier, + "content_md5": hash, + "source": normalizedSource, + }, + IdempotencyKey: fmt.Sprintf("collect-skill-package-%d-%s", instanceID, hash), + TimeoutSeconds: 600, + }) + } + if skill == nil { + if normalizedSource == "discovered_in_instance" { + skillKey = s.nextDiscoveredSkillKey(ownerUserID, skillKey, hash) + } + skill = &models.Skill{ + UserID: ownerUserID, SkillKey: skillKey, Name: strings.TrimSpace(record.Identifier), + SourceType: skillSourceDiscovered, Status: "active", RiskLevel: blob.RiskLevel, + LastScannedAt: blob.LastScannedAt, LastScanResultID: blob.LastScanResultID, + } + if skill.Name == "" { + skill.Name = skillKey + } + if err := s.repo.CreateSkill(skill); err != nil { + return err + } + } + if version == nil { + version, err = s.repo.GetVersionBySkillAndBlob(skill.ID, blob.ID) + if err != nil { + return err + } + } + if version == nil && !(strings.EqualFold(skill.SourceType, skillSourceUploaded) && normalizedSource == "injected_by_clawmanager") { + latest, err := s.repo.GetLatestVersionBySkillID(skill.ID) + if err != nil { + return err + } + versionNo := 1 + if latest != nil { + versionNo = latest.VersionNo + 1 + } + version = &models.SkillVersion{SkillID: skill.ID, BlobID: blob.ID, VersionNo: versionNo, SourceType: skillSourceDiscovered} + if err := s.repo.CreateVersion(version); err != nil { + return err + } + } + if version != nil && !strings.EqualFold(skill.SourceType, skillSourceUploaded) { + skill.CurrentVersionID = &version.ID + skill.RiskLevel = blob.RiskLevel + skill.LastScannedAt = blob.LastScannedAt + skill.LastScanResultID = blob.LastScanResultID + if err := s.repo.UpdateSkill(skill); err != nil { + return err + } + } + existingInstanceSkill, err := s.repo.GetInstanceSkill(instanceID, skill.ID) + if err != nil { + return err + } + if isRemovedInstanceSkill(existingInstanceSkill) { + continue + } + + active = append(active, skill.ID) + instanceSkill := &models.InstanceSkill{ + InstanceID: instanceID, SkillID: skill.ID, SkillVersionID: optionalVersionID(version), SourceType: normalizedSource, + InstallPath: optionalString(strings.TrimSpace(record.InstallPath)), ObservedHash: optionalString(hash), + Status: "active", LastSeenAt: &reportedAt, UpdatedAt: reportedAt, + } + if err := s.repo.UpsertInstanceSkill(instanceSkill); err != nil { + return err + } + } + if strings.EqualFold(strings.TrimSpace(req.Mode), "full") || !strings.EqualFold(strings.TrimSpace(req.Mode), "incremental") { + if err := s.repo.MarkMissingInstanceSkills(instanceID, active, reportedAt); err != nil { + return err + } + } + return nil +} + +func (s *skillService) UploadAgentSkillPackage(ctx context.Context, instanceID int, req AgentSkillPackageUploadRequest, fileHeader *multipart.FileHeader) (*SkillPayload, error) { + if !strings.HasSuffix(strings.ToLower(strings.TrimSpace(fileHeader.Filename)), ".zip") { + return nil, fmt.Errorf("only .zip skill archives are supported") + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, err + } + if instance == nil { + return nil, fmt.Errorf("instance not found") + } + file, err := fileHeader.Open() + if err != nil { + return nil, fmt.Errorf("failed to open uploaded skill package: %w", err) + } + defer file.Close() + + raw, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("failed to read uploaded skill package: %w", err) + } + directories, err := extractSkillDirectories(fileHeader.Filename, raw) + if err != nil { + return nil, err + } + if len(directories) != 1 { + return nil, fmt.Errorf("agent skill package must contain exactly one skill directory") + } + dir := directories[0] + contentMD5 := hashDirectory(dir.Files) + expectedMD5 := strings.TrimSpace(req.ContentMD5) + if expectedMD5 != "" && !strings.EqualFold(contentMD5, expectedMD5) { + return nil, fmt.Errorf("skill package md5 mismatch: expected %s got %s", expectedMD5, contentMD5) + } + + archiveBytes, archiveHash, err := buildNormalizedZip(dir) + if err != nil { + return nil, err + } + + blob, err := s.repo.GetBlobByContentHash(contentMD5) + if err != nil { + return nil, err + } + if blob == nil { + blob = &models.SkillBlob{ + ContentHash: contentMD5, + ArchiveHash: archiveHash, + ObjectKey: fmt.Sprintf("discovered/%d/%s/%s.zip", instanceID, sanitizeSkillKey(dir.Name), contentMD5), + FileName: fmt.Sprintf("%s.zip", sanitizeSkillKey(dir.Name)), + MediaType: "application/zip", + SizeBytes: int64(len(archiveBytes)), + ScanStatus: "pending", + RiskLevel: skillRiskUnknown, + } + if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil { + return nil, err + } + if err := s.repo.CreateBlob(blob); err != nil { + return nil, err + } + } else if strings.TrimSpace(blob.ObjectKey) == "" { + blob.ObjectKey = fmt.Sprintf("discovered/%d/%s/%s.zip", instanceID, sanitizeSkillKey(dir.Name), contentMD5) + blob.FileName = fmt.Sprintf("%s.zip", sanitizeSkillKey(dir.Name)) + blob.MediaType = "application/zip" + blob.SizeBytes = int64(len(archiveBytes)) + if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil { + return nil, err + } + if err := s.repo.UpdateBlob(blob); err != nil { + return nil, err + } + } + + if blob.LastScanResultID == nil || blob.ScanStatus != "completed" { + if err := s.recordScan(blob, &dir); err != nil { + blob.ScanStatus = "failed" + blob.UpdatedAt = time.Now().UTC() + _ = s.repo.UpdateBlob(blob) + return nil, err + } + } + + normalizedSource := normalizeSkillSource(req.Source) + skillKey := sanitizeSkillKey(req.Identifier) + if skillKey == "" { + skillKey = sanitizeSkillKey(dir.Name) + } + if skillKey == "" { + skillKey = contentMD5[:skillMin(16, len(contentMD5))] + } + + var skill *models.Skill + if skillID, err := parseExternalSkillID(req.SkillID); err == nil { + item, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if item != nil && item.UserID == instance.UserID { + skill = item + } + } + if skill == nil { + item, err := s.repo.GetSkillByUserKey(instance.UserID, skillKey) + if err != nil { + return nil, err + } + if item != nil { + skill = item + } + } + if skill == nil { + skill = &models.Skill{ + UserID: instance.UserID, SkillKey: skillKey, Name: strings.TrimSpace(req.Identifier), + SourceType: skillSourceDiscovered, Status: "active", RiskLevel: blob.RiskLevel, + LastScannedAt: blob.LastScannedAt, LastScanResultID: blob.LastScanResultID, + } + if strings.TrimSpace(skill.Name) == "" { + skill.Name = dir.Name + } + if err := s.repo.CreateSkill(skill); err != nil { + return nil, err + } + } + + version, err := s.repo.GetVersionBySkillAndBlob(skill.ID, blob.ID) + if err != nil { + return nil, err + } + if version == nil { + latest, err := s.repo.GetLatestVersionBySkillID(skill.ID) + if err != nil { + return nil, err + } + versionNo := 1 + if latest != nil { + versionNo = latest.VersionNo + 1 + } + version = &models.SkillVersion{SkillID: skill.ID, BlobID: blob.ID, VersionNo: versionNo, SourceType: skillSourceDiscovered} + if err := s.repo.CreateVersion(version); err != nil { + return nil, err + } + } + + skill.CurrentVersionID = &version.ID + skill.RiskLevel = blob.RiskLevel + skill.LastScannedAt = blob.LastScannedAt + skill.LastScanResultID = blob.LastScanResultID + skill.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateSkill(skill); err != nil { + return nil, err + } + + now := time.Now().UTC() + instanceSkill := &models.InstanceSkill{ + InstanceID: instanceID, + SkillID: skill.ID, + SkillVersionID: &version.ID, + SourceType: normalizedSource, + InstallPath: nil, + ObservedHash: optionalString(contentMD5), + Status: "active", + LastSeenAt: &now, + UpdatedAt: now, + } + if err := s.repo.UpsertInstanceSkill(instanceSkill); err != nil { + return nil, err + } + return s.toSkillPayload(*skill) +} + +func (s *skillService) ListScanResults(userID, skillID int) ([]SkillScanResultPayload, error) { + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil || (skill.UserID != userID && userID != 0) { + return nil, fmt.Errorf("skill not found") + } + if skill.CurrentVersionID == nil { + return nil, nil + } + version, err := s.repo.GetVersionByID(*skill.CurrentVersionID) + if err != nil { + return nil, err + } + items, err := s.repo.ListScanResultsByBlobID(version.BlobID) + if err != nil { + return nil, err + } + result := make([]SkillScanResultPayload, 0, len(items)) + for _, item := range items { + payload := SkillScanResultPayload{ + ID: item.ID, BlobID: item.BlobID, Engine: item.Engine, RiskLevel: item.RiskLevel, + Status: item.Status, Summary: item.Summary, ScannedAt: item.ScannedAt, + } + if item.FindingsJSON != nil && strings.TrimSpace(*item.FindingsJSON) != "" { + _ = json.Unmarshal([]byte(*item.FindingsJSON), &payload.Findings) + } + payload.ParsedFindings = parseSkillFindings(&item) + result = append(result, payload) + } + return result, nil +} + +type extractedSkillDirectory struct { + Name string + Files map[string][]byte +} + +func extractSkillDirectories(filename string, raw []byte) ([]extractedSkillDirectory, error) { + fileMap, err := extractArchiveFileMap(filename, raw) + if err != nil { + return nil, err + } + + normalized := map[string][]byte{} + for name, content := range fileMap { + clean := normalizeArchiveEntryPath(name) + if clean == "" || isArchiveMetadataEntry(clean) { + continue + } + normalized[clean] = content + } + if len(normalized) == 0 { + return nil, nil + } + + if hasSkillManifest(normalized) { + return []extractedSkillDirectory{{ + Name: archiveSkillName(filename), + Files: normalized, + }}, nil + } + + grouped := map[string]map[string][]byte{} + for clean, content := range normalized { + parts := strings.Split(clean, "/") + if len(parts) < 2 { + return nil, fmt.Errorf("archive must contain SKILL.md at the root or top-level skill directories; found loose file %s", clean) + } + root := parts[0] + if _, ok := grouped[root]; !ok { + grouped[root] = map[string][]byte{} + } + grouped[root][strings.Join(parts[1:], "/")] = content + } + + keys := make([]string, 0, len(grouped)) + for key, files := range grouped { + if !hasSkillManifest(files) { + return nil, fmt.Errorf("skill directory %s must contain SKILL.md", key) + } + keys = append(keys, key) + } + sort.Strings(keys) + result := make([]extractedSkillDirectory, 0, len(keys)) + for _, key := range keys { + result = append(result, extractedSkillDirectory{Name: key, Files: grouped[key]}) + } + return result, nil +} + +func normalizeArchiveEntryPath(value string) string { + value = strings.ReplaceAll(value, "\\", "/") + value = path.Clean(strings.TrimPrefix(strings.TrimSpace(value), "./")) + if value == "." || value == "" || strings.HasPrefix(value, "..") { + return "" + } + return value +} + +func hasSkillManifest(files map[string][]byte) bool { + for key := range files { + if strings.EqualFold(normalizeArchiveEntryPath(key), "SKILL.md") { + return true + } + } + return false +} + +func isArchiveMetadataEntry(value string) bool { + clean := normalizeArchiveEntryPath(value) + if clean == "" { + return true + } + parts := strings.Split(clean, "/") + if parts[0] == "__MACOSX" { + return true + } + base := parts[len(parts)-1] + return base == ".DS_Store" || base == "Thumbs.db" || strings.HasPrefix(base, "._") +} + +func archiveSkillName(filename string) string { + name := path.Base(strings.ReplaceAll(strings.TrimSpace(filename), "\\", "/")) + if ext := path.Ext(name); ext != "" { + name = strings.TrimSuffix(name, ext) + } + if sanitizeSkillKey(name) == "" { + return "skill" + } + return name +} + +func extractArchiveFileMap(filename string, raw []byte) (map[string][]byte, error) { + lower := strings.ToLower(strings.TrimSpace(filename)) + fileMap := map[string][]byte{} + switch { + case strings.HasSuffix(lower, ".zip"): + reader, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw))) + if err != nil { + return nil, fmt.Errorf("failed to read zip archive: %w", err) + } + for _, entry := range reader.File { + if entry.FileInfo().IsDir() { + continue + } + rc, err := entry.Open() + if err != nil { + return nil, fmt.Errorf("failed to open zip entry: %w", err) + } + content, err := io.ReadAll(rc) + rc.Close() + if err != nil { + return nil, fmt.Errorf("failed to read zip entry: %w", err) + } + fileMap[entry.Name] = content + } + default: + return nil, fmt.Errorf("only .zip skill archives are supported") + } + return fileMap, nil +} + +func (s *skillService) importDirectory(ctx context.Context, userID int, dir extractedSkillDirectory, originalName string) (*SkillPayload, error) { + skillKey := sanitizeSkillKey(dir.Name) + if skillKey == "" { + return nil, fmt.Errorf("skill directory name %q is invalid", dir.Name) + } + contentHash := hashDirectory(dir.Files) + archiveBytes, archiveHash, err := buildNormalizedZip(dir) + if err != nil { + return nil, err + } + + blob, err := s.repo.GetBlobByContentHash(contentHash) + if err != nil { + return nil, err + } + if blob == nil { + blob = &models.SkillBlob{ + ContentHash: contentHash, ArchiveHash: archiveHash, + ObjectKey: fmt.Sprintf("%d/%s/%s.zip", userID, skillKey, contentHash), + FileName: fmt.Sprintf("%s.zip", skillKey), + MediaType: "application/zip", SizeBytes: int64(len(archiveBytes)), + ScanStatus: "pending", RiskLevel: skillRiskUnknown, + } + if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil { + return nil, err + } + if err := s.repo.CreateBlob(blob); err != nil { + return nil, err + } + if err := s.recordScan(blob, &dir); err != nil { + return nil, err + } + } + + skill, err := s.repo.GetSkillByUserKey(userID, skillKey) + if err != nil { + return nil, err + } + if skill == nil { + description := fmt.Sprintf("Imported from %s", originalName) + skill = &models.Skill{ + UserID: userID, SkillKey: skillKey, Name: dir.Name, Description: &description, + SourceType: skillSourceUploaded, Status: "active", RiskLevel: blob.RiskLevel, + LastScannedAt: blob.LastScannedAt, LastScanResultID: blob.LastScanResultID, + } + if err := s.repo.CreateSkill(skill); err != nil { + return nil, err + } + } + version, err := s.repo.GetVersionBySkillAndBlob(skill.ID, blob.ID) + if err != nil { + return nil, err + } + if version == nil { + latest, err := s.repo.GetLatestVersionBySkillID(skill.ID) + if err != nil { + return nil, err + } + versionNo := 1 + if latest != nil { + versionNo = latest.VersionNo + 1 + } + manifest, _ := json.Marshal(map[string]interface{}{"root_dir": dir.Name, "files": len(dir.Files)}) + manifestJSON := string(manifest) + version = &models.SkillVersion{ + SkillID: skill.ID, BlobID: blob.ID, VersionNo: versionNo, ManifestJSON: &manifestJSON, SourceType: skillSourceUploaded, + } + if err := s.repo.CreateVersion(version); err != nil { + return nil, err + } + } + skill.CurrentVersionID = &version.ID + skill.RiskLevel = blob.RiskLevel + skill.LastScannedAt = blob.LastScannedAt + skill.LastScanResultID = blob.LastScanResultID + skill.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateSkill(skill); err != nil { + return nil, err + } + return s.toSkillPayload(*skill) +} + +func (s *skillService) recordScan(blob *models.SkillBlob, dir *extractedSkillDirectory) error { + if s.scanner == nil { + return fmt.Errorf("skill scanner is not configured") + } + if dir == nil { + return fmt.Errorf("skill scanner requires real skill package content") + } + archiveBytes, _, err := buildNormalizedZip(*dir) + if err != nil { + return fmt.Errorf("failed to prepare skill archive for scanning: %w", err) + } + riskLevel, findings, summary, err := s.scanner.ScanArchive(context.Background(), blob.FileName, archiveBytes, nil) + if err != nil { + return fmt.Errorf("skill scanner failed: %w", err) + } + if strings.TrimSpace(summary) == "" { + summary = "Skill scanned by external skill-scanner service" + } + scannedAt := time.Now().UTC() + findingsJSON, _ := json.Marshal(findings) + result := &models.SkillScanResult{ + BlobID: blob.ID, Engine: "skill-scanner", RiskLevel: riskLevel, Status: "completed", + Summary: &summary, FindingsJSON: optionalString(string(findingsJSON)), ScannedAt: &scannedAt, + } + if err := s.repo.CreateScanResult(result); err != nil { + return err + } + blob.ScanStatus = "completed" + blob.RiskLevel = riskLevel + blob.LastScannedAt = &scannedAt + blob.LastScanResultID = &result.ID + if err := s.repo.UpdateBlob(blob); err != nil { + return err + } + return nil +} + +func buildNormalizedZip(dir extractedSkillDirectory) ([]byte, string, error) { + var buffer bytes.Buffer + zipWriter := zip.NewWriter(&buffer) + keys := make([]string, 0, len(dir.Files)) + for key := range dir.Files { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + content := dir.Files[key] + writer, err := zipWriter.Create(path.Join(dir.Name, key)) + if err != nil { + return nil, "", fmt.Errorf("failed to create normalized zip entry: %w", err) + } + if _, err := writer.Write(content); err != nil { + return nil, "", fmt.Errorf("failed to write normalized zip content: %w", err) + } + } + if err := zipWriter.Close(); err != nil { + return nil, "", fmt.Errorf("failed to finalize zip archive: %w", err) + } + hash := sha256.Sum256(buffer.Bytes()) + return buffer.Bytes(), hex.EncodeToString(hash[:]), nil +} + +func hashDirectory(files map[string][]byte) string { + digest := md5.New() + entryKinds := map[string]string{} + fileMap := map[string][]byte{} + for key, body := range files { + clean := normalizeSkillRelPath(key) + if clean == "" || hasHiddenPathSegment(clean) { + continue + } + fileMap[clean] = body + entryKinds[clean] = "file" + for _, dir := range parentDirs(clean) { + if dir == "" || hasHiddenPathSegment(dir) { + continue + } + entryKinds[dir] = "dir" + } + } + entryKeys := make([]string, 0, len(entryKinds)) + for key := range entryKinds { + entryKeys = append(entryKeys, key) + } + sort.Strings(entryKeys) + for _, key := range entryKeys { + _, _ = digest.Write([]byte(key)) + _, _ = digest.Write([]byte("\n")) + if entryKinds[key] == "dir" { + _, _ = digest.Write([]byte("dir\n")) + continue + } + _, _ = digest.Write([]byte("file\n")) + _, _ = digest.Write(fileMap[key]) + _, _ = digest.Write([]byte("\n")) + } + return hex.EncodeToString(digest.Sum(nil)) +} + +func (s *skillService) resolveContentMD5(blob *models.SkillBlob) string { + if blob == nil { + return "" + } + contentHash := strings.TrimSpace(blob.ContentHash) + if len(contentHash) == 32 { + return contentHash + } + content, err := s.storage.GetObject(context.Background(), blob.ObjectKey) + if err != nil { + return contentHash + } + files, err := extractArchiveFileMap(blob.FileName, content) + if err != nil { + sum := md5.Sum(content) + return hex.EncodeToString(sum[:]) + } + return hashDirectory(flattenSingleTopLevelDir(files)) +} + +func normalizeSkillRelPath(value string) string { + value = path.Clean(strings.TrimPrefix(strings.TrimSpace(value), "./")) + if value == "." || value == "" || strings.HasPrefix(value, "..") { + return "" + } + return value +} + +func hasHiddenPathSegment(value string) bool { + for _, part := range strings.Split(value, "/") { + if strings.HasPrefix(part, ".") { + return true + } + } + return false +} + +func parentDirs(value string) []string { + parts := strings.Split(value, "/") + if len(parts) <= 1 { + return nil + } + dirs := make([]string, 0, len(parts)-1) + for i := 1; i < len(parts); i++ { + dir := strings.Join(parts[:i], "/") + if dir != "" { + dirs = append(dirs, dir) + } + } + return dirs +} + +func flattenSingleTopLevelDir(files map[string][]byte) map[string][]byte { + normalized := map[string][]byte{} + topLevel := map[string]struct{}{} + for key, body := range files { + clean := normalizeSkillRelPath(key) + if clean == "" || hasHiddenPathSegment(clean) { + continue + } + normalized[clean] = body + part := clean + if slash := strings.IndexByte(clean, '/'); slash >= 0 { + part = clean[:slash] + } + topLevel[part] = struct{}{} + } + if len(topLevel) != 1 { + return normalized + } + var root string + for key := range topLevel { + root = key + } + prefix := root + "/" + flattened := map[string][]byte{} + for key, body := range normalized { + if strings.HasPrefix(key, prefix) { + flattened[strings.TrimPrefix(key, prefix)] = body + continue + } + flattened[key] = body + } + return flattened +} + +func sanitizeSkillKey(value string) string { + value = strings.ToLower(strings.TrimSpace(value)) + var builder strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + builder.WriteRune(r) + case r == '-' || r == '_' || r == ' ' || r == '.': + builder.WriteRune('-') + } + } + result := strings.Trim(builder.String(), "-") + for strings.Contains(result, "--") { + result = strings.ReplaceAll(result, "--", "-") + } + return result +} + +func (s *skillService) toSkillPayloads(items []models.Skill) ([]SkillPayload, error) { + result := make([]SkillPayload, 0, len(items)) + for _, item := range items { + payload, err := s.toSkillPayload(item) + if err != nil { + return nil, err + } + result = append(result, *payload) + } + return result, nil +} + +func (s *skillService) toSkillPayload(item models.Skill) (*SkillPayload, error) { + payload := &SkillPayload{ + ID: item.ID, ExternalSkillID: formatExternalSkillID(item.ID), UserID: item.UserID, SkillKey: item.SkillKey, Name: item.Name, Description: item.Description, + Status: item.Status, SourceType: item.SourceType, RiskLevel: item.RiskLevel, ScanStatus: "pending", + LastScannedAt: item.LastScannedAt, CurrentVersionID: item.CurrentVersionID, CreatedAt: item.CreatedAt, UpdatedAt: item.UpdatedAt, + } + if item.CurrentVersionID != nil { + version, err := s.repo.GetVersionByID(*item.CurrentVersionID) + if err != nil { + return nil, err + } + if version != nil { + payload.CurrentVersionNo = &version.VersionNo + blob, err := s.repo.GetBlobByID(version.BlobID) + if err != nil { + return nil, err + } + if blob != nil { + contentMD5 := s.resolveContentMD5(blob) + payload.ContentHash = &blob.ContentHash + payload.ContentMD5 = &contentMD5 + payload.ArchiveHash = &blob.ArchiveHash + payload.ScanStatus = blob.ScanStatus + payload.LastScannedAt = blob.LastScannedAt + } + } + } + if item.LastScanResultID != nil { + scanResult, err := s.repo.GetScanResultByID(*item.LastScanResultID) + if err != nil { + return nil, err + } + findings := parseSkillFindings(scanResult) + payload.TopFindings = topRiskFindings(findings, 3) + payload.RiskReason = summarizeRiskReason(payload.TopFindings) + } + instanceSkills, err := s.findInstanceRefs(item.ID) + if err != nil { + return nil, err + } + payload.InstanceCount = instanceSkills + return payload, nil +} + +func parseSkillFindings(result *models.SkillScanResult) []SkillFindingPayload { + if result == nil || result.FindingsJSON == nil || strings.TrimSpace(*result.FindingsJSON) == "" { + return []SkillFindingPayload{} + } + var raw struct { + Findings []struct { + Analyzer string `json:"analyzer"` + Severity string `json:"severity"` + Category string `json:"category"` + RuleID string `json:"rule_id"` + Title string `json:"title"` + Description string `json:"description"` + FilePath *string `json:"file_path"` + LineNumber *int `json:"line_number"` + Remediation string `json:"remediation"` + Snippet *string `json:"snippet"` + } `json:"findings"` + } + if err := json.Unmarshal([]byte(*result.FindingsJSON), &raw); err != nil { + return []SkillFindingPayload{} + } + items := make([]SkillFindingPayload, 0, len(raw.Findings)) + for _, item := range raw.Findings { + items = append(items, SkillFindingPayload{ + Analyzer: item.Analyzer, + Severity: item.Severity, + Category: item.Category, + RuleID: item.RuleID, + Title: item.Title, + Description: item.Description, + FilePath: item.FilePath, + LineNumber: item.LineNumber, + Remediation: item.Remediation, + Snippet: item.Snippet, + }) + } + sort.SliceStable(items, func(i, j int) bool { + return severityRank(items[i].Severity) > severityRank(items[j].Severity) + }) + return items +} + +func topRiskFindings(items []SkillFindingPayload, limit int) []SkillFindingPayload { + if limit <= 0 || len(items) == 0 { + return []SkillFindingPayload{} + } + if len(items) <= limit { + return items + } + return items[:limit] +} + +func summarizeRiskReason(items []SkillFindingPayload) *string { + if len(items) == 0 { + return nil + } + first := items[0] + summary := strings.TrimSpace(first.Title) + if summary == "" { + summary = strings.TrimSpace(first.Description) + } + if summary == "" { + return nil + } + if first.FilePath != nil && strings.TrimSpace(*first.FilePath) != "" { + summary = fmt.Sprintf("%s (%s)", summary, strings.TrimSpace(*first.FilePath)) + } + return &summary +} + +func severityRank(value string) int { + switch strings.ToUpper(strings.TrimSpace(value)) { + case "CRITICAL": + return 5 + case "HIGH": + return 4 + case "MEDIUM", "MODERATE": + return 3 + case "LOW", "WARNING": + return 2 + case "INFO", "SAFE", "NONE": + return 1 + default: + return 0 + } +} + +func (s *skillService) findInstanceRefs(skillID int) (int, error) { + all, err := s.repo.ListAllSkills() + if err != nil { + return 0, err + } + _ = all + count := 0 + for instanceID := 1; instanceID <= 0; instanceID++ { + _ = instanceID + } + instances, err := s.instanceRepo.GetAll(0, 100000) + if err != nil { + return 0, err + } + for _, instance := range instances { + items, err := s.repo.ListInstanceSkills(instance.ID) + if err != nil { + return 0, err + } + for _, item := range items { + if item.SkillID == skillID && item.Status != "removed" { + count++ + } + } + } + return count, nil +} + +func normalizeSkillSource(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "discovered_in_instance" + } + return value +} + +func canActorAttachSkillToInstance(instance *models.Instance, skill models.Skill, userID int, userRole string) bool { + if instance == nil { + return false + } + if strings.EqualFold(userRole, "admin") { + return true + } + return instance.UserID == userID +} + +func isUserManagedSkill(skill models.Skill) bool { + return strings.EqualFold(strings.TrimSpace(skill.SourceType), skillSourceUploaded) +} + +func optionalVersionID(version *models.SkillVersion) *int { + if version == nil { + return nil + } + return &version.ID +} + +func (s *skillService) findVersionByContentMD5(skillID int, contentMD5 string) (*models.SkillVersion, *models.SkillBlob, error) { + versions, err := s.repo.ListVersionsBySkillID(skillID) + if err != nil { + return nil, nil, err + } + for _, candidate := range versions { + blob, err := s.repo.GetBlobByID(candidate.BlobID) + if err != nil { + return nil, nil, err + } + if blob != nil && s.resolveContentMD5(blob) == contentMD5 { + return &candidate, blob, nil + } + } + return nil, nil, nil +} + +func (s *skillService) nextDiscoveredSkillKey(userID int, baseKey, hash string) string { + candidate := baseKey + if candidate == "" { + candidate = "discovered-skill" + } + existing, err := s.repo.GetSkillByUserKey(userID, candidate) + if err == nil && existing == nil { + return candidate + } + suffix := hash + if len(suffix) > 8 { + suffix = suffix[:8] + } + candidate = fmt.Sprintf("%s-%s", candidate, suffix) + existing, err = s.repo.GetSkillByUserKey(userID, candidate) + if err == nil && existing == nil { + return candidate + } + return fmt.Sprintf("%s-%d", candidate, time.Now().UTC().Unix()) +} + +func formatExternalSkillID(id int) string { + return fmt.Sprintf("skill_%d", id) +} + +func formatExternalVersionID(id int) string { + return fmt.Sprintf("ver_%d", id) +} + +func parseExternalVersionID(value string) (int, error) { + value = strings.TrimSpace(strings.TrimPrefix(value, "ver_")) + if value == "" { + return 0, fmt.Errorf("invalid skill version") + } + var id int + if _, err := fmt.Sscanf(value, "%d", &id); err != nil || id <= 0 { + return 0, fmt.Errorf("invalid skill version") + } + return id, nil +} + +func parseExternalSkillID(value string) (int, error) { + value = strings.TrimSpace(strings.TrimPrefix(value, "skill_")) + if value == "" { + return 0, fmt.Errorf("invalid skill id") + } + var id int + if _, err := fmt.Sscanf(value, "%d", &id); err != nil || id <= 0 { + return 0, fmt.Errorf("invalid skill id") + } + return id, nil +} + +func skillKeyForRemoval(item *models.InstanceSkill) string { + if item == nil { + return "" + } + if item.InstallPath != nil && strings.TrimSpace(*item.InstallPath) != "" { + parts := strings.Split(strings.TrimSpace(*item.InstallPath), "/") + return parts[len(parts)-1] + } + return fmt.Sprintf("skill-%d", item.SkillID) +} + +func skillMin(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/backend/internal/services/skill_service_test.go b/backend/internal/services/skill_service_test.go new file mode 100644 index 0000000..91db8a8 --- /dev/null +++ b/backend/internal/services/skill_service_test.go @@ -0,0 +1,482 @@ +package services + +import ( + "archive/zip" + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "os" + "path" + "path/filepath" + "sort" + "strings" + "testing" + + "clawreef/internal/models" +) + +func TestHashDirectoryPreservesSingleTopLevelSubdirectory(t *testing.T) { + files := map[string][]byte{ + "src/main.py": []byte("print('hello')\n"), + } + + got := hashDirectory(files) + want := referenceSkillContentMD5(map[string][]byte{ + "src/main.py": []byte("print('hello')\n"), + }) + if got != want { + t.Fatalf("hashDirectory() = %s, want %s", got, want) + } + + flattened := referenceSkillContentMD5(map[string][]byte{ + "main.py": []byte("print('hello')\n"), + }) + if got == flattened { + t.Fatalf("hashDirectory stripped the skill's internal src/ directory") + } +} + +func TestExtractSkillDirectoriesStripsArchiveRootOnlyOnce(t *testing.T) { + archive := buildTestZip(t, map[string][]byte{ + "weather/SKILL.md": []byte("# Weather\n"), + "weather/src/main.py": []byte("print('weather')\n"), + }) + + dirs, err := extractSkillDirectories("weather.zip", archive) + if err != nil { + t.Fatalf("extractSkillDirectories() error = %v", err) + } + if len(dirs) != 1 { + t.Fatalf("extractSkillDirectories() returned %d dirs, want 1", len(dirs)) + } + if _, ok := dirs[0].Files["src/main.py"]; !ok { + t.Fatalf("expected skill files to preserve src/main.py after stripping archive root once: %#v", dirs[0].Files) + } + + got := hashDirectory(dirs[0].Files) + want := referenceSkillContentMD5(map[string][]byte{ + "SKILL.md": []byte("# Weather\n"), + "src/main.py": []byte("print('weather')\n"), + }) + if got != want { + t.Fatalf("hashDirectory(extracted files) = %s, want %s", got, want) + } +} + +func TestExtractSkillDirectoriesAcceptsRootSkillPackage(t *testing.T) { + archive := buildTestZip(t, map[string][]byte{ + ".env": []byte("TOKEN=example\n"), + "SKILL.md": []byte("# Weather\n"), + "__MACOSX/._SKILL.md": []byte("metadata\n"), + "scripts/run.sh": []byte("python src/main.py\n"), + "src/main.py": []byte("print('weather')\n"), + }) + + dirs, err := extractSkillDirectories("weather.zip", archive) + if err != nil { + t.Fatalf("extractSkillDirectories() error = %v", err) + } + if len(dirs) != 1 { + t.Fatalf("extractSkillDirectories() returned %d dirs, want 1", len(dirs)) + } + if dirs[0].Name != "weather" { + t.Fatalf("root skill package name = %q, want weather", dirs[0].Name) + } + if _, ok := dirs[0].Files["SKILL.md"]; !ok { + t.Fatalf("expected root SKILL.md to be preserved: %#v", dirs[0].Files) + } + if _, ok := dirs[0].Files["scripts/run.sh"]; !ok { + t.Fatalf("expected root scripts/run.sh to be preserved: %#v", dirs[0].Files) + } + if _, ok := dirs[0].Files[".env"]; !ok { + t.Fatalf("expected root .env to be preserved for scanning: %#v", dirs[0].Files) + } + if _, ok := dirs[0].Files["__MACOSX/._SKILL.md"]; ok { + t.Fatalf("expected archive metadata to be ignored: %#v", dirs[0].Files) + } +} + +func TestExtractSkillDirectoriesImportsMultipleManifestDirs(t *testing.T) { + archive := buildTestZip(t, map[string][]byte{ + "alpha/SKILL.md": []byte("# Alpha\n"), + "beta/SKILL.md": []byte("# Beta\n"), + }) + + dirs, err := extractSkillDirectories("skills.zip", archive) + if err != nil { + t.Fatalf("extractSkillDirectories() error = %v", err) + } + if len(dirs) != 2 { + t.Fatalf("extractSkillDirectories() returned %d dirs, want 2", len(dirs)) + } + if dirs[0].Name != "alpha" || dirs[1].Name != "beta" { + t.Fatalf("skill directory names = %q, %q; want alpha, beta", dirs[0].Name, dirs[1].Name) + } +} + +func TestExtractSkillDirectoriesRejectsTopLevelDirWithoutManifest(t *testing.T) { + archive := buildTestZip(t, map[string][]byte{ + "weather/src/main.py": []byte("print('weather')\n"), + }) + + _, err := extractSkillDirectories("weather.zip", archive) + if err == nil { + t.Fatal("extractSkillDirectories() error = nil, want SKILL.md error") + } + if !strings.Contains(err.Error(), "SKILL.md") { + t.Fatalf("extractSkillDirectories() error = %v, want SKILL.md error", err) + } +} + +func TestExtractSkillDirectoriesRejectsLooseFileWithoutRootManifest(t *testing.T) { + archive := buildTestZip(t, map[string][]byte{ + "README.md": []byte("not a skill manifest\n"), + "weather/SKILL.md": []byte("# Weather\n"), + }) + + _, err := extractSkillDirectories("weather.zip", archive) + if err == nil { + t.Fatal("extractSkillDirectories() error = nil, want loose file error") + } + if !strings.Contains(err.Error(), "loose file README.md") { + t.Fatalf("extractSkillDirectories() error = %v, want loose README.md error", err) + } +} + +func TestFlattenSingleTopLevelDirForArchiveRoot(t *testing.T) { + files := map[string][]byte{ + "weather/src/main.py": []byte("print('weather')\n"), + } + + got := hashDirectory(flattenSingleTopLevelDir(files)) + want := referenceSkillContentMD5(map[string][]byte{ + "src/main.py": []byte("print('weather')\n"), + }) + if got != want { + t.Fatalf("hashDirectory(flattenSingleTopLevelDir(files)) = %s, want %s", got, want) + } +} + +func TestMaterializeLiteInstanceSkillWritesOpenClawWorkspaceSkill(t *testing.T) { + archive := buildTestZip(t, map[string][]byte{ + "paper-ranker/SKILL.md": []byte("# Paper Ranker\n"), + "paper-ranker/src/rank.py": []byte("print('rank')\n"), + "paper-ranker/.ignored-file": []byte("local secret\n"), + }) + workspacePath := filepath.Join(t.TempDir(), "openclaw", "user-45", "instance-77") + if err := os.MkdirAll(workspacePath, 0750); err != nil { + t.Fatalf("MkdirAll(workspacePath): %v", err) + } + + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[77] = &models.Instance{ + ID: 0, + UserID: 45, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: &workspacePath, + } + service := &skillService{ + instanceRepo: instanceRepo, + storage: fakeObjectStorage{"skills/paper-ranker.zip": archive}, + } + + err := service.materializeLiteInstanceSkill(context.Background(), 77, &models.Skill{ + SkillKey: "paper-ranker", + }, &models.SkillBlob{ + ObjectKey: "skills/paper-ranker.zip", + FileName: "paper-ranker.zip", + }) + if err != nil { + t.Fatalf("materializeLiteInstanceSkill() error = %v", err) + } + + target := filepath.Join(workspacePath, "home", ".openclaw", "workspace", "skills", "paper-ranker") + assertFileEquals(t, filepath.Join(target, "SKILL.md"), "# Paper Ranker\n") + assertFileEquals(t, filepath.Join(target, "src", "rank.py"), "print('rank')\n") + if _, err := os.Stat(filepath.Join(target, ".ignored-file")); !os.IsNotExist(err) { + t.Fatalf("hidden archive entry was materialized, stat err = %v", err) + } +} + +func TestMaterializeLiteInstanceSkillWritesHermesHomeSkill(t *testing.T) { + archive := buildTestZip(t, map[string][]byte{ + "paper-ranker/SKILL.md": []byte("# Paper Ranker\n"), + }) + workspacePath := filepath.Join(t.TempDir(), "hermes", "user-45", "instance-90") + if err := os.MkdirAll(workspacePath, 0750); err != nil { + t.Fatalf("MkdirAll(workspacePath): %v", err) + } + + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[90] = &models.Instance{ + ID: 90, + UserID: 45, + Type: RuntimeTypeHermes, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: &workspacePath, + } + service := &skillService{ + instanceRepo: instanceRepo, + storage: fakeObjectStorage{"skills/paper-ranker.zip": archive}, + } + + err := service.materializeLiteInstanceSkill(context.Background(), 90, &models.Skill{ + SkillKey: "paper-ranker", + }, &models.SkillBlob{ + ObjectKey: "skills/paper-ranker.zip", + FileName: "paper-ranker.zip", + }) + if err != nil { + t.Fatalf("materializeLiteInstanceSkill() error = %v", err) + } + + target := filepath.Join(workspacePath, "home", ".hermes", "skills", "paper-ranker") + assertFileEquals(t, filepath.Join(target, "SKILL.md"), "# Paper Ranker\n") +} +func TestChownRuntimePathToleratesNonRootPermissionDenied(t *testing.T) { + target := filepath.Join(t.TempDir(), "skill-file") + if err := os.WriteFile(target, []byte("skill"), 0644); err != nil { + t.Fatalf("WriteFile(target): %v", err) + } + + oldChown := chownRuntimePathOwner + oldEffectiveUID := currentEffectiveUID + t.Cleanup(func() { + chownRuntimePathOwner = oldChown + currentEffectiveUID = oldEffectiveUID + }) + chownRuntimePathOwner = func(string, int, int) error { + return os.ErrPermission + } + currentEffectiveUID = func() int { + return 1000 + } + + if err := chownRuntimePath(target, RuntimeLinuxID(77), RuntimeLinuxID(77), 0600); err != nil { + t.Fatalf("chownRuntimePath() error = %v", err) + } + if os.PathSeparator == '/' { + info, err := os.Stat(target) + if err != nil { + t.Fatalf("Stat(target): %v", err) + } + if got := info.Mode().Perm(); got != 0600 { + t.Fatalf("target mode = %v, want 0600", got) + } + } +} + +func TestChownRuntimePathReportsRootPermissionDenied(t *testing.T) { + target := filepath.Join(t.TempDir(), "skill-file") + if err := os.WriteFile(target, []byte("skill"), 0644); err != nil { + t.Fatalf("WriteFile(target): %v", err) + } + + oldChown := chownRuntimePathOwner + oldEffectiveUID := currentEffectiveUID + t.Cleanup(func() { + chownRuntimePathOwner = oldChown + currentEffectiveUID = oldEffectiveUID + }) + chownRuntimePathOwner = func(string, int, int) error { + return os.ErrPermission + } + currentEffectiveUID = func() int { + return 0 + } + + err := chownRuntimePath(target, RuntimeLinuxID(90), RuntimeLinuxID(90), 0600) + if err == nil || !strings.Contains(err.Error(), "failed to set lite runtime owner") { + t.Fatalf("chownRuntimePath() error = %v, want owner error", err) + } +} +func TestWriteSkillDirectoryAtomicallyUsesNestedTempRoot(t *testing.T) { + targetRoot := t.TempDir() + err := writeSkillDirectoryAtomically(targetRoot, "marker-pdf-ingest", map[string][]byte{ + "SKILL.md": []byte("# Marker PDF Ingest\n"), + "scripts/parse_pdf.py": []byte("print('parse')\n"), + "scripts/helpers/io.py": []byte("print('io')\n"), + }) + if err != nil { + t.Fatalf("writeSkillDirectoryAtomically() error = %v", err) + } + + target := filepath.Join(targetRoot, "marker-pdf-ingest") + assertFileEquals(t, filepath.Join(target, "scripts", "parse_pdf.py"), "print('parse')\n") + if _, err := os.Stat(filepath.Join(targetRoot, ".tmp-skill-marker-pdf-ingest")); !os.IsNotExist(err) { + t.Fatalf("temporary skill directory leaked at root, stat err = %v", err) + } + if _, err := os.Stat(filepath.Join(targetRoot, ".tmp")); err != nil { + t.Fatalf("expected nested temporary root to remain available, stat err = %v", err) + } +} +func TestRemoveLiteInstanceSkillDeletesOpenClawWorkspaceSkill(t *testing.T) { + workspacePath := filepath.Join(t.TempDir(), "openclaw", "user-45", "instance-77") + target := filepath.Join(workspacePath, "home", ".openclaw", "workspace", "skills", "paper-ranker") + if err := os.MkdirAll(target, 0750); err != nil { + t.Fatalf("MkdirAll(target): %v", err) + } + if err := os.WriteFile(filepath.Join(target, "SKILL.md"), []byte("# Paper Ranker\n"), 0640); err != nil { + t.Fatalf("WriteFile(SKILL.md): %v", err) + } + + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[77] = &models.Instance{ + ID: 77, + UserID: 45, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: &workspacePath, + } + installPath := "home/.openclaw/workspace/skills/paper-ranker" + service := &skillService{instanceRepo: instanceRepo} + + if err := service.removeLiteInstanceSkillDirectory(77, &models.InstanceSkill{SkillID: 12, InstallPath: &installPath}); err != nil { + t.Fatalf("removeLiteInstanceSkillDirectory() error = %v", err) + } + if _, err := os.Stat(target); !os.IsNotExist(err) { + t.Fatalf("expected skill directory to be removed, stat err = %v", err) + } +} +func TestRemoveLiteInstanceSkillDeletesHermesHomeSkill(t *testing.T) { + workspacePath := filepath.Join(t.TempDir(), "hermes", "user-45", "instance-90") + target := filepath.Join(workspacePath, "home", ".hermes", "skills", "paper-ranker") + if err := os.MkdirAll(target, 0750); err != nil { + t.Fatalf("MkdirAll(target): %v", err) + } + if err := os.WriteFile(filepath.Join(target, "SKILL.md"), []byte("# Paper Ranker\n"), 0640); err != nil { + t.Fatalf("WriteFile(SKILL.md): %v", err) + } + + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[90] = &models.Instance{ + ID: 90, + UserID: 45, + Type: RuntimeTypeHermes, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: &workspacePath, + } + installPath := "home/.hermes/skills/paper-ranker" + service := &skillService{instanceRepo: instanceRepo} + + if err := service.removeLiteInstanceSkillDirectory(90, &models.InstanceSkill{SkillID: 12, InstallPath: &installPath}); err != nil { + t.Fatalf("removeLiteInstanceSkillDirectory() error = %v", err) + } + if _, err := os.Stat(target); !os.IsNotExist(err) { + t.Fatalf("expected skill directory to be removed, stat err = %v", err) + } +} +func TestLiteRuntimePersistentAncestorsIncludeOpenClawHome(t *testing.T) { + workspacePath := filepath.Join(t.TempDir(), "openclaw", "user-1", "instance-89") + persistentRoot := filepath.Join(workspacePath, "home", ".openclaw") + + got := liteRuntimePersistentAncestors(workspacePath, persistentRoot) + want := []string{ + workspacePath, + filepath.Join(workspacePath, "home"), + filepath.Join(workspacePath, "home", ".openclaw"), + } + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("liteRuntimePersistentAncestors() = %#v, want %#v", got, want) + } +} + +type fakeObjectStorage map[string][]byte + +func (f fakeObjectStorage) PutObject(context.Context, string, []byte, string) error { + return nil +} + +func (f fakeObjectStorage) GetObject(_ context.Context, objectKey string) ([]byte, error) { + return f[objectKey], nil +} + +func assertFileEquals(t *testing.T, filePath string, want string) { + t.Helper() + + body, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("ReadFile(%q): %v", filePath, err) + } + if got := string(body); got != want { + t.Fatalf("ReadFile(%q) = %q, want %q", filePath, got, want) + } +} + +func buildTestZip(t *testing.T, files map[string][]byte) []byte { + t.Helper() + + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + keys := make([]string, 0, len(files)) + for key := range files { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + entry, err := writer.Create(key) + if err != nil { + t.Fatalf("Create(%q): %v", key, err) + } + if _, err := entry.Write(files[key]); err != nil { + t.Fatalf("Write(%q): %v", key, err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("Close(): %v", err) + } + return buffer.Bytes() +} + +func referenceSkillContentMD5(files map[string][]byte) string { + entryKinds := map[string]string{} + fileMap := map[string][]byte{} + for key, body := range files { + clean := path.Clean(key) + if clean == "." || clean == "" { + continue + } + fileMap[clean] = body + entryKinds[clean] = "file" + parts := splitTestPath(clean) + for i := 1; i < len(parts); i++ { + entryKinds[path.Join(parts[:i]...)] = "dir" + } + } + + keys := make([]string, 0, len(entryKinds)) + for key := range entryKinds { + keys = append(keys, key) + } + sort.Strings(keys) + + digest := md5.New() + for _, key := range keys { + _, _ = digest.Write([]byte(key)) + _, _ = digest.Write([]byte("\n")) + if entryKinds[key] == "dir" { + _, _ = digest.Write([]byte("dir\n")) + continue + } + _, _ = digest.Write([]byte("file\n")) + _, _ = digest.Write(fileMap[key]) + _, _ = digest.Write([]byte("\n")) + } + return hex.EncodeToString(digest.Sum(nil)) +} + +func splitTestPath(value string) []string { + result := []string{} + for _, part := range bytes.Split([]byte(value), []byte("/")) { + if len(part) > 0 { + result = append(result, string(part)) + } + } + return result +} diff --git a/backend/internal/services/sync_service.go b/backend/internal/services/sync_service.go new file mode 100644 index 0000000..9911e4d --- /dev/null +++ b/backend/internal/services/sync_service.go @@ -0,0 +1,378 @@ +package services + +import ( + "context" + "fmt" + "sync" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +// SyncService handles synchronization between database and K8s state +type SyncService struct { + instanceRepo repository.InstanceRepository + runtimeStatusService InstanceRuntimeStatusService + podService *k8s.PodService + deploymentService *k8s.InstanceDeploymentService + interval time.Duration + + mu sync.Mutex + running bool + stopChan chan struct{} +} + +// NewSyncService creates a new sync service +func NewSyncService(instanceRepo repository.InstanceRepository, runtimeStatusService InstanceRuntimeStatusService) *SyncService { + return &SyncService{ + instanceRepo: instanceRepo, + runtimeStatusService: runtimeStatusService, + podService: k8s.NewPodService(), + deploymentService: k8s.NewInstanceDeploymentService(), + interval: 5 * time.Second, // Sync every 5 seconds for more responsive status updates + } +} + +// Start starts the sync loop. It is safe to call repeatedly: a second call +// while already running is a no-op, and after Stop the service can be started +// again (used by leader-election re-acquisition). +func (s *SyncService) Start() { + s.mu.Lock() + defer s.mu.Unlock() + if s.running { + return + } + s.stopChan = make(chan struct{}) + s.running = true + fmt.Println("Starting K8s state sync service...") + go s.syncLoop(s.stopChan) +} + +// Stop stops the sync loop. It is idempotent: calling Stop when not running is +// a no-op, and it never closes the same channel twice. +func (s *SyncService) Stop() { + s.mu.Lock() + defer s.mu.Unlock() + if !s.running { + return + } + close(s.stopChan) + s.running = false +} + +// syncLoop runs the synchronization loop until its stop channel is closed. The +// channel is passed in (rather than read from the struct) so a restarted loop +// never races with a previously stopped one. +func (s *SyncService) syncLoop(stop <-chan struct{}) { + fmt.Printf("[SyncService] Starting sync loop with interval %v\n", s.interval) + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + + // Run immediately on start + fmt.Println("[SyncService] Running initial sync...") + s.syncAllInstances() + fmt.Println("[SyncService] Initial sync complete") + + for { + select { + case <-ticker.C: + fmt.Println("[SyncService] Tick - running scheduled sync...") + s.syncAllInstances() + case <-stop: + fmt.Println("[SyncService] Stopping K8s state sync service...") + return + } + } +} + +// syncAllInstances synchronizes the state of all instances +func (s *SyncService) syncAllInstances() { + ctx := context.Background() + + fmt.Println("[SyncService] Fetching all instances from database...") + // Get all running instances from database + instances, err := s.instanceRepo.GetAllRunning() + if err != nil { + fmt.Printf("[SyncService] Error getting running instances: %v\n", err) + return + } + + fmt.Printf("[SyncService] Found %d instances to sync\n", len(instances)) + + if len(instances) == 0 { + fmt.Println("[SyncService] No instances found, skipping sync") + return + } + + for i, instance := range instances { + fmt.Printf("[SyncService] Syncing instance %d/%d: ID=%d, Status=%s\n", + i+1, len(instances), instance.ID, instance.Status) + s.syncInstance(ctx, &instance) + } + + fmt.Println("[SyncService] Sync complete") +} + +// syncInstance synchronizes a single instance's state +func (s *SyncService) syncInstance(ctx context.Context, instance *models.Instance) { + if _, ok := v2RuntimeTypeForInstance(instance); ok { + s.updateInfraStatus(instance.ID, instance.Status) + return + } + if instanceUsesDesktopRuntime(instance) { + s.syncDeploymentInstance(ctx, instance) + return + } + + // Check if pod exists in K8s + pod, err := s.podService.GetPod(ctx, instance.UserID, instance.ID) + if err != nil { + deploymentExists, deploymentErr := s.podService.DeploymentExists(ctx, instance.UserID, instance.ID) + if deploymentErr != nil { + fmt.Printf("Instance %d: failed to check deployment while pod was missing: %v\n", instance.ID, deploymentErr) + } + if deploymentExists { + if instance.Status != "creating" { + fmt.Printf("Instance %d has deployment but no pod yet, updating status to creating\n", instance.ID) + instance.Status = "creating" + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = time.Now() + + if err := s.instanceRepo.Update(instance); err != nil { + fmt.Printf("Error updating instance %d status: %v\n", instance.ID, err) + } else { + s.updateInfraStatus(instance.ID, "creating") + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + } else { + s.updateInfraStatus(instance.ID, "creating") + } + return + } + + // Pod doesn't exist in K8s + if instance.Status == "running" || instance.Status == "creating" { + nextStatus := "stopped" + if instance.Status == "creating" { + nextStatus = "error" + } + + fmt.Printf("Instance %d marked as %s but pod not found in K8s, updating status to %s\n", + instance.ID, instance.Status, nextStatus) + instance.Status = nextStatus + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = time.Now() + + if err := s.instanceRepo.Update(instance); err != nil { + fmt.Printf("Error updating instance %d status: %v\n", instance.ID, err) + } else { + s.updateInfraStatus(instance.ID, nextStatus) + // Broadcast status update + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + } + return + } + + // Pod exists, update instance info + needsUpdate := false + + // Check pod status and update instance accordingly + desiredStatus := mapPodToInstanceStatus(pod) + if instance.Status != desiredStatus { + fmt.Printf("Instance %d: Pod status %s/ready=%v but instance status is %s, updating to %s\n", + instance.ID, pod.Status.Phase, isPodReady(pod), instance.Status, desiredStatus) + instance.Status = desiredStatus + needsUpdate = true + } + s.updateInfraStatus(instance.ID, desiredStatus) + + // Update Pod IP if changed + if pod.Status.PodIP != "" { + if instance.PodIP == nil || *instance.PodIP != pod.Status.PodIP { + instance.PodIP = &pod.Status.PodIP + needsUpdate = true + } + } + + // Update Pod name if changed + if instance.PodName == nil || *instance.PodName != pod.Name { + instance.PodName = &pod.Name + needsUpdate = true + } + + // Update namespace if changed + if instance.PodNamespace == nil || *instance.PodNamespace != pod.Namespace { + instance.PodNamespace = &pod.Namespace + needsUpdate = true + } + + if needsUpdate { + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + fmt.Printf("Error updating instance %d: %v\n", instance.ID, err) + } else { + fmt.Printf("Instance %d status synced: %s (Pod: %s, IP: %s)\n", + instance.ID, instance.Status, pod.Name, pod.Status.PodIP) + // Broadcast status update + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + } +} + +func (s *SyncService) syncDeploymentInstance(ctx context.Context, instance *models.Instance) { + deployment, err := s.deploymentService.GetDeployment(ctx, instance.UserID, instance.ID) + if err != nil { + if instance.Status == "running" || instance.Status == "creating" { + nextStatus := "stopped" + if instance.Status == "creating" { + nextStatus = "error" + } + fmt.Printf("Instance %d marked as %s but deployment not found in K8s, updating status to %s\n", + instance.ID, instance.Status, nextStatus) + instance.Status = nextStatus + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + fmt.Printf("Error updating instance %d status: %v\n", instance.ID, err) + } else { + s.updateInfraStatus(instance.ID, nextStatus) + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + } + return + } + + desiredStatus := mapDeploymentToInstanceStatus(deployment) + needsUpdate := false + if instance.Status != desiredStatus { + fmt.Printf("Instance %d: Deployment replicas=%d available=%d but instance status is %s, updating to %s\n", + instance.ID, desiredReplicas(deployment), deployment.Status.AvailableReplicas, instance.Status, desiredStatus) + instance.Status = desiredStatus + needsUpdate = true + } + s.updateInfraStatus(instance.ID, desiredStatus) + + if pod, podErr := s.deploymentService.GetActivePod(ctx, instance.UserID, instance.ID); podErr == nil && pod != nil { + if pod.Status.PodIP != "" && (instance.PodIP == nil || *instance.PodIP != pod.Status.PodIP) { + instance.PodIP = &pod.Status.PodIP + needsUpdate = true + } + if instance.PodName == nil || *instance.PodName != pod.Name { + instance.PodName = &pod.Name + needsUpdate = true + } + if instance.PodNamespace == nil || *instance.PodNamespace != pod.Namespace { + instance.PodNamespace = &pod.Namespace + needsUpdate = true + } + } else if desiredStatus == "stopped" { + if instance.PodName != nil || instance.PodNamespace != nil || instance.PodIP != nil { + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + needsUpdate = true + } + } + + if needsUpdate { + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + fmt.Printf("Error updating instance %d: %v\n", instance.ID, err) + } else { + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + } +} + +func (s *SyncService) updateInfraStatus(instanceID int, instanceStatus string) { + if s.runtimeStatusService == nil { + return + } + infraStatus := mapInstanceStatusToInfraStatus(instanceStatus) + if err := s.runtimeStatusService.UpsertInfraStatus(instanceID, infraStatus); err != nil { + fmt.Printf("Error updating runtime infra status for instance %d: %v\n", instanceID, err) + } +} + +func mapInstanceStatusToInfraStatus(instanceStatus string) string { + switch instanceStatus { + case "running": + return "ready" + case "stopped": + return "stopped" + case "error": + return "error" + case "creating": + return "creating" + default: + return "creating" + } +} + +func mapPodToInstanceStatus(pod *corev1.Pod) string { + if pod == nil { + return "error" + } + + switch pod.Status.Phase { + case corev1.PodRunning: + if isPodReady(pod) { + return "running" + } + return "creating" + case corev1.PodPending: + return "creating" + case corev1.PodSucceeded: + return "stopped" + case corev1.PodFailed, corev1.PodUnknown: + return "error" + default: + return "creating" + } +} + +func mapDeploymentToInstanceStatus(deployment *appsv1.Deployment) string { + if deployment == nil { + return "error" + } + if desiredReplicas(deployment) == 0 { + return "stopped" + } + if deployment.Status.AvailableReplicas > 0 { + return "running" + } + return "creating" +} + +func desiredReplicas(deployment *appsv1.Deployment) int32 { + if deployment == nil || deployment.Spec.Replicas == nil { + return 1 + } + return *deployment.Spec.Replicas +} + +func isPodReady(pod *corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} + +// ForceSync forces an immediate sync of all instances +func (s *SyncService) ForceSync() { + s.syncAllInstances() +} diff --git a/backend/internal/services/system_image_setting_service.go b/backend/internal/services/system_image_setting_service.go new file mode 100644 index 0000000..9af2f58 --- /dev/null +++ b/backend/internal/services/system_image_setting_service.go @@ -0,0 +1,403 @@ +package services + +import ( + "errors" + "fmt" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +var orderedSystemImageTypes = []string{ + "openclaw", + "ubuntu", + "webtop", + "hermes", + "debian", + "centos", + "custom", +} + +var supportedSystemImageTypes = map[string]string{ + "openclaw": "OpenClaw Pro", + "ubuntu": "Ubuntu Desktop", + "webtop": "Webtop Desktop", + "hermes": "Hermes Pro", + "debian": "Debian Desktop", + "centos": "CentOS Desktop", + "custom": "Custom Image", +} + +var defaultSystemImageSettings = map[string]string{ + "openclaw": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest", + "ubuntu": "lscr.io/linuxserver/webtop:ubuntu-xfce", + "webtop": "lscr.io/linuxserver/webtop:ubuntu-xfce", + "hermes": "ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest", + "debian": "docker.io/clawreef/debian-desktop:12", + "centos": "docker.io/clawreef/centos-desktop:9", + "custom": "registry.example.com/your-custom-image:latest", +} + +var defaultGatewaySystemImageSettings = map[string]string{ + "openclaw": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest", + "ubuntu": "ubuntu:22.04", + "webtop": "ubuntu:22.04", + "hermes": "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest", + "debian": "debian:12", + "centos": "quay.io/centos/centos:stream9", + "custom": "registry.example.com/your-custom-shell-image:latest", +} + +var defaultEnabledSystemImageTypes = map[string]bool{ + "openclaw": true, + "ubuntu": true, + "hermes": true, +} + +var defaultEnabledGatewaySystemImageTypes = map[string]bool{ + "openclaw": true, + "hermes": true, +} + +// RuntimeImageConfig is the runtime card selected for an instance type. +type RuntimeImageConfig struct { + Image string + RuntimeType string +} + +// RuntimeImageSettingsProvider exposes runtime image lookup for instance types. +type RuntimeImageSettingsProvider interface { + GetRuntimeImage(instanceType string) (RuntimeImageConfig, bool) + GetRuntimeImageForImage(instanceType, image string) (RuntimeImageConfig, bool) +} + +var runtimeImageSettingsProvider RuntimeImageSettingsProvider + +// SetRuntimeImageSettingsProvider configures the global runtime image provider used by runtime resolution. +func SetRuntimeImageSettingsProvider(provider RuntimeImageSettingsProvider) { + runtimeImageSettingsProvider = provider +} + +type SystemImageSettingService interface { + List() ([]models.SystemImageSetting, error) + Save(setting *models.SystemImageSetting) (*models.SystemImageSetting, error) + DeleteByID(id int) error + DisableType(instanceType string) error + GetRuntimeImage(instanceType string) (RuntimeImageConfig, bool) + GetRuntimeImageForImage(instanceType, image string) (RuntimeImageConfig, bool) +} + +type systemImageSettingService struct { + repo repository.SystemImageSettingRepository +} + +// NewSystemImageSettingService creates a new system image setting service. +func NewSystemImageSettingService(repo repository.SystemImageSettingRepository) SystemImageSettingService { + return &systemImageSettingService{repo: repo} +} + +func (s *systemImageSettingService) List() ([]models.SystemImageSetting, error) { + stored, err := s.repo.List() + if err != nil { + return nil, err + } + + byType := make(map[string][]models.SystemImageSetting, len(orderedSystemImageTypes)) + for _, item := range stored { + normalizedType := strings.TrimSpace(strings.ToLower(item.InstanceType)) + item.InstanceType = normalizedType + item.RuntimeType = normalizeSystemImageRuntimeType(item.RuntimeType) + if strings.TrimSpace(item.DisplayName) == "" { + item.DisplayName = displayNameForSystemImageType(normalizedType) + } + byType[normalizedType] = append(byType[normalizedType], item) + } + + settings := make([]models.SystemImageSetting, 0, len(stored)+len(orderedSystemImageTypes)) + for _, instanceType := range orderedSystemImageTypes { + items := byType[instanceType] + if len(items) == 0 { + settings = append(settings, defaultSystemImagePresetsForType(instanceType)...) + continue + } + + settings = append(settings, enabledSystemImageSettingsForType(instanceType, items)...) + delete(byType, instanceType) + } + + for instanceType, items := range byType { + settings = append(settings, enabledSystemImageSettingsForType(instanceType, items)...) + } + + return settings, nil +} + +func (s *systemImageSettingService) Save(setting *models.SystemImageSetting) (*models.SystemImageSetting, error) { + normalizedType := strings.TrimSpace(strings.ToLower(setting.InstanceType)) + if _, ok := supportedSystemImageTypes[normalizedType]; !ok { + return nil, errors.New("unsupported instance type") + } + + runtimeType, err := validateSystemImageRuntimeType(setting.RuntimeType) + if err != nil { + return nil, err + } + + image := strings.TrimSpace(setting.Image) + if image == "" { + return nil, errors.New("image is required") + } + + setting.InstanceType = normalizedType + setting.RuntimeType = runtimeType + setting.Image = image + setting.DisplayName = strings.TrimSpace(setting.DisplayName) + if setting.DisplayName == "" { + setting.DisplayName = displayNameForSystemImagePreset(normalizedType, runtimeType) + } + setting.IsEnabled = true + + if err := s.repo.Save(setting); err != nil { + return nil, err + } + + return setting, nil +} + +func (s *systemImageSettingService) DeleteByID(id int) error { + if id <= 0 { + return errors.New("invalid image setting id") + } + + existing, err := s.repo.GetByID(id) + if err != nil { + return err + } + if existing == nil { + return nil + } + + if err := s.repo.DeleteByID(id); err != nil { + return err + } + + remaining, err := s.repo.ListByInstanceType(existing.InstanceType) + if err != nil { + return err + } + if len(remaining) > 0 || !isSupportedSystemImageType(existing.InstanceType) { + return nil + } + + return s.disableTypeWithFallback(existing.InstanceType) +} + +func (s *systemImageSettingService) DisableType(instanceType string) error { + normalizedType := strings.TrimSpace(strings.ToLower(instanceType)) + if !isSupportedSystemImageType(normalizedType) { + return errors.New("unsupported instance type") + } + + return s.disableTypeWithFallback(normalizedType) +} + +func (s *systemImageSettingService) disableTypeWithFallback(instanceType string) error { + if err := s.repo.DeleteByInstanceType(instanceType); err != nil { + return err + } + + return s.repo.Save(&models.SystemImageSetting{ + InstanceType: instanceType, + RuntimeType: "desktop", + DisplayName: displayNameForSystemImagePreset(instanceType, "desktop"), + Image: defaultSystemImageSettings[instanceType], + IsEnabled: false, + }) +} + +func (s *systemImageSettingService) GetRuntimeImage(instanceType string) (RuntimeImageConfig, bool) { + normalizedType := strings.TrimSpace(strings.ToLower(instanceType)) + items, err := s.repo.ListByInstanceType(normalizedType) + if err != nil { + return RuntimeImageConfig{}, false + } + + if len(items) == 0 { + for _, item := range defaultSystemImagePresetsForType(normalizedType) { + image := strings.TrimSpace(item.Image) + if item.IsEnabled && image != "" { + return RuntimeImageConfig{Image: image, RuntimeType: item.RuntimeType}, true + } + } + return RuntimeImageConfig{}, false + } + + for _, item := range enabledSystemImageSettingsForType(normalizedType, items) { + image := strings.TrimSpace(item.Image) + if image != "" { + return RuntimeImageConfig{ + Image: image, + RuntimeType: normalizeSystemImageRuntimeType(item.RuntimeType), + }, true + } + } + + return RuntimeImageConfig{}, false +} + +func (s *systemImageSettingService) GetRuntimeImageForImage(instanceType, image string) (RuntimeImageConfig, bool) { + normalizedType := strings.TrimSpace(strings.ToLower(instanceType)) + normalizedImage := strings.TrimSpace(image) + if normalizedType == "" || normalizedImage == "" { + return RuntimeImageConfig{}, false + } + + items, err := s.repo.ListByInstanceType(normalizedType) + if err != nil { + return RuntimeImageConfig{}, false + } + + if len(items) == 0 { + for _, item := range defaultSystemImagePresetsForType(normalizedType) { + defaultImage := strings.TrimSpace(item.Image) + if item.IsEnabled && defaultImage == normalizedImage { + return RuntimeImageConfig{Image: defaultImage, RuntimeType: item.RuntimeType}, true + } + } + return RuntimeImageConfig{}, false + } + + for _, item := range enabledSystemImageSettingsForType(normalizedType, items) { + if strings.TrimSpace(item.Image) == normalizedImage { + return RuntimeImageConfig{ + Image: normalizedImage, + RuntimeType: normalizeSystemImageRuntimeType(item.RuntimeType), + }, true + } + } + + return RuntimeImageConfig{}, false +} + +func runtimeImageOverride(instanceType string) (RuntimeImageConfig, bool) { + if runtimeImageSettingsProvider == nil { + return RuntimeImageConfig{}, false + } + return runtimeImageSettingsProvider.GetRuntimeImage(instanceType) +} + +func runtimeImageOverrideForImage(instanceType, image string) (RuntimeImageConfig, bool) { + if runtimeImageSettingsProvider == nil { + return RuntimeImageConfig{}, false + } + return runtimeImageSettingsProvider.GetRuntimeImageForImage(instanceType, image) +} + +func displayNameForSystemImageType(instanceType string) string { + if name, ok := supportedSystemImageTypes[instanceType]; ok { + return name + } + return fmt.Sprintf("%s Image", instanceType) +} + +func displayNameForSystemImagePreset(instanceType, runtimeType string) string { + normalizedRuntimeType := normalizeSystemImageRuntimeType(runtimeType) + if instanceType == "openclaw" { + if normalizedRuntimeType == "gateway" { + return "OpenClaw Lite" + } + return "OpenClaw Pro" + } + if instanceType == "hermes" { + if normalizedRuntimeType == "gateway" { + return "Hermes Lite" + } + return "Hermes Pro" + } + return displayNameForSystemImageType(instanceType) +} + +func defaultSystemImagePresetsForType(instanceType string) []models.SystemImageSetting { + settings := []models.SystemImageSetting{{ + InstanceType: instanceType, + RuntimeType: "desktop", + DisplayName: displayNameForSystemImagePreset(instanceType, "desktop"), + Image: defaultSystemImageSettings[instanceType], + IsEnabled: defaultEnabledSystemImageTypes[instanceType], + }} + + if image := strings.TrimSpace(defaultGatewaySystemImageSettings[instanceType]); image != "" { + if defaultEnabledGatewaySystemImageTypes[instanceType] { + settings = append(settings, models.SystemImageSetting{ + InstanceType: instanceType, + RuntimeType: "gateway", + DisplayName: displayNameForSystemImagePreset(instanceType, "gateway"), + Image: image, + IsEnabled: true, + }) + } + } + + return settings +} + +func enabledSystemImageSettingsForType(instanceType string, stored []models.SystemImageSetting) []models.SystemImageSetting { + if len(stored) == 0 { + return defaultSystemImagePresetsForType(instanceType) + } + + hasEnabled := false + runtimeTypesWithRows := map[string]bool{} + result := make([]models.SystemImageSetting, 0, len(stored)+2) + for _, item := range stored { + item.RuntimeType = normalizeSystemImageRuntimeType(item.RuntimeType) + runtimeTypesWithRows[item.RuntimeType] = true + if strings.TrimSpace(item.DisplayName) == "" { + item.DisplayName = displayNameForSystemImagePreset(instanceType, item.RuntimeType) + } + if item.IsEnabled { + hasEnabled = true + result = append(result, item) + } + } + + if !hasEnabled { + return result + } + + for _, preset := range defaultSystemImagePresetsForType(instanceType) { + if runtimeTypesWithRows[preset.RuntimeType] || !preset.IsEnabled || strings.TrimSpace(preset.Image) == "" { + continue + } + result = append(result, preset) + } + return result +} + +func isSupportedSystemImageType(instanceType string) bool { + _, ok := supportedSystemImageTypes[instanceType] + return ok +} + +func normalizeSystemImageRuntimeType(runtimeType string) string { + normalized := strings.TrimSpace(strings.ToLower(runtimeType)) + if normalized == "gateway" || normalized == "shell" { + return "gateway" + } + return "desktop" +} + +func validateSystemImageRuntimeType(runtimeType string) (string, error) { + normalized := strings.TrimSpace(strings.ToLower(runtimeType)) + if normalized == "" { + return "desktop", nil + } + if normalized == "shell" { + return "gateway", nil + } + if normalized != "desktop" && normalized != "gateway" { + return "", errors.New("unsupported runtime type") + } + return normalized, nil +} diff --git a/backend/internal/services/system_image_setting_service_test.go b/backend/internal/services/system_image_setting_service_test.go new file mode 100644 index 0000000..60abab6 --- /dev/null +++ b/backend/internal/services/system_image_setting_service_test.go @@ -0,0 +1,377 @@ +package services + +import ( + "testing" + + "clawreef/internal/models" +) + +type stubSystemImageSettingRepository struct { + items []models.SystemImageSetting + nextID int +} + +func (r *stubSystemImageSettingRepository) List() ([]models.SystemImageSetting, error) { + out := make([]models.SystemImageSetting, len(r.items)) + copy(out, r.items) + return out, nil +} + +func (r *stubSystemImageSettingRepository) GetByID(id int) (*models.SystemImageSetting, error) { + for _, item := range r.items { + if item.ID == id { + copyItem := item + return ©Item, nil + } + } + return nil, nil +} + +func (r *stubSystemImageSettingRepository) ListByInstanceType(instanceType string) ([]models.SystemImageSetting, error) { + var out []models.SystemImageSetting + for _, item := range r.items { + if item.InstanceType == instanceType { + out = append(out, item) + } + } + return out, nil +} + +func (r *stubSystemImageSettingRepository) Save(setting *models.SystemImageSetting) error { + if setting.ID > 0 { + for i := range r.items { + if r.items[i].ID == setting.ID { + r.items[i] = *setting + return nil + } + } + } + + r.nextID++ + copyItem := *setting + copyItem.ID = r.nextID + *setting = copyItem + r.items = append(r.items, copyItem) + return nil +} + +func (r *stubSystemImageSettingRepository) DeleteByID(id int) error { + filtered := r.items[:0] + for _, item := range r.items { + if item.ID != id { + filtered = append(filtered, item) + } + } + r.items = filtered + return nil +} + +func (r *stubSystemImageSettingRepository) DeleteByInstanceType(instanceType string) error { + filtered := r.items[:0] + for _, item := range r.items { + if item.InstanceType != instanceType { + filtered = append(filtered, item) + } + } + r.items = filtered + return nil +} + +func TestSystemImageSettingServiceListAllowsMultipleImagesPerType(t *testing.T) { + repo := &stubSystemImageSettingRepository{ + items: []models.SystemImageSetting{ + {ID: 1, InstanceType: "openclaw", DisplayName: "OpenClaw Stable", Image: "registry/openclaw:stable", IsEnabled: true}, + {ID: 2, InstanceType: "openclaw", DisplayName: "OpenClaw Canary", Image: "registry/openclaw:canary", IsEnabled: true}, + {ID: 3, InstanceType: "ubuntu", DisplayName: "Ubuntu Desktop", Image: "lscr.io/linuxserver/webtop:ubuntu-xfce", IsEnabled: false}, + }, + nextID: 3, + } + + service := NewSystemImageSettingService(repo) + items, err := service.List() + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + openClawDesktopCount := 0 + openClawGatewayCount := 0 + for _, item := range items { + if item.InstanceType != "openclaw" { + continue + } + switch item.RuntimeType { + case "desktop": + openClawDesktopCount++ + case "gateway": + openClawGatewayCount++ + } + } + + if openClawDesktopCount != 2 { + t.Fatalf("expected 2 openclaw desktop images, got %d", openClawDesktopCount) + } + if openClawGatewayCount != 1 { + t.Fatalf("expected openclaw gateway default to be backfilled, got %d", openClawGatewayCount) + } +} + +func TestSystemImageSettingServiceDeleteByIDCreatesDisabledFallback(t *testing.T) { + repo := &stubSystemImageSettingRepository{ + items: []models.SystemImageSetting{ + {ID: 1, InstanceType: "openclaw", DisplayName: "OpenClaw Stable", Image: "registry/openclaw:stable", IsEnabled: true}, + }, + nextID: 1, + } + + service := NewSystemImageSettingService(repo) + if err := service.DeleteByID(1); err != nil { + t.Fatalf("DeleteByID returned error: %v", err) + } + + items, err := repo.ListByInstanceType("openclaw") + if err != nil { + t.Fatalf("ListByInstanceType returned error: %v", err) + } + if len(items) != 1 { + t.Fatalf("expected 1 fallback row after deleting last image, got %d", len(items)) + } + if items[0].IsEnabled { + t.Fatalf("expected fallback row to be disabled") + } + + selection, ok := service.GetRuntimeImage("openclaw") + if ok || selection.Image != "" { + t.Fatalf("expected runtime image lookup to be disabled after fallback, got %q %v", selection.Image, ok) + } +} + +func TestSystemImageSettingServiceGetRuntimeImageFallsBackToDefaultWhenNoRowsExist(t *testing.T) { + service := NewSystemImageSettingService(&stubSystemImageSettingRepository{}) + + selection, ok := service.GetRuntimeImage("openclaw") + if !ok { + t.Fatalf("expected default openclaw runtime image to be available") + } + if selection.Image != defaultSystemImageSettings["openclaw"] { + t.Fatalf("expected default openclaw image %q, got %q", defaultSystemImageSettings["openclaw"], selection.Image) + } + if selection.RuntimeType != "desktop" { + t.Fatalf("expected default openclaw runtime type desktop, got %q", selection.RuntimeType) + } +} + +func TestSystemImageSettingServiceListIncludesOpenClawGatewayDefault(t *testing.T) { + service := NewSystemImageSettingService(&stubSystemImageSettingRepository{}) + + items, err := service.List() + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + found := false + for _, item := range items { + if item.InstanceType == "openclaw" && item.RuntimeType == "gateway" { + found = true + if item.Image != defaultGatewaySystemImageSettings["openclaw"] { + t.Fatalf("expected default openclaw gateway image %q, got %q", defaultGatewaySystemImageSettings["openclaw"], item.Image) + } + if !item.IsEnabled { + t.Fatalf("expected default openclaw gateway image to be enabled") + } + } + } + + if !found { + t.Fatalf("expected default openclaw gateway runtime image") + } +} + +func TestSystemImageSettingServiceListIncludesHermesLiteDefault(t *testing.T) { + service := NewSystemImageSettingService(&stubSystemImageSettingRepository{}) + + items, err := service.List() + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + found := false + for _, item := range items { + if item.InstanceType == "hermes" && item.RuntimeType == "gateway" { + found = true + if item.Image != defaultGatewaySystemImageSettings["hermes"] { + t.Fatalf("expected default hermes lite image %q, got %q", defaultGatewaySystemImageSettings["hermes"], item.Image) + } + if item.DisplayName != "Hermes Lite" { + t.Fatalf("expected Hermes Lite display name, got %q", item.DisplayName) + } + if !item.IsEnabled { + t.Fatalf("expected default hermes lite image to be enabled") + } + } + } + + if !found { + t.Fatalf("expected default hermes lite runtime image") + } +} + +func TestSystemImageSettingServiceListBackfillsHermesProDefaultWhenLiteStored(t *testing.T) { + repo := &stubSystemImageSettingRepository{ + items: []models.SystemImageSetting{ + { + ID: 1, + InstanceType: "hermes", + RuntimeType: "gateway", + DisplayName: "Hermes Lite", + Image: "registry/hermes-lite:stored", + IsEnabled: true, + }, + }, + nextID: 1, + } + service := NewSystemImageSettingService(repo) + + items, err := service.List() + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + foundLite := false + foundPro := false + for _, item := range items { + if item.InstanceType != "hermes" { + continue + } + if item.RuntimeType == "gateway" && item.Image == "registry/hermes-lite:stored" { + foundLite = true + } + if item.RuntimeType == "desktop" && item.Image == defaultSystemImageSettings["hermes"] { + foundPro = true + if item.DisplayName != "Hermes Pro" { + t.Fatalf("expected Hermes Pro display name, got %q", item.DisplayName) + } + } + } + + if !foundLite { + t.Fatalf("expected stored hermes lite image to remain available") + } + if !foundPro { + t.Fatalf("expected missing hermes pro default image to be backfilled") + } +} + +func TestSystemImageSettingServiceGetRuntimeImageForImageFallsBackToOpenClawGatewayDefault(t *testing.T) { + service := NewSystemImageSettingService(&stubSystemImageSettingRepository{}) + + selection, ok := service.GetRuntimeImageForImage("openclaw", defaultGatewaySystemImageSettings["openclaw"]) + if !ok { + t.Fatalf("expected default openclaw gateway runtime image to resolve") + } + if selection.RuntimeType != "gateway" { + t.Fatalf("expected runtime type gateway, got %q", selection.RuntimeType) + } + if selection.Image != defaultGatewaySystemImageSettings["openclaw"] { + t.Fatalf("expected gateway image %q, got %q", defaultGatewaySystemImageSettings["openclaw"], selection.Image) + } +} + +func TestSystemImageSettingServiceGetRuntimeImageForImageBackfillsHermesProDefaultWhenLiteStored(t *testing.T) { + repo := &stubSystemImageSettingRepository{ + items: []models.SystemImageSetting{ + { + ID: 1, + InstanceType: "hermes", + RuntimeType: "gateway", + DisplayName: "Hermes Lite", + Image: "registry/hermes-lite:stored", + IsEnabled: true, + }, + }, + nextID: 1, + } + service := NewSystemImageSettingService(repo) + + selection, ok := service.GetRuntimeImageForImage("hermes", defaultSystemImageSettings["hermes"]) + if !ok { + t.Fatalf("expected default hermes pro image to resolve") + } + if selection.RuntimeType != "desktop" { + t.Fatalf("expected runtime type desktop, got %q", selection.RuntimeType) + } + if selection.Image != defaultSystemImageSettings["hermes"] { + t.Fatalf("expected hermes pro image %q, got %q", defaultSystemImageSettings["hermes"], selection.Image) + } +} + +func TestSystemImageSettingServiceGetRuntimeImageForImageUsesCardRuntimeType(t *testing.T) { + repo := &stubSystemImageSettingRepository{ + items: []models.SystemImageSetting{ + {ID: 1, InstanceType: "openclaw", RuntimeType: "desktop", DisplayName: "OpenClaw Desktop", Image: "registry/openclaw-desktop:latest", IsEnabled: true}, + {ID: 2, InstanceType: "openclaw", RuntimeType: "gateway", DisplayName: "OpenClaw Lite", Image: "registry/openclaw-lite:latest", IsEnabled: true}, + }, + nextID: 2, + } + + service := NewSystemImageSettingService(repo) + selection, ok := service.GetRuntimeImageForImage("openclaw", "registry/openclaw-lite:latest") + if !ok { + t.Fatalf("expected selected gateway runtime image to resolve") + } + if selection.RuntimeType != "gateway" { + t.Fatalf("expected runtime type gateway, got %q", selection.RuntimeType) + } + if selection.Image != "registry/openclaw-lite:latest" { + t.Fatalf("expected gateway image to resolve, got %q", selection.Image) + } +} + +func TestSystemImageSettingServiceSaveAcceptsGatewayRuntimeType(t *testing.T) { + repo := &stubSystemImageSettingRepository{} + service := NewSystemImageSettingService(repo) + + saved, err := service.Save(&models.SystemImageSetting{ + InstanceType: "OpenClaw", + RuntimeType: "gateway", + DisplayName: "OpenClaw Lite", + Image: "registry/openclaw-lite:v2", + }) + if err != nil { + t.Fatalf("Save returned error: %v", err) + } + + if saved.InstanceType != "openclaw" { + t.Fatalf("expected normalized instance type openclaw, got %q", saved.InstanceType) + } + if saved.RuntimeType != "gateway" { + t.Fatalf("expected gateway runtime type, got %q", saved.RuntimeType) + } +} + +func TestSystemImageSettingServiceNormalizesLegacyShellRuntimeTypeToGateway(t *testing.T) { + repo := &stubSystemImageSettingRepository{ + items: []models.SystemImageSetting{ + {ID: 1, InstanceType: "hermes", RuntimeType: "shell", DisplayName: "Hermes Lite", Image: "registry/hermes-lite:legacy", IsEnabled: true}, + }, + nextID: 1, + } + service := NewSystemImageSettingService(repo) + + items, err := service.List() + if err != nil { + t.Fatalf("List returned error: %v", err) + } + for _, item := range items { + if item.ID == 1 && item.RuntimeType != "gateway" { + t.Fatalf("expected legacy shell row to be exposed as gateway, got %q", item.RuntimeType) + } + } + + selection, ok := service.GetRuntimeImageForImage("hermes", "registry/hermes-lite:legacy") + if !ok { + t.Fatalf("expected legacy shell image to resolve") + } + if selection.RuntimeType != "gateway" { + t.Fatalf("expected legacy shell image to resolve as gateway, got %q", selection.RuntimeType) + } +} diff --git a/backend/internal/services/team_redis.go b/backend/internal/services/team_redis.go new file mode 100644 index 0000000..928de5b --- /dev/null +++ b/backend/internal/services/team_redis.go @@ -0,0 +1,331 @@ +package services + +import ( + "bufio" + "context" + "crypto/tls" + "fmt" + "io" + "net" + "net/url" + "strconv" + "strings" + "time" +) + +type redisStreamMessage struct { + ID string + Fields map[string]string +} + +type redisBus struct { + address string + password string + db int + useTLS bool +} + +func newRedisBus(rawURL string) (*redisBus, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return nil, fmt.Errorf("redis url is invalid: %w", err) + } + if parsed.Scheme != "redis" && parsed.Scheme != "rediss" { + return nil, fmt.Errorf("redis url scheme must be redis or rediss") + } + address := parsed.Host + if !strings.Contains(address, ":") { + address += ":6379" + } + password, _ := parsed.User.Password() + dbIndex := 0 + if path := strings.Trim(parsed.Path, "/"); path != "" { + parsedDB, err := strconv.Atoi(path) + if err != nil { + return nil, fmt.Errorf("redis db index is invalid: %w", err) + } + dbIndex = parsedDB + } + return &redisBus{ + address: address, + password: password, + db: dbIndex, + useTLS: parsed.Scheme == "rediss", + }, nil +} + +func (b *redisBus) XAdd(ctx context.Context, key string, fields map[string]string) (string, error) { + args := []string{"XADD", key, "*"} + for field, value := range fields { + args = append(args, field, value) + } + reply, err := b.do(ctx, args...) + if err != nil { + return "", err + } + id, ok := reply.(string) + if !ok || strings.TrimSpace(id) == "" { + return "", fmt.Errorf("unexpected redis XADD response") + } + return id, nil +} + +func (b *redisBus) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) { + if ttl <= 0 { + ttl = time.Second + } + reply, err := b.do(ctx, "SET", key, value, "NX", "PX", fmt.Sprintf("%d", ttl.Milliseconds())) + if err != nil { + return false, err + } + if reply == nil { + return false, nil + } + return reply == "OK", nil +} + +func (b *redisBus) Set(ctx context.Context, key, value string, ttl time.Duration) error { + args := []string{"SET", key, value} + if ttl > 0 { + args = append(args, "PX", fmt.Sprintf("%d", ttl.Milliseconds())) + } + _, err := b.do(ctx, args...) + return err +} + +func (b *redisBus) Del(ctx context.Context, key string) error { + _, err := b.do(ctx, "DEL", key) + return err +} + +func (b *redisBus) XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) { + blockMillis := int(block / time.Millisecond) + if blockMillis <= 0 { + blockMillis = 5000 + } + reply, err := b.do(ctx, "XREAD", "BLOCK", strconv.Itoa(blockMillis), "STREAMS", key, lastID) + if err != nil { + return nil, err + } + if reply == nil { + return nil, nil + } + root, ok := reply.([]interface{}) + if !ok { + return nil, fmt.Errorf("unexpected redis XREAD response") + } + var messages []redisStreamMessage + for _, streamRaw := range root { + stream, ok := streamRaw.([]interface{}) + if !ok || len(stream) != 2 { + continue + } + messageList, ok := stream[1].([]interface{}) + if !ok { + continue + } + for _, messageRaw := range messageList { + message, ok := messageRaw.([]interface{}) + if !ok || len(message) != 2 { + continue + } + id, ok := message[0].(string) + if !ok { + continue + } + fieldList, ok := message[1].([]interface{}) + if !ok { + continue + } + fields := map[string]string{} + for i := 0; i+1 < len(fieldList); i += 2 { + field, okField := fieldList[i].(string) + value, okValue := fieldList[i+1].(string) + if okField && okValue { + fields[field] = value + } + } + messages = append(messages, redisStreamMessage{ID: id, Fields: fields}) + } + } + return messages, nil +} + +func (b *redisBus) XRevRange(ctx context.Context, key string, count int) ([]redisStreamMessage, error) { + if count <= 0 { + count = 100 + } + reply, err := b.do(ctx, "XREVRANGE", key, "+", "-", "COUNT", strconv.Itoa(count)) + if err != nil { + return nil, err + } + root, ok := reply.([]interface{}) + if !ok { + return nil, fmt.Errorf("unexpected redis XREVRANGE response") + } + return parseRedisStreamEntries(root), nil +} + +func parseRedisStreamEntries(entries []interface{}) []redisStreamMessage { + messages := make([]redisStreamMessage, 0, len(entries)) + for _, messageRaw := range entries { + message, ok := messageRaw.([]interface{}) + if !ok || len(message) != 2 { + continue + } + id, ok := message[0].(string) + if !ok { + continue + } + fieldList, ok := message[1].([]interface{}) + if !ok { + continue + } + fields := map[string]string{} + for i := 0; i+1 < len(fieldList); i += 2 { + field, okField := fieldList[i].(string) + value, okValue := fieldList[i+1].(string) + if okField && okValue { + fields[field] = value + } + } + messages = append(messages, redisStreamMessage{ID: id, Fields: fields}) + } + return messages +} + +func (b *redisBus) do(ctx context.Context, args ...string) (interface{}, error) { + conn, reader, err := b.connect(ctx) + if err != nil { + return nil, err + } + defer conn.Close() + + if err := writeRedisCommand(conn, args...); err != nil { + return nil, err + } + return readRedisReply(reader) +} + +func (b *redisBus) connect(ctx context.Context) (net.Conn, *bufio.Reader, error) { + dialer := &net.Dialer{Timeout: 5 * time.Second} + var conn net.Conn + var err error + if b.useTLS { + conn, err = tls.DialWithDialer(dialer, "tcp", b.address, &tls.Config{MinVersion: tls.VersionTLS12}) + } else { + conn, err = dialer.DialContext(ctx, "tcp", b.address) + } + if err != nil { + return nil, nil, fmt.Errorf("failed to connect redis: %w", err) + } + reader := bufio.NewReader(conn) + if b.password != "" { + if err := writeRedisCommand(conn, "AUTH", b.password); err != nil { + _ = conn.Close() + return nil, nil, err + } + if _, err := readRedisReply(reader); err != nil { + _ = conn.Close() + return nil, nil, fmt.Errorf("redis auth failed: %w", err) + } + } + if b.db > 0 { + if err := writeRedisCommand(conn, "SELECT", strconv.Itoa(b.db)); err != nil { + _ = conn.Close() + return nil, nil, err + } + if _, err := readRedisReply(reader); err != nil { + _ = conn.Close() + return nil, nil, fmt.Errorf("redis select db failed: %w", err) + } + } + return conn, reader, nil +} + +func writeRedisCommand(conn net.Conn, args ...string) error { + var builder strings.Builder + builder.WriteString("*") + builder.WriteString(strconv.Itoa(len(args))) + builder.WriteString("\r\n") + for _, arg := range args { + builder.WriteString("$") + builder.WriteString(strconv.Itoa(len(arg))) + builder.WriteString("\r\n") + builder.WriteString(arg) + builder.WriteString("\r\n") + } + if _, err := conn.Write([]byte(builder.String())); err != nil { + return fmt.Errorf("failed to write redis command: %w", err) + } + return nil +} + +func readRedisReply(reader *bufio.Reader) (interface{}, error) { + prefix, err := reader.ReadByte() + if err != nil { + return nil, err + } + switch prefix { + case '+': + line, err := readRedisLine(reader) + return line, err + case '-': + line, _ := readRedisLine(reader) + return nil, fmt.Errorf("redis error: %s", line) + case ':': + line, err := readRedisLine(reader) + if err != nil { + return nil, err + } + return strconv.ParseInt(line, 10, 64) + case '$': + line, err := readRedisLine(reader) + if err != nil { + return nil, err + } + size, err := strconv.Atoi(line) + if err != nil { + return nil, err + } + if size < 0 { + return nil, nil + } + buf := make([]byte, size+2) + if _, err := io.ReadFull(reader, buf); err != nil { + return nil, err + } + return string(buf[:size]), nil + case '*': + line, err := readRedisLine(reader) + if err != nil { + return nil, err + } + count, err := strconv.Atoi(line) + if err != nil { + return nil, err + } + if count < 0 { + return nil, nil + } + items := make([]interface{}, 0, count) + for i := 0; i < count; i++ { + item, err := readRedisReply(reader) + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, nil + default: + return nil, fmt.Errorf("unexpected redis reply prefix %q", prefix) + } +} + +func readRedisLine(reader *bufio.Reader) (string, error) { + line, err := reader.ReadString('\n') + if err != nil { + return "", err + } + return strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r"), nil +} diff --git a/backend/internal/services/team_service.go b/backend/internal/services/team_service.go new file mode 100644 index 0000000..95f826e --- /dev/null +++ b/backend/internal/services/team_service.go @@ -0,0 +1,9819 @@ +package services + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "os" + posixpath "path" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" +) + +const ( + teamSharedMountPath = "/team" + teamConfigFileName = "team.json" + teamAgentsFileName = "AGENTS.md" + teamSoulFileName = "SOUL.md" + teamConfigMountDirPath = "/etc/clawmanager/team" + teamConfigMountPath = teamConfigMountDirPath + "/" + teamConfigFileName + teamHermesSoulMountPath = "/config/.hermes/SOUL.md" + teamSharedUID = 1000 + teamSharedGID = 1000 + teamSharedUmask = "0002" + teamRedisURLSecretKey = "CLAWMANAGER_TEAM_REDIS_URL" + teamTokenSecretKey = "CLAWMANAGER_TEAM_TOKEN" + + defaultTeamTaskStaleTimeout = 30 * time.Minute + teamTaskStaleSweepInterval = 30 * time.Second + teamConsumerScanInterval = 10 * time.Second + teamAssignmentMonitorEvery = 3 * time.Minute + teamEventOutboxBatchSize = 100 + + initialLeaderTaskIntent = "team_bootstrap_introduction" + teamTaskCompletionTool = "team_complete_task" + teamTaskReplyTarget = "clawmanager" +) + +const ( + teamCommunicationModeLeaderMediated = "leader_mediated" + teamCommunicationModePeerAssisted = "peer_assisted" + teamCommunicationModeFullMesh = "full_mesh" +) + +const ( + teamWorkflowStatePlanning = "planning" + teamWorkflowStateExecuting = "executing" + teamWorkflowStateAwaitingPhaseResults = "awaiting_phase_results" + teamWorkflowStateAwaitingLeaderDecision = "awaiting_leader_decision" + teamWorkflowStateSynthesizing = "synthesizing" + teamWorkflowStateCompletionPending = "completion_pending" + teamWorkflowStateCompleted = "completed" + teamWorkflowStateFailed = "failed" + + teamPhaseStatusPlanned = "planned" + teamPhaseStatusActive = "active" + teamPhaseStatusAwaitingResults = "awaiting_results" + teamPhaseStatusAwaitingLeaderDecision = "awaiting_leader_decision" + teamPhaseStatusCompleted = "completed" + teamPhaseStatusCancelled = "cancelled" + teamPhaseStatusSuperseded = "superseded" + + teamCompletionDecisionAccepted = "accepted" + teamCompletionDecisionDeferred = "deferred" + teamCompletionDecisionNeedsConfirmation = "needs_confirmation" + teamCompletionDecisionRejected = "rejected" +) + +var ( + teamMemberKeyPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + teamMemberInstanceNameInvalidChars = regexp.MustCompile(`[^a-z0-9-]+`) + teamMemberInstanceNameRepeatedDashs = regexp.MustCompile(`-+`) +) + +type TeamService interface { + StartBackground(ctx context.Context) + StopBackground() + CreateTeam(userID int, req CreateTeamRequest) (*TeamDetailsPayload, error) + ListTeams(userID, offset, limit int) (*TeamListPayload, error) + GetTeam(userID, teamID int) (*TeamDetailsPayload, error) + ListTeamTasks(userID, teamID, beforeID, limit int) (*TeamTasksHistoryPayload, error) + ListTeamEvents(userID, teamID, beforeID, limit int) (*TeamEventsHistoryPayload, error) + ListWorkspaceFiles(ctx context.Context, userID, teamID int, relPath string) (*TeamWorkspaceListPayload, error) + PreviewWorkspaceFile(ctx context.Context, userID, teamID int, relPath string) (*TeamWorkspacePreviewPayload, error) + DownloadWorkspaceFile(ctx context.Context, userID, teamID int, relPath string) (*TeamWorkspaceDownloadPayload, error) + CreateWorkspaceFolder(ctx context.Context, userID, teamID int, req TeamWorkspaceFolderRequest) error + RenameWorkspaceEntry(ctx context.Context, userID, teamID int, req TeamWorkspaceRenameRequest) error + DeleteWorkspaceEntry(ctx context.Context, userID, teamID int, relPath string) error + UploadWorkspaceFiles(ctx context.Context, userID, teamID int, targetPath string, files []*multipart.FileHeader, relativePaths []string) error + DispatchTask(userID, teamID int, req DispatchTeamTaskRequest) (*TeamTaskPayload, error) + DeleteTeam(userID, teamID int) error + DeleteMember(userID, teamID int, memberID string) error +} + +type CreateTeamRequest struct { + Name string `json:"name"` + Description *string `json:"description,omitempty"` + CommunicationMode string `json:"communication_mode,omitempty"` + RedisURL string `json:"redis_url,omitempty"` + SharedStorageGB int `json:"shared_storage_gb,omitempty"` + StorageClass string `json:"storage_class,omitempty"` + Members []CreateTeamMemberRequest `json:"members"` +} + +type CreateTeamMemberRequest struct { + MemberID string `json:"member_id,omitempty"` + Name string `json:"name,omitempty"` + Role string `json:"role"` + Mode string `json:"mode,omitempty"` + InstanceMode string `json:"instance_mode,omitempty"` + RuntimeType string `json:"runtime_type,omitempty"` + Description *string `json:"description,omitempty"` + CPUCores float64 `json:"cpu_cores,omitempty"` + MemoryGB int `json:"memory_gb,omitempty"` + DiskGB int `json:"disk_gb,omitempty"` + GPUEnabled bool `json:"gpu_enabled,omitempty"` + GPUCount int `json:"gpu_count,omitempty"` + ImageRegistry *string `json:"image_registry,omitempty"` + ImageTag *string `json:"image_tag,omitempty"` + EnvironmentOverrides map[string]string `json:"environment_overrides,omitempty"` + OpenClawConfigPlan *OpenClawConfigPlan `json:"openclaw_config_plan,omitempty"` + IsLeader bool `json:"is_leader,omitempty"` +} + +type DispatchTeamTaskRequest struct { + TargetMemberID string `json:"target_member_id"` + MessageID string `json:"message_id,omitempty"` + Payload map[string]interface{} `json:"payload"` +} + +type TeamListPayload struct { + Teams []models.Team `json:"teams"` + Total int `json:"total"` +} + +type TeamDetailsPayload struct { + Team *models.Team `json:"team"` + LeaderMemberID string `json:"leader_member_id,omitempty"` + Leader *models.TeamMember `json:"leader,omitempty"` + Members []models.TeamMember `json:"members"` + Tasks []TeamTaskPayload `json:"tasks,omitempty"` + Events []TeamEventPayload `json:"events,omitempty"` + WorkItems []TeamWorkItemPayload `json:"work_items,omitempty"` +} + +type TeamTasksHistoryPayload struct { + Tasks []TeamTaskPayload `json:"tasks"` + HasMore bool `json:"has_more"` + NextBeforeID *int `json:"next_before_id,omitempty"` +} + +type TeamEventsHistoryPayload struct { + Events []TeamEventPayload `json:"events"` + HasMore bool `json:"has_more"` + NextBeforeID *int `json:"next_before_id,omitempty"` +} + +type TeamWorkspaceFileEntry struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + Size int64 `json:"size"` + ModifiedAt string `json:"modified_at,omitempty"` + Previewable bool `json:"previewable"` +} + +type TeamWorkspaceListPayload struct { + Path string `json:"path"` + Root string `json:"root"` + Entries []TeamWorkspaceFileEntry `json:"entries"` +} + +type TeamWorkspacePreviewPayload struct { + Path string `json:"path"` + Name string `json:"name"` + Content string `json:"content"` +} + +type TeamWorkspaceDownloadPayload struct { + Path string + Name string + ContentType string + Data []byte +} + +type TeamWorkspaceFolderRequest struct { + Path string `json:"path"` + Name string `json:"name"` +} + +type TeamWorkspaceRenameRequest struct { + Path string `json:"path"` + NewName string `json:"new_name"` +} + +type TeamTaskPayload struct { + models.TeamTask + Payload map[string]interface{} `json:"payload,omitempty"` + Result map[string]interface{} `json:"result,omitempty"` +} + +type TeamEventPayload struct { + models.TeamEvent + Payload map[string]interface{} `json:"payload,omitempty"` +} + +type TeamWorkItemPayload struct { + models.TeamWorkItem + DependsOn []string `json:"depends_on,omitempty"` + Result map[string]interface{} `json:"result,omitempty"` + ArtifactRefs []string `json:"artifact_refs,omitempty"` +} + +type teamService struct { + repo repository.TeamRepository + instanceService InstanceService + openClawConfigPlanner teamOpenClawConfigPlanner + pvcService *k8s.PVCService + secretService *k8s.SecretService + configMapService *k8s.ConfigMapService + podService *k8s.PodService + + ctx context.Context + cancel context.CancelFunc + mu sync.Mutex + running bool + wg sync.WaitGroup + consumers map[int]struct{} + assignmentMonitorLast map[string]time.Time + staleMonitorStarted bool + runtimeWorkspaceRoot string +} + +type teamOpenClawConfigPlanner interface { + PlanWithoutTeamMemberLeaderOnlyChannels(userID int, plan *OpenClawConfigPlan) (*OpenClawConfigPlan, error) +} + +type plannedTeamMember struct { + Request CreateTeamMemberRequest + MemberKey string + DisplayName string + Role string + ProfileKey string + ProfileName string + EffectiveRole string + RuntimeType string + InstanceMode string + IsLeader bool +} + +type teamRuntimeSecrets struct { + RedisURL string + Token string +} + +type TeamServiceOption func(*teamService) + +func WithTeamRuntimeWorkspaceRoot(root string) TeamServiceOption { + return func(s *teamService) { + if strings.TrimSpace(root) != "" { + s.runtimeWorkspaceRoot = strings.TrimSpace(root) + } + } +} + +func WithTeamOpenClawConfigService(service OpenClawConfigService) TeamServiceOption { + return func(s *teamService) { + s.openClawConfigPlanner = service + } +} + +func NewTeamService(repo repository.TeamRepository, instanceService InstanceService, opts ...TeamServiceOption) TeamService { + ctx, cancel := context.WithCancel(context.Background()) + service := &teamService{ + repo: repo, + instanceService: instanceService, + pvcService: k8s.NewPVCService(), + secretService: k8s.NewSecretService(), + configMapService: k8s.NewConfigMapService(), + podService: k8s.NewPodService(), + ctx: ctx, + cancel: cancel, + consumers: map[int]struct{}{}, + assignmentMonitorLast: map[string]time.Time{}, + runtimeWorkspaceRoot: "/workspaces", + } + for _, opt := range opts { + if opt != nil { + opt(service) + } + } + return service +} + +// StartBackground starts the leader-only background workers: a periodic scan +// that ensures a Redis event consumer is running for every active team, and +// the stale-task monitor. It is safe to call repeatedly (a second call while +// running is a no-op) and can be called again after StopBackground, which is +// required for leader-election re-acquisition. HTTP request handling does not +// depend on these workers, so followers can still serve the API and the in-pod +// nginx data plane while only the leader runs them. +func (s *teamService) StartBackground(parent context.Context) { + s.mu.Lock() + if s.running { + s.mu.Unlock() + return + } + if parent == nil { + parent = context.Background() + } + ctx, cancel := context.WithCancel(parent) + s.ctx = ctx + s.cancel = cancel + s.running = true + s.staleMonitorStarted = false + s.consumers = map[int]struct{}{} + s.assignmentMonitorLast = map[string]time.Time{} + s.wg.Add(1) + go s.consumerScanLoop(ctx) + s.mu.Unlock() + + fmt.Println("[TeamService] Starting leader-only background workers...") + s.ensureStaleTaskMonitor(ctx) +} + +// StopBackground stops all background workers and blocks until they have fully +// exited, so a subsequent StartBackground starts from a clean state with no +// goroutines from the previous generation still touching shared maps. It is +// idempotent. +func (s *teamService) StopBackground() { + s.mu.Lock() + if !s.running { + s.mu.Unlock() + return + } + s.running = false + cancel := s.cancel + s.mu.Unlock() + + fmt.Println("[TeamService] Stopping leader-only background workers...") + if cancel != nil { + cancel() + } + s.wg.Wait() + + s.mu.Lock() + s.consumers = map[int]struct{}{} + s.staleMonitorStarted = false + s.mu.Unlock() +} + +// consumerScanLoop periodically ensures a consumer goroutine exists for every +// active team. Team creation no longer starts consumers inline (that would run +// on whichever replica served the request); the leader picks up newly active +// teams here within teamConsumerScanInterval. +func (s *teamService) consumerScanLoop(ctx context.Context) { + defer s.wg.Done() + + s.ensureConsumersForActiveTeams(ctx) + + ticker := time.NewTicker(teamConsumerScanInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.ensureConsumersForActiveTeams(ctx) + } + } +} + +func (s *teamService) ensureConsumersForActiveTeams(ctx context.Context) { + teams, err := s.repo.ListActiveTeams() + if err != nil { + fmt.Printf("Warning: failed to list active teams for consumer scan: %v\n", err) + return + } + for i := range teams { + s.ensureConsumer(ctx, teams[i].ID) + } +} + +func (s *teamService) CreateTeam(userID int, req CreateTeamRequest) (*TeamDetailsPayload, error) { + req.Name = strings.TrimSpace(req.Name) + if req.Name == "" { + return nil, fmt.Errorf("team name is required") + } + if len(req.Members) == 0 { + return nil, fmt.Errorf("team must include at least one member") + } + memberPlans, err := planTeamMembers(req.Name, req.Members) + if err != nil { + return nil, err + } + existingTeam, err := s.repo.GetTeamByUserIDAndName(userID, req.Name) + if err != nil { + return nil, err + } + if existingTeam != nil { + if existingTeam.Status == models.TeamStatusFailed { + if err := s.DeleteTeam(userID, existingTeam.ID); err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("team name already exists") + } + } + + communicationMode, err := normalizeTeamCommunicationMode(req.CommunicationMode) + if err != nil { + return nil, err + } + redisURL := strings.TrimSpace(req.RedisURL) + if redisURL == "" { + redisURL = defaultTeamRedisURL() + } + if redisURL == "" { + return nil, fmt.Errorf("team redis url is required") + } + if _, err := newRedisBus(redisURL); err != nil { + return nil, err + } + + sharedStorageGB := req.SharedStorageGB + if sharedStorageGB <= 0 { + sharedStorageGB = 10 + } + preflightTeam := &models.Team{ + ID: 0, + Name: req.Name, + StorageClass: optionalString(strings.TrimSpace(req.StorageClass)), + SharedMountPath: teamSharedMountPath, + } + if err := s.instanceService.ValidateCreateRequests(userID, s.buildTeamMemberInstanceRequests(preflightTeam, memberPlans)); err != nil { + return nil, err + } + + now := time.Now().UTC() + storageClass := optionalString(strings.TrimSpace(req.StorageClass)) + team := &models.Team{ + UserID: userID, + Name: req.Name, + Description: req.Description, + Status: models.TeamStatusCreating, + CommunicationMode: communicationMode, + RedisEventsLastID: "0-0", + SharedMountPath: teamSharedMountPath, + StorageClass: storageClass, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.repo.CreateTeam(team); err != nil { + return nil, err + } + if err := s.instanceService.ValidateCreateRequests(userID, s.buildTeamMemberInstanceRequests(team, memberPlans)); err != nil { + return nil, s.rollbackTeamCreation(userID, team, err) + } + + runtimeSecrets, err := s.provisionTeamK8s(userID, team, redisURL, sharedStorageGB, strings.TrimSpace(req.StorageClass)) + if err != nil { + return nil, s.rollbackTeamCreation(userID, team, err) + } + rosterJSON, err := s.upsertTeamRosterConfig(userID, team, memberPlans) + if err != nil { + return nil, s.rollbackTeamCreation(userID, team, err) + } + + for _, memberPlan := range memberPlans { + member, err := s.createTeamMemberInstance(userID, team, memberPlan, runtimeSecrets, rosterJSON) + if err != nil { + return nil, s.rollbackTeamCreation(userID, team, err) + } + member.Status = models.TeamMemberStatusIdle + member.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateMember(member); err != nil { + return nil, s.rollbackTeamCreation(userID, team, err) + } + } + + team.Status = models.TeamStatusRunning + team.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateTeam(team); err != nil { + return nil, err + } + // Background consumers / stale-task monitor are leader-only and started by + // the leader's periodic scan (consumerScanLoop). Starting them here would + // run them on whichever replica served the create request, bypassing + // leader election, so we intentionally do not call ensureConsumer here. + if err := s.dispatchInitialLeaderTask(userID, team); err != nil { + fmt.Printf("Warning: failed to dispatch initial Team %d leader task: %v\n", team.ID, err) + if recordErr := s.recordInitialLeaderTaskDispatchFailure(team.ID, err); recordErr != nil { + fmt.Printf("Warning: failed to record Team %d initial leader task dispatch failure: %v\n", team.ID, recordErr) + } + } + return s.GetTeam(userID, team.ID) +} + +func (s *teamService) dispatchInitialLeaderTask(userID int, team *models.Team) error { + if team == nil { + return fmt.Errorf("team is required") + } + members, err := s.repo.ListMembersByTeamID(team.ID) + if err != nil { + return err + } + leader := findTeamLeader(activeTeamMembers(members)) + if leader == nil { + return fmt.Errorf("team leader not found") + } + _, err = s.DispatchTask(userID, team.ID, DispatchTeamTaskRequest{ + TargetMemberID: leader.MemberKey, + MessageID: initialLeaderTaskMessageID(team.ID), + Payload: buildInitialLeaderTaskPayload(team.Name), + }) + return err +} + +func initialLeaderTaskMessageID(teamID int) string { + return fmt.Sprintf("team-%d-bootstrap-introduction", teamID) +} + +func (s *teamService) recordInitialLeaderTaskDispatchFailure(teamID int, cause error) error { + now := time.Now().UTC() + payload := map[string]interface{}{ + "v": 1, + "event": "bootstrap_dispatch_failed", + "teamId": strconv.Itoa(teamID), + "intent": initialLeaderTaskIntent, + "messageId": initialLeaderTaskMessageID(teamID), + "source": "clawmanager", + } + if cause != nil { + payload["diagnostic"] = cause.Error() + } + payloadJSON, err := marshalOptionalJSON(payload) + if err != nil { + return err + } + messageID := initialLeaderTaskMessageID(teamID) + return s.repo.CreateEvent(&models.TeamEvent{ + TeamID: teamID, + MessageID: &messageID, + EventType: "bootstrap_dispatch_failed", + PayloadJSON: payloadJSON, + OccurredAt: &now, + CreatedAt: now, + }) +} + +func buildTeamTaskEnvelope(teamID int, memberKey string, task *models.TeamTask, messageID string, taskPayload map[string]interface{}, memberContext map[string]string, now time.Time) map[string]interface{} { + if taskPayload == nil { + taskPayload = map[string]interface{}{} + } + taskID := 0 + workflowState := teamWorkflowStatePlanning + planVersion := int64(0) + ledgerVersion := int64(0) + if task != nil { + taskID = task.ID + workflowState = task.WorkflowState + if workflowState == "" { + workflowState = teamWorkflowStatePlanning + } + planVersion = task.PlanVersion + ledgerVersion = task.LedgerVersion + } + taskRef := fmt.Sprintf("team-%d-task-%d", teamID, taskID) + prompt := eventString(taskPayload, "prompt", "goal", "instruction", "instructions") + if prompt == "" { + rawPayload, _ := marshalJSON(taskPayload) + prompt = rawPayload + } + rawPrompt := prompt + prompt = buildTeamRuntimePrompt(prompt, memberContext) + intent := eventString(taskPayload, "intent") + responseLocale := eventString(taskPayload, "responseLocale", "response_locale") + if responseLocale == "" { + responseLocale = inferTeamResponseLocale(rawPrompt) + } + envelope := map[string]interface{}{ + "v": 1, + "protocolVersion": 3, + "messageId": messageID, + "teamId": strconv.Itoa(teamID), + "from": "clawmanager", + "to": memberKey, + "replyTo": teamTaskReplyTarget, + "requiresCompletion": true, + "completionTool": teamTaskCompletionTool, + "resultSink": map[string]interface{}{ + "type": "redis_stream", + "eventsKey": teamEventsKey(teamID), + "successEvent": "completion_proposed", + "failureEvent": "task_failed", + "replyEvent": "reply", + "resultField": "resultMarkdown", + "summaryField": "summary", + "artifactField": "artifactRefs", + "completionTool": teamTaskCompletionTool, + }, + "intent": intent, + "taskId": taskRef, + "title": eventString(taskPayload, "title"), + "prompt": appendTeamResponseLocaleInstruction(appendTeamTaskCompletionInstruction(prompt, memberContext["communicationMode"], intent), responseLocale), + "rawPrompt": rawPrompt, + "responseLocale": responseLocale, + "workflowState": workflowState, + "planVersion": planVersion, + "ledgerVersion": ledgerVersion, + "contextRefs": normalizeContextRefs(taskPayload["contextRefs"]), + "memberContext": memberContext, + "systemPrompt": memberContext["systemPrompt"], + "monitorPolicy": defaultTeamMonitorPolicy(), + "metadata": taskPayload, + "createdAt": now.Format(time.RFC3339Nano), + } + if workspaceContract, ok := taskPayload["workspaceContract"]; ok { + envelope["workspaceContract"] = workspaceContract + if contract, ok := workspaceContract.(map[string]interface{}); ok { + physicalRoot := eventString(contract, "physicalSharedDir") + taskRef := eventString(contract, "taskRef") + memberArtifactPhysicalRoot := "" + if physicalRoot != "" && taskRef != "" { + memberArtifactPhysicalRoot = filepath.ToSlash(filepath.Join(physicalRoot, "artifacts", taskRef, "members", normalizeTeamMemberRouteKey(memberKey))) + } + envelope["sharedWorkspace"] = map[string]interface{}{ + "physicalPath": physicalRoot, + "canonicalPrefix": "/team", + "memberArtifactPhysicalRoot": memberArtifactPhysicalRoot, + "memberArtifactCanonicalRoot": "/team/artifacts/" + taskRef + "/members/" + normalizeTeamMemberRouteKey(memberKey), + } + } + } + if bootstrapSnapshot, ok := taskPayload["bootstrapSnapshot"]; ok { + envelope["bootstrapSnapshot"] = bootstrapSnapshot + } + if teamConfigJSON, ok := taskPayload["teamConfigJson"]; ok { + envelope["teamConfigJson"] = teamConfigJSON + } else if teamConfigJSON, ok := taskPayload["teamConfigJSON"]; ok { + envelope["teamConfigJson"] = teamConfigJSON + } + if envelope["intent"] == "" { + envelope["intent"] = "run_task" + } + if envelope["title"] == "" { + envelope["title"] = fmt.Sprintf("Team task %d", taskID) + } + return envelope +} + +func inferTeamResponseLocale(text string) string { + for _, r := range text { + if r >= '\u3400' && r <= '\u9fff' { + return "zh-CN" + } + } + return "en-US" +} + +func appendTeamResponseLocaleInstruction(prompt, locale string) string { + locale = strings.TrimSpace(locale) + if locale == "" { + locale = "zh-CN" + } + instruction := fmt.Sprintf("Response locale contract: use %s for every user-visible plan, assignment, progress summary, status-check summary, resultMarkdown, and final synthesis. Keep source code, API names, file names, and necessary technical terms in their original form. Do not rely on frontend translation.", locale) + if strings.TrimSpace(prompt) == "" { + return instruction + } + return strings.TrimSpace(prompt) + "\n\n" + instruction +} + +func applyTeamTaskEnvelopeContext(envelope map[string]interface{}, task *models.TeamTask, targetMemberKey string) { + if envelope == nil || task == nil { + return + } + payload := map[string]interface{}{} + if strings.TrimSpace(task.PayloadJSON) != "" { + _ = json.Unmarshal([]byte(task.PayloadJSON), &payload) + } + locale := eventString(payload, "responseLocale", "response_locale") + if locale == "" { + locale = inferTeamResponseLocale(eventString(payload, "prompt", "goal", "instruction", "instructions")) + } + envelope["responseLocale"] = locale + envelope["workflowState"] = task.WorkflowState + envelope["planVersion"] = task.PlanVersion + envelope["ledgerVersion"] = task.LedgerVersion + if task.CurrentPhaseID != nil { + envelope["currentPhaseId"] = *task.CurrentPhaseID + } + contract, _ := payload["workspaceContract"].(map[string]interface{}) + if contract == nil { + return + } + envelope["workspaceContract"] = contract + physicalRoot := eventString(contract, "physicalSharedDir") + taskRef := eventString(contract, "taskRef") + memberKey := normalizeTeamMemberRouteKey(targetMemberKey) + memberPhysicalRoot := "" + memberCanonicalRoot := "" + if physicalRoot != "" && taskRef != "" && memberKey != "" { + memberPhysicalRoot = filepath.ToSlash(filepath.Join(physicalRoot, "artifacts", taskRef, "members", memberKey)) + memberCanonicalRoot = "/team/artifacts/" + taskRef + "/members/" + memberKey + } + envelope["sharedWorkspace"] = map[string]interface{}{ + "physicalPath": physicalRoot, + "canonicalPrefix": "/team", + "memberArtifactPhysicalRoot": memberPhysicalRoot, + "memberArtifactCanonicalRoot": memberCanonicalRoot, + } +} + +func defaultTeamMonitorPolicy() map[string]interface{} { + return map[string]interface{}{ + "enabled": true, + "heartbeatEverySec": 30, + "visibleHeartbeatEverySec": int(teamAssignmentMonitorEvery.Seconds()), + "checkEverySec": int(teamAssignmentMonitorEvery.Seconds()), + "softTimeoutSec": int((2 * teamAssignmentMonitorEvery).Seconds()), + "visibleToChat": true, + } +} + +func appendTeamTaskCompletionInstruction(prompt string, communicationMode, intent string) string { + base := strings.TrimSpace(prompt) + if strings.TrimSpace(intent) == initialLeaderTaskIntent { + instruction := strings.Join([]string{ + "Bootstrap completion contract:", + "- This is a control-plane Team snapshot assigned only to the Leader. Do not delegate it, create worker assignments, or wait for member replies.", + "- Use the injected metadata.bootstrapSnapshot, metadata.teamConfigJson, and metadata.workspaceContract first. They are the authoritative roster, runtime, role, and workspace facts for this snapshot.", + "- If injected metadata is insufficient, read /team/team.json from the shared workspace. $CLAWMANAGER_TEAM_CONFIG_PATH and /etc/clawmanager/team/team.json are optional system fallbacks and may be blocked by tool sandboxes. Never look for /team/members.", + "- Do not probe member HTTP health endpoints, Redis CLI, or worker desktops for this bootstrap. If a runtime status source is unavailable in the injected snapshot, report that field as unavailable instead of blocking.", + "- Summarize every member's identity, role, runtime, responsibilities, capability boundaries, and the configured collaboration mode.", + "- Explain task routing, Team Redis event synchronization, shared workspace usage, and the available Team methods without asking other members to restate their own roles.", + "- If you write a detailed report, write it under \"$CLAWMANAGER_TEAM_SHARED_DIR/results//\" and report it only as /team/results//.", + "- Complete this bootstrap in the current turn by calling team_complete_task with status=\"succeeded\", summary, and resultMarkdown.", + "- Successful bootstrap completion is recorded as the task_completed event.", + "- Do not finish with tool calls only and do not wait for QA/review evidence for this bootstrap snapshot.", + }, "\n") + if base == "" { + return instruction + } + return base + "\n\n" + instruction + } + if strings.Contains(base, teamTaskCompletionTool) && strings.Contains(base, "task_completed") { + return base + } + mode := normalizedTeamCommunicationMode(communicationMode) + modeInstructions := []string{ + "- Leader-mediated mode is a strict hub-and-spoke workflow: user root task -> Leader -> assigned workers -> Leader -> final user-facing result.", + "- If you are the Leader and the user names a non-Leader member or role, you MUST delegate the work to that exact member with team_send, wait for the assigned workers' actual results, then provide a final synthesis before completing the root task.", + "- If you are the Leader and the user gives a broad task without naming one member, first create a compact plan, decompose the work into owner/member_id assignments, send those assignments, wait for the assigned workers' actual results, verify them, then complete the root task.", + "- If you are the Leader, answer self-contained control-plane or simple tasks directly only when the request clearly stays within the Leader/control-plane scope and does not require a named worker or multi-member evidence.", + "- If you are a Worker, execute only the assignment addressed to you and report the result, evidence, artifact paths, or blocker back to the Leader. Do not hand off directly to another Worker.", + "- A Leader dispatch, plan, or handoff is not a final result and must not call team_complete_task. Worker completion never closes the user root task. Only the Leader may finalize the root task after reconciling all required member outputs.", + } + switch mode { + case teamCommunicationModePeerAssisted: + modeInstructions = []string{ + "- Worker-direct mode: if the root task or collaboration plan names a downstream member, you MUST hand off to that exact member with team_send before completing your own step. This handoff is required, not optional.", + "- In worker-direct mode, do not send a completed step only to the Leader when a downstream owner is specified. The Leader is the fallback only when no downstream owner is specified, when you are blocked, or when final synthesis is explicitly requested.", + "- A worker-to-worker handoff must include rootTaskId/rootMessageId when available, artifact paths, the requested next action, acceptance criteria, and whether a reply is required.", + } + case teamCommunicationModeFullMesh: + modeInstructions = []string{ + "- Full-mesh mode: coordinate directly with the named downstream owners. If a member is specified as the next owner, hand off to that exact member before completing your own step.", + "- Preserve rootTaskId/rootMessageId, artifact paths, requested next action, acceptance criteria, and reply requirements in every peer handoff.", + } + } + instruction := strings.Join([]string{ + "Completion contract:", + "- For multi-member Teams, first write a compact collaboration plan: subtasks, owner member_id, dependency, expected artifact, and verification rule.", + "- Publish that plan to ClawManager with team_update_progress status=\"running\" and eventKind=\"leader_plan\" before dispatching worker assignments. Plans are process visibility, not completion.", + }, "\n") + instruction += "\n" + strings.Join(modeInstructions, "\n") + instruction += "\n" + strings.Join([]string{ + "- Publish meaningful process updates with team_update_progress. Use eventKind=\"worker_plan\" for worker execution plans, \"worker_progress\" for milestones, and \"leader_synthesis\" while reconciling member outputs. Use \"assignment_check_result\" only when replying to a ClawManager Monitor envelope carrying a monitor checkId; ordinary progress must remain worker_progress.", + "- Prefer team_artifact_write, team_artifact_read, team_artifact_list, and team_artifact_mkdir for shared artifacts. These tools enforce current-Team path isolation and cooperative permissions.", + "- If a worker is still executing a long step, report concise progress and continue. If context was lost or an artifact path is wrong, report a recoverable blocker to the Leader instead of treating the root task as failed.", + "- Every Team message must preserve rootTaskId/messageId context when available and must clearly state whether it is an assignment, peer request, progress update, result, review, blocker, or final synthesis.", + "- For multi-stage work, publish a structured leader_plan with planVersion and phases. Every team_send must carry a stable phaseId, assignmentId, workId, revision, required flag, and dependencies. Completing one phase never completes the user root task.", + "- The Leader may call team_complete_task for the root only after the workflow is sealed, remainingActions is empty, every required latest assignment and review is complete, and finalAnswerReady is true. Worker completion closes only that assignment.", + "- A failed or stale required assignment blocks root success unless the Leader supplies a structured waiver containing assignmentId, reason, and accepted risk. Never waive running/pending work or omit the risk record.", + "- Optional work does not need to succeed, but every omitted optional assignment must be listed in skippedAssignments with assignmentId and a concrete reason.", + "- Report verification truthfully. If browser/DOM verification did not run or failed, label it as unverified; a hand-written simulator or static inspection is not a browser pass. Any artifact change after review invalidates that review and requires fresh validation.", + "- The Leader must not mark the root task succeeded after merely dispatching work. Final success requires returned member evidence plus Leader synthesis, or a truly direct self-contained answer.", + "- Write shared artifacts under the exact directory in CLAWMANAGER_TEAM_SHARED_DIR. When using shell commands, always create files under \"$CLAWMANAGER_TEAM_SHARED_DIR/\".", + "- Never create or report a relative team/... folder. The path team/... is invalid because ClawManager file browsing only resolves shared artifacts through the Team shared directory.", + "- Report shared artifact links using the canonical UI path /team/, even when a Lite runtime uses a different physical shared directory.", + "- Members must report produced artifact paths and concrete outcomes through the Team channel before completing their assigned task.", + "- When the final result is ready, call team_complete_task with status=\"succeeded\", summary, and resultMarkdown.", + "- If the task fails, call team_complete_task with status=\"failed\" and an error message.", + "- Do not send the final answer as a normal message to clawmanager; ClawManager consumes task_completed/task_failed events from the Team Redis event stream.", + }, "\n") + if base == "" { + return instruction + } + return base + "\n\n" + instruction +} + +func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL string, sharedStorageGB int, storageClass string) (*teamRuntimeSecrets, error) { + ctx := context.Background() + pvc, err := s.pvcService.CreateTeamSharedPVC(ctx, userID, team.ID, sharedStorageGB, storageClass) + if err != nil { + return nil, err + } + secretName := s.pvcService.GetClient().GetTeamSecretName(team.ID) + teamToken, err := generatePrefixedToken("team") + if err != nil { + return nil, fmt.Errorf("failed to generate Team token: %w", err) + } + if err := s.secretService.UpsertSecret(ctx, userID, secretName, map[string]string{ + teamRedisURLSecretKey: redisURL, + teamTokenSecretKey: teamToken, + }, map[string]string{ + "app": "clawreef", + "managed-by": "clawreef", + "team-id": strconv.Itoa(team.ID), + }); err != nil { + return nil, err + } + + team.RedisURLSecretName = &secretName + team.RedisURLSecretKey = optionalString(teamRedisURLSecretKey) + team.TeamTokenSecretName = &secretName + team.TeamTokenSecretKey = optionalString(teamTokenSecretKey) + team.SharedPVCName = &pvc.Name + team.SharedPVCNamespace = &pvc.Namespace + team.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateTeam(team); err != nil { + return nil, err + } + return &teamRuntimeSecrets{RedisURL: redisURL, Token: teamToken}, nil +} + +func (s *teamService) upsertTeamRosterConfig(userID int, team *models.Team, members []plannedTeamMember) (string, error) { + rosterJSON, err := buildTeamRosterConfig(team, members) + if err != nil { + return "", err + } + data := buildTeamRosterConfigData(rosterJSON, team, members) + if err := s.configMapService.UpsertConfigMap(context.Background(), userID, s.teamConfigMapName(team.ID), data, map[string]string{ + "app": "clawreef", + "managed-by": "clawreef", + "team-id": strconv.Itoa(team.ID), + }); err != nil { + return "", err + } + if err := s.writeSharedTeamRosterConfig(userID, team, rosterJSON); err != nil { + return "", err + } + return rosterJSON, nil +} + +func (s *teamService) writeSharedTeamRosterConfig(userID int, team *models.Team, rosterJSON string) error { + if s == nil || team == nil || strings.TrimSpace(rosterJSON) == "" { + return nil + } + root := filepath.Clean(s.teamRuntimeSharedPathFor(userID, team.ID)) + if root == "." || root == string(filepath.Separator) { + return fmt.Errorf("invalid Team shared workspace root for Team %d: %q", team.ID, root) + } + for _, rel := range []string{"", "results", "tasks", "inbox", "status", "artifacts", "tmp"} { + target := root + if rel != "" { + target = filepath.Join(root, rel) + } + if err := os.MkdirAll(target, 0o2775); err != nil { + return fmt.Errorf("failed to prepare Team shared workspace %s: %w", target, err) + } + _ = os.Chmod(target, 0o2775) + } + path := filepath.Join(root, teamConfigFileName) + if err := os.WriteFile(path, []byte(rosterJSON), 0o664); err != nil { + return fmt.Errorf("failed to write shared Team roster %s: %w", path, err) + } + _ = os.Chmod(path, 0o664) + return nil +} + +func buildTeamRosterConfigData(rosterJSON string, team *models.Team, members []plannedTeamMember) map[string]string { + data := map[string]string{ + teamConfigFileName: rosterJSON, + } + for _, member := range members { + if member.RuntimeType != "hermes" { + continue + } + data[teamMemberSoulConfigKey(member.MemberKey)] = buildTeamMemberSoulMarkdown(member, normalizedTeamCommunicationMode(team.CommunicationMode)) + } + return data +} + +func (s *teamService) teamConfigMapName(teamID int) string { + client := k8s.GetClient() + if client == nil { + return fmt.Sprintf("clawreef-team-%d-config", teamID) + } + return client.GetTeamConfigMapName(teamID) +} + +func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, memberPlan plannedTeamMember, runtimeSecrets *teamRuntimeSecrets, rosterJSON string) (*models.TeamMember, error) { + now := time.Now().UTC() + member := &models.TeamMember{ + TeamID: team.ID, + UserID: userID, + MemberKey: memberPlan.MemberKey, + DisplayName: memberPlan.DisplayName, + Role: memberPlan.Role, + RuntimeType: memberPlan.RuntimeType, + InstanceMode: memberPlan.InstanceMode, + Description: optionalString(plannedTeamMemberDescription(memberPlan)), + Status: models.TeamMemberStatusCreating, + Availability: models.TeamMemberAvailabilityUnknown, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.repo.CreateMember(member); err != nil { + return nil, err + } + + createReq := s.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, runtimeSecrets, rosterJSON) + memberOpenClawPlan, err := s.openClawConfigPlanForTeamMember(userID, memberPlan) + if err != nil { + member.Status = models.TeamMemberStatusFailed + member.UpdatedAt = time.Now().UTC() + _ = s.repo.UpdateMember(member) + return nil, err + } + createReq.OpenClawConfigPlan = memberOpenClawPlan + instance, err := s.instanceService.Create(userID, createReq) + if err != nil { + member.Status = models.TeamMemberStatusFailed + member.UpdatedAt = time.Now().UTC() + _ = s.repo.UpdateMember(member) + return nil, err + } + if err := s.writeLiteTeamMemberIdentityFiles(instance, team, memberPlan, rosterJSON); err != nil { + member.Status = models.TeamMemberStatusFailed + member.UpdatedAt = time.Now().UTC() + _ = s.repo.UpdateMember(member) + return nil, err + } + member.InstanceID = &instance.ID + member.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateMember(member); err != nil { + return nil, err + } + return member, nil +} + +func (s *teamService) openClawConfigPlanForTeamMember(userID int, memberPlan plannedTeamMember) (*OpenClawConfigPlan, error) { + plan := memberPlan.Request.OpenClawConfigPlan + if plan == nil || memberPlan.IsLeader || s.openClawConfigPlanner == nil { + return plan, nil + } + return s.openClawConfigPlanner.PlanWithoutTeamMemberLeaderOnlyChannels(userID, plan) +} + +func (s *teamService) buildTeamMemberInstanceRequests(team *models.Team, memberPlans []plannedTeamMember) []CreateInstanceRequest { + requests := make([]CreateInstanceRequest, 0, len(memberPlans)) + for _, memberPlan := range memberPlans { + requests = append(requests, s.buildTeamMemberInstanceRequest(team, memberPlan)) + } + return requests +} + +func (s *teamService) buildTeamMemberInstanceRequest(team *models.Team, memberPlan plannedTeamMember) CreateInstanceRequest { + return s.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, nil, "") +} + +func (s *teamService) buildTeamMemberInstanceRequestWithSecrets(team *models.Team, memberPlan plannedTeamMember, runtimeSecrets *teamRuntimeSecrets, rosterJSON string) CreateInstanceRequest { + req := memberPlan.Request + instanceMode := memberPlan.InstanceMode + if instanceMode == "" { + instanceMode = InstanceModeLite + } + runtimeBackendType, _ := RuntimeTypeForInstanceMode(instanceMode) + memberEnv := s.teamMemberEnv(team, memberPlan) + if instanceMode == InstanceModeLite { + memberEnv["CLAWMANAGER_TEAM_SHARED_DIR"] = s.teamRuntimeSharedPath(team) + } + environmentOverrides := mergeEnvMaps(req.EnvironmentOverrides, memberEnv) + if instanceMode == InstanceModeLite && runtimeSecrets != nil { + environmentOverrides = mergeEnvMaps(environmentOverrides, map[string]string{ + teamRedisURLSecretKey: runtimeSecrets.RedisURL, + teamTokenSecretKey: runtimeSecrets.Token, + }) + if strings.TrimSpace(rosterJSON) != "" { + environmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"] = rosterJSON + } + } + return CreateInstanceRequest{ + Name: teamMemberInstanceName(team.Name, team.ID, memberPlan.MemberKey), + Type: memberPlan.RuntimeType, + Mode: instanceMode, + InstanceMode: instanceMode, + RuntimeType: runtimeBackendType, + CPUCores: defaultFloat(req.CPUCores, 2), + MemoryGB: defaultInt(req.MemoryGB, 4), + DiskGB: defaultInt(req.DiskGB, 20), + GPUEnabled: req.GPUEnabled, + GPUCount: req.GPUCount, + OSType: memberPlan.RuntimeType, + OSVersion: "latest", + ImageRegistry: req.ImageRegistry, + ImageTag: req.ImageTag, + EnvironmentOverrides: environmentOverrides, + StorageClass: derefTeamString(team.StorageClass), + OpenClawConfigPlan: req.OpenClawConfigPlan, + Team: &TeamInstanceConfig{ + Environment: memberEnv, + SecretName: derefTeamString(team.TeamTokenSecretName), + SharedPVCName: derefTeamString(team.SharedPVCName), + SharedMountPath: team.SharedMountPath, + ConfigMapName: s.teamConfigMapName(team.ID), + ConfigMountPath: teamConfigMountDirPath, + PersonaConfigKey: teamMemberPersonaConfigKey(memberPlan), + SharedUID: teamSharedUID, + SharedGID: teamSharedGID, + SharedUmask: teamSharedUmask, + }, + } +} + +func (s *teamService) teamRuntimeSharedPath(team *models.Team) string { + if team == nil { + return k8s.TeamSharedWorkspacePath(s.runtimeWorkspaceRoot, 0, 0) + } + return k8s.TeamSharedWorkspacePath(s.runtimeWorkspaceRoot, team.UserID, team.ID) +} + +func (s *teamService) teamRuntimeSharedPathFor(userID, teamID int) string { + return k8s.TeamSharedWorkspacePath(s.runtimeWorkspaceRoot, userID, teamID) +} + +func (s *teamService) teamMemberEnv(team *models.Team, member plannedTeamMember) map[string]string { + managerBaseURL, _ := defaultTeamManagerBaseURL() + memberContext := buildPlannedTeamMemberTaskContext(member) + communicationMode := normalizedTeamCommunicationMode(team.CommunicationMode) + collaborationPolicy := buildTeamCollaborationPolicy(communicationMode) + collaborationPolicyJSON, _ := json.Marshal(collaborationPolicy) + env := map[string]string{ + "CLAWMANAGER_TEAM_ENABLED": "true", + "CLAWMANAGER_TEAM_ID": strconv.Itoa(team.ID), + "CLAWMANAGER_TEAM_MEMBER_ID": member.MemberKey, + "CLAWMANAGER_TEAM_ROLE": effectiveTeamMemberRole(member), + "CLAWMANAGER_TEAM_EFFECTIVE_ROLE": effectiveTeamMemberRole(member), + "CLAWMANAGER_TEAM_COMMUNICATION_MODE": communicationMode, + "CLAWMANAGER_TEAM_SHARED_DIR": team.SharedMountPath, + "CLAWMANAGER_TEAM_SHARED_UID": strconv.Itoa(teamSharedUID), + "CLAWMANAGER_TEAM_SHARED_GID": strconv.Itoa(teamSharedGID), + "CLAWMANAGER_TEAM_UMASK": teamSharedUmask, + "PUID": strconv.Itoa(teamSharedUID), + "PGID": strconv.Itoa(teamSharedGID), + "UMASK": teamSharedUmask, + "CLAWMANAGER_TEAM_CONFIG_PATH": teamConfigMountPath, + "CLAWMANAGER_TEAM_AUTORUN": "true", + "CLAWMANAGER_TEAM_CONSUMER_GROUP": "team-members", + "CLAWMANAGER_TEAM_INBOX_KEY": teamInboxKey(team.ID, member.MemberKey), + "CLAWMANAGER_TEAM_EVENTS_KEY": teamEventsKey(team.ID), + "CLAWMANAGER_TEAM_PRESENCE_KEY": teamPresenceKey(team.ID), + "CLAWMANAGER_TEAM_DLQ_KEY": teamDLQKey(team.ID), + "CLAWMANAGER_TEAM_MANAGER_URL": managerBaseURL, + "GATEWAY_ALLOW_ALL_USERS": "true", + } + if len(collaborationPolicyJSON) > 0 { + env["CLAWMANAGER_TEAM_COLLABORATION_POLICY_JSON"] = string(collaborationPolicyJSON) + } + if profileKey := strings.TrimSpace(member.ProfileKey); profileKey != "" { + env["CLAWMANAGER_TEAM_PROFILE_KEY"] = profileKey + } + if profileName := strings.TrimSpace(member.ProfileName); profileName != "" { + env["CLAWMANAGER_TEAM_PROFILE_NAME"] = profileName + } + if description := strings.TrimSpace(memberContext["description"]); description != "" { + env["CLAWMANAGER_TEAM_MEMBER_DESCRIPTION"] = description + } + if systemPrompt := strings.TrimSpace(memberContext["systemPrompt"]); systemPrompt != "" { + systemPrompt = appendTeamCollaborationGuidance(systemPrompt, communicationMode) + systemPrompt = appendTeamWorkspaceGuidance(systemPrompt) + env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"] = systemPrompt + env["HERMES_AGENT_HELP_GUIDANCE"] = systemPrompt + } + return env +} + +func teamMemberPersonaConfigKey(member plannedTeamMember) string { + if member.RuntimeType != "hermes" { + return "" + } + return teamMemberSoulConfigKey(member.MemberKey) +} + +func teamMemberSoulConfigKey(memberKey string) string { + key := normalizeTeamMemberKeyForInstanceName(memberKey) + if key == "" { + key = "member" + } + return fmt.Sprintf("hermes-soul-%s.md", key) +} + +func (s *teamService) ListTeams(userID, offset, limit int) (*TeamListPayload, error) { + teams, err := s.repo.ListTeamsByUserID(userID, offset, limit) + if err != nil { + return nil, err + } + teams = activeTeams(teams) + total, err := s.repo.CountTeamsByUserID(userID) + if err != nil { + return nil, err + } + return &TeamListPayload{Teams: teams, Total: total}, nil +} + +func (s *teamService) GetTeam(userID, teamID int) (*TeamDetailsPayload, error) { + team, err := s.requireOwnedTeam(userID, teamID) + if err != nil { + return nil, err + } + members, err := s.repo.ListMembersByTeamID(teamID) + if err != nil { + return nil, err + } + members = activeTeamMembers(members) + tasks, err := s.repo.ListTasksByTeamID(teamID, 20) + if err != nil { + return nil, err + } + events, err := s.repo.ListEventsByTeamID(teamID, 200) + if err != nil { + return nil, err + } + workItems, err := s.repo.ListWorkItemsByTeamID(teamID, 200) + if err != nil { + return nil, err + } + leader := findTeamLeader(members) + return &TeamDetailsPayload{ + Team: team, + LeaderMemberID: leaderMemberKey(leader), + Leader: leader, + Members: members, + Tasks: teamTaskPayloads(tasks), + Events: teamEventPayloads(events), + WorkItems: teamWorkItemPayloads(workItems), + }, nil +} + +func (s *teamService) ListTeamTasks(userID, teamID, beforeID, limit int) (*TeamTasksHistoryPayload, error) { + if _, err := s.requireOwnedTeam(userID, teamID); err != nil { + return nil, err + } + limit = normalizeTeamHistoryLimit(limit, 20, 100) + tasks, err := s.repo.ListTasksBeforeID(teamID, beforeID, limit+1) + if err != nil { + return nil, err + } + hasMore := len(tasks) > limit + if hasMore { + tasks = tasks[:limit] + } + payload := teamTaskPayloads(tasks) + return &TeamTasksHistoryPayload{ + Tasks: payload, + HasMore: hasMore, + NextBeforeID: nextTeamTaskBeforeID(payload), + }, nil +} + +func (s *teamService) ListTeamEvents(userID, teamID, beforeID, limit int) (*TeamEventsHistoryPayload, error) { + if _, err := s.requireOwnedTeam(userID, teamID); err != nil { + return nil, err + } + limit = normalizeTeamHistoryLimit(limit, 50, 200) + events, err := s.repo.ListEventsBeforeID(teamID, beforeID, limit+1) + if err != nil { + return nil, err + } + hasMore := len(events) > limit + if hasMore { + events = events[:limit] + } + var nextBeforeID *int + if len(events) > 0 { + value := events[len(events)-1].ID + nextBeforeID = &value + } + payload := teamEventPayloads(events) + return &TeamEventsHistoryPayload{ + Events: payload, + HasMore: hasMore, + NextBeforeID: nextBeforeID, + }, nil +} + +func (s *teamService) ListWorkspaceFiles(ctx context.Context, userID, teamID int, relPath string) (*TeamWorkspaceListPayload, error) { + cleanPath, err := cleanTeamWorkspacePath(relPath) + if err != nil { + return nil, err + } + team, root, target, err := s.resolveTeamWorkspacePath(ctx, userID, teamID, cleanPath) + if err != nil { + return nil, err + } + info, err := os.Stat(target) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("Team workspace path not found") + } + return nil, fmt.Errorf("failed to inspect Team workspace: %w", err) + } + if !info.IsDir() { + return nil, fmt.Errorf("Team workspace path is not a folder") + } + dirEntries, err := os.ReadDir(target) + if err != nil { + return nil, fmt.Errorf("failed to list Team workspace: %w", err) + } + entries := make([]TeamWorkspaceFileEntry, 0, len(dirEntries)) + for _, dirEntry := range dirEntries { + if err := ctx.Err(); err != nil { + return nil, err + } + info, err := dirEntry.Info() + if err != nil { + return nil, fmt.Errorf("failed to inspect Team workspace entry %q: %w", dirEntry.Name(), err) + } + entries = append(entries, teamWorkspaceFileEntryFromInfo(cleanPath, info)) + } + sortTeamWorkspaceEntries(entries) + return &TeamWorkspaceListPayload{ + Path: cleanPath, + Root: teamWorkspaceDisplayRoot(team, root), + Entries: entries, + }, nil +} + +func (s *teamService) PreviewWorkspaceFile(ctx context.Context, userID, teamID int, relPath string) (*TeamWorkspacePreviewPayload, error) { + cleanPath, err := cleanTeamWorkspacePath(relPath) + if err != nil { + return nil, err + } + if cleanPath == "" || !isPreviewableWorkspaceFile(cleanPath) { + return nil, fmt.Errorf("only md, txt, and json files can be previewed") + } + _, _, target, err := s.resolveTeamWorkspacePath(ctx, userID, teamID, cleanPath) + if err != nil { + return nil, err + } + info, err := os.Stat(target) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("Team workspace file not found") + } + return nil, fmt.Errorf("failed to inspect Team workspace file: %w", err) + } + if info.IsDir() { + return nil, fmt.Errorf("Team workspace entry is a folder") + } + if info.Size() > 1048576 { + return nil, fmt.Errorf("Team workspace file is too large to preview") + } + raw, err := os.ReadFile(target) + if err != nil { + return nil, fmt.Errorf("failed to preview Team workspace file: %w", err) + } + return &TeamWorkspacePreviewPayload{ + Path: cleanPath, + Name: posixpath.Base(cleanPath), + Content: string(raw), + }, nil +} + +func (s *teamService) DownloadWorkspaceFile(ctx context.Context, userID, teamID int, relPath string) (*TeamWorkspaceDownloadPayload, error) { + cleanPath, err := cleanTeamWorkspacePath(relPath) + if err != nil { + return nil, err + } + if cleanPath == "" { + return nil, fmt.Errorf("file path is required") + } + _, _, target, err := s.resolveTeamWorkspacePath(ctx, userID, teamID, cleanPath) + if err != nil { + return nil, err + } + info, err := os.Stat(target) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("Team workspace entry not found") + } + return nil, fmt.Errorf("failed to inspect Team workspace entry: %w", err) + } + if info.IsDir() { + data, err := zipTeamWorkspaceDirectory(ctx, target) + if err != nil { + return nil, err + } + return &TeamWorkspaceDownloadPayload{ + Path: cleanPath, + Name: posixpath.Base(cleanPath) + ".zip", + ContentType: "application/zip", + Data: data, + }, nil + } + data, err := os.ReadFile(target) + if err != nil { + return nil, fmt.Errorf("failed to download Team workspace file: %w", err) + } + return &TeamWorkspaceDownloadPayload{ + Path: cleanPath, + Name: posixpath.Base(cleanPath), + ContentType: "application/octet-stream", + Data: data, + }, nil +} + +func (s *teamService) CreateWorkspaceFolder(ctx context.Context, userID, teamID int, req TeamWorkspaceFolderRequest) error { + parent, err := cleanTeamWorkspacePath(req.Path) + if err != nil { + return err + } + name, err := cleanWorkspaceEntryName(req.Name) + if err != nil { + return err + } + _, _, target, err := s.resolveTeamWorkspacePath(ctx, userID, teamID, joinTeamWorkspacePath(parent, name)) + if err != nil { + return err + } + if err := ensureTeamWorkspaceDirectory(target); err != nil { + return fmt.Errorf("failed to create Team workspace folder: %w", err) + } + return nil +} + +func (s *teamService) RenameWorkspaceEntry(ctx context.Context, userID, teamID int, req TeamWorkspaceRenameRequest) error { + cleanPath, err := cleanTeamWorkspacePath(req.Path) + if err != nil { + return err + } + if cleanPath == "" { + return fmt.Errorf("path is required") + } + newName, err := cleanWorkspaceEntryName(req.NewName) + if err != nil { + return err + } + parent := posixpath.Dir(cleanPath) + if parent == "." { + parent = "" + } + _, _, source, err := s.resolveTeamWorkspacePath(ctx, userID, teamID, cleanPath) + if err != nil { + return err + } + _, _, target, err := s.resolveTeamWorkspacePath(ctx, userID, teamID, joinTeamWorkspacePath(parent, newName)) + if err != nil { + return err + } + if _, err := os.Stat(source); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("Team workspace entry not found") + } + return fmt.Errorf("failed to inspect Team workspace entry: %w", err) + } + if _, err := os.Stat(target); err == nil { + return fmt.Errorf("target Team workspace entry already exists") + } else if !os.IsNotExist(err) { + return fmt.Errorf("failed to inspect target Team workspace entry: %w", err) + } + if err := os.Rename(source, target); err != nil { + return fmt.Errorf("failed to rename Team workspace entry: %w", err) + } + return nil +} + +func (s *teamService) DeleteWorkspaceEntry(ctx context.Context, userID, teamID int, relPath string) error { + cleanPath, err := cleanTeamWorkspacePath(relPath) + if err != nil { + return err + } + if cleanPath == "" { + return fmt.Errorf("cannot delete workspace root") + } + _, _, target, err := s.resolveTeamWorkspacePath(ctx, userID, teamID, cleanPath) + if err != nil { + return err + } + if _, err := os.Stat(target); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("Team workspace entry not found") + } + return fmt.Errorf("failed to inspect Team workspace entry: %w", err) + } + if err := os.RemoveAll(target); err != nil { + return fmt.Errorf("failed to delete Team workspace entry: %w", err) + } + return nil +} + +func (s *teamService) UploadWorkspaceFiles(ctx context.Context, userID, teamID int, targetPath string, files []*multipart.FileHeader, relativePaths []string) error { + if len(files) == 0 { + return fmt.Errorf("no files uploaded") + } + basePath, err := cleanTeamWorkspacePath(targetPath) + if err != nil { + return err + } + for index, fileHeader := range files { + if fileHeader == nil { + continue + } + uploadName := fileHeader.Filename + if index < len(relativePaths) && strings.TrimSpace(relativePaths[index]) != "" { + uploadName = relativePaths[index] + } + uploadPath, err := cleanTeamWorkspacePath(uploadName) + if err != nil { + return err + } + if uploadPath == "" { + return fmt.Errorf("uploaded file name is required") + } + _, _, destination, err := s.resolveTeamWorkspacePath(ctx, userID, teamID, joinTeamWorkspacePath(basePath, uploadPath)) + if err != nil { + return err + } + file, err := fileHeader.Open() + if err != nil { + return fmt.Errorf("failed to open uploaded file: %w", err) + } + if err := ensureTeamWorkspaceDirectory(filepath.Dir(destination)); err != nil { + _ = file.Close() + return fmt.Errorf("failed to create Team workspace upload folder: %w", err) + } + out, err := os.OpenFile(destination, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0664) + if err != nil { + _ = file.Close() + return fmt.Errorf("failed to create Team workspace upload file: %w", err) + } + _, copyErr := io.Copy(out, file) + closeErr := out.Close() + _ = file.Close() + if copyErr != nil { + return fmt.Errorf("failed to upload Team workspace file: %w", copyErr) + } + if closeErr != nil { + return fmt.Errorf("failed to close Team workspace upload file: %w", closeErr) + } + chownTeamWorkspacePath(destination) + } + return nil +} + +func (s *teamService) DispatchTask(userID, teamID int, req DispatchTeamTaskRequest) (*TeamTaskPayload, error) { + team, err := s.requireOwnedTeam(userID, teamID) + if err != nil { + return nil, err + } + memberKey := strings.TrimSpace(req.TargetMemberID) + if memberKey == "" { + members, err := s.repo.ListMembersByTeamID(teamID) + if err != nil { + return nil, err + } + memberKey = leaderMemberKey(findTeamLeader(activeTeamMembers(members))) + } + if memberKey == "" { + return nil, fmt.Errorf("target member id is required") + } + if req.Payload == nil { + return nil, fmt.Errorf("task payload is required") + } + if strings.TrimSpace(eventString(req.Payload, "responseLocale", "response_locale")) == "" { + prompt := eventString(req.Payload, "prompt", "goal", "instruction", "instructions") + req.Payload["responseLocale"] = inferTeamResponseLocale(prompt) + } + if _, exists := req.Payload["origin"]; !exists { + req.Payload["origin"] = "user_query" + } + if _, exists := req.Payload["anchorEligible"]; !exists { + req.Payload["anchorEligible"] = true + } + if strings.TrimSpace(eventString(req.Payload, "intent")) == initialLeaderTaskIntent { + if err := s.enrichBootstrapTaskPayload(userID, team, req.Payload); err != nil { + return nil, err + } + } + member, err := s.repo.GetMemberByTeamKey(teamID, memberKey) + if err != nil { + return nil, err + } + if member == nil { + return nil, fmt.Errorf("team member not found") + } + + messageID := strings.TrimSpace(req.MessageID) + if messageID == "" { + messageID = fmt.Sprintf("team-%d-task-%d", teamID, time.Now().UTC().UnixNano()) + } + existing, err := s.repo.GetTaskByMessageID(teamID, messageID) + if err != nil { + return nil, err + } + if existing != nil { + if existing.TargetMemberID != member.ID { + return nil, fmt.Errorf("team task message id already exists") + } + if existing.Status != models.TeamTaskStatusPending || existing.RedisStreamID != nil { + return teamTaskPayload(*existing) + } + } else { + payloadJSON, err := marshalJSON(req.Payload) + if err != nil { + return nil, fmt.Errorf("failed to encode task payload: %w", err) + } + now := time.Now().UTC() + existing = &models.TeamTask{ + TeamID: teamID, + TargetMemberID: member.ID, + CreatedBy: &userID, + MessageID: messageID, + Status: models.TeamTaskStatusPending, + WorkflowState: teamWorkflowStatePlanning, + PlanVersion: 0, + LedgerVersion: 0, + PayloadJSON: payloadJSON, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.repo.CreateTask(existing); err != nil { + return nil, err + } + } + task := existing + if err := s.prepareTeamTaskWorkspace(userID, team.ID, task.ID); err != nil { + return nil, err + } + + taskPayload := map[string]interface{}{} + if strings.TrimSpace(task.PayloadJSON) != "" { + if err := json.Unmarshal([]byte(task.PayloadJSON), &taskPayload); err != nil { + return nil, fmt.Errorf("failed to decode task payload: %w", err) + } + } + s.enrichTaskWorkspaceContract(userID, team, task, taskPayload) + enrichedPayloadJSON, err := marshalJSON(taskPayload) + if err != nil { + return nil, fmt.Errorf("failed to encode enriched task payload: %w", err) + } + if enrichedPayloadJSON != task.PayloadJSON { + task.PayloadJSON = enrichedPayloadJSON + task.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateTask(task); err != nil { + return nil, err + } + } + if strings.TrimSpace(eventString(taskPayload, "intent")) == initialLeaderTaskIntent && backendGeneratedBootstrapEnabled() { + return s.completeInitialLeaderTaskFromSnapshot(userID, team, task, member, taskPayload) + } + + bus, err := s.redisBusForTeam(context.Background(), team) + if err != nil { + return nil, err + } + now := time.Now().UTC() + memberInstance, _ := s.teamMemberInstance(member) + memberContext := buildTeamMemberTaskContext(member, memberInstance) + communicationMode := normalizedTeamCommunicationMode(team.CommunicationMode) + memberContext["communicationMode"] = communicationMode + memberContext["systemPrompt"] = appendTeamCollaborationGuidance(memberContext["systemPrompt"], communicationMode) + memberContext["systemPrompt"] = appendTeamWorkspaceGuidance(memberContext["systemPrompt"]) + envelope := buildTeamTaskEnvelope(teamID, member.MemberKey, task, messageID, taskPayload, memberContext, now) + envelopeJSON, err := marshalJSON(envelope) + if err != nil { + return nil, fmt.Errorf("failed to encode task envelope: %w", err) + } + streamID, err := bus.XAdd(context.Background(), teamInboxKey(team.ID, member.MemberKey), map[string]string{ + "payload": envelopeJSON, + "team_id": strconv.Itoa(team.ID), + "task_id": strconv.Itoa(task.ID), + "message_id": messageID, + "member_id": member.MemberKey, + }) + if err != nil { + return nil, err + } + task.Status = models.TeamTaskStatusDispatched + task.RedisStreamID = &streamID + task.DispatchedAt = &now + task.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateTask(task); err != nil { + return nil, err + } + return teamTaskPayload(*task) +} + +func (s *teamService) enrichBootstrapTaskPayload(userID int, team *models.Team, payload map[string]interface{}) error { + if s == nil || team == nil || payload == nil { + return nil + } + members, err := s.repo.ListMembersByTeamID(team.ID) + if err != nil { + return err + } + activeMembers := activeTeamMembers(members) + rosterJSON, err := buildTeamRosterConfigFromMembersWithSharedDir(team, activeMembers, team.SharedMountPath) + if err != nil { + return err + } + roster := map[string]interface{}{} + _ = json.Unmarshal([]byte(rosterJSON), &roster) + physicalSharedDir := s.teamRuntimeSharedPathFor(userID, team.ID) + workspaceContract := map[string]interface{}{ + "configPath": teamConfigMountPath, + "configEnv": "CLAWMANAGER_TEAM_CONFIG_PATH", + "sharedDir": team.SharedMountPath, + "sharedDirEnv": "CLAWMANAGER_TEAM_SHARED_DIR", + "physicalSharedDir": physicalSharedDir, + "sharedConfigPath": "/team/" + teamConfigFileName, + "fallbackConfigPaths": []string{"/team/" + teamConfigFileName}, + "writeRoot": "$CLAWMANAGER_TEAM_SHARED_DIR", + "canonicalReportPrefix": "/team", + "validReportPathPattern": "/team/", + "invalidConfigPaths": []string{"/team/members"}, + "invalidArtifactPrefixes": []string{"team/"}, + } + memberSnapshots := make([]map[string]interface{}, 0, len(activeMembers)) + for _, member := range activeMembers { + memberSnapshots = append(memberSnapshots, map[string]interface{}{ + "memberId": member.MemberKey, + "displayName": member.DisplayName, + "role": member.Role, + "runtimeType": member.RuntimeType, + "instanceMode": member.InstanceMode, + "description": derefTeamString(member.Description), + "status": member.Status, + "availability": member.Availability, + "progress": member.Progress, + "runtimeStatus": derefTeamString(member.RuntimeStatus), + "runtimeTaskId": derefTeamString(member.RuntimeTaskID), + "lastSummary": derefTeamString(member.LastSummary), + "isLeader": isTeamLeaderRole(member.Role), + }) + } + payload["workspaceContract"] = workspaceContract + payload["teamConfigJson"] = rosterJSON + payload["bootstrapSnapshot"] = map[string]interface{}{ + "teamId": team.ID, + "teamName": team.Name, + "communicationMode": normalizedTeamCommunicationMode(team.CommunicationMode), + "sharedDir": team.SharedMountPath, + "configPath": teamConfigMountPath, + "sharedConfigPath": "/team/" + teamConfigFileName, + "teamConfigJson": rosterJSON, + "roster": roster, + "members": memberSnapshots, + } + return nil +} + +func backendGeneratedBootstrapEnabled() bool { + raw := strings.TrimSpace(os.Getenv("TEAM_BOOTSTRAP_BACKEND_GENERATED")) + if raw == "" { + raw = strings.TrimSpace(os.Getenv("CLAWMANAGER_TEAM_BACKEND_BOOTSTRAP")) + } + if raw == "" { + return true + } + switch strings.ToLower(raw) { + case "0", "false", "no", "off", "disabled": + return false + default: + return true + } +} + +func (s *teamService) completeInitialLeaderTaskFromSnapshot(userID int, team *models.Team, task *models.TeamTask, leader *models.TeamMember, taskPayload map[string]interface{}) (*TeamTaskPayload, error) { + if s == nil || team == nil || task == nil || leader == nil { + return nil, fmt.Errorf("bootstrap completion requires team, task and leader") + } + members, err := s.repo.ListMembersByTeamID(team.ID) + if err != nil { + return nil, err + } + activeMembers := activeTeamMembers(members) + now := time.Now().UTC() + taskRef := fmt.Sprintf("team-%d-task-%d", team.ID, task.ID) + reportRel := filepath.ToSlash(filepath.Join("results", taskRef, "team-introduction.md")) + reportPath := filepath.Join(filepath.Clean(s.teamRuntimeSharedPathFor(userID, team.ID)), filepath.FromSlash(reportRel)) + if err := ensureTeamWorkspaceDirectory(filepath.Dir(reportPath)); err != nil { + return nil, err + } + resultMarkdown := buildBackendBootstrapReport(team, activeMembers, taskRef, taskPayload) + if err := os.WriteFile(reportPath, []byte(resultMarkdown), 0o664); err != nil { + return nil, fmt.Errorf("failed to write bootstrap report: %w", err) + } + _ = os.Chmod(reportPath, 0o664) + chownTeamWorkspacePath(reportPath) + artifactRef := "/team/" + reportRel + summary := fmt.Sprintf("Team %s 启动快照完成,已生成成员与协作机制介绍。", team.Name) + completionID := fmt.Sprintf("clawmanager-bootstrap:%s", taskRef) + eventID := fmt.Sprintf("%s:completed", completionID) + eventPayload := map[string]interface{}{ + "v": 2, + "protocolVersion": 2, + "event": "task_completed", + "type": "task_completed", + "eventKind": "leader_synthesis", + "intent": initialLeaderTaskIntent, + "origin": "system_bootstrap", + "source": "clawmanager", + "completionSource": "clawmanager_backend", + "completionId": completionID, + "explicitCompletion": true, + "rootTaskTerminal": true, + "teamId": strconv.Itoa(team.ID), + "taskId": taskRef, + "rootTaskId": taskRef, + "messageId": task.MessageID, + "rootMessageId": task.MessageID, + "from": leader.MemberKey, + "memberId": leader.MemberKey, + "status": models.TeamTaskStatusSucceeded, + "runtimeStatus": models.TeamTaskStatusSucceeded, + "availability": models.TeamMemberAvailabilityIdle, + "summary": summary, + "result": resultMarkdown, + "resultMarkdown": resultMarkdown, + "artifactRefs": []string{artifactRef}, + "visibleToChat": true, + "backendGenerated": true, + } + payloadJSON, err := marshalOptionalJSON(eventPayload) + if err != nil { + return nil, err + } + task.Status = models.TeamTaskStatusSucceeded + task.StartedAt = &now + task.FinishedAt = &now + task.ResultJSON = payloadJSON + task.ErrorMessage = nil + task.UpdatedAt = now + if err := s.repo.UpdateTask(task); err != nil { + return nil, err + } + leader.Status = models.TeamMemberStatusIdle + leader.CurrentTaskID = nil + leader.Progress = 100 + leader.Availability = models.TeamMemberAvailabilityIdle + runtimeStatus := models.TeamTaskStatusSucceeded + leader.RuntimeStatus = &runtimeStatus + leader.RuntimeTaskID = &task.MessageID + runtimeIntent := initialLeaderTaskIntent + leader.RuntimeIntent = &runtimeIntent + leader.BlockedReason = nil + leader.LastSummary = &summary + leader.LastSeenAt = &now + leader.UpdatedAt = now + if err := s.repo.UpdateMember(leader); err != nil { + return nil, err + } + messageID := task.MessageID + event := &models.TeamEvent{ + EventID: &eventID, + CompletionID: &completionID, + TeamID: team.ID, + MemberID: &leader.ID, + TaskID: &task.ID, + MessageID: &messageID, + EventType: "task_completed", + PayloadJSON: payloadJSON, + OccurredAt: &now, + CreatedAt: now, + } + if err := s.repo.CreateEvent(event); err != nil && !errors.Is(err, repository.ErrDuplicateTeamEvent) { + return nil, err + } + return teamTaskPayload(*task) +} + +func buildBackendBootstrapReport(team *models.Team, members []models.TeamMember, taskRef string, taskPayload map[string]interface{}) string { + var b strings.Builder + mode := normalizedTeamCommunicationMode(team.CommunicationMode) + b.WriteString("# Team 启动介绍\n\n") + b.WriteString(fmt.Sprintf("- Team:%s(ID %d)\n", team.Name, team.ID)) + b.WriteString(fmt.Sprintf("- 任务:%s\n", taskRef)) + b.WriteString(fmt.Sprintf("- 协作模式:%s\n", mode)) + b.WriteString(fmt.Sprintf("- 共享目录:%s\n\n", team.SharedMountPath)) + b.WriteString("## 成员\n\n") + b.WriteString("| 成员 ID | 显示名称 | 角色 | Runtime | Mode | 状态 | 职责 |\n") + b.WriteString("| --- | --- | --- | --- | --- | --- | --- |\n") + for _, member := range members { + b.WriteString(fmt.Sprintf( + "| %s | %s | %s | %s | %s | %s/%s | %s |\n", + markdownTableCell(member.MemberKey), + markdownTableCell(member.DisplayName), + markdownTableCell(member.Role), + markdownTableCell(member.RuntimeType), + markdownTableCell(member.InstanceMode), + markdownTableCell(member.Status), + markdownTableCell(member.Availability), + markdownTableCell(derefTeamString(member.Description)), + )) + } + b.WriteString("\n## 协作机制\n\n") + b.WriteString("- Leader 负责拆解任务、派发成员、整合结果,并最终关闭根任务。\n") + b.WriteString("- 成员结果应回到 Leader,由 Leader 进行最终汇总。\n") + b.WriteString("- Redis Streams 用于任务分发、成员回报、过程事件和完成事件同步。\n") + b.WriteString("- NFS 共享目录用于耐久化产物、任务快照和可恢复状态,不作为实时状态唯一事实源。\n\n") + b.WriteString("## 共享工作区\n\n") + b.WriteString("- 规范路径前缀:`/team/`\n") + if contract, ok := taskPayload["workspaceContract"].(map[string]interface{}); ok { + if sharedDir := eventString(contract, "sharedDir"); sharedDir != "" { + b.WriteString(fmt.Sprintf("- 容器共享目录:`%s`\n", sharedDir)) + } + if physicalDir := eventString(contract, "physicalSharedDir"); physicalDir != "" { + b.WriteString(fmt.Sprintf("- 物理共享目录:`%s`\n", physicalDir)) + } + } + b.WriteString("- 成员产物建议写入 `/team/artifacts//members//`。\n") + b.WriteString("- 根任务最终报告建议写入 `/team/results//`。\n\n") + b.WriteString("## 团队可用方法\n\n") + b.WriteString("- `team_send`:Leader 向成员派发任务。\n") + b.WriteString("- `team_update_progress`:记录业务计划、阶段进度、长任务状态和检查反馈。\n") + b.WriteString("- `team_status`:查询成员和任务状态。\n") + b.WriteString("- `team_complete_task`:仅用于成员提交分配结果或 Leader 关闭根任务。\n") + b.WriteString("- `team_artifact_write/read/list/mkdir`:在当前 Team 共享目录内安全读写产物,自动限制路径并使用协作权限。\n") + return b.String() +} + +func markdownTableCell(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "-" + } + value = strings.ReplaceAll(value, "\r\n", " ") + value = strings.ReplaceAll(value, "\n", " ") + value = strings.ReplaceAll(value, "|", "\\|") + return value +} + +func (s *teamService) prepareTeamTaskWorkspace(userID, teamID, taskID int) error { + if s == nil || taskID <= 0 { + return nil + } + root := filepath.Clean(s.teamRuntimeSharedPathFor(userID, teamID)) + taskRef := fmt.Sprintf("team-%d-task-%d", teamID, taskID) + standardDirs := []string{ + "", + "artifacts", + filepath.ToSlash(filepath.Join("artifacts", taskRef)), + filepath.ToSlash(filepath.Join("artifacts", taskRef, "members")), + "results", + filepath.ToSlash(filepath.Join("results", taskRef)), + filepath.ToSlash(filepath.Join("results", taskRef, "members")), + "tasks", + filepath.ToSlash(filepath.Join("tasks", taskRef)), + "inbox", + "status", + filepath.ToSlash(filepath.Join("status", taskRef)), + "tmp", + } + if s.repo != nil { + members, err := s.repo.ListMembersByTeamID(teamID) + if err != nil { + return err + } + for _, member := range activeTeamMembers(members) { + memberKey := normalizeTeamMemberRouteKey(member.MemberKey) + if memberKey == "" { + continue + } + standardDirs = append(standardDirs, + filepath.ToSlash(filepath.Join("artifacts", taskRef, "members", memberKey)), + filepath.ToSlash(filepath.Join("results", taskRef, "members", memberKey)), + filepath.ToSlash(filepath.Join("tmp", memberKey)), + ) + } + } + for _, rel := range standardDirs { + target := root + if rel != "" { + target = filepath.Join(root, filepath.FromSlash(rel)) + } + if err := ensureTeamWorkspaceDirectory(target); err != nil { + return err + } + } + return nil +} + +func (s *teamService) enrichTaskWorkspaceContract(userID int, team *models.Team, task *models.TeamTask, payload map[string]interface{}) { + if s == nil || team == nil || task == nil || payload == nil { + return + } + taskRef := fmt.Sprintf("team-%d-task-%d", team.ID, task.ID) + physicalSharedDir := s.teamRuntimeSharedPathFor(userID, team.ID) + contract := map[string]interface{}{ + "configPath": teamConfigMountPath, + "configEnv": "CLAWMANAGER_TEAM_CONFIG_PATH", + "sharedDir": team.SharedMountPath, + "sharedDirEnv": "CLAWMANAGER_TEAM_SHARED_DIR", + "physicalSharedDir": physicalSharedDir, + "sharedConfigPath": "/team/" + teamConfigFileName, + "fallbackConfigPaths": []string{"/team/" + teamConfigFileName}, + "writeRoot": "$CLAWMANAGER_TEAM_SHARED_DIR", + "canonicalReportPrefix": "/team", + "validReportPathPattern": "/team/", + "invalidConfigPaths": []string{"/team/members"}, + "invalidArtifactPrefixes": []string{"team/"}, + "taskRef": taskRef, + "artifactRoot": "/team/artifacts/" + taskRef, + "memberArtifactRoot": "/team/artifacts/" + taskRef + "/members/${memberId}", + "memberArtifactPhysicalRoot": filepath.ToSlash(filepath.Join(physicalSharedDir, "artifacts", taskRef, "members", "${memberId}")), + "memberResultRoot": "/team/results/" + taskRef + "/members/${memberId}", + "leaderResultRoot": "/team/results/" + taskRef, + "statusRoot": "/team/status/" + taskRef, + "tmpRoot": "/team/tmp/${memberId}", + "statusFilesAreAdvisory": true, + "stateAuthority": "clawmanager_event_ledger", + "rules": []string{ + "Use /team/artifacts//members/// for member deliverables.", + "Use /team/results//members// for member result summaries.", + "Only the Leader writes the root final synthesis under /team/results//.", + "Shared status JSON files are compatibility snapshots; do not treat them as the task truth source.", + }, + } + if existing, ok := payload["workspaceContract"].(map[string]interface{}); ok { + for key, value := range contract { + if _, exists := existing[key]; !exists { + existing[key] = value + } + } + return + } + payload["workspaceContract"] = contract +} + +func (s *teamService) teamMemberInstance(member *models.TeamMember) (*models.Instance, error) { + if s == nil || s.instanceService == nil || member == nil || member.InstanceID == nil || *member.InstanceID <= 0 { + return nil, nil + } + return s.instanceService.GetByID(*member.InstanceID) +} + +func buildTeamMemberTaskContext(member *models.TeamMember, instance *models.Instance) map[string]string { + if member == nil { + return map[string]string{} + } + displayName := strings.TrimSpace(member.DisplayName) + if displayName == "" { + displayName = member.MemberKey + } + role := strings.TrimSpace(member.Role) + if role == "" { + role = "member" + } + description := derefTeamString(member.Description) + personaSystemPrompt, personaDescription := teamMemberPersonaFromInstance(instance) + if description == "" { + description = personaDescription + } + systemPrompt := buildTeamMemberSystemPrompt(displayName, member.MemberKey, role, description, personaSystemPrompt) + return map[string]string{ + "memberId": member.MemberKey, + "displayName": displayName, + "role": role, + "description": description, + "systemPrompt": systemPrompt, + } +} + +func buildPlannedTeamMemberTaskContext(member plannedTeamMember) map[string]string { + displayName := strings.TrimSpace(member.DisplayName) + if displayName == "" { + displayName = member.MemberKey + } + role := strings.TrimSpace(member.Role) + if role == "" { + role = "member" + } + description := strings.TrimSpace(derefTeamString(member.Request.Description)) + personaSystemPrompt, personaDescription := teamMemberPersonaFromEnv(member.Request.EnvironmentOverrides) + if description == "" { + description = personaDescription + } + systemPrompt := buildTeamMemberSystemPrompt(displayName, member.MemberKey, role, description, personaSystemPrompt) + return map[string]string{ + "memberId": member.MemberKey, + "displayName": displayName, + "role": role, + "description": description, + "systemPrompt": systemPrompt, + } +} + +func buildTeamMemberSystemPrompt(displayName, memberID, role, description, personaSystemPrompt string) string { + systemPrompt := strings.TrimSpace(fmt.Sprintf( + "You are Team member %q (member_id=%s, role=%s). Follow this role for this task. Role responsibilities: %s", + displayName, + memberID, + role, + description, + )) + if strings.TrimSpace(description) == "" { + systemPrompt = fmt.Sprintf( + "You are Team member %q (member_id=%s, role=%s). Follow this role for this task.", + displayName, + memberID, + role, + ) + } + if personaSystemPrompt != "" { + systemPrompt = personaSystemPrompt + "\n\n" + systemPrompt + } + return systemPrompt +} + +func buildTeamMemberSoulMarkdown(member plannedTeamMember, communicationMode string) string { + context := buildPlannedTeamMemberTaskContext(member) + lines := []string{ + fmt.Sprintf("# %s", strings.TrimSpace(context["displayName"])), + "", + "You are running as a ClawManager Team member. Treat this file as persistent identity and role guidance.", + "", + "## Team Identity", + fmt.Sprintf("- Member ID: %s", context["memberId"]), + fmt.Sprintf("- Display name: %s", context["displayName"]), + fmt.Sprintf("- Role: %s", context["role"]), + fmt.Sprintf("- Effective role: %s", effectiveTeamMemberRole(member)), + } + if profileKey := strings.TrimSpace(member.ProfileKey); profileKey != "" { + lines = append(lines, fmt.Sprintf("- Profile key: %s", profileKey)) + } + if profileName := strings.TrimSpace(member.ProfileName); profileName != "" { + lines = append(lines, fmt.Sprintf("- Profile name: %s", profileName)) + } + if description := strings.TrimSpace(context["description"]); description != "" { + lines = append(lines, fmt.Sprintf("- Responsibilities: %s", description)) + } + lines = append(lines, + "", + "## Role Instructions", + strings.TrimSpace(context["systemPrompt"]), + "", + "## Collaboration Rules", + teamCollaborationGuidance(communicationMode), + "- Only handle tasks addressed to your Team member inbox.", + "- If team.json contains effectiveRole/profileName, use those fields when describing your role instead of falling back to a generic roster role.", + "- Use the exact CLAWMANAGER_TEAM_SHARED_DIR value for shared context, durable notes, and handoff artifacts. When using shell commands, always create files under \"$CLAWMANAGER_TEAM_SHARED_DIR/\".", + "- Never create or report a relative team/... folder. The path team/... is invalid because ClawManager file browsing only resolves shared artifacts through the Team shared directory.", + "- Report shared artifact links as /team/. /team is the canonical ClawManager UI path even when a Lite runtime uses a different physical directory.", + "- Report progress, blockers, verification evidence, and final results through the Team channel.", + "- If asked about your role, answer from this Team Identity and Role Instructions section.", + "", + ) + return strings.Join(lines, "\n") +} + +func buildTeamMemberAgentsMarkdown(team *models.Team, member plannedTeamMember) string { + communicationMode := teamCommunicationModeLeaderMediated + teamID := "" + if team != nil { + communicationMode = normalizedTeamCommunicationMode(team.CommunicationMode) + teamID = strconv.Itoa(team.ID) + } + lines := []string{ + "# ClawManager Team Runtime", + "", + "This file defines the stable Team runtime contract. Treat SOUL.md as the member-specific identity and role file.", + "", + "## Runtime Capabilities", + "- You are running inside a ClawManager managed OpenClaw/Hermes runtime.", + "- Use the available runtime tools normally, but coordinate Team work through the ClawManager Team channel.", + "- Use team_send for assignments, handoffs, clarifying questions, blockers, and final delivery messages.", + "- Use team_status / progress updates to report work state when available.", + "- Use team_complete_task only when the assigned task is actually complete and evidence has been reported.", + "", + "## Workspace Contract", + "- CLAWMANAGER_TEAM_SHARED_DIR is the only writable shared artifact directory.", + "- Create shared artifacts under \"$CLAWMANAGER_TEAM_SHARED_DIR/\".", + "- Report shared artifact links as /team/.", + "- Do not report bare filenames or relative team/... paths as final artifact links.", + "", + "## Team Identity Source Order", + "- Prefer SOUL.md for your member identity, role, profile, and collaboration rules.", + "- Then use CLAWMANAGER_TEAM_CONFIG_JSON / team.json for roster and communication mode.", + "- Environment variables are compatibility fallbacks, not a reason to ignore SOUL.md.", + "", + "## Collaboration Contract", + teamCollaborationGuidance(communicationMode), + "- Never invent another member's reply or treat your own assumptions as a peer response.", + "- Keep role answers consistent with SOUL.md and the effectiveRole/profileName fields in team.json.", + "", + "## Current Member", + fmt.Sprintf("- Team ID: %s", teamID), + fmt.Sprintf("- Member ID: %s", member.MemberKey), + fmt.Sprintf("- Display name: %s", member.DisplayName), + fmt.Sprintf("- Runtime: %s", member.RuntimeType), + fmt.Sprintf("- Effective role: %s", effectiveTeamMemberRole(member)), + } + if member.ProfileKey != "" { + lines = append(lines, fmt.Sprintf("- Profile key: %s", member.ProfileKey)) + } + if member.ProfileName != "" { + lines = append(lines, fmt.Sprintf("- Profile name: %s", member.ProfileName)) + } + return strings.Join(lines, "\n") + "\n" +} + +func plannedTeamMemberDescription(member plannedTeamMember) string { + if description := strings.TrimSpace(derefTeamString(member.Request.Description)); description != "" { + return description + } + _, description := teamMemberPersonaFromEnv(member.Request.EnvironmentOverrides) + return strings.TrimSpace(description) +} + +func effectiveTeamMemberRole(member plannedTeamMember) string { + if role := strings.TrimSpace(member.EffectiveRole); role != "" { + return role + } + if role := strings.TrimSpace(member.Role); role != "" { + return role + } + return "member" +} + +func (s *teamService) writeLiteTeamMemberIdentityFiles(instance *models.Instance, team *models.Team, member plannedTeamMember, rosterJSON string) error { + if instance == nil || modeForExistingInstance(instance) != InstanceModeLite { + return nil + } + workspacePath := "" + if instance.WorkspacePath != nil { + workspacePath = strings.TrimSpace(*instance.WorkspacePath) + } + if workspacePath == "" { + return fmt.Errorf("failed to write Lite Team identity files: workspace path is empty") + } + if err := os.MkdirAll(workspacePath, 0755); err != nil { + return fmt.Errorf("failed to prepare Lite Team identity workspace: %w", err) + } + files := map[string]string{ + teamAgentsFileName: buildTeamMemberAgentsMarkdown(team, member), + teamSoulFileName: buildTeamMemberSoulMarkdown(member, normalizedTeamCommunicationMode(team.CommunicationMode)), + } + if strings.TrimSpace(rosterJSON) != "" { + files[teamConfigFileName] = rosterJSON + } + for name, content := range files { + target := filepath.Join(workspacePath, name) + if err := os.WriteFile(target, []byte(content), 0644); err != nil { + return fmt.Errorf("failed to write Lite Team identity file %s: %w", name, err) + } + chownTeamWorkspacePath(target) + } + if strings.EqualFold(member.RuntimeType, "hermes") { + hermesDir := filepath.Join(workspacePath, ".hermes") + if err := os.MkdirAll(hermesDir, 0755); err != nil { + return fmt.Errorf("failed to prepare Hermes identity directory: %w", err) + } + chownTeamWorkspacePath(hermesDir) + target := filepath.Join(hermesDir, teamSoulFileName) + if err := os.WriteFile(target, []byte(files[teamSoulFileName]), 0644); err != nil { + return fmt.Errorf("failed to write Hermes Lite SOUL.md: %w", err) + } + chownTeamWorkspacePath(target) + } + return nil +} + +func appendTeamCollaborationGuidance(systemPrompt, communicationMode string) string { + guidance := teamCollaborationGuidance(communicationMode) + if strings.Contains(systemPrompt, guidance) { + return systemPrompt + } + return strings.TrimSpace(systemPrompt) + "\n\n" + guidance +} + +func appendTeamWorkspaceGuidance(systemPrompt string) string { + guidance := "Shared workspace contract: write every shared artifact under the exact CLAWMANAGER_TEAM_SHARED_DIR value. When using shell commands, always create files under \"$CLAWMANAGER_TEAM_SHARED_DIR/\". Never list, search, resolve, or scan the parent of CLAWMANAGER_TEAM_SHARED_DIR, and never inspect sibling Team directories. Never create or report a relative team/... directory; team/... is invalid and may not be visible in ClawManager. When reporting an artifact to ClawManager or another member, use the canonical link /team/. The /team prefix is a UI/logical alias and may map to a different physical directory in Lite runtimes." + if strings.Contains(systemPrompt, guidance) { + return systemPrompt + } + return strings.TrimSpace(systemPrompt) + "\n\n" + guidance +} + +func teamCollaborationGuidance(communicationMode string) string { + switch normalizedTeamCommunicationMode(communicationMode) { + case teamCommunicationModePeerAssisted: + return "Collaboration mode: peer_assisted / worker-direct. This mode is isolated from leader_mediated flow: the Leader still owns final user-facing synthesis, but members must hand off directly to the named downstream owner when the root task, collaboration plan, or current instruction specifies one. Direct handoff is mandatory, not optional; sending only to the Leader is allowed only when there is no named downstream owner, when blocked, or when final synthesis is explicitly required. Preserve rootTaskId/rootMessageId, artifact paths, requested next action, acceptance criteria, and reply-required status in every peer message. Ask peer questions through the Team channel, then wait for the addressed member's real reply before continuing dependent work; never simulate, invent, or reinterpret another member's answer as if the user said it. Write durable artifacts under CLAWMANAGER_TEAM_SHARED_DIR, report peer outcomes through the Team channel, and let the Leader close the root task only after final synthesis. When receiving a peer request, respond with explicit evidence, artifact paths, blockers, or review findings so the requester can finish its own task." + case teamCommunicationModeFullMesh: + return "Collaboration mode: full_mesh. Team members coordinate directly with each other while preserving rootTaskId/rootMessageId context, shared artifacts under CLAWMANAGER_TEAM_SHARED_DIR, and final user-facing synthesis. If a downstream owner is named, hand off to that exact member before completing your own step. Use direct member-to-member messages for parallel research, design, implementation, review, and verification. Wait for real addressed-member replies before continuing dependent work; do not simulate peer answers or label peer messages as user replies. Keep each peer exchange bounded, evidence based, and visible in the Team channel." + default: + return "Collaboration mode: leader_mediated. This is a strict hub-and-spoke workflow isolated from worker-direct flow. User root tasks enter through the Leader. If the user names a non-Leader member or role, the Leader must delegate to that exact member with team_send, wait for that member's real result, then synthesize the final answer. If the user gives a broad task without naming one member, the Leader must create a compact plan, decompose work by owner/member_id, send assignments, wait for required member results, verify them, and produce final synthesis. Every delegated assignment must carry a stable workId and assignmentId; reuse that workId for all progress, result, and review messages belonging to the assignment. The Leader may answer directly only for self-contained control-plane or simple tasks that do not require a named worker or multi-member evidence. A dispatch, plan, or handoff is not a final result and must not close the root task. Workers execute only assignments addressed to them, preserve rootTaskId/rootMessageId/workId and artifact paths, and report results or blockers back to the Leader. Workers must not hand off directly to other workers. A worker completion never closes the user root task; only the Leader may finalize it after all required outputs are reconciled." + } +} + +func teamMemberPersonaFromInstance(instance *models.Instance) (string, string) { + if instance == nil { + return "", "" + } + overrides, err := parseEnvironmentOverridesJSON(instance.EnvironmentOverridesJSON) + if err != nil || len(overrides) == 0 { + return "", "" + } + return teamMemberPersonaFromEnv(overrides) +} + +func teamMemberPersonaFromEnv(overrides map[string]string) (string, string) { + if len(overrides) == 0 { + return "", "" + } + systemPrompt := strings.TrimSpace(firstNonEmptyEnv(overrides, + "CLAWMANAGER_AGENT_SYSTEM_PROMPT", + "CLAWMANAGER_HERMES_SYSTEM_PROMPT", + "CLAWMANAGER_RUNTIME_SYSTEM_PROMPT", + "HERMES_SYSTEM_PROMPT", + )) + description := "" + for _, key := range []string{ + "CLAWMANAGER_AGENT_PERSONA_JSON", + "CLAWMANAGER_HERMES_PERSONA_JSON", + "CLAWMANAGER_RUNTIME_PERSONA_JSON", + } { + persona := parseTeamPersonaEnv(overrides[key]) + if persona == nil { + continue + } + if systemPrompt == "" { + systemPrompt = strings.TrimSpace(persona.SystemPrompt) + } + if description == "" { + description = strings.TrimSpace(persona.Summary) + } + } + for _, key := range []string{ + "CLAWMANAGER_HERMES_AGENTS_JSON", + "CLAWMANAGER_RUNTIME_AGENTS_JSON", + "CLAWMANAGER_OPENCLAW_AGENTS_JSON", + } { + agents := parseTeamAgentsEnv(overrides[key]) + if agents == nil { + continue + } + if systemPrompt == "" { + systemPrompt = strings.TrimSpace(agents.SystemPrompt) + } + if description == "" { + description = strings.TrimSpace(agents.Summary) + } + } + return systemPrompt, description +} + +type teamPersonaEnv struct { + ProfileKey string `json:"profileKey"` + Name string `json:"name"` + DisplayName string `json:"displayName"` + RoleHint string `json:"roleHint"` + SystemPrompt string `json:"systemPrompt"` + Summary string `json:"summary"` +} + +type teamProfileEnv struct { + ProfileKey string + ProfileName string + RoleHint string + Summary string +} + +func teamMemberProfileFromEnv(overrides map[string]string) teamProfileEnv { + if len(overrides) == 0 { + return teamProfileEnv{} + } + for _, key := range []string{ + "CLAWMANAGER_AGENT_PERSONA_JSON", + "CLAWMANAGER_HERMES_PERSONA_JSON", + "CLAWMANAGER_RUNTIME_PERSONA_JSON", + } { + persona := parseTeamPersonaEnv(overrides[key]) + if persona == nil { + continue + } + profile := teamProfileEnv{ + ProfileKey: strings.TrimSpace(persona.ProfileKey), + ProfileName: strings.TrimSpace(firstNonEmptyString(persona.DisplayName, persona.Name)), + RoleHint: strings.TrimSpace(persona.RoleHint), + Summary: strings.TrimSpace(persona.Summary), + } + if profile.ProfileKey != "" || profile.ProfileName != "" || profile.RoleHint != "" || profile.Summary != "" { + return profile + } + } + for _, key := range []string{ + "CLAWMANAGER_HERMES_AGENTS_JSON", + "CLAWMANAGER_RUNTIME_AGENTS_JSON", + "CLAWMANAGER_OPENCLAW_AGENTS_JSON", + } { + agents := parseTeamAgentsEnv(overrides[key]) + if agents == nil { + continue + } + profile := teamProfileEnv{ + ProfileKey: strings.TrimSpace(agents.ProfileKey), + ProfileName: strings.TrimSpace(agents.Name), + RoleHint: strings.TrimSpace(agents.RoleHint), + Summary: strings.TrimSpace(agents.Summary), + } + if profile.ProfileKey != "" || profile.ProfileName != "" || profile.RoleHint != "" || profile.Summary != "" { + return profile + } + } + return teamProfileEnv{} +} + +func parseTeamPersonaEnv(raw string) *teamPersonaEnv { + if strings.TrimSpace(raw) == "" { + return nil + } + var persona teamPersonaEnv + if err := json.Unmarshal([]byte(raw), &persona); err != nil { + return nil + } + return &persona +} + +type teamAgentsEnv struct { + Items []struct { + Content struct { + Config struct { + ProfileKey string `json:"profileKey"` + Name string `json:"name"` + DisplayName string `json:"displayName"` + RoleHint string `json:"roleHint"` + SystemPrompt string `json:"systemPrompt"` + Summary string `json:"summary"` + } `json:"config"` + } `json:"content"` + } `json:"items"` +} + +func parseTeamAgentsEnv(raw string) *teamPersonaEnv { + if strings.TrimSpace(raw) == "" { + return nil + } + var payload teamAgentsEnv + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + return nil + } + for _, item := range payload.Items { + systemPrompt := strings.TrimSpace(item.Content.Config.SystemPrompt) + summary := strings.TrimSpace(item.Content.Config.Summary) + profileKey := strings.TrimSpace(item.Content.Config.ProfileKey) + name := strings.TrimSpace(firstNonEmptyString(item.Content.Config.DisplayName, item.Content.Config.Name)) + roleHint := strings.TrimSpace(item.Content.Config.RoleHint) + if systemPrompt != "" || summary != "" || profileKey != "" || name != "" || roleHint != "" { + return &teamPersonaEnv{ + ProfileKey: profileKey, + Name: name, + RoleHint: roleHint, + SystemPrompt: systemPrompt, + Summary: summary, + } + } + } + return nil +} + +func firstNonEmptyString(values ...string) string { + for _, value := range values { + if trimmed := strings.TrimSpace(value); trimmed != "" { + return trimmed + } + } + return "" +} + +func firstNonEmptyEnv(values map[string]string, keys ...string) string { + for _, key := range keys { + if value := strings.TrimSpace(values[key]); value != "" { + return value + } + } + return "" +} + +func buildTeamRuntimePrompt(rawPrompt string, memberContext map[string]string) string { + prompt := strings.TrimSpace(rawPrompt) + if len(memberContext) == 0 { + return prompt + } + systemPrompt := strings.TrimSpace(memberContext["systemPrompt"]) + if systemPrompt == "" { + return prompt + } + if prompt == "" { + return systemPrompt + } + return fmt.Sprintf("%s\n\nUser task:\n%s", systemPrompt, prompt) +} + +func (s *teamService) DeleteTeam(userID, teamID int) error { + team, err := s.requireOwnedTeam(userID, teamID) + if err != nil { + return err + } + if team.Status == models.TeamStatusDeleted { + return nil + } + + now := time.Now().UTC() + team.Status = models.TeamStatusDeleting + team.UpdatedAt = now + if err := s.repo.UpdateTeam(team); err != nil { + return err + } + + members, err := s.repo.ListMembersByTeamID(teamID) + if err != nil { + return err + } + for idx := range members { + member := members[idx] + if member.Status == models.TeamMemberStatusDeleted { + continue + } + member.Status = models.TeamMemberStatusDeleting + member.UpdatedAt = time.Now().UTC() + _ = s.repo.UpdateMember(&member) + if member.InstanceID != nil && *member.InstanceID > 0 { + if err := s.instanceService.Delete(*member.InstanceID); err != nil { + fmt.Printf("Warning: failed to delete Team %d member %s instance %d: %v\n", teamID, member.MemberKey, *member.InstanceID, err) + } + } + member.Status = models.TeamMemberStatusDeleted + member.CurrentTaskID = nil + member.UpdatedAt = time.Now().UTC() + _ = s.repo.UpdateMember(&member) + } + + ctx := context.Background() + if strings.TrimSpace(derefTeamString(team.TeamTokenSecretName)) != "" { + if err := s.secretService.DeleteSecret(ctx, userID, derefTeamString(team.TeamTokenSecretName)); err != nil { + fmt.Printf("Warning: failed to delete Team %d secret: %v\n", teamID, err) + } + } + if err := s.configMapService.DeleteConfigMap(ctx, userID, s.teamConfigMapName(teamID)); err != nil { + fmt.Printf("Warning: failed to delete Team %d configmap: %v\n", teamID, err) + } + if err := s.pvcService.DeleteTeamSharedPVC(ctx, userID, teamID); err != nil { + fmt.Printf("Warning: failed to delete Team %d shared PVC: %v\n", teamID, err) + } + + team.Name = deletedTeamName(team.Name, team.ID) + team.Status = models.TeamStatusDeleted + team.UpdatedAt = time.Now().UTC() + return s.repo.UpdateTeam(team) +} + +func (s *teamService) DeleteMember(userID, teamID int, memberID string) error { + team, err := s.requireOwnedTeam(userID, teamID) + if err != nil { + return err + } + member, err := s.findTeamMemberForDelete(teamID, memberID) + if err != nil { + return err + } + if member == nil { + return fmt.Errorf("team member not found") + } + if member.UserID != userID || member.TeamID != teamID { + return fmt.Errorf("access denied") + } + if member.Status == models.TeamMemberStatusDeleted { + return nil + } + if isTeamLeaderRole(member.Role) { + return fmt.Errorf("team leader cannot be deleted before assigning a new leader") + } + + now := time.Now().UTC() + member.Status = models.TeamMemberStatusDeleting + member.UpdatedAt = now + if err := s.repo.UpdateMember(member); err != nil { + return err + } + if member.InstanceID != nil && *member.InstanceID > 0 { + if err := s.instanceService.Delete(*member.InstanceID); err != nil { + return err + } + } + member.Status = models.TeamMemberStatusDeleted + member.CurrentTaskID = nil + member.Progress = 0 + member.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateMember(member); err != nil { + return err + } + return s.refreshTeamRosterConfig(userID, team) +} + +func (s *teamService) findTeamMemberForDelete(teamID int, memberID string) (*models.TeamMember, error) { + value := strings.TrimSpace(memberID) + if value == "" { + return nil, fmt.Errorf("team member id is required") + } + if numericID, err := strconv.Atoi(value); err == nil && numericID > 0 { + member, err := s.repo.GetMemberByID(numericID) + if err != nil || member == nil || member.TeamID != teamID { + return member, err + } + return member, nil + } + return s.repo.GetMemberByTeamKey(teamID, value) +} + +func (s *teamService) refreshTeamRosterConfig(userID int, team *models.Team) error { + members, err := s.repo.ListMembersByTeamID(team.ID) + if err != nil { + return err + } + rosterJSON, err := buildTeamRosterConfigFromMembers(team, activeTeamMembers(members)) + if err != nil { + return err + } + configData := map[string]string{ + teamConfigFileName: rosterJSON, + } + for _, member := range activeTeamMembers(members) { + if member.RuntimeType != "hermes" { + continue + } + configData[teamMemberSoulConfigKey(member.MemberKey)] = buildTeamMemberSoulMarkdown(plannedTeamMember{ + Request: CreateTeamMemberRequest{ + Description: member.Description, + }, + MemberKey: member.MemberKey, + DisplayName: member.DisplayName, + Role: member.Role, + RuntimeType: member.RuntimeType, + InstanceMode: member.InstanceMode, + IsLeader: isTeamLeaderRole(member.Role), + }, normalizedTeamCommunicationMode(team.CommunicationMode)) + } + if err := s.configMapService.UpsertConfigMap(context.Background(), userID, s.teamConfigMapName(team.ID), configData, map[string]string{ + "app": "clawreef", + "managed-by": "clawreef", + "team-id": strconv.Itoa(team.ID), + }); err != nil { + return err + } + return s.writeSharedTeamRosterConfig(userID, team, rosterJSON) +} + +func (s *teamService) requireOwnedTeam(userID, teamID int) (*models.Team, error) { + team, err := s.repo.GetTeamByID(teamID) + if err != nil { + return nil, err + } + if team == nil { + return nil, fmt.Errorf("team not found") + } + if team.UserID != userID { + return nil, fmt.Errorf("access denied") + } + return team, nil +} + +func (s *teamService) ensureConsumer(ctx context.Context, teamID int) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.running { + return + } + if _, exists := s.consumers[teamID]; exists { + return + } + s.consumers[teamID] = struct{}{} + s.wg.Add(1) + go s.consumeTeamEvents(ctx, teamID) +} + +func (s *teamService) consumeTeamEvents(ctx context.Context, teamID int) { + defer s.wg.Done() + defer func() { + s.mu.Lock() + delete(s.consumers, teamID) + s.mu.Unlock() + }() + + for { + select { + case <-ctx.Done(): + return + default: + } + + team, err := s.repo.GetTeamByID(teamID) + if err != nil || team == nil { + time.Sleep(5 * time.Second) + continue + } + bus, err := s.redisBusForTeam(ctx, team) + if err != nil { + time.Sleep(5 * time.Second) + continue + } + lastID := strings.TrimSpace(team.RedisEventsLastID) + if lastID == "" { + lastID = "0-0" + } + messages, err := bus.XRead(ctx, teamEventsKey(teamID), lastID, 5*time.Second) + if err != nil { + time.Sleep(2 * time.Second) + continue + } + for _, message := range messages { + if err := s.projectTeamEvent(team, bus, message); err != nil { + fmt.Printf("Warning: failed to project Team %d event %s: %v\n", teamID, message.ID, err) + } + team.RedisEventsLastID = message.ID + team.UpdatedAt = time.Now().UTC() + _ = s.repo.UpdateTeam(team) + } + } +} + +func (s *teamService) ensureStaleTaskMonitor(ctx context.Context) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.running { + return + } + if s.staleMonitorStarted { + return + } + s.staleMonitorStarted = true + s.wg.Add(1) + go s.monitorStaleTasks(ctx) +} + +func (s *teamService) monitorStaleTasks(ctx context.Context) { + defer s.wg.Done() + ticker := time.NewTicker(teamTaskStaleSweepInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := s.sweepTeamEventOutbox(); err != nil { + fmt.Printf("Warning: failed to deliver Team event outbox: %v\n", err) + } + if err := s.sweepStaleTasks(); err != nil { + fmt.Printf("Warning: failed to sweep stale Team tasks: %v\n", err) + } + if err := s.sweepAssignmentStatusChecks(); err != nil { + fmt.Printf("Warning: failed to sweep Team assignment status checks: %v\n", err) + } + } + } +} + +func (s *teamService) sweepTeamEventOutbox() error { + if s == nil || s.repo == nil { + return nil + } + now := time.Now().UTC() + rows, err := s.repo.ListPendingEventOutbox(now, teamEventOutboxBatchSize) + if err != nil { + return err + } + var errs []error + for idx := range rows { + row := rows[idx] + team, getErr := s.repo.GetTeamByID(row.TeamID) + if getErr != nil { + errs = append(errs, getErr) + continue + } + if team == nil || team.Status == models.TeamStatusDeleted || team.Status == models.TeamStatusDeleting { + continue + } + bus, busErr := s.redisBusForTeam(context.Background(), team) + if busErr != nil { + _ = s.repo.MarkEventOutboxFailed(row.ID, now.Add(teamOutboxRetryDelay(row.Attempts)), busErr.Error()) + errs = append(errs, busErr) + continue + } + if deliverErr := s.deliverTeamEventOutbox(team, bus, &row); deliverErr != nil { + _ = s.repo.MarkEventOutboxFailed(row.ID, now.Add(teamOutboxRetryDelay(row.Attempts)), deliverErr.Error()) + errs = append(errs, deliverErr) + continue + } + if markErr := s.repo.MarkEventOutboxDelivered(row.ID, time.Now().UTC()); markErr != nil { + errs = append(errs, markErr) + } + } + return errors.Join(errs...) +} + +func teamOutboxRetryDelay(attempts int) time.Duration { + if attempts < 0 { + attempts = 0 + } + if attempts > 6 { + attempts = 6 + } + return time.Duration(1< 0 { + taskIDs[items[idx].RootTaskID] = struct{}{} + } + } + var errs []error + for taskID := range taskIDs { + task, err := s.repo.GetTaskByID(taskID) + if err != nil { + errs = append(errs, err) + continue + } + if task == nil || task.TeamID != team.ID || isTerminalTeamTaskStatus(task.Status) { + continue + } + leader := membersByID[task.TargetMemberID] + if leader == nil || !isLeaderTeamMember(leader) || !isActiveTeamMember(leader) { + continue + } + // Phase state is derived data. Repair it before applying the normal + // reminder age gate so a stuck workflow can advance without another + // Leader model round. + ledgerRepaired, repairErr := s.reconcileTeamWorkflowLedger(task, false, now) + if repairErr != nil { + errs = append(errs, repairErr) + continue + } + reconciled, reconcileErr := s.reconcileDeferredTeamCompletion(team, bus, task, leader) + if reconcileErr != nil { + errs = append(errs, reconcileErr) + continue + } + if reconciled || isTerminalTeamTaskStatus(task.Status) { + continue + } + if !ledgerRepaired && !task.UpdatedAt.IsZero() && task.UpdatedAt.After(cutoff) { + continue + } + ready, resultItems := leaderMediatedRootNeedsSynthesisReminder(task, items, membersByID) + if !ready { + continue + } + monitorKey := fmt.Sprintf("%d:%d:%s:%d", team.ID, task.ID, task.WorkflowState, task.LedgerVersion) + if !s.claimAssignmentMonitorSlot(monitorKey, now) { + continue + } + teamBus := bus + if teamBus == nil { + teamBus, err = s.redisBusForTeam(context.Background(), team) + if err != nil { + errs = append(errs, err) + continue + } + } + if err := s.createLeaderSynthesisReminder(team, teamBus, task, leader, resultItems, now); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (s *teamService) reconcileDeferredTeamCompletion(team *models.Team, bus *redisBus, task *models.TeamTask, leader *models.TeamMember) (bool, error) { + if s == nil || team == nil || task == nil || leader == nil || isTerminalTeamTaskStatus(task.Status) { + return false, nil + } + events, err := s.repo.ListEventsByTeamID(team.ID, 500) + if err != nil { + return false, err + } + for idx := range events { + event := events[idx] + if event.TaskID == nil || *event.TaskID != task.ID || event.EventType != "completion_deferred" { + continue + } + payload := teamEventPayloadMap(event) + if !eventBool(payload, "explicitCompletion", "explicit_completion") || eventString(payload, "completionId", "completion_id") == "" { + continue + } + proposalPlanVersion := int64(eventInt(payload, "planVersion", "plan_version")) + if proposalPlanVersion > 0 && task.PlanVersion > 0 && proposalPlanVersion != task.PlanVersion { + // A completion report produced for an older plan cannot summarize work + // introduced by a later plan. Ask the Leader to synthesize again. + continue + } + if teamRedisProtocolVersion(payload) >= 3 && (!eventBool(payload, "workflowFinal", "workflow_final", "sealWorkflow", "seal_workflow") || + !eventBool(payload, "finalAnswerReady", "final_answer_ready") || + len(normalizeContextRefs(firstTeamValue(payload, "remainingActions", "remaining_actions", "nextActions", "next_actions"))) > 0) { + continue + } + // Only a current, explicitly sealed completion proposal may retire an + // unused planned phase. An older report must not mutate a newer plan. + if _, err := s.reconcileTeamWorkflowLedger(task, true, time.Now().UTC()); err != nil { + return false, err + } + payload["event"] = "completion_proposed" + payload["type"] = "completion_proposed" + payload["rootTaskTerminal"] = true + payload["status"] = models.TeamTaskStatusSucceeded + payload["runtimeStatus"] = models.TeamTaskStatusSucceeded + payload["eventId"] = fmt.Sprintf("completion-reconcile:%d:%d:%s:%d", team.ID, task.ID, normalizeTeamRedisKeyPart(eventString(payload, "completionId", "completion_id")), task.LedgerVersion) + payload["attemptId"] = fmt.Sprintf("reconcile:%d", task.LedgerVersion) + payload["ledgerVersion"] = task.LedgerVersion + payload["planVersion"] = task.PlanVersion + payload["memberId"] = leader.MemberKey + payload["taskId"] = fmt.Sprintf("team-%d-task-%d", team.ID, task.ID) + payload["rootTaskId"] = fmt.Sprintf("team-%d-task-%d", team.ID, task.ID) + delete(payload, "completionDecision") + delete(payload, "completionDecisionReason") + delete(payload, "pendingAssignments") + delete(payload, "pendingPhases") + encoded, err := json.Marshal(payload) + if err != nil { + return false, err + } + streamID := fmt.Sprintf("reconcile-%d-%d", task.ID, task.LedgerVersion) + if err := s.projectTeamEvent(team, bus, redisStreamMessage{ID: streamID, Fields: map[string]string{"payload": string(encoded)}}); err != nil { + return false, err + } + updated, err := s.repo.GetTaskByID(task.ID) + if err != nil { + return false, err + } + if updated != nil && updated.Status == models.TeamTaskStatusSucceeded { + *task = *updated + return true, nil + } + return false, nil + } + return false, nil +} + +func leaderMediatedRootNeedsSynthesisReminder(task *models.TeamTask, items []models.TeamWorkItem, membersByID map[int]*models.TeamMember) (bool, []models.TeamWorkItem) { + if task == nil || isTerminalTeamTaskStatus(task.Status) { + return false, nil + } + latest := map[string]models.TeamWorkItem{} + for idx := range items { + item := items[idx] + if item.RootTaskID != task.ID || item.SupersededBy != nil { + continue + } + key := derefTeamString(item.AssignmentID) + if key == "" { + key = item.WorkID + } + if current, ok := latest[key]; !ok || teamMaxInt(item.Revision, 1) >= teamMaxInt(current.Revision, 1) { + latest[key] = item + } + } + var resultItems []models.TeamWorkItem + for _, item := range latest { + owner := memberForWorkItem(item, membersByID) + if owner == nil { + continue + } + if isLeaderTeamMember(owner) { + if item.Status == models.TeamTaskStatusSucceeded && isLeaderFinalSynthesisWorkItem(item) { + return false, nil + } + continue + } + if !isActiveTeamMember(owner) { + continue + } + if !(item.RequiredForRoot || item.AssignmentID == nil) { + continue + } + switch item.Status { + case models.TeamTaskStatusSucceeded: + resultItems = append(resultItems, item) + case models.TeamTaskStatusFailed, models.TeamTaskStatusStale: + return false, nil + default: + return false, nil + } + } + sort.Slice(resultItems, func(i, j int) bool { + return resultItems[i].WorkID < resultItems[j].WorkID + }) + return len(resultItems) > 0, resultItems +} + +func memberForWorkItem(item models.TeamWorkItem, membersByID map[int]*models.TeamMember) *models.TeamMember { + if item.OwnerMemberID == nil || membersByID == nil { + return nil + } + return membersByID[*item.OwnerMemberID] +} + +func isLeaderFinalSynthesisWorkItem(item models.TeamWorkItem) bool { + text := strings.ToLower(strings.TrimSpace(item.WorkID + " " + item.Title)) + return strings.Contains(text, "leader") && + (strings.Contains(text, "final") || + strings.Contains(text, "synthesis") || + strings.Contains(text, "result") || + strings.Contains(text, "complete")) +} + +func (s *teamService) createLeaderSynthesisReminder(team *models.Team, bus *redisBus, task *models.TeamTask, leader *models.TeamMember, resultItems []models.TeamWorkItem, now time.Time) error { + if s == nil || s.repo == nil || team == nil || task == nil || leader == nil || len(resultItems) == 0 { + return nil + } + eventID := fmt.Sprintf("leader-workflow-reminder:%d:%d:%s:%d", team.ID, task.ID, normalizeTeamRedisKeyPart(task.WorkflowState), task.LedgerVersion) + exists, err := s.repo.EventExistsByEventID(team.ID, eventID) + if err != nil || exists { + return err + } + rootTaskRef := fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID) + memberResults := make([]map[string]interface{}, 0, len(resultItems)) + for idx := range resultItems { + item := resultItems[idx] + memberResult := map[string]interface{}{ + "workId": item.WorkID, + "title": item.Title, + "status": item.Status, + "summary": workItemResultSummary(item), + } + if item.OwnerMemberID != nil { + memberResult["ownerMemberId"] = *item.OwnerMemberID + } + if refs := workItemArtifactRefs(item); len(refs) > 0 { + memberResult["artifactRefs"] = refs + } + memberResults = append(memberResults, memberResult) + } + decisionReminder := task.WorkflowState == teamWorkflowStateAwaitingLeaderDecision || task.WorkflowState == teamWorkflowStateExecuting + summary := "All tracked member assignments have delivered results; Leader should synthesize the final answer and close the root task." + eventKind := "leader_synthesis_reminder" + intent := "leader_synthesis_reminder" + title := "Leader final synthesis requested" + if decisionReminder { + summary = "The current phase has delivered its results. Leader must decide whether to create the next phase or explicitly seal the workflow before final synthesis." + eventKind = "leader_decision_reminder" + intent = "leader_workflow_decision" + title = "Leader workflow decision requested" + } + payload := map[string]interface{}{ + "event": eventKind, + "type": eventKind, + "eventKind": eventKind, + "protocolVersion": 3, + "source": "clawmanager_monitor", + "nonAuthoritative": true, + "rootTaskTerminal": false, + "teamId": strconv.Itoa(team.ID), + "taskId": rootTaskRef, + "rootTaskId": rootTaskRef, + "rootMessageId": task.MessageID, + "messageId": eventID, + "memberId": leader.MemberKey, + "from": "clawmanager-monitor", + "to": leader.MemberKey, + "target": leader.MemberKey, + "workId": "leader-final-synthesis", + "assignmentId": "leader-final-synthesis", + "status": models.TeamTaskStatusRunning, + "runtimeStatus": models.TeamTaskStatusRunning, + "availability": models.TeamMemberAvailabilityBusy, + "summary": summary, + "workflowState": task.WorkflowState, + "planVersion": task.PlanVersion, + "ledgerVersion": task.LedgerVersion, + "visibleToChat": true, + "chatDigestEligible": true, + "dedupeKey": fmt.Sprintf("leader-synthesis:%d:%d", team.ID, task.ID), + "monitor": true, + "monitorType": eventKind, + "memberResults": memberResults, + "collaborationStep": map[string]interface{}{ + "type": "progress", + "status": models.TeamTaskStatusRunning, + "actor": "clawmanager-monitor", + "target": leader.MemberKey, + "rootTaskId": rootTaskRef, + "rootMessageId": task.MessageID, + "workId": "leader-final-synthesis", + "title": title, + "summary": summary, + "content": summary, + "source": "clawmanager_monitor", + }, + } + payloadJSON, err := marshalOptionalJSON(payload) + if err != nil { + return err + } + event := &models.TeamEvent{ + TeamID: team.ID, + TaskID: &task.ID, + MemberID: &leader.ID, + MessageID: &eventID, + EventID: &eventID, + EventType: eventKind, + PayloadJSON: payloadJSON, + OccurredAt: &now, + CreatedAt: now, + } + if err := s.repo.CreateEvent(event); err != nil && !errors.Is(err, repository.ErrDuplicateTeamEvent) { + return err + } + if bus == nil { + return nil + } + prompt := buildLeaderSynthesisReminderPrompt(task, resultItems, rootTaskRef) + if decisionReminder { + prompt = "Review the confirmed results for the current phase. If the root task requires another implementation, verification, review, or refinement phase, publish a new planVersion and dispatch those assignments now. If no required action remains, explicitly seal the workflow and then submit the final user-facing result. Do not call team_complete_task merely because the current workers have finished.\n\n" + prompt + } + envelope := map[string]interface{}{ + "v": 1, + "protocolVersion": 3, + "messageId": eventID, + "teamId": strconv.Itoa(team.ID), + "from": "clawmanager-monitor", + "to": leader.MemberKey, + "replyTo": teamTaskReplyTarget, + "requiresCompletion": false, + "completionTool": teamTaskCompletionTool, + "intent": intent, + "taskId": rootTaskRef, + "rootTaskId": rootTaskRef, + "rootMessageId": task.MessageID, + "workId": "leader-final-synthesis", + "assignmentId": "leader-final-synthesis", + "title": title, + "prompt": prompt, + "rawPrompt": prompt, + "monitorPolicy": defaultTeamMonitorPolicy(), + "metadata": payload, + "createdAt": now.Format(time.RFC3339Nano), + } + applyTeamTaskEnvelopeContext(envelope, task, leader.MemberKey) + envelopeJSON, err := marshalJSON(envelope) + if err != nil { + return err + } + _, err = bus.XAdd(context.Background(), teamInboxKey(team.ID, leader.MemberKey), map[string]string{ + "payload": envelopeJSON, + "team_id": strconv.Itoa(team.ID), + "task_id": strconv.Itoa(task.ID), + "message_id": eventID, + "member_id": leader.MemberKey, + }) + return err +} + +func buildLeaderSynthesisReminderPrompt(task *models.TeamTask, resultItems []models.TeamWorkItem, rootTaskRef string) string { + lines := []string{ + "[LEADER_SYNTHESIS_REMINDER] All tracked member assignments for this root task have delivered terminal results in the ClawManager ledger.", + fmt.Sprintf("rootTaskId=%s rootMessageId=%s workId=leader-final-synthesis assignmentId=leader-final-synthesis", rootTaskRef, task.MessageID), + "Synthesize the final user-facing answer from the confirmed member results below, then call team_complete_task for the root task. Do not re-dispatch finished assignments unless the evidence below is actually insufficient.", + "", + "Confirmed member results:", + } + for idx := range resultItems { + item := resultItems[idx] + summary := workItemResultSummary(item) + if summary == "" { + summary = item.Title + } + lines = append(lines, fmt.Sprintf("- %s: %s", item.WorkID, summary)) + } + return strings.Join(lines, "\n") +} + +func workItemResultPayload(item models.TeamWorkItem) map[string]interface{} { + payload := map[string]interface{}{} + if item.ResultJSON != nil && strings.TrimSpace(*item.ResultJSON) != "" { + _ = json.Unmarshal([]byte(*item.ResultJSON), &payload) + } + return payload +} + +func workItemResultSummary(item models.TeamWorkItem) string { + payload := workItemResultPayload(item) + summary := eventString(payload, "summary", "title", "resultMarkdown", "result_markdown", "result", "answer", "text", "message") + if summary == "" { + summary = item.Title + } + return truncateForSummary(summary, 240) +} + +func workItemArtifactRefs(item models.TeamWorkItem) []string { + payload := workItemResultPayload(item) + refs := explicitTeamArtifactReferences(payload) + if len(refs) > 0 { + return refs + } + if item.ArtifactRefsJSON == nil || strings.TrimSpace(*item.ArtifactRefsJSON) == "" { + return nil + } + var parsed []string + if err := json.Unmarshal([]byte(*item.ArtifactRefsJSON), &parsed); err == nil { + return parsed + } + return nil +} + +func shouldMonitorTeamWorkItem(item models.TeamWorkItem, cutoff time.Time) bool { + if item.OwnerMemberID == nil || strings.TrimSpace(item.WorkID) == "" { + return false + } + switch item.Status { + case models.TeamTaskStatusDispatched, models.TeamTaskStatusRunning: + default: + return false + } + return item.UpdatedAt.IsZero() || item.UpdatedAt.Before(cutoff) +} + +func (s *teamService) claimAssignmentMonitorSlot(key string, now time.Time) bool { + if s == nil || strings.TrimSpace(key) == "" { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if s.assignmentMonitorLast == nil { + s.assignmentMonitorLast = map[string]time.Time{} + } + if last, ok := s.assignmentMonitorLast[key]; ok && now.Sub(last) < teamAssignmentMonitorEvery { + return false + } + s.assignmentMonitorLast[key] = now + return true +} + +func (s *teamService) dispatchAssignmentStatusCheck(team *models.Team, bus *redisBus, task *models.TeamTask, item *models.TeamWorkItem, owner *models.TeamMember, now time.Time) error { + if team == nil || bus == nil || task == nil || item == nil || owner == nil { + return nil + } + envelope, messageID := buildAssignmentStatusCheckEnvelope(team, task, item, owner, now) + if envelope == nil || strings.TrimSpace(messageID) == "" { + return nil + } + envelopeJSON, err := marshalJSON(envelope) + if err != nil { + return err + } + _, err = bus.XAdd(context.Background(), teamInboxKey(team.ID, owner.MemberKey), map[string]string{ + "payload": envelopeJSON, + "team_id": strconv.Itoa(team.ID), + "task_id": strconv.Itoa(task.ID), + "message_id": messageID, + "member_id": owner.MemberKey, + }) + if err != nil { + return err + } + payload := map[string]interface{}{ + "v": 1, + "protocolVersion": 2, + "event": "assignment_check_requested", + "type": "assignment_check_requested", + "eventKind": "assignment_check_requested", + "teamId": strconv.Itoa(team.ID), + "taskId": fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID), + "rootTaskId": fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID), + "rootMessageId": task.MessageID, + "messageId": messageID, + "memberId": owner.MemberKey, + "from": "clawmanager-monitor", + "to": owner.MemberKey, + "workId": item.WorkID, + "assignmentId": item.WorkID, + "status": models.TeamTaskStatusRunning, + "runtimeStatus": models.TeamTaskStatusRunning, + "availability": models.TeamMemberAvailabilityBusy, + "summary": "ClawManager requested an automatic assignment status check.", + "visibleToChat": false, + "nonAuthoritative": true, + "rootTaskTerminal": false, + "monitor": true, + "monitorType": "assignment_status_check", + "workItemId": item.ID, + "workItemTitle": item.Title, + "lastWorkUpdatedAt": item.UpdatedAt.Format(time.RFC3339Nano), + } + payloadJSON, err := marshalOptionalJSON(payload) + if err != nil { + return err + } + event := &models.TeamEvent{ + TeamID: team.ID, + TaskID: &task.ID, + MemberID: &owner.ID, + EventType: "assignment_check_requested", + MessageID: &messageID, + PayloadJSON: payloadJSON, + OccurredAt: &now, + CreatedAt: now, + } + if err := s.repo.CreateEvent(event); err != nil && !errors.Is(err, repository.ErrDuplicateTeamEvent) { + return err + } + return nil +} + +func buildAssignmentStatusCheckEnvelope(team *models.Team, task *models.TeamTask, item *models.TeamWorkItem, owner *models.TeamMember, now time.Time) (map[string]interface{}, string) { + if team == nil || task == nil || item == nil || owner == nil { + return nil, "" + } + taskRef := fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID) + checkSequence := now.Unix() + if seconds := int64(teamAssignmentMonitorEvery.Seconds()); seconds > 0 { + checkSequence = now.Unix() / seconds + } + messageID := fmt.Sprintf("monitor:%s:%s:%d", taskRef, item.WorkID, checkSequence) + prompt := strings.Join([]string{ + "[STATUS_CHECK] This is an automatic ClawManager assignment monitor.", + fmt.Sprintf("rootTaskId=%s rootMessageId=%s workId=%s assignmentId=%s", taskRef, task.MessageID, item.WorkID, item.WorkID), + "Check the current assignment state. If you are still working, call team_update_progress with status=\"running\", eventKind=\"assignment_check_result\", a concise progress summary, and continue the same assignment. If you stopped or lost context, resume from the assignment evidence and report the recoverable blocker or needed retry to the Leader. Do not mark the assignment complete unless the deliverable is actually ready.", + }, "\n") + envelope := map[string]interface{}{ + "v": 1, + "protocolVersion": 2, + "messageId": messageID, + "teamId": strconv.Itoa(team.ID), + "from": "clawmanager-monitor", + "to": owner.MemberKey, + "replyTo": teamTaskReplyTarget, + "requiresCompletion": false, + "completionTool": teamTaskCompletionTool, + "intent": "assignment_status_check", + "taskId": taskRef, + "rootTaskId": taskRef, + "rootMessageId": task.MessageID, + "workId": item.WorkID, + "assignmentId": item.WorkID, + "checkId": messageID, + "checkSequence": checkSequence, + "requestedAt": now.Format(time.RFC3339Nano), + "title": "Assignment status check", + "prompt": prompt, + "rawPrompt": prompt, + "monitorPolicy": defaultTeamMonitorPolicy(), + "metadata": map[string]interface{}{ + "monitor": true, + "monitorType": "assignment_status_check", + "eventKind": "assignment_check_requested", + "visibleToChat": false, + "workItemId": item.ID, + "workItemTitle": item.Title, + "lastUpdatedAt": item.UpdatedAt.Format(time.RFC3339Nano), + "checkId": messageID, + "checkSequence": checkSequence, + "requestedAt": now.Format(time.RFC3339Nano), + }, + "createdAt": now.Format(time.RFC3339Nano), + } + applyTeamTaskEnvelopeContext(envelope, task, owner.MemberKey) + return envelope, messageID +} + +func (s *teamService) markTaskStale(task *models.TeamTask, timeout time.Duration) error { + if task == nil { + return nil + } + if task.Status != models.TeamTaskStatusDispatched && task.Status != models.TeamTaskStatusRunning { + return nil + } + lastUpdatedAt := task.UpdatedAt + team, err := s.repo.GetTeamByID(task.TeamID) + if err != nil { + return err + } + if team == nil || team.Status == models.TeamStatusDeleted || team.Status == models.TeamStatusDeleting { + return nil + } + if payloadJSON, terminal, err := s.taskHasTerminalCompletionEvidence(team, task); err != nil { + return err + } else if terminal { + now := time.Now().UTC() + task.Status = models.TeamTaskStatusSucceeded + task.FinishedAt = &now + task.UpdatedAt = now + task.ErrorMessage = nil + if payloadJSON != nil { + task.ResultJSON = payloadJSON + } + if err := s.repo.UpdateTask(task); err != nil { + return err + } + if member, err := s.repo.GetMemberByID(task.TargetMemberID); err != nil { + return err + } else if member != nil && member.TeamID == task.TeamID && member.CurrentTaskID != nil && *member.CurrentTaskID == task.ID { + member.Status = models.TeamMemberStatusIdle + member.CurrentTaskID = nil + member.Availability = models.TeamMemberAvailabilityIdle + member.BlockedReason = nil + member.Progress = 100 + member.UpdatedAt = now + if err := s.repo.UpdateMember(member); err != nil { + return err + } + } + return nil + } + cutoff := time.Now().UTC().Add(-timeout) + active, err := s.taskHasRecentActivity(team, task, cutoff) + if err != nil { + return err + } + if active { + return nil + } + + now := time.Now().UTC() + previousStatus := task.Status + task.Status = models.TeamTaskStatusStale + task.FinishedAt = &now + message := fmt.Sprintf("Team task stale: no runtime event for %s since %s", timeout.String(), task.UpdatedAt.Format(time.RFC3339)) + task.ErrorMessage = &message + task.UpdatedAt = now + if err := s.repo.UpdateTask(task); err != nil { + return err + } + + member, err := s.repo.GetMemberByID(task.TargetMemberID) + if err != nil { + return err + } + if member != nil && member.TeamID == task.TeamID && member.CurrentTaskID != nil && *member.CurrentTaskID == task.ID { + member.Status = models.TeamMemberStatusIdle + member.CurrentTaskID = nil + member.Availability = models.TeamMemberAvailabilityBlocked + member.RuntimeTaskID = &task.MessageID + member.RuntimeIntent = nil + member.BlockedReason = &message + member.LastSummary = &message + member.Progress = 0 + member.UpdatedAt = now + if err := s.repo.UpdateMember(member); err != nil { + return err + } + } + + payload := map[string]interface{}{ + "v": 1, + "event": "task_stale", + "teamId": strconv.Itoa(task.TeamID), + "taskId": fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID), + "messageId": task.MessageID, + "previousStatus": previousStatus, + "staleAfterSeconds": int(timeout.Seconds()), + "lastTaskUpdatedAt": lastUpdatedAt.Format(time.RFC3339Nano), + "diagnostic": message, + "source": "clawmanager", + } + payloadJSON, err := marshalOptionalJSON(payload) + if err != nil { + return err + } + event := &models.TeamEvent{ + TeamID: task.TeamID, + TaskID: &task.ID, + EventType: "task_stale", + MessageID: &task.MessageID, + PayloadJSON: payloadJSON, + OccurredAt: &now, + CreatedAt: now, + } + if member != nil && member.TeamID == task.TeamID { + event.MemberID = &member.ID + } + return s.repo.CreateEvent(event) +} + +func (s *teamService) taskHasRecentActivity(team *models.Team, task *models.TeamTask, cutoff time.Time) (bool, error) { + if s == nil || team == nil || task == nil { + return false, nil + } + events, err := s.repo.ListEventsByTeamID(team.ID, 500) + if err != nil { + return false, err + } + for idx := range events { + event := events[idx] + // CreatedAt is assigned by ClawManager when the event is accepted. Runtime + // clocks can drift and must not keep a dead task alive or mark an active + // task stale merely because occurredAt arrived out of order. + eventTime := event.CreatedAt + if !eventTime.After(cutoff) { + continue + } + payload := teamEventPayloadMap(event) + if teamEventMatchesRootTask(event, payload, task) { + return true, nil + } + } + workItems, err := s.repo.ListWorkItemsByRootTaskID(task.ID) + if err != nil { + return false, err + } + for idx := range workItems { + item := workItems[idx] + if item.Status != models.TeamTaskStatusDispatched && item.Status != models.TeamTaskStatusRunning { + continue + } + if item.UpdatedAt.After(cutoff) { + return true, nil + } + } + members, err := s.repo.ListMembersByTeamID(team.ID) + if err != nil { + return false, err + } + for idx := range members { + member := members[idx] + ownsRootTask := member.CurrentTaskID != nil && *member.CurrentTaskID == task.ID + runtimeTracksRoot := member.RuntimeTaskID != nil && strings.TrimSpace(*member.RuntimeTaskID) == task.MessageID + if !ownsRootTask && !runtimeTracksRoot { + continue + } + if member.UpdatedAt.After(cutoff) { + return true, nil + } + if member.LastSeenAt != nil && member.LastSeenAt.After(cutoff) { + return true, nil + } + } + return false, nil +} + +func (s *teamService) taskHasTerminalCompletionEvidence(team *models.Team, task *models.TeamTask) (*string, bool, error) { + if s == nil || team == nil || task == nil { + return nil, false, nil + } + events, err := s.repo.ListEventsByTeamID(team.ID, 1000) + if err != nil { + return nil, false, err + } + for idx := range events { + event := events[idx] + payload := teamEventPayloadMap(event) + if !teamEventMatchesRootTask(event, payload, task) { + continue + } + if eventBool(payload, "artifactValidationFailed", "artifact_validation_failed") { + continue + } + member := (*models.TeamMember)(nil) + if event.MemberID != nil { + found, err := s.repo.GetMemberByID(*event.MemberID) + if err != nil { + return nil, false, err + } + if found != nil && found.TeamID == team.ID { + member = found + } + } + eventType := event.EventType + if markedType := markLegacyRuntimeCompletionCandidate(eventType, payload, task, member); markedType != eventType { + eventType = markedType + } + if isTeamTaskCompletionSignal(eventType, normalizedTeamTaskEventStatus(payload), payload) { + payloadJSON, err := marshalOptionalJSON(payload) + if err != nil { + return nil, false, err + } + return payloadJSON, true, nil + } + } + return nil, false, nil +} + +func (s *teamService) redisBusForTeam(ctx context.Context, team *models.Team) (*redisBus, error) { + redisURL := "" + if team.RedisURLSecretName != nil && team.RedisURLSecretKey != nil { + client := k8s.GetClient() + if client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + value, err := s.secretService.GetSecretValue(ctx, client.GetNamespace(team.UserID), *team.RedisURLSecretName, *team.RedisURLSecretKey) + if err != nil { + return nil, err + } + redisURL = strings.TrimSpace(value) + } + if redisURL == "" { + redisURL = defaultTeamRedisURL() + } + if redisURL == "" { + return nil, fmt.Errorf("team redis url is required") + } + return newRedisBus(redisURL) +} + +type teamTaskProjectionResult struct { + status string + changed bool +} + +func projectTeamTaskRuntimeState(task *models.TeamTask, payload map[string]interface{}, eventType string, payloadJSON *string, now time.Time) teamTaskProjectionResult { + if task == nil { + return teamTaskProjectionResult{} + } + status := normalizedTeamTaskEventStatus(payload) + completed := isTeamTaskCompletionSignal(eventType, status, payload) + failed := isTeamTaskFailureSignal(eventType, status, payload) + running := isTeamTaskRunningSignal(eventType, status, payload) + + result := teamTaskProjectionResult{} + setStatus := func(next string) { + result.status = next + if task.Status != next { + task.Status = next + result.changed = true + } + } + setStarted := func() { + if task.StartedAt == nil { + task.StartedAt = &now + result.changed = true + } + } + setFinished := func() { + if task.FinishedAt == nil || !task.FinishedAt.Equal(now) { + task.FinishedAt = &now + result.changed = true + } + } + + if eventBool(payload, "artifactValidationFailed", "artifact_validation_failed") { + setStatus(models.TeamTaskStatusRunning) + setStarted() + if task.FinishedAt != nil { + task.FinishedAt = nil + result.changed = true + } + // Keep the member's final body available while the artifact contract is + // unresolved. A later valid completion replaces this payload and closes + // the task normally. + if payloadJSON != nil && (task.ResultJSON == nil || *task.ResultJSON != *payloadJSON) { + task.ResultJSON = payloadJSON + result.changed = true + } + return result + } + + if completed { + setStatus(models.TeamTaskStatusSucceeded) + setFinished() + if payloadJSON != nil && (task.ResultJSON == nil || *task.ResultJSON != *payloadJSON) { + task.ResultJSON = payloadJSON + result.changed = true + } + if task.ErrorMessage != nil { + task.ErrorMessage = nil + result.changed = true + } + return result + } + + if failed && task.Status != models.TeamTaskStatusSucceeded { + setStatus(models.TeamTaskStatusFailed) + setFinished() + if errText := eventString(payload, "error_message", "error", "reason", "diagnostic", "lastSummary", "last_summary", "summary"); errText != "" { + if task.ErrorMessage == nil || *task.ErrorMessage != errText { + task.ErrorMessage = &errText + result.changed = true + } + } + return result + } + + if isTerminalTeamTaskStatus(task.Status) { + return result + } + + switch eventType { + case "task_received": + if task.Status == models.TeamTaskStatusPending { + setStatus(models.TeamTaskStatusDispatched) + } + case "task_started": + setStatus(models.TeamTaskStatusRunning) + setStarted() + case "outbound", "task_assigned", "team_send", "peer_request", "peer_handoff", "peer_review_request": + if task.Status == models.TeamTaskStatusPending { + setStatus(models.TeamTaskStatusDispatched) + } + setStarted() + default: + if running { + setStatus(models.TeamTaskStatusRunning) + setStarted() + } + } + return result +} + +func normalizedTeamTaskEventStatus(payload map[string]interface{}) string { + raw := eventString(payload, "task_status", "taskStatus", "result_status", "resultStatus", "status", "state") + raw = strings.ToLower(strings.TrimSpace(raw)) + raw = strings.ReplaceAll(raw, "-", "_") + raw = strings.ReplaceAll(raw, " ", "_") + return raw +} + +func isTeamTaskCompletionSignal(eventType, status string, payload map[string]interface{}) bool { + if isFailedTeamTaskEventStatus(status) || isDispatchOnlyCompletionPayload(payload) { + return false + } + if teamRedisProtocolVersion(payload) >= 2 { + return (eventType == "task_completed" || eventType == "completion_proposed") && + isSuccessfulTeamTaskEventStatus(status) && + isExplicitTeamTaskCompletion(payload) && + hasStrictTeamCompletionEnvelope(payload) + } + if eventBool(payload, "legacyCompletionCandidate", "legacy_completion_candidate") { + return hasTeamCompletionResultBody(payload) && (status == "" || isSuccessfulTeamTaskEventStatus(status)) + } + if !hasAuthoritativeTeamCompletionPayload(eventType, status, payload) { + return false + } + switch eventType { + case "task_completed", "completion", "task_failed", "message_failed": + return true + } + return hasTeamTaskCompletionToolCall(payload) && isSuccessfulTeamTaskEventStatus(status) +} + +func hasAuthoritativeTeamCompletionPayload(eventType, status string, payload map[string]interface{}) bool { + if payload == nil { + return false + } + if isDispatchOnlyCompletionPayload(payload) { + return false + } + source := strings.ToLower(strings.TrimSpace(eventString(payload, "completionSource", "completion_source"))) + explicitCompletion := hasTeamTaskCompletionToolCall(payload) || + source == teamTaskCompletionTool || + eventBool(payload, "explicitCompletion", "explicit_completion", "rootTaskTerminal") + if !explicitCompletion { + return false + } + if step, ok := payload["collaborationStep"].(map[string]interface{}); ok { + switch strings.ToLower(strings.TrimSpace(eventString(step, "type"))) { + case "assignment", "progress", "ack", "peer_request": + return false + } + } + if eventType != "task_completed" && eventType != "completion" && eventType != "task_failed" && eventType != "message_failed" { + return false + } + if !hasTeamCompletionResultBody(payload) { + return false + } + return status == "" || isSuccessfulTeamTaskEventStatus(status) +} + +func hasTeamCompletionResultBody(payload map[string]interface{}) bool { + if payload == nil { + return false + } + if body := eventString(payload, "resultMarkdown", "result_markdown", "result", "answer"); body != "" { + return true + } + if summary := eventString(payload, "summary"); summary != "" { + return true + } + if step, ok := payload["collaborationStep"].(map[string]interface{}); ok { + if body := eventString(step, "resultMarkdown", "result_markdown", "result", "answer"); body != "" { + return true + } + if summary := eventString(step, "summary"); summary != "" { + return true + } + } + for _, record := range eventRecordCandidates(payload) { + if body := eventString(record, "resultMarkdown", "result_markdown", "result", "answer"); body != "" { + return true + } + if summary := eventString(record, "summary"); summary != "" { + return true + } + } + return false +} + +func isTeamTaskFailureSignal(eventType, status string, payload map[string]interface{}) bool { + if isSuccessfulTeamTaskEventStatus(status) || isNonAuthoritativeDispatchFailure(eventType, payload) { + return false + } + if teamRedisProtocolVersion(payload) >= 2 { + if eventType != "task_failed" && eventType != "completion_proposed" { + return false + } + source := strings.ToLower(strings.TrimSpace(eventString(payload, "completionSource", "completion_source"))) + if source != teamTaskCompletionTool && source != "runtime_error" && source != "runtime_processing" { + return false + } + return hasStrictTeamFailureEnvelope(payload) + } + switch eventType { + case "task_failed", "message_failed": + return true + } + if !hasTeamTaskCompletionToolCall(payload) { + return false + } + switch status { + case "failed", "failure", "error", "errored", "blocked": + return true + default: + return false + } +} + +func hasStrictTeamFailureEnvelope(payload map[string]interface{}) bool { + if payload == nil { + return false + } + for _, keys := range [][]string{ + {"eventId", "event_id"}, + {"completionId", "completion_id"}, + {"taskId", "task_id"}, + {"rootTaskId", "root_task_id"}, + {"memberId", "member_id"}, + {"summary"}, + } { + if eventString(payload, keys...) == "" { + return false + } + } + return len(explicitTeamArtifactReferences(payload)) > 0 +} + +func teamRedisProtocolVersion(payload map[string]interface{}) int { + if payload == nil { + return 1 + } + if version := eventInt(payload, "protocolVersion"); version > 0 { + return version + } + if version := eventInt(payload, "protocol_version"); version > 0 { + return version + } + return 1 +} + +func isExplicitTeamTaskCompletion(payload map[string]interface{}) bool { + if payload == nil || eventString(payload, "completionId", "completion_id") == "" { + return false + } + if !eventBool(payload, "explicitCompletion", "explicit_completion") { + return false + } + return strings.EqualFold( + strings.TrimSpace(eventString(payload, "completionSource", "completion_source")), + teamTaskCompletionTool, + ) +} + +func hasStrictTeamCompletionEnvelope(payload map[string]interface{}) bool { + if payload == nil || !hasTeamCompletionResultBody(payload) { + return false + } + for _, keys := range [][]string{ + {"eventId", "event_id"}, + {"completionId", "completion_id"}, + {"taskId", "task_id"}, + {"rootTaskId", "root_task_id"}, + {"memberId", "member_id"}, + {"summary"}, + {"resultMarkdown", "result_markdown"}, + } { + if eventString(payload, keys...) == "" { + return false + } + } + // Artifact references are validated separately when they are present. + // Some valid control-plane/bootstrap tasks only return resultMarkdown and + // rely on the completion tool/runtime to persist the standard result files. + // Requiring artifactRefs here leaves those completed root tasks stuck in a + // dispatched/running Kanban state even though the explicit completion + // envelope is otherwise authoritative. + return true +} + +func (s *teamService) hasAcceptedTeamCompletionID(teamID int, completionID string) (bool, error) { + completionID = strings.TrimSpace(completionID) + if s == nil || completionID == "" { + return false, nil + } + return s.repo.EventExistsByCompletionID(teamID, completionID) +} + +func isUnauthoritativeCompletionEvent(eventType string, completion, failure bool) bool { + if completion || failure { + return false + } + switch strings.ToLower(strings.TrimSpace(eventType)) { + case "task_completed", "completion": + return true + default: + return false + } +} + +func isSuccessfulTeamTaskEventStatus(status string) bool { + switch status { + case "succeeded", "success", "completed", "complete", "done", "finished", "ok": + return true + default: + return false + } +} + +func isFailedTeamTaskEventStatus(status string) bool { + switch status { + case "failed", "failure", "error", "errored", "blocked": + return true + default: + return false + } +} + +func isDispatchOnlyCompletionPayload(payload map[string]interface{}) bool { + for _, key := range []string{"resultMarkdown", "result_markdown", "result", "answer"} { + if body := eventString(payload, key); looksLikeSubstantiveResultDocument(body) { + return false + } + } + text := strings.ToLower(strings.Join(strings.Fields(strings.Join([]string{ + eventString(payload, "summary", "lastSummary", "last_summary", "message", "text", "diagnostic"), + eventString(payload, "title", "intent"), + }, " ")), " ")) + if text == "" { + return false + } + if text == "redis team task completed" || text == "redis team task processing completed" { + return true + } + if strings.Contains(text, "result already delivered") || strings.Contains(text, "\u7ed3\u679c\u5df2\u53cd\u9988") { + return true + } + compact := strings.ReplaceAll(text, " ", "") + if len([]rune(compact)) > 120 { + return false + } + if strings.Contains(text, "dispatch") && (strings.Contains(text, "worker") || strings.Contains(text, "member")) { + return true + } + return strings.Contains(compact, "\u5728\u7ebf\u7a7a\u95f2") || + strings.Contains(compact, "\u6d3e\u5355") || + strings.Contains(compact, "\u5df2\u6d3e\u53d1") || + strings.Contains(compact, "\u7b49\u5f85\u5176") || + strings.Contains(compact, "\u4efb\u52a1\u5206\u6d3e") +} + +func looksLikeSubstantiveResultDocument(value string) bool { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return false + } + compact := strings.ReplaceAll(strings.ToLower(strings.Join(strings.Fields(trimmed), "")), " ", "") + if compact == "redisteamtaskcompleted" || compact == "redisteamtaskprocessingcompleted" { + return false + } + if len([]rune(compact)) >= 180 { + return true + } + return strings.Contains(trimmed, "\n## ") || + strings.Contains(trimmed, "\n|") || + strings.Contains(trimmed, "\n- ") || + strings.Contains(trimmed, "\n1.") || + strings.Contains(trimmed, "\n### ") +} +func isTeamTaskRunningSignal(eventType, status string, payload map[string]interface{}) bool { + switch eventType { + case "task_started", "task_progress", "progress": + return true + } + switch status { + case "running", "in_progress", "processing", "busy", "working": + return true + } + progress := eventInt(payload, "progress") + return progress > 0 && progress < 100 +} + +func isTerminalTeamTaskStatus(status string) bool { + return status == models.TeamTaskStatusSucceeded || + status == models.TeamTaskStatusFailed || + status == models.TeamTaskStatusStale +} + +func isAssignmentHeartbeatEvent(eventType string, payload map[string]interface{}) bool { + eventType = strings.ToLower(strings.TrimSpace(eventType)) + if eventType == "assignment_heartbeat" { + return true + } + eventKind := strings.ToLower(strings.TrimSpace(eventString(payload, "eventKind", "event_kind", "kind"))) + return eventKind == "assignment_heartbeat" +} + +func isPassiveAssignmentMonitorEvent(eventType string, payload map[string]interface{}) bool { + eventKind := strings.ToLower(strings.TrimSpace(eventString(payload, "eventKind", "event_kind", "kind"))) + switch strings.ToLower(strings.TrimSpace(eventType)) { + case "assignment_heartbeat", "assignment_check_requested": + return true + } + switch eventKind { + case "assignment_heartbeat", "assignment_check_requested", "assignment_check_result": + return true + } + monitorType := strings.ToLower(strings.TrimSpace(eventString(payload, "monitorType", "monitor_type"))) + return monitorType == "assignment_status_check" && eventBool(payload, "nonAuthoritative", "non_authoritative") +} + +func isTeamPresenceEvent(eventType string) bool { + switch strings.ToLower(strings.TrimSpace(eventType)) { + case "presence", "member_presence", "status", "member_status": + return true + default: + return false + } +} + +func normalizePassiveAssignmentMonitorPayload(payload map[string]interface{}) { + if payload == nil { + return + } + payload["visibleToChat"] = false + payload["visible_to_chat"] = false + payload["chatDigestEligible"] = true + payload["chat_digest_eligible"] = true + payload["nonAuthoritative"] = true + payload["rootTaskTerminal"] = false +} + +func normalizeAssignmentHeartbeatPayload(payload map[string]interface{}) { + if payload == nil { + return + } + normalizePassiveAssignmentMonitorPayload(payload) + summary := strings.TrimSpace(eventString(payload, "summary", "message", "text", "diagnostic")) + if summary == "" || strings.EqualFold(summary, "Agent turn is still running") { + payload["summary"] = "\u4efb\u52a1\u4ecd\u5728\u6267\u884c\uff0cAgent \u6b63\u5728\u7ee7\u7eed\u5904\u7406\u5f53\u524d\u56de\u5408\u3002" + } +} + +func normalizeUnauthorizedAssignmentCheckResult(payload map[string]interface{}) { + if payload == nil { + return + } + eventKind := strings.ToLower(strings.TrimSpace(eventString(payload, "eventKind", "event_kind", "kind"))) + if eventKind != "assignment_check_result" && eventKind != "assignment_check_requested" { + return + } + checkID := strings.TrimSpace(eventString(payload, "checkId", "check_id")) + if strings.HasPrefix(checkID, "monitor:") { + return + } + payload["originalEventKind"] = eventKind + payload["eventKind"] = "worker_progress" + payload["checkSemanticCorrected"] = true + delete(payload, "checkId") + delete(payload, "check_id") +} + +func (s *teamService) leaderMediatedMonitorTargetsTerminalWorkItem(team *models.Team, task *models.TeamTask, member *models.TeamMember, payload map[string]interface{}) (bool, error) { + if s == nil || s.repo == nil || team == nil || task == nil || member == nil || payload == nil { + return false, nil + } + if isLeaderTeamMember(member) { + return false, nil + } + items, err := s.repo.ListWorkItemsByRootTaskID(task.ID) + if err != nil { + return false, err + } + workID := eventString(payload, "workId", "work_id", "assignmentId", "assignment_id") + canonicalWorkID := "member-" + normalizeTeamMemberRouteKey(member.MemberKey) + found := false + for idx := range items { + item := items[idx] + if item.OwnerMemberID == nil || *item.OwnerMemberID != member.ID { + continue + } + if workID != "" && item.WorkID != workID && item.WorkID != canonicalWorkID { + continue + } + found = true + if !isTerminalTeamTaskStatus(item.Status) { + return false, nil + } + } + return found, nil +} + +func isLeaderControlPlaneSnapshotTask(task *models.TeamTask, payload map[string]interface{}) bool { + if payload != nil { + if strings.TrimSpace(eventString(payload, "intent")) == initialLeaderTaskIntent { + return true + } + if strings.EqualFold(strings.TrimSpace(eventString(payload, "origin")), "system_bootstrap") { + return true + } + if strings.EqualFold(strings.TrimSpace(eventString(payload, "executionMode", "execution_mode")), "leader_control_plane_snapshot") { + return true + } + } + if task == nil { + return false + } + return strings.Contains(strings.ToLower(strings.TrimSpace(task.MessageID)), "bootstrap-introduction") +} + +func shouldAssociateEventWithCurrentMemberTask(eventType string, payload map[string]interface{}) bool { + switch eventType { + case "reply", "completion", "task_completed", "task_failed", "message_failed", "message_warning", "task_started", "task_progress", "progress": + return true + case "outbound", "task_assigned", "team_send", "peer_request", "peer_handoff", "peer_review_request", "peer_reply": + return true + } + if eventString(payload, "taskId", "task_id", "runtimeTaskId", "runtime_task_id", "messageId", "message_id") != "" { + return true + } + return teamEventHasBody(payload) +} + +func isNonAuthoritativeDispatchFailure(eventType string, payload map[string]interface{}) bool { + if eventType != "task_failed" && eventType != "message_failed" { + return false + } + text := strings.ToLower(strings.Join(strings.Fields(strings.Join([]string{ + eventString(payload, "error_message", "error", "reason", "diagnostic", "lastSummary", "last_summary", "summary", "text", "message"), + }, " ")), " ")) + if text == "redis team task failed" { + return true + } + return strings.Contains(text, "dispatch finished without reply/completion") || + strings.Contains(text, "without reply/completion") +} + +func isNonAuthoritativeDispatchWarning(eventType string, payload map[string]interface{}) bool { + return eventType == "message_warning" && isNonAuthoritativeDispatchFailure(eventString(payload, "originalEvent"), payload) +} + +func isLeaderMediatedLeaderDispatchOnlyCompletion(team *models.Team, eventType string, payload map[string]interface{}, member *models.TeamMember, task *models.TeamTask, completion bool) bool { + return completion && isLeaderMediatedLeaderDispatchOnlyMessage(team, eventType, payload, member, task) +} + +func isLeaderMediatedLeaderDispatchOnlyMessage(team *models.Team, eventType string, payload map[string]interface{}, member *models.TeamMember, task *models.TeamTask) bool { + if team == nil || payload == nil || member == nil || task == nil { + return false + } + if normalizedTeamCommunicationMode(team.CommunicationMode) != teamCommunicationModeLeaderMediated { + return false + } + if member.ID != task.TargetMemberID || !isLeaderTeamMember(member) { + return false + } + if strings.TrimSpace(eventString(payload, "intent")) == initialLeaderTaskIntent { + return false + } + if eventBool(payload, "rootTaskTerminal", "root_task_terminal", "finalSynthesis", "final_synthesis") { + return false + } + return looksLikeLeaderDispatchOnlyText(eventString(payload, "resultMarkdown", "result_markdown", "result", "answer", "text", "message", "summary")) +} + +func (s *teamService) leaderMediatedRootCompletionReady(team *models.Team, task *models.TeamTask, member *models.TeamMember) (bool, error) { + if !isLeaderMediatedTeam(team) || task == nil || member == nil || member.ID != task.TargetMemberID || !isLeaderTeamMember(member) { + return true, nil + } + if isLeaderControlPlaneSnapshotTask(task, nil) { + return true, nil + } + workItems, err := s.repo.ListWorkItemsByRootTaskID(task.ID) + if err != nil { + return false, err + } + requiredOwners := map[int]struct{}{} + deliveredOwners := map[int]struct{}{} + for idx := range workItems { + item := workItems[idx] + if item.OwnerMemberID == nil || *item.OwnerMemberID == member.ID || item.WorkID == "leader-final-synthesis" { + continue + } + ownerID := *item.OwnerMemberID + requiredOwners[ownerID] = struct{}{} + if item.Status == models.TeamTaskStatusSucceeded { + deliveredOwners[ownerID] = struct{}{} + } + } + if len(requiredOwners) > 0 { + for ownerID := range requiredOwners { + if _, ok := deliveredOwners[ownerID]; !ok { + return false, nil + } + } + return true, nil + } + events, err := s.repo.ListEventsByTeamID(team.ID, 500) + if err != nil { + return false, err + } + required := map[string]struct{}{} + delivered := map[string]struct{}{} + for idx := range events { + event := events[idx] + payload := teamEventPayloadMap(event) + if !teamEventMatchesRootTask(event, payload, task) { + continue + } + step, _ := payload["collaborationStep"].(map[string]interface{}) + stepType := eventString(step, "type") + actor := normalizeTeamMemberRouteKey(eventString(step, "actor")) + target := normalizeTeamMemberRouteKey(eventString(step, "target")) + if actor == "" && event.MemberID != nil { + // The event table keeps member_id numeric; fall back to payload fields only here. + actor = normalizeTeamMemberRouteKey(eventString(payload, "memberId", "member_id", "from", "sourceMemberId", "senderMemberId")) + } + if target == "" { + target = normalizeTeamMemberRouteKey(leaderMediatedRouteTarget(payload)) + } + if stepType == "assignment" && isLeaderRouteTarget(actor) && target != "" && !isLeaderRouteTarget(target) { + required[target] = struct{}{} + continue + } + if eventBool(payload, "assignmentResultOnly", "assignment_result_only") && actor != "" && !isLeaderRouteTarget(actor) { + delivered[actor] = struct{}{} + continue + } + if stepType == "result" && actor != "" && !isLeaderRouteTarget(actor) && (target == "" || isLeaderRouteTarget(target)) { + delivered[actor] = struct{}{} + } + } + if len(required) == 0 { + return true, nil + } + for memberKey := range required { + if !leaderMediatedDeliveredForRequiredMember(memberKey, delivered) { + return false, nil + } + } + return true, nil +} + +type teamCompletionEvaluation struct { + Decision string + Reason string + PendingAssignments []string + PendingPhases []string + WaivedAssignments []string + SkippedAssignments []string + StrongContradictions []string + LedgerVersion int64 + PlanVersion int64 +} + +type teamCompletionWaiver struct { + AssignmentID string + Reason string + Risk string +} + +func structuredTeamCompletionWaivers(payload map[string]interface{}) map[string]teamCompletionWaiver { + result := map[string]teamCompletionWaiver{} + raw := firstTeamValue(payload, "waivers", "assignmentWaivers", "assignment_waivers") + values, ok := raw.([]interface{}) + if !ok { + return result + } + for _, value := range values { + entry, ok := value.(map[string]interface{}) + if !ok { + continue + } + waiver := teamCompletionWaiver{ + AssignmentID: eventString(entry, "assignmentId", "assignment_id", "workId", "work_id"), + Reason: eventString(entry, "reason", "waiverReason", "waiver_reason"), + Risk: eventString(entry, "risk", "riskAccepted", "risk_accepted"), + } + if waiver.AssignmentID != "" && waiver.Reason != "" && waiver.Risk != "" { + result[waiver.AssignmentID] = waiver + } + } + return result +} + +func structuredTeamSkippedAssignments(payload map[string]interface{}) map[string]string { + result := map[string]string{} + raw := firstTeamValue(payload, "skippedAssignments", "skipped_assignments") + values, ok := raw.([]interface{}) + if !ok { + return result + } + for _, value := range values { + entry, ok := value.(map[string]interface{}) + if !ok { + continue + } + assignmentID := eventString(entry, "assignmentId", "assignment_id", "workId", "work_id") + reason := eventString(entry, "reason", "skipReason", "skip_reason") + if assignmentID != "" && reason != "" { + result[assignmentID] = reason + } + } + return result +} + +func acceptedTeamCompletionEvaluation(task *models.TeamTask) teamCompletionEvaluation { + result := teamCompletionEvaluation{Decision: teamCompletionDecisionAccepted} + if task != nil { + result.LedgerVersion = task.LedgerVersion + result.PlanVersion = task.PlanVersion + } + return result +} + +func (s *teamService) evaluateLeaderRootCompletion(team *models.Team, task *models.TeamTask, member *models.TeamMember, payload map[string]interface{}) (teamCompletionEvaluation, error) { + if !isLeaderMediatedTeam(team) || task == nil || member == nil || payload == nil || isLeaderControlPlaneSnapshotTask(task, payload) { + return acceptedTeamCompletionEvaluation(task), nil + } + result := acceptedTeamCompletionEvaluation(task) + if member.ID != task.TargetMemberID || !isLeaderTeamMember(member) { + result.Decision = teamCompletionDecisionRejected + result.Reason = "invalid_root_completion_owner" + return result, nil + } + + protocolVersion := teamRedisProtocolVersion(payload) + if protocolVersion >= 2 && (!isExplicitTeamTaskCompletion(payload) || !eventBool(payload, "rootTaskTerminal", "root_task_terminal")) { + result.Decision = teamCompletionDecisionRejected + result.Reason = "invalid_completion_envelope" + return result, nil + } + proposalPlanVersion := int64(eventInt(payload, "planVersion", "plan_version")) + proposalLedgerVersion := int64(eventInt(payload, "ledgerVersion", "ledger_version")) + if proposalPlanVersion > 0 && task.PlanVersion > 0 && proposalPlanVersion != task.PlanVersion { + result.Decision = teamCompletionDecisionDeferred + result.Reason = "stale_plan" + return result, nil + } + if proposalLedgerVersion > 0 && task.LedgerVersion > 0 && proposalLedgerVersion != task.LedgerVersion { + result.Decision = teamCompletionDecisionDeferred + result.Reason = "stale_ledger" + return result, nil + } + + items, err := s.repo.ListWorkItemsByRootTaskID(task.ID) + if err != nil { + return result, err + } + if len(items) == 0 && protocolVersion < 3 { + legacyReady, legacyErr := s.leaderMediatedRootCompletionReady(team, task, member) + if legacyErr != nil { + return result, legacyErr + } + if !legacyReady { + result.Decision = teamCompletionDecisionDeferred + result.Reason = "pending_legacy_assignments" + return result, nil + } + } + byBusinessID := make(map[string]models.TeamWorkItem, len(items)) + waivers := structuredTeamCompletionWaivers(payload) + skippedAssignments := structuredTeamSkippedAssignments(payload) + for idx := range items { + item := items[idx] + key := strings.TrimSpace(derefTeamString(item.AssignmentID)) + if key == "" { + key = strings.TrimSpace(item.WorkID) + } + if key != "" { + if current, ok := byBusinessID[key]; !ok || item.Revision >= current.Revision { + byBusinessID[key] = item + } + } + } + for key, item := range byBusinessID { + if item.OwnerMemberID == nil || *item.OwnerMemberID == member.ID || item.WorkID == "leader-final-synthesis" || item.SupersededBy != nil { + continue + } + required := item.RequiredForRoot || item.AssignmentID == nil + if !required { + if item.Status != models.TeamTaskStatusSucceeded { + if skippedAssignments[key] == "" { + result.PendingAssignments = append(result.PendingAssignments, key+":skip_reason") + } else { + result.SkippedAssignments = append(result.SkippedAssignments, key) + } + } + continue + } + if item.Status != models.TeamTaskStatusSucceeded { + if (item.Status == models.TeamTaskStatusFailed || item.Status == models.TeamTaskStatusStale) && waivers[key].AssignmentID != "" { + result.WaivedAssignments = append(result.WaivedAssignments, key) + continue + } + result.PendingAssignments = append(result.PendingAssignments, key) + continue + } + if item.ReviewRequired && (item.ValidatedRevision == nil || *item.ValidatedRevision < teamMaxInt(item.Revision, 1)) { + result.PendingAssignments = append(result.PendingAssignments, key+":review") + continue + } + for _, dependency := range teamWorkItemDependencies(item) { + dependencyItem, ok := byBusinessID[dependency] + if !ok || dependencyItem.Status != models.TeamTaskStatusSucceeded { + result.PendingAssignments = append(result.PendingAssignments, key+":depends_on:"+dependency) + } + } + } + sort.Strings(result.PendingAssignments) + result.PendingAssignments = uniqueTeamStrings(result.PendingAssignments) + sort.Strings(result.WaivedAssignments) + result.WaivedAssignments = uniqueTeamStrings(result.WaivedAssignments) + sort.Strings(result.SkippedAssignments) + result.SkippedAssignments = uniqueTeamStrings(result.SkippedAssignments) + if len(result.PendingAssignments) > 0 { + result.Decision = teamCompletionDecisionDeferred + result.Reason = "pending_assignments" + return result, nil + } + + phases, err := s.repo.ListWorkflowPhasesByRootTaskID(task.ID) + if err != nil { + return result, err + } + latestPlanVersion := task.PlanVersion + if latestPlanVersion <= 0 { + for idx := range phases { + if phases[idx].PlanVersion > latestPlanVersion { + latestPlanVersion = phases[idx].PlanVersion + } + } + } + workflowFinal := eventBool(payload, "workflowFinal", "workflow_final", "sealWorkflow", "seal_workflow") + finalAnswerReady := eventBool(payload, "finalAnswerReady", "final_answer_ready") + remainingActions := normalizeContextRefs(firstTeamValue(payload, "remainingActions", "remaining_actions", "nextActions", "next_actions")) + if protocolVersion >= 3 && (!workflowFinal || !finalAnswerReady || len(remainingActions) > 0) { + result.Decision = teamCompletionDecisionDeferred + result.Reason = "workflow_not_sealed" + result.PendingAssignments = append(result.PendingAssignments, remainingActions...) + return result, nil + } + for idx := range phases { + phase := phases[idx] + if latestPlanVersion > 0 && phase.PlanVersion != latestPlanVersion { + continue + } + if !phase.RequiredForRoot || phase.Status == teamPhaseStatusCompleted || phase.Status == teamPhaseStatusCancelled || phase.Status == teamPhaseStatusSuperseded { + continue + } + // The phase ledger is a projection, not an additional source of work. A + // planned phase with no dispatched required work must never keep an + // explicitly sealed workflow open (Team 58 was stuck exactly this way). + // Actual current work items and their dependencies were checked above. + if phaseHasIncompleteRequiredWork(phase.PhaseID, byBusinessID, member.ID, waivers) { + result.PendingPhases = append(result.PendingPhases, phase.PhaseID) + continue + } + if phase.DecisionRequired && !workflowFinal { + result.PendingPhases = append(result.PendingPhases, phase.PhaseID+":leader_decision") + } + } + sort.Strings(result.PendingPhases) + result.PendingPhases = uniqueTeamStrings(result.PendingPhases) + if len(result.PendingPhases) > 0 { + result.Decision = teamCompletionDecisionDeferred + result.Reason = "open_workflow_phases" + return result, nil + } + + result.StrongContradictions = analyzeCompletionNarrativeContradictions(payload) + if len(result.StrongContradictions) > 0 && !eventBool(payload, "confirmFinal", "confirm_final") { + result.Decision = teamCompletionDecisionNeedsConfirmation + result.Reason = "narrative_indicates_remaining_work" + return result, nil + } + return result, nil +} + +func firstTeamValue(payload map[string]interface{}, keys ...string) interface{} { + for _, key := range keys { + if value, ok := payload[key]; ok && value != nil { + return value + } + } + return nil +} + +func teamWorkItemDependencies(item models.TeamWorkItem) []string { + if item.DependsOnJSON == nil || strings.TrimSpace(*item.DependsOnJSON) == "" { + return nil + } + var values []string + if err := json.Unmarshal([]byte(*item.DependsOnJSON), &values); err != nil { + return nil + } + return uniqueTeamStrings(values) +} + +func uniqueTeamStrings(values []string) []string { + seen := map[string]struct{}{} + result := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + return result +} + +func teamMaxInt(left, right int) int { + if left > right { + return left + } + return right +} + +func (s *teamService) invalidateModifiedTeamArtifactReviews(task *models.TeamTask, payload map[string]interface{}, now time.Time) (bool, error) { + if s == nil || s.repo == nil || task == nil || payload == nil || + !(eventBool(payload, "artifactChanged", "artifact_changed") || strings.EqualFold(eventString(payload, "eventKind", "event_kind"), "artifact_changed")) { + return false, nil + } + changedRefs := explicitTeamArtifactReferences(payload) + if len(changedRefs) == 0 { + return false, nil + } + changedSet := make(map[string]struct{}, len(changedRefs)) + for _, ref := range changedRefs { + changedSet[strings.TrimSpace(ref)] = struct{}{} + } + items, err := s.repo.ListWorkItemsByRootTaskID(task.ID) + if err != nil { + return false, err + } + invalidated := false + for idx := range items { + item := items[idx] + if item.ID <= 0 || item.SupersededBy != nil || !item.ReviewRequired || item.ValidatedRevision == nil { + continue + } + matches := false + for _, ref := range workItemArtifactRefs(item) { + if _, ok := changedSet[strings.TrimSpace(ref)]; ok { + matches = true + break + } + } + if !matches { + continue + } + if err := s.repo.InvalidateWorkItemReview(item.ID, now); err != nil { + return invalidated, err + } + invalidated = true + } + if invalidated { + task.LedgerVersion++ + task.UpdatedAt = now + } + return invalidated, nil +} + +func phaseHasIncompleteRequiredWork(phaseID string, items map[string]models.TeamWorkItem, leaderID int, waivers map[string]teamCompletionWaiver) bool { + for key, item := range items { + if strings.TrimSpace(derefTeamString(item.PhaseID)) != strings.TrimSpace(phaseID) || item.SupersededBy != nil { + continue + } + if item.OwnerMemberID == nil || *item.OwnerMemberID == leaderID { + continue + } + if !(item.RequiredForRoot || item.AssignmentID == nil) { + continue + } + if (item.Status == models.TeamTaskStatusFailed || item.Status == models.TeamTaskStatusStale) && waivers[key].AssignmentID != "" { + continue + } + if item.Status != models.TeamTaskStatusSucceeded { + return true + } + } + return false +} + +// reconcileTeamWorkflowLedger repairs the derived phase view from the current +// work-item ledger. It deliberately does not invent or complete work: a phase +// moves to completed only when all of its actual required assignments succeeded. +// A future planned phase without assignments remains planned until the Leader +// explicitly seals the workflow, at which point it is cancelled as unused. +func (s *teamService) reconcileTeamWorkflowLedger(task *models.TeamTask, workflowFinal bool, now time.Time) (bool, error) { + if s == nil || s.repo == nil || task == nil || task.ID <= 0 || isTerminalTeamTaskStatus(task.Status) { + return false, nil + } + phases, err := s.repo.ListWorkflowPhasesByRootTaskID(task.ID) + if err != nil { + return false, err + } + items, err := s.repo.ListWorkItemsByRootTaskID(task.ID) + if err != nil { + return false, err + } + latestPlanVersion := task.PlanVersion + if latestPlanVersion <= 0 { + for _, phase := range phases { + if phase.PlanVersion > latestPlanVersion { + latestPlanVersion = phase.PlanVersion + } + } + } + if latestPlanVersion <= 0 { + return false, nil + } + + latestItems := map[string]models.TeamWorkItem{} + for _, item := range items { + if item.SupersededBy != nil { + continue + } + key := strings.TrimSpace(derefTeamString(item.AssignmentID)) + if key == "" { + key = strings.TrimSpace(item.WorkID) + } + if key == "" { + continue + } + if current, ok := latestItems[key]; !ok || teamMaxInt(item.Revision, 1) >= teamMaxInt(current.Revision, 1) { + latestItems[key] = item + } + } + + changed := false + anyIncomplete := false + anyDecision := false + var currentPhase string + for idx := range phases { + phase := phases[idx] + if phase.PlanVersion != latestPlanVersion || !phase.RequiredForRoot || + phase.Status == teamPhaseStatusCancelled || phase.Status == teamPhaseStatusSuperseded || phase.Status == teamPhaseStatusCompleted { + continue + } + phaseHasWork := false + phaseComplete := true + for _, item := range latestItems { + if strings.TrimSpace(derefTeamString(item.PhaseID)) != strings.TrimSpace(phase.PhaseID) || + item.OwnerMemberID == nil || *item.OwnerMemberID == task.TargetMemberID || + !(item.RequiredForRoot || item.AssignmentID == nil) { + continue + } + phaseHasWork = true + if item.Status != models.TeamTaskStatusSucceeded { + phaseComplete = false + } + } + previousStatus := phase.Status + switch { + case !phaseHasWork && workflowFinal && phase.Status == teamPhaseStatusPlanned: + phase.Status = teamPhaseStatusCancelled + case phaseHasWork && phaseComplete && phase.DecisionRequired && !workflowFinal: + phase.Status = teamPhaseStatusAwaitingLeaderDecision + anyDecision = true + case phaseHasWork && phaseComplete: + phase.Status = teamPhaseStatusCompleted + phase.CompletedAt = &now + case phaseHasWork: + phase.Status = teamPhaseStatusAwaitingResults + anyIncomplete = true + currentPhase = phase.PhaseID + case phase.Status == teamPhaseStatusAwaitingLeaderDecision: + anyDecision = true + } + if phase.Status != previousStatus { + phase.UpdatedAt = now + if err := s.repo.UpsertWorkflowPhase(&phase); err != nil { + return changed, err + } + changed = true + } + } + newState := task.WorkflowState + switch { + case anyIncomplete: + newState = teamWorkflowStateAwaitingPhaseResults + case anyDecision: + newState = teamWorkflowStateAwaitingLeaderDecision + case workflowFinal: + newState = teamWorkflowStateSynthesizing + default: + newState = teamWorkflowStateAwaitingLeaderDecision + } + if task.WorkflowState != newState { + task.WorkflowState = newState + changed = true + } + if currentPhase == "" && !anyIncomplete { + if task.CurrentPhaseID != nil { + task.CurrentPhaseID = nil + changed = true + } + } else if currentPhase != "" && (task.CurrentPhaseID == nil || *task.CurrentPhaseID != currentPhase) { + task.CurrentPhaseID = ¤tPhase + changed = true + } + if changed { + task.LedgerVersion++ + task.UpdatedAt = now + if err := s.repo.UpdateTask(task); err != nil { + return false, err + } + } + return changed, nil +} + +func analyzeCompletionNarrativeContradictions(payload map[string]interface{}) []string { + if payload == nil { + return nil + } + texts := []string{ + eventString(payload, "summary"), + eventString(payload, "resultMarkdown", "result_markdown", "result", "answer"), + } + if step, ok := payload["collaborationStep"].(map[string]interface{}); ok { + texts = append(texts, eventString(step, "summary"), eventString(step, "content", "detail")) + } + patterns := []struct { + name string + pattern *regexp.Regexp + }{ + {"waiting_for_member", regexp.MustCompile(`(?i)(仍在等待|继续等待|等待\s*(pm|architect|designer|developer|reviewer|worker)|still waiting|waiting for\s+(pm|architect|designer|developer|reviewer|worker))`)}, + {"future_dispatch", regexp.MustCompile(`(?i)((下一步|接下来|随后|然后).{0,10}(将|需要|必须|准备|继续).{0,24}(派发|下发|交给|发送给|实现|评审|验证)|(next|then).{0,12}(will|must|need to|going to|continue to).{0,24}(dispatch|assign|send to|implement|review|verify))`)}, + {"phase_not_final", regexp.MustCompile(`(?i)(完成|结束|汇总).{0,12}(第一阶段|阶段一|phase\s*1).{0,30}(继续|下一阶段|phase\s*2)`)}, + } + result := []string{} + for _, text := range texts { + text = strings.TrimSpace(text) + if text == "" { + continue + } + for _, candidate := range patterns { + if candidate.pattern.MatchString(text) { + result = append(result, candidate.name) + } + } + } + return uniqueTeamStrings(result) +} + +func leaderMediatedDeliveredForRequiredMember(required string, delivered map[string]struct{}) bool { + if _, ok := delivered[required]; ok { + return true + } + for candidate := range delivered { + if teamMemberRouteEquivalent(required, candidate) { + return true + } + } + return false +} + +func teamMemberRouteEquivalent(a, b string) bool { + left := normalizeTeamMemberRouteKey(a) + right := normalizeTeamMemberRouteKey(b) + if left == "" || right == "" { + return false + } + if left == right { + return true + } + aliasGroups := [][]string{ + {"pm", "product-manager", "product"}, + {"designer", "ui-designer", "ui-ux-designer", "ux-designer"}, + {"architect", "solution-architect", "software-architect"}, + {"developer", "worker", "senior-developer", "frontend-developer", "backend-developer"}, + } + for _, group := range aliasGroups { + leftMatch := false + rightMatch := false + for _, alias := range group { + if left == alias || strings.Contains(left, "-"+alias) || strings.Contains(left, alias+"-") { + leftMatch = true + } + if right == alias || strings.Contains(right, "-"+alias) || strings.Contains(right, alias+"-") { + rightMatch = true + } + } + if leftMatch && rightMatch { + return true + } + } + return false +} + +func markLeaderMediatedPrematureCompletion(eventType string, payload map[string]interface{}) string { + if eventString(payload, "originalEvent") == "" { + payload["originalEvent"] = eventType + } + payload["event"] = "reply" + payload["type"] = "reply" + payload["status"] = models.TeamTaskStatusRunning + payload["runtimeStatus"] = models.TeamTaskStatusRunning + payload["availability"] = models.TeamMemberAvailabilityBusy + payload["rootTaskTerminal"] = false + payload["leaderPrematureCompletion"] = true + if eventString(payload, "summary") == "" { + payload["summary"] = "Leader is waiting for assigned member results before final synthesis." + } + return "reply" +} + +func markStructuredCompletionDecision(eventType string, payload map[string]interface{}, evaluation teamCompletionEvaluation) string { + if eventString(payload, "originalEvent") == "" { + payload["originalEvent"] = eventType + } + decisionType := "completion_" + evaluation.Decision + payload["event"] = decisionType + payload["type"] = decisionType + payload["completionDecision"] = evaluation.Decision + payload["completionDecisionReason"] = evaluation.Reason + payload["pendingAssignments"] = evaluation.PendingAssignments + payload["pendingPhases"] = evaluation.PendingPhases + payload["waivedAssignments"] = evaluation.WaivedAssignments + payload["skippedAssignmentsAccepted"] = evaluation.SkippedAssignments + payload["strongNarrativeContradictions"] = evaluation.StrongContradictions + payload["ledgerVersion"] = evaluation.LedgerVersion + payload["planVersion"] = evaluation.PlanVersion + payload["status"] = models.TeamTaskStatusRunning + payload["runtimeStatus"] = models.TeamTaskStatusRunning + payload["availability"] = models.TeamMemberAvailabilityBusy + payload["rootTaskTerminal"] = false + payload["completionProposalPreserved"] = true + // A deferred completion is business information: it contains the Leader's + // delivery proposal plus the exact reason the system did not accept it. + // Keep it in the group chat; only the transport ACK is hidden. + payload["visibleToChat"] = true + payload["visible_to_chat"] = true + payload["chatPolicy"] = "warning" + payload["chatKind"] = "completion_" + evaluation.Decision + if completionID := eventString(payload, "completionId", "completion_id"); completionID != "" { + payload["displayKey"] = fmt.Sprintf("completion:%s:%d", completionID, evaluation.LedgerVersion) + } + return decisionType +} + +func applyTeamChatPolicy(eventType string, payload map[string]interface{}, task *models.TeamTask, member *models.TeamMember) { + if payload == nil { + return + } + normalizedEvent := strings.ToLower(strings.TrimSpace(eventType)) + eventKind := strings.ToLower(strings.TrimSpace(eventString(payload, "eventKind", "event_kind", "kind"))) + // Runtime-provided visibility is advisory. A real plan, assignment, + // narrative, delivery, review, or synthesis must not be hidden simply + // because an older adapter labelled it as transport traffic. + if teamChatEventIsBusinessContent(normalizedEvent, eventKind, payload) { + payload["chatPolicy"] = "visible" + payload["visibleToChat"] = true + payload["visible_to_chat"] = true + if eventKind == "assignment_check_result" { + payload["chatBusinessKind"] = "worker_progress" + } + if eventString(payload, "chatKind", "chat_kind") != "final_delivery" && + eventString(payload, "completionDecision", "completion_decision") != teamCompletionDecisionAccepted { + // Legacy display keys can be shared by multiple workers when the + // assignment id was absent. Let content-based dedupe handle business + // messages instead of suppressing a different worker's narrative. + delete(payload, "displayKey") + delete(payload, "display_key") + } + return + } + if eventString(payload, "chatPolicy", "chat_policy") != "" { + return + } + rootKey := eventString(payload, "rootTaskId", "root_task_id") + if rootKey == "" && task != nil { + rootKey = fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID) + } + assignmentID := teamChatAssignmentIdentity(payload) + actor := eventString(payload, "from", "memberId", "member_id") + if actor == "" && member != nil { + actor = member.MemberKey + } + displayKey := "" + switch { + case normalizedEvent == "member_result_confirmed": + payload["chatPolicy"] = "hidden" + payload["visibleToChat"] = false + return + case eventKind == "assignment_heartbeat" || normalizedEvent == "assignment_heartbeat" || eventKind == "assignment_check_requested" || normalizedEvent == "assignment_check_requested": + payload["chatPolicy"] = "digest" + payload["visibleToChat"] = false + return + case eventKind == "assignment_check_result" || normalizedEvent == "assignment_check_result": + if teamMonitorEventHasBusinessChange(payload) { + payload["chatPolicy"] = "visible" + payload["chatBusinessKind"] = "worker_progress" + payload["visibleToChat"] = true + } else { + payload["chatPolicy"] = "digest" + payload["visibleToChat"] = false + return + } + case eventKind == "leader_plan" || normalizedEvent == "leader_plan": + payload["chatPolicy"] = "visible" + payload["visibleToChat"] = true + displayKey = "leader-plan:" + rootKey + ":" + strconv.FormatInt(int64(eventInt(payload, "planVersion", "plan_version")), 10) + case eventKind == "worker_plan" || normalizedEvent == "worker_plan": + payload["chatPolicy"] = "visible" + payload["visibleToChat"] = true + case eventKind == "worker_progress" || normalizedEvent == "worker_progress": + payload["chatPolicy"] = "visible" + payload["visibleToChat"] = true + case eventKind == "artifact_changed" || normalizedEvent == "artifact_changed": + payload["chatPolicy"] = "visible" + payload["chatBusinessKind"] = "worker_progress" + payload["visibleToChat"] = true + case eventBool(payload, "leaderDispatchOnly", "leader_dispatch_only") || normalizedEvent == "team_send" || normalizedEvent == "task_assigned" || normalizedEvent == "outbound": + payload["chatPolicy"] = "visible" + payload["visibleToChat"] = true + displayKey = "assignment:" + rootKey + ":" + assignmentID + case eventBool(payload, "assignmentResultOnly", "assignment_result_only"): + payload["chatPolicy"] = "visible" + payload["visibleToChat"] = true + displayKey = "worker-result:" + rootKey + ":" + assignmentID + ":" + strconv.Itoa(teamMaxInt(eventInt(payload, "revision"), 1)) + case normalizedEvent == "task_completed" && eventString(payload, "completionDecision", "completion_decision") == teamCompletionDecisionAccepted: + payload["chatPolicy"] = "visible" + payload["visibleToChat"] = true + displayKey = "root-final:" + rootKey + case normalizedEvent == "message_warning" || strings.Contains(normalizedEvent, "rejected") || strings.Contains(normalizedEvent, "needs_confirmation"): + payload["chatPolicy"] = "warning" + payload["visibleToChat"] = true + case normalizedEvent == "task_received" || normalizedEvent == "task_started" || normalizedEvent == "presence": + payload["chatPolicy"] = "hidden" + payload["visibleToChat"] = false + } + if displayKey != "" { + payload["displayKey"] = displayKey + } +} + +func teamChatAssignmentIdentity(payload map[string]interface{}) string { + if payload == nil { + return "" + } + if identity := eventString(payload, "assignmentId", "assignment_id", "canonicalWorkId", "canonical_work_id", "workId", "work_id"); identity != "" { + return identity + } + if step, ok := payload["collaborationStep"].(map[string]interface{}); ok { + if identity := eventString(step, "assignmentId", "assignment_id", "workId", "work_id", "id"); identity != "" { + return identity + } + } + actor := eventString(payload, "from", "memberId", "member_id") + phase := eventString(payload, "phaseId", "phase_id", "phase", "stage") + if actor != "" || phase != "" { + return "member:" + normalizeTeamMemberRouteKey(actor) + ":phase:" + normalizeTeamRedisKeyPart(phase) + } + return "" +} + +func teamChatEventIsBusinessContent(eventType, eventKind string, payload map[string]interface{}) bool { + if payload == nil || teamChatEventIsTransportNoise(eventType, eventKind, payload) { + return false + } + switch eventKind { + case "leader_plan", "worker_plan", "worker_progress", "leader_synthesis", "leader_synthesis_reminder", "leader_decision_reminder", + "agent_narrative", "agent_plan", "agent_assignment", "agent_handoff", "agent_progress", "agent_delivery", "agent_review", "agent_synthesis", + "completion_deferred", "completion_candidate", "completion_validation_warning", "assignment_recovery_started", "assignment_reissued", "assignment_recovery_exhausted": + return teamChatHasMeaningfulBody(payload) + } + if eventBool(payload, "leaderDispatchOnly", "leader_dispatch_only", "assignmentResultOnly", "assignment_result_only") { + return teamChatHasMeaningfulBody(payload) + } + switch eventType { + case "outbound", "team_send", "task_assigned", "peer_request", "peer_handoff", "peer_review_request", "peer_reply", "reply", "completion_proposed", "task_completed", "completion", "message_warning": + return teamChatHasMeaningfulBody(payload) + case "task_progress", "progress": + return teamChatHasMeaningfulBody(payload) + default: + return false + } +} + +func teamChatEventIsTransportNoise(eventType, eventKind string, payload map[string]interface{}) bool { + if eventType == "member_result_confirmed" || eventType == "inbound" { + return true + } + switch eventKind { + case "assignment_heartbeat", "assignment_check_requested": + return true + case "assignment_check_result": + return !teamMonitorEventHasBusinessChange(payload) + } + switch eventType { + case "assignment_heartbeat", "assignment_check_requested", "task_received", "task_started", "presence": + return true + } + return false +} + +func teamChatHasMeaningfulBody(payload map[string]interface{}) bool { + if payload == nil { + return false + } + values := []string{ + eventString(payload, "content", "detail", "resultMarkdown", "result_markdown", "result", "answer", "text", "message", "summary"), + } + if step, ok := payload["collaborationStep"].(map[string]interface{}); ok { + values = append(values, eventString(step, "content", "detail", "resultMarkdown", "result_markdown", "result", "answer", "text", "message", "summary")) + } + for _, value := range values { + normalized := strings.ToLower(strings.TrimSpace(strings.Join(strings.Fields(value), " "))) + if normalized == "" { + continue + } + switch normalized { + case "task_received", "task_started", "redis team task received", "redis team task started", "redis team task processing completed", "agent turn is still running", "still running", "status unchanged", "no change": + continue + } + return true + } + return false +} + +func teamMonitorEventHasBusinessChange(payload map[string]interface{}) bool { + if payload == nil { + return false + } + if eventBool(payload, "blocked", "recovered", "artifactChanged", "artifact_changed", "phaseChanged", "phase_changed") || + eventInt(payload, "progress") > 0 || len(explicitTeamArtifactReferences(payload)) > 0 { + return true + } + summary := strings.ToLower(strings.TrimSpace(eventString(payload, "summary", "detail", "message", "text"))) + if summary == "" { + return false + } + for _, generic := range []string{"agent turn is still running", "still running", "status unchanged", "no change", "仍在执行", "状态无变化"} { + if summary == generic { + return false + } + } + return len([]rune(strings.Join(strings.Fields(summary), ""))) >= 12 +} + +func buildCompletionAcknowledgement(team *models.Team, task *models.TeamTask, member *models.TeamMember, payload map[string]interface{}, decision, reason string) (map[string]interface{}, string, string) { + completionID := eventString(payload, "completionId", "completion_id") + attemptID := eventString(payload, "attemptId", "attempt_id") + if attemptID == "" { + attemptID = eventString(payload, "eventId", "event_id") + } + if attemptID == "" { + attemptID = completionID + } + memberKey := "leader" + if member != nil && strings.TrimSpace(member.MemberKey) != "" { + memberKey = member.MemberKey + } + rootTaskID := "" + ledgerVersion := int64(0) + planVersion := int64(0) + if task != nil { + rootTaskID = fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID) + ledgerVersion = task.LedgerVersion + planVersion = task.PlanVersion + } + ack := map[string]interface{}{ + "v": 1, + "protocolVersion": 3, + "event": "completion_ack", + "type": "completion_ack", + "intent": "completion_ack", + "teamId": strconv.Itoa(team.ID), + "memberId": memberKey, + "rootTaskId": rootTaskID, + "rootMessageId": eventString(payload, "rootMessageId", "root_message_id"), + "completionId": completionID, + "attemptId": attemptID, + "decision": decision, + "reason": reason, + "pendingAssignments": firstTeamValue(payload, "pendingAssignments", "pending_assignments"), + "pendingPhases": firstTeamValue(payload, "pendingPhases", "pending_phases"), + "ledgerVersion": ledgerVersion, + "planVersion": planVersion, + "requiresCompletion": false, + "visibleToChat": false, + "chatPolicy": "hidden", + "acknowledgedAt": time.Now().UTC().Format(time.RFC3339Nano), + } + return ack, completionID, attemptID +} + +func (s *teamService) emitCompletionAcknowledgement(team *models.Team, bus *redisBus, task *models.TeamTask, member *models.TeamMember, payload map[string]interface{}, decision, reason string) error { + if team == nil || payload == nil { + return nil + } + ack, completionID, attemptID := buildCompletionAcknowledgement(team, task, member, payload, decision, reason) + if completionID == "" || attemptID == "" || bus == nil { + return nil + } + encoded, err := json.Marshal(ack) + if err != nil { + return err + } + if err := bus.Set(context.Background(), teamCompletionAckKey(team.ID, completionID, attemptID), string(encoded), 24*time.Hour); err != nil { + return err + } + return bus.Set(context.Background(), teamCompletionStateKey(team.ID, completionID), string(encoded), 7*24*time.Hour) +} + +func (s *teamService) completeWorkflowPhases(task *models.TeamTask, now time.Time) error { + if s == nil || task == nil || task.ID <= 0 { + return nil + } + phases, err := s.repo.ListWorkflowPhasesByRootTaskID(task.ID) + if err != nil { + return err + } + for idx := range phases { + phase := phases[idx] + if task.PlanVersion > 0 && phase.PlanVersion != task.PlanVersion { + continue + } + if phase.Status == teamPhaseStatusCancelled || phase.Status == teamPhaseStatusSuperseded { + continue + } + phase.Status = teamPhaseStatusCompleted + phase.CompletedAt = &now + phase.UpdatedAt = now + if err := s.repo.UpsertWorkflowPhase(&phase); err != nil { + return err + } + } + return nil +} + +func completionAcknowledgementOutbox(team *models.Team, task *models.TeamTask, member *models.TeamMember, payload map[string]interface{}, decision, reason, sourceEventID string, now time.Time) (*models.TeamEventOutbox, error) { + ack, completionID, attemptID := buildCompletionAcknowledgement(team, task, member, payload, decision, reason) + if completionID == "" || attemptID == "" { + return nil, fmt.Errorf("completion acknowledgement requires completionId and attemptId") + } + encoded, err := json.Marshal(ack) + if err != nil { + return nil, err + } + memberKey := "leader" + if member != nil && strings.TrimSpace(member.MemberKey) != "" { + memberKey = member.MemberKey + } + return &models.TeamEventOutbox{ + TeamID: team.ID, + SourceEventID: sourceEventID, + Destination: teamCompletionAckStreamKey(team.ID, memberKey), + MessageID: fmt.Sprintf("completion-ack:%s:%s", completionID, attemptID), + PayloadJSON: string(encoded), + Status: "pending", + AvailableAt: now, + CreatedAt: now, + UpdatedAt: now, + }, nil +} + +func isLeaderMediatedInterimRootCompletion(team *models.Team, task *models.TeamTask, member *models.TeamMember, payload map[string]interface{}) bool { + if !isLeaderMediatedTeam(team) || task == nil || member == nil || payload == nil { + return false + } + if member.ID != task.TargetMemberID || !isLeaderTeamMember(member) { + return false + } + if isLeaderControlPlaneSnapshotTask(task, payload) { + return false + } + text := eventString(payload, "resultMarkdown", "result_markdown", "result", "answer", "text", "message", "summary") + if text == "" { + return false + } + return isInterimOrDelegationReplyText(text) +} + +func teamEventPayloadMap(event models.TeamEvent) map[string]interface{} { + payload := map[string]interface{}{} + if event.PayloadJSON != nil && strings.TrimSpace(*event.PayloadJSON) != "" { + _ = json.Unmarshal([]byte(*event.PayloadJSON), &payload) + } + return payload +} + +func teamEventMatchesRootTask(event models.TeamEvent, payload map[string]interface{}, task *models.TeamTask) bool { + if task == nil { + return false + } + if event.TaskID != nil && *event.TaskID == task.ID { + return true + } + for _, key := range []string{"taskId", "task_id", "rootTaskId", "root_task_id", "parentTaskId", "parent_task_id", "currentTaskId", "current_task_id", "runtimeTaskId", "runtime_task_id"} { + if parseClawManagerTeamTaskRef(task.TeamID, eventString(payload, key)) == task.ID { + return true + } + } + for _, key := range []string{"messageId", "message_id", "rootMessageId", "root_message_id", "parentMessageId", "parent_message_id", "inReplyTo", "in_reply_to", "replyTo", "reply_to"} { + if task.MessageID != "" && eventString(payload, key) == task.MessageID { + return true + } + } + if step, ok := payload["collaborationStep"].(map[string]interface{}); ok { + return teamEventMatchesRootTask(event, step, task) + } + return false +} + +func normalizeTeamMemberRouteKey(value string) string { + return strings.ToLower(strings.TrimSpace(value)) +} + +func isLeaderTeamMember(member *models.TeamMember) bool { + if member == nil { + return false + } + normalizedKey := strings.ToLower(strings.TrimSpace(member.MemberKey)) + normalizedRole := strings.ToLower(strings.TrimSpace(member.Role)) + return normalizedKey == "leader" || strings.Contains(normalizedKey, "leader") || + normalizedRole == "leader" || strings.Contains(normalizedRole, "leader") +} + +func isLeaderMediatedTeam(team *models.Team) bool { + return team != nil && normalizedTeamCommunicationMode(team.CommunicationMode) == teamCommunicationModeLeaderMediated +} + +func leaderMediatedRouteTarget(payload map[string]interface{}) string { + target := eventString(payload, "to", "recipient", "target", "targetMemberId", "target_member_id") + if target != "" { + return target + } + return eventString(payload, "assignee", "owner", "targetMember") +} + +func isLeaderRouteTarget(target string) bool { + normalized := strings.ToLower(strings.TrimSpace(target)) + if normalized == "" { + return false + } + return normalized == "leader" || + normalized == teamTaskReplyTarget || + strings.Contains(normalized, "leader") || + strings.Contains(normalized, "clawmanager") +} + +func isLeaderMediatedOutboundLikeEvent(eventType string) bool { + switch strings.ToLower(strings.TrimSpace(eventType)) { + case "outbound", "task_assigned", "team_send", "reply", "task_completed", "completion", "completion_proposed": + return true + default: + return false + } +} + +func isLeaderMediatedInvalidWorkerRoute(team *models.Team, eventType string, payload map[string]interface{}, member *models.TeamMember) bool { + if !isLeaderMediatedTeam(team) || payload == nil || member == nil || isLeaderTeamMember(member) { + return false + } + if !isLeaderMediatedOutboundLikeEvent(eventType) { + return false + } + target := leaderMediatedRouteTarget(payload) + if target == "" { + return false + } + return !isLeaderRouteTarget(target) +} + +func markLeaderMediatedRouteViolation(eventType string, payload map[string]interface{}, member *models.TeamMember) string { + if eventString(payload, "originalEvent") == "" { + payload["originalEvent"] = eventType + } + target := leaderMediatedRouteTarget(payload) + payload["event"] = "message_warning" + payload["type"] = "message_warning" + payload["status"] = "warning" + payload["runtimeStatus"] = "warning" + payload["availability"] = models.TeamMemberAvailabilityIdle + payload["leaderMediatedRouteViolation"] = true + payload["nonAuthoritative"] = true + payload["rootTaskTerminal"] = false + payload["summary"] = "Leader-mediated mode ignores worker-to-worker/self routing; workers must return assignment results to the Leader." + if target != "" { + payload["target"] = target + payload["to"] = target + } + if member != nil { + payload["from"] = member.MemberKey + } + return "message_warning" +} + +func isLeaderMediatedMonitorBlockerCandidate(team *models.Team, eventType string, payload map[string]interface{}, member *models.TeamMember, task *models.TeamTask) bool { + if !isLeaderMediatedTeam(team) || payload == nil || member == nil || task == nil || member.ID != task.TargetMemberID || !isLeaderTeamMember(member) { + return false + } + if hasTeamTaskCompletionToolCall(payload) || + strings.EqualFold(strings.TrimSpace(eventString(payload, "completionSource", "completion_source")), teamTaskCompletionTool) || + eventBool(payload, "explicitCompletion", "explicit_completion", "rootTaskTerminal") { + return false + } + if eventType != "task_failed" && eventType != "message_failed" { + status := normalizedTeamTaskEventStatus(payload) + if !isFailedTeamTaskEventStatus(status) { + return false + } + } + return looksLikeNonAuthoritativeMonitorBlockerText(eventString(payload, "resultMarkdown", "result_markdown", "result", "answer", "text", "message", "summary", "error_message", "error", "reason", "diagnostic", "lastSummary", "last_summary")) +} + +func markLeaderMediatedMonitorBlockerCandidate(eventType string, payload map[string]interface{}) string { + if eventString(payload, "originalEvent") == "" { + payload["originalEvent"] = eventType + } + payload["event"] = "message_warning" + payload["type"] = "message_warning" + payload["status"] = "attention_required" + payload["runtimeStatus"] = models.TeamTaskStatusRunning + payload["availability"] = models.TeamMemberAvailabilityBusy + payload["rootTaskTerminal"] = false + payload["nonAuthoritative"] = true + payload["blockerCandidate"] = true + payload["monitorBlockerCandidate"] = true + if eventString(payload, "summary") == "" { + payload["summary"] = "Leader monitor reported a possible blocker; waiting for member result or explicit Leader finalization." + } + return "message_warning" +} + +func looksLikeNonAuthoritativeMonitorBlockerText(text string) bool { + normalized := strings.TrimSpace(text) + if normalized == "" { + return false + } + lower := strings.ToLower(normalized) + compact := strings.ToLower(strings.Join(strings.Fields(normalized), "")) + if strings.Contains(lower, "monitoring agent") || + strings.Contains(lower, "status check") || + strings.Contains(lower, "status checks") || + strings.Contains(lower, "unresponsive") || + strings.Contains(lower, "no summary created") || + strings.Contains(lower, "zero paper files") || + strings.Contains(lower, "artifacts not found") || + strings.Contains(lower, "artifact not found") || + strings.Contains(lower, "blocker report sent") { + return true + } + return strings.Contains(compact, "监控") || + strings.Contains(compact, "状态检查") || + strings.Contains(compact, "无响应") || + strings.Contains(compact, "未找到产物") || + strings.Contains(compact, "找不到产物") +} + +func isLeaderMediatedWorkerToLeaderResult(team *models.Team, eventType string, payload map[string]interface{}, member *models.TeamMember, task *models.TeamTask) bool { + if !isLeaderMediatedTeam(team) || payload == nil || member == nil || task == nil || isLeaderTeamMember(member) { + return false + } + if member.ID == task.TargetMemberID { + return false + } + if isFailedTeamTaskEventStatus(normalizedTeamTaskEventStatus(payload)) { + return false + } + if !isLeaderMediatedOutboundLikeEvent(eventType) || !teamEventHasBody(payload) { + return false + } + if isNonAuthoritativeDispatchFailure(eventType, payload) || + isDispatchOnlyCompletionPayload(payload) || + eventBool(payload, "leaderMediatedRouteViolation") { + return false + } + target := leaderMediatedRouteTarget(payload) + if target == "" { + target = eventString(payload, "replyTarget", "reply_target") + } + if target != "" && !isLeaderRouteTarget(target) { + return false + } + if teamRedisProtocolVersion(payload) >= 2 && isExplicitTeamTaskCompletion(payload) { + return true + } + body := eventString(payload, "resultMarkdown", "result_markdown", "result", "answer", "text", "message", "summary") + if looksLikeLeaderDispatchOnlyText(body) { + return false + } + if looksLikeNonTerminalProgressText(body) { + return false + } + return true +} + +func markLeaderMediatedAssignmentResult(eventType string, payload map[string]interface{}, member *models.TeamMember) { + payload["assignmentResultOnly"] = true + payload["memberResultConfirmed"] = true + payload["rootTaskTerminal"] = false + payload["status"] = models.TeamTaskStatusSucceeded + payload["runtimeStatus"] = models.TeamTaskStatusSucceeded + payload["availability"] = models.TeamMemberAvailabilityIdle + if eventString(payload, "normalizedResultSource") == "" { + if teamRedisProtocolVersion(payload) >= 2 { + payload["normalizedResultSource"] = "explicit_completion" + } else { + payload["normalizedResultSource"] = "legacy_normalized_reply" + } + } + if eventString(payload, "originalEvent") == "" { + payload["originalEvent"] = eventType + } + if eventString(payload, "contentHash", "content_hash") == "" { + payload["contentHash"] = teamResultContentHash(payload) + } + if member != nil { + payload["from"] = member.MemberKey + payload["memberId"] = member.MemberKey + } + if leaderMediatedRouteTarget(payload) == "" { + payload["to"] = "leader" + payload["target"] = "leader" + } +} + +func teamResultContentHash(payload map[string]interface{}) string { + content := eventString(payload, "resultMarkdown", "result_markdown", "result", "answer", "text", "message", "summary") + normalized := strings.Join(strings.Fields(strings.TrimSpace(content)), " ") + refs := explicitTeamArtifactReferences(payload) + sort.Strings(refs) + if normalized == "" && len(refs) == 0 { + return "" + } + fingerprint := normalized + "\nrefs=" + strings.Join(refs, "|") + digest := sha256.Sum256([]byte(fingerprint)) + return fmt.Sprintf("sha256:%x", digest[:]) +} + +func (s *teamService) reopenLeaderMediatedRootAfterMemberResult(team *models.Team, task *models.TeamTask, payload map[string]interface{}, now time.Time) error { + if s == nil || task == nil || !isLeaderMediatedTeam(team) || !isTerminalTeamTaskStatus(task.Status) { + return nil + } + if task.Status == models.TeamTaskStatusSucceeded { + return nil + } + errText := derefTeamString(task.ErrorMessage) + if errText == "" { + errText = eventString(payload, "previousError", "previous_error") + } + if !looksLikeNonAuthoritativeMonitorBlockerText(errText) { + return nil + } + task.Status = models.TeamTaskStatusRunning + task.FinishedAt = nil + task.ErrorMessage = nil + task.UpdatedAt = now + return s.repo.UpdateTask(task) +} + +func looksLikeNonTerminalProgressText(text string) bool { + normalized := strings.TrimSpace(text) + if normalized == "" { + return false + } + if looksLikeFinalResultText(normalized) || + containsAnyTeamTextMarker(normalized, []string{ + "final delivery", + "final confirmation", + "files are at", + "qa verdict", + "pass with", + "pass:", + "needs revision", + "ready for review", + "delivered", + "交付", + "最终", + "已归档", + }) { + return false + } + lower := strings.ToLower(normalized) + compact := strings.ToLower(strings.Join(strings.Fields(normalized), "")) + if strings.Contains(lower, "progress update") || + strings.Contains(lower, "status update") || + strings.Contains(lower, "phase 1") || + strings.Contains(lower, "phase 2") || + strings.Contains(lower, "starting ") || + strings.Contains(lower, "now extracting") || + strings.Contains(lower, "now processing") || + strings.Contains(lower, "still working") || + strings.Contains(lower, "in progress") || + strings.Contains(lower, "currently ") || + strings.Contains(lower, "processing all") || + strings.Contains(lower, "downloaded successfully") || + strings.Contains(lower, "curating to") { + return true + } + return strings.Contains(compact, "进度更新") || + strings.Contains(compact, "状态更新") || + strings.Contains(compact, "正在") || + strings.Contains(compact, "处理中") || + strings.Contains(compact, "继续执行") || + strings.Contains(compact, "阶段一") || + strings.Contains(compact, "阶段二") +} + +func looksLikeLeaderDispatchOnlyText(text string) bool { + normalized := strings.TrimSpace(text) + if normalized == "" { + return false + } + if containsAnyTeamTextMarker(normalized, finalResultTextMarkers()) { + return false + } + if containsAnyTeamTextMarker(normalized, leaderDispatchTextMarkers()) { + return true + } + compact := strings.ToLower(strings.Join(strings.Fields(normalized), "")) + lower := strings.ToLower(normalized) + if strings.Contains(normalized, "$CLAWMANAGER_TEAM_SHARED_DIR") || + (containsTeamTextMarker(normalized, "\u5171\u4eab\u76ee\u5f55") && containsTeamTextMarker(normalized, "\u89c4\u8303\u8def\u5f84")) || + (strings.Contains(lower, "shared directory") && strings.Contains(lower, "canonical path")) { + return true + } + if containsTeamTextMarker(normalized, "\u5b8c\u6210\u540e") && + (containsTeamTextMarker(normalized, "\u56de\u4f20") || containsTeamTextMarker(normalized, "\u8fd4\u56de\u7ed9\u6211") || containsTeamTextMarker(normalized, "\u901a\u77e5\u6211") || containsTeamTextMarker(normalized, "\u4ea4\u4ed8\u7ed9\u6211")) { + return true + } + if containsTeamTextMarker(normalized, "\u7528\u6237\u60f3\u8ba9\u4f60") && containsTeamTextMarker(normalized, "\u8bf7") { + return true + } + for _, target := range []string{"pm", "designer", "ui-designer", "architect", "worker", "product-manager", "solution-architect"} { + if strings.Contains(compact, target+"\u4f60\u597d") || strings.Contains(compact, "@"+target) { + return true + } + } + if strings.Contains(lower, "please ") && + (strings.Contains(lower, "write") || strings.Contains(lower, "complete") || strings.Contains(lower, "return") || strings.Contains(lower, "report back")) && + (strings.Contains(lower, "designer") || strings.Contains(lower, "pm") || strings.Contains(lower, "architect") || strings.Contains(lower, "worker")) { + return true + } + return false +} + +func looksLikeFinalResultText(text string) bool { + normalized := strings.TrimSpace(text) + if normalized == "" { + return false + } + return containsAnyTeamTextMarker(normalized, finalResultTextMarkers()) +} + +func finalResultTextMarkers() []string { + return []string{ + "\u4efb\u52a1\u7ed3\u679c\u53cd\u9988", + "\u4efb\u52a1\u8f93\u51fa", + "\u6700\u7ec8\u56de\u7b54", + "\u6700\u7ec8\u65b9\u6848", + "\u6700\u7ec8\u603b\u7ed3", + "\u6700\u7ec8\u4ea4\u4ed8", + "\u6700\u7ec8\u4ea4\u4ed8\u62a5\u544a", + "\u6c47\u603b\u5982\u4e0b", + "\u7ed3\u679c\u5982\u4e0b", + "\u5df2\u5b8c\u6210\u5e76", + "\u5df2\u5b8c\u6210\uff0c", + "\u5df2\u5b8c\u6210", + "\u5df2\u5b8c\u6210\uff1a", + "\u4ea7\u51fa\u6458\u8981", + "final answer", + "final synthesis", + "final delivery", + "result summary", + "task result", + "completed with evidence", + } +} + +func leaderDispatchTextMarkers() []string { + return []string{ + "\u4efb\u52a1\u5206\u6d3e", + "\u4efb\u52a1\u4e0b\u53d1", + "\u5206\u6d3e\u7ed9", + "\u6d3e\u53d1\u7ed9", + "\u4ea4\u7ed9", + "\u8bf7\u4f60", + "\u8bf7\u5199", + "\u8bf7\u5b8c\u6210", + "assignment", + "assigned to", + "handoff", + } +} + +func containsAnyTeamTextMarker(text string, markers []string) bool { + for _, marker := range markers { + if containsTeamTextMarker(text, marker) { + return true + } + } + return false +} + +func containsTeamTextMarker(text, marker string) bool { + lowerText := strings.ToLower(text) + lowerMarker := strings.ToLower(marker) + compactText := strings.ToLower(strings.Join(strings.Fields(text), "")) + compactMarker := strings.ToLower(strings.Join(strings.Fields(marker), "")) + return strings.Contains(lowerText, lowerMarker) || strings.Contains(compactText, compactMarker) +} +func inferLeaderDispatchTarget(payload map[string]interface{}) string { + if payload == nil { + return "" + } + if target := eventString(payload, "to", "recipient", "target", "targetMemberId", "target_member_id", "assignee", "owner"); target != "" { + return target + } + text := strings.ToLower(eventString(payload, "resultMarkdown", "result_markdown", "result", "answer", "text", "message", "summary")) + for _, target := range []string{"ui-designer", "designer", "product-manager", "pm", "solution-architect", "architect", "worker"} { + if strings.Contains(text, target) { + return target + } + } + return "" +} + +func (s *teamService) projectTeamEvent(team *models.Team, bus *redisBus, message redisStreamMessage) error { + if exists, err := s.repo.EventExistsByStreamID(team.ID, message.ID); err != nil || exists { + return err + } + payload := mergeRedisEventPayload(message.Fields) + eventID := eventString(payload, "eventId", "event_id") + if eventID != "" { + if exists, err := s.repo.EventExistsByEventID(team.ID, eventID); err != nil || exists { + return err + } + } + eventType := eventString(payload, "event_type", "event", "type") + if eventType == "" { + eventType = "message" + } + incomingEventType := strings.ToLower(strings.TrimSpace(eventType)) + isCompletionProposal := incomingEventType == "completion_proposed" + if strings.EqualFold(eventType, "member_result_confirmed") { + payload["visibleToChat"] = false + payload["visible_to_chat"] = false + payload["assignmentResultOnly"] = true + payload["rootTaskTerminal"] = false + } + messageID := eventString(payload, "message_id", "messageId") + memberKey := eventString(payload, "member_id", "memberId", "member_key") + if isOutboundTeamEvent(eventType) && messageID != "" && !teamEventHasBody(payload) && bus != nil { + enriched, err := s.enrichOutboundEventFromInbox(team.ID, bus, payload, messageID) + if err != nil { + fmt.Printf("Warning: failed to enrich Team %d outbound event %s from inbox: %v\n", team.ID, messageID, err) + } else { + payload = enriched + } + } + + var member *models.TeamMember + if memberKey != "" { + found, err := s.repo.GetMemberByTeamKey(team.ID, memberKey) + if err != nil { + return err + } + member = found + } + var err error + eventType, member, err = s.normalizePeerAssistedTeamEvent(team, eventType, payload, member) + if err != nil { + return err + } + + var task *models.TeamTask + if taskID := eventInt(payload, "task_id", "taskId"); taskID > 0 { + found, err := s.repo.GetTaskByID(taskID) + if err != nil { + return err + } + if found != nil && found.TeamID == team.ID { + task = found + } + } + if task == nil && messageID != "" { + found, err := s.repo.GetTaskByMessageID(team.ID, messageID) + if err != nil { + return err + } + task = found + } + if task == nil { + found, err := s.resolveTeamTaskFromEventReferences(team.ID, payload) + if err != nil { + return err + } + task = found + } + if task == nil && isLeaderMediatedTeam(team) && member != nil { + found, err := s.resolveLeaderMediatedTaskFromWorkItem(team.ID, payload, member) + if err != nil { + return err + } + task = found + } + if task == nil && member != nil && member.CurrentTaskID != nil && shouldAssociateEventWithCurrentMemberTask(eventType, payload) { + found, err := s.activeTaskForMember(team.ID, member, true) + if err != nil { + return err + } + task = found + } + if task == nil && shouldAssociateEventWithCurrentMemberTask(eventType, payload) { + found, err := s.activeTaskFromPeerContext(team.ID, payload, member) + if err != nil { + return err + } + task = found + } + normalizeUnauthorizedAssignmentCheckResult(payload) + passiveMonitorEvent := isPassiveAssignmentMonitorEvent(eventType, payload) + if isAssignmentHeartbeatEvent(eventType, payload) { + normalizeAssignmentHeartbeatPayload(payload) + if task != nil && isTerminalTeamTaskStatus(task.Status) { + return nil + } + passiveMonitorEvent = true + } else if passiveMonitorEvent { + normalizePassiveAssignmentMonitorPayload(payload) + } + if passiveMonitorEvent && task != nil && member != nil { + targetsTerminalWorkItem, err := s.leaderMediatedMonitorTargetsTerminalWorkItem(team, task, member, payload) + if err != nil { + return err + } + if targetsTerminalWorkItem { + return nil + } + } + if isTeamPresenceEvent(eventType) && task != nil && member != nil { + terminalWorkItem, err := s.leaderMediatedMonitorTargetsTerminalWorkItem(team, task, member, payload) + if err != nil { + return err + } + if terminalWorkItem { + payload["availability"] = models.TeamMemberAvailabilityIdle + payload["runtimeStatus"] = models.TeamTaskStatusSucceeded + payload["status"] = models.TeamTaskStatusSucceeded + payload["staleRunningSuppressed"] = true + } + } + if isNonAuthoritativeDispatchFailure(eventType, payload) { + if eventString(payload, "originalEvent") == "" { + payload["originalEvent"] = eventType + } + payload["event"] = "message_warning" + payload["type"] = "message_warning" + payload["status"] = "warning" + payload["availability"] = "idle" + payload["nonAuthoritative"] = true + eventType = "message_warning" + } + leaderMediatedRouteViolation := isLeaderMediatedInvalidWorkerRoute(team, eventType, payload, member) + if leaderMediatedRouteViolation { + eventType = markLeaderMediatedRouteViolation(eventType, payload, member) + } + assignmentResultOnly := isLeaderMediatedWorkerToLeaderResult(team, eventType, payload, member, task) + if assignmentResultOnly { + markLeaderMediatedAssignmentResult(eventType, payload, member) + if task != nil && member != nil { + assignmentID := eventString(payload, "assignmentId", "assignment_id", "canonicalWorkId", "canonical_work_id", "workId", "work_id") + contentHash := eventString(payload, "contentHash", "content_hash") + alreadyProjected, confirmErr := s.hasLeaderMediatedResultConfirmation(team.ID, task.ID, member.MemberKey, assignmentID, contentHash) + if confirmErr != nil { + return confirmErr + } + if alreadyProjected { + payload["visibleToChat"] = false + payload["visible_to_chat"] = false + payload["duplicateResultProjection"] = true + } + } + } + if !assignmentResultOnly && isLeaderMediatedMonitorBlockerCandidate(team, eventType, payload, member, task) { + eventType = markLeaderMediatedMonitorBlockerCandidate(eventType, payload) + } + eventType = normalizeFinalReplyTaskEvent(eventType, payload, task, member) + eventType = markLegacyRuntimeCompletionCandidate(eventType, payload, task, member) + eventStatus := normalizedTeamTaskEventStatus(payload) + eventSignalsCompletion := isTeamTaskCompletionSignal(eventType, eventStatus, payload) + eventSignalsFailure := isTeamTaskFailureSignal(eventType, eventStatus, payload) + if isCompletionProposal && !eventSignalsCompletion && !eventSignalsFailure { + evaluation := teamCompletionEvaluation{Decision: teamCompletionDecisionRejected, Reason: "invalid_completion_envelope"} + if task != nil { + evaluation.LedgerVersion = task.LedgerVersion + evaluation.PlanVersion = task.PlanVersion + } + eventType = markStructuredCompletionDecision(eventType, payload, evaluation) + eventStatus = normalizedTeamTaskEventStatus(payload) + } + if isUnauthoritativeCompletionEvent(eventType, eventSignalsCompletion, eventSignalsFailure) { + if eventString(payload, "originalEvent") == "" { + payload["originalEvent"] = eventType + } + payload["event"] = "reply" + payload["type"] = "reply" + payload["status"] = models.TeamTaskStatusRunning + payload["runtimeStatus"] = models.TeamTaskStatusRunning + payload["availability"] = models.TeamMemberAvailabilityBusy + payload["rootTaskTerminal"] = false + payload["nonAuthoritativeCompletion"] = true + eventType = "reply" + eventStatus = normalizedTeamTaskEventStatus(payload) + eventSignalsCompletion = false + eventSignalsFailure = false + } + leaderDispatchOnly := false + if !eventSignalsCompletion && !eventSignalsFailure { + leaderDispatchOnly = isLeaderMediatedLeaderDispatchOnlyMessage(team, eventType, payload, member, task) + } + if leaderDispatchOnly { + if eventString(payload, "originalEvent") == "" { + payload["originalEvent"] = eventType + } + payload["event"] = "reply" + payload["type"] = "reply" + payload["status"] = models.TeamTaskStatusDispatched + payload["runtimeStatus"] = models.TeamTaskStatusRunning + payload["availability"] = models.TeamMemberAvailabilityBusy + payload["rootTaskTerminal"] = false + payload["leaderDispatchOnly"] = true + if target := inferLeaderDispatchTarget(payload); target != "" { + payload["target"] = target + payload["to"] = target + } + eventType = "reply" + eventStatus = normalizedTeamTaskEventStatus(payload) + eventSignalsCompletion = false + eventSignalsFailure = false + } + if eventSignalsCompletion && !leaderDispatchOnly && !assignmentResultOnly && !leaderMediatedRouteViolation { + // Repair a stale derived phase state before judging an explicit final + // delivery. This is local projection repair, not a concurrent-plan change, + // so carry the repaired ledger version into this evaluation. + if task != nil { + workflowFinal := eventBool(payload, "workflowFinal", "workflow_final", "sealWorkflow", "seal_workflow") + reconciled, reconcileErr := s.reconcileTeamWorkflowLedger(task, workflowFinal, time.Now().UTC()) + if reconcileErr != nil { + return reconcileErr + } + if reconciled { + payload["ledgerVersion"] = task.LedgerVersion + payload["planVersion"] = task.PlanVersion + payload["workflowLedgerReconciled"] = true + } + } + narrative := eventString(payload, "summary") + "\n" + eventString(payload, "resultMarkdown", "result_markdown", "result", "answer") + payload["interimNarrativeSignal"] = isInterimOrDelegationReplyText(narrative) + evaluation, err := s.evaluateLeaderRootCompletion(team, task, member, payload) + if err != nil { + return err + } + payload["completionEvaluation"] = evaluation.Decision + if len(evaluation.WaivedAssignments) > 0 { + payload["waivedAssignments"] = evaluation.WaivedAssignments + } + if len(evaluation.SkippedAssignments) > 0 { + payload["skippedAssignmentsAccepted"] = evaluation.SkippedAssignments + } + if evaluation.Decision != teamCompletionDecisionAccepted { + eventType = markStructuredCompletionDecision(eventType, payload, evaluation) + eventStatus = normalizedTeamTaskEventStatus(payload) + eventSignalsCompletion = false + eventSignalsFailure = false + } else { + payload["event"] = "task_completed" + payload["type"] = "task_completed" + payload["completionDecision"] = teamCompletionDecisionAccepted + payload["chatKind"] = "final_delivery" + payload["chatPolicy"] = "visible" + payload["visibleToChat"] = true + payload["visible_to_chat"] = true + payload["displayKey"] = fmt.Sprintf("root-final:%d", task.ID) + } + } + s.normalizeTeamArtifactReferences(team, payload) + if eventSignalsCompletion { + if missing := s.missingTeamArtifactReferences(team, payload); len(missing) > 0 { + if eventString(payload, "originalEvent") == "" { + payload["originalEvent"] = eventType + } + payload["event"] = "message_warning" + payload["type"] = "message_warning" + payload["status"] = "blocked" + payload["runtimeStatus"] = models.TeamTaskStatusRunning + payload["availability"] = models.TeamMemberAvailabilityBusy + payload["rootTaskTerminal"] = false + payload["artifactValidationFailed"] = true + payload["missingArtifactRefs"] = missing + if isCompletionProposal || isExplicitTeamTaskCompletion(payload) { + payload["completionDecision"] = teamCompletionDecisionRejected + payload["completionDecisionReason"] = "missing_artifacts" + payload["completionEvaluation"] = teamCompletionDecisionRejected + } + // Preserve the final answer. Artifact validation is workflow state, not a + // replacement for the result the member already returned. + payload["artifactValidationMessage"] = "Team artifact references are not readable from the shared workspace; task remains open." + eventType = "message_warning" + eventStatus = normalizedTeamTaskEventStatus(payload) + eventSignalsCompletion = false + eventSignalsFailure = false + } + } + if (eventSignalsCompletion || eventSignalsFailure) && teamRedisProtocolVersion(payload) >= 2 { + duplicate, err := s.hasAcceptedTeamCompletionID( + team.ID, + eventString(payload, "completionId", "completion_id"), + ) + if err != nil { + return err + } + if duplicate { + if isCompletionProposal { + if task != nil && task.Status == models.TeamTaskStatusSucceeded { + if err := s.completeWorkflowPhases(task, time.Now().UTC()); err != nil { + return err + } + } + if err := s.emitCompletionAcknowledgement(team, bus, task, member, payload, teamCompletionDecisionAccepted, "already_accepted"); err != nil { + return err + } + } + return nil + } + } + memberTerminalOnly := task != nil && + member != nil && + member.ID != task.TargetMemberID && + (eventSignalsCompletion || eventSignalsFailure) + if memberTerminalOnly { + payload["memberTerminalOnly"] = true + payload["rootTaskTerminal"] = false + } + applyTeamChatPolicy(eventType, payload, task, member) + enrichTeamCollaborationStep(team, eventType, payload, member, task) + + payloadJSON, err := marshalOptionalJSON(payload) + if err != nil { + return err + } + streamID := message.ID + event := &models.TeamEvent{ + TeamID: team.ID, + EventType: eventType, + PayloadJSON: payloadJSON, + RedisStreamID: &streamID, + OccurredAt: eventTime(payload), + } + if eventID != "" { + event.EventID = &eventID + } + if eventSignalsCompletion || eventSignalsFailure { + completionID := eventString(payload, "completionId", "completion_id") + if completionID != "" { + event.CompletionID = &completionID + } + } + if member != nil { + event.MemberID = &member.ID + } + if task != nil { + event.TaskID = &task.ID + } + if messageID != "" { + event.MessageID = &messageID + } + atomicRootCompletionAccepted := false + if eventSignalsCompletion && task != nil && member != nil && member.ID == task.TargetMemberID && isLeaderTeamMember(member) { + now := time.Now().UTC() + expectedLedgerVersion := task.LedgerVersion + completionID := eventString(payload, "completionId", "completion_id") + if completionID == "" { + seed := eventID + if seed == "" { + seed = message.ID + } + completionID = fmt.Sprintf("legacy:%d:%s", task.ID, normalizeTeamRedisKeyPart(seed)) + payload["completionId"] = completionID + payloadJSON, err = marshalOptionalJSON(payload) + if err != nil { + return err + } + event.PayloadJSON = payloadJSON + event.CompletionID = &completionID + } + task.Status = models.TeamTaskStatusSucceeded + task.WorkflowState = teamWorkflowStateCompleted + task.LedgerVersion = expectedLedgerVersion + 1 + task.CurrentPhaseID = nil + task.AcceptedCompletionID = &completionID + task.ResultJSON = payloadJSON + task.ErrorMessage = nil + task.FinishedAt = &now + task.UpdatedAt = now + sourceEventID := eventID + if sourceEventID == "" { + sourceEventID = message.ID + } + ackOutbox, ackErr := completionAcknowledgementOutbox(team, task, member, payload, teamCompletionDecisionAccepted, "workflow_closed", sourceEventID, now) + if ackErr != nil { + return ackErr + } + accepted, acceptErr := s.repo.AcceptRootCompletion(task, expectedLedgerVersion, event, ackOutbox) + if acceptErr != nil { + return acceptErr + } + if accepted { + atomicRootCompletionAccepted = true + if err := s.emitCompletionAcknowledgement(team, bus, task, member, payload, teamCompletionDecisionAccepted, "workflow_closed"); err != nil { + return err + } + if ackOutbox.ID > 0 && bus != nil { + if err := s.repo.MarkEventOutboxDelivered(ackOutbox.ID, time.Now().UTC()); err != nil { + return err + } + } + } else { + task.LedgerVersion = expectedLedgerVersion + task.Status = models.TeamTaskStatusRunning + task.WorkflowState = teamWorkflowStateCompletionPending + task.AcceptedCompletionID = nil + task.FinishedAt = nil + evaluation := teamCompletionEvaluation{ + Decision: teamCompletionDecisionDeferred, + Reason: "stale_ledger", + LedgerVersion: expectedLedgerVersion, + PlanVersion: task.PlanVersion, + } + eventType = markStructuredCompletionDecision(eventType, payload, evaluation) + eventSignalsCompletion = false + event.CompletionID = nil + event.EventType = eventType + payloadJSON, err = marshalOptionalJSON(payload) + if err != nil { + return err + } + event.PayloadJSON = payloadJSON + } + } + if !atomicRootCompletionAccepted { + if err := s.repo.CreateEvent(event); err != nil { + if errors.Is(err, repository.ErrDuplicateTeamEvent) { + return nil + } + return err + } + } + reviewInvalidated, err := s.invalidateModifiedTeamArtifactReviews(task, payload, time.Now().UTC()) + if err != nil { + return err + } + workflowChanged := reviewInvalidated + if err := s.projectTeamWorkItem(team, task, member, eventType, payload, event); err != nil { + return err + } + if !atomicRootCompletionAccepted { + ledgerChanged, ledgerErr := s.projectTeamWorkflowLedger(team, task, member, eventType, payload, time.Now().UTC()) + err = ledgerErr + if err != nil { + return err + } + workflowChanged = workflowChanged || ledgerChanged + } + now := time.Now().UTC() + if assignmentResultOnly { + if err := s.createLeaderMediatedResultNotification(team, bus, task, member, payload, event); err != nil { + return err + } + if err := s.reopenLeaderMediatedRootAfterMemberResult(team, task, payload, now); err != nil { + return err + } + } + if !assignmentResultOnly && !passiveMonitorEvent && isLeaderMediatedRecoverableWarning(team, eventType, payload, member, task) { + if err := s.createLeaderMediatedRecoveryRequest(team, bus, task, member, payload, event); err != nil { + return err + } + } + + taskProjection := teamTaskProjectionResult{} + if atomicRootCompletionAccepted { + taskProjection = teamTaskProjectionResult{changed: true, status: models.TeamTaskStatusSucceeded} + } else if task != nil && !passiveMonitorEvent && !memberTerminalOnly && !assignmentResultOnly && !leaderMediatedRouteViolation { + taskProjection = projectTeamTaskRuntimeState(task, payload, eventType, payloadJSON, now) + if taskProjection.changed || workflowChanged { + task.UpdatedAt = now + if err := s.repo.UpdateTask(task); err != nil { + return err + } + } + } + if task != nil && (memberTerminalOnly || leaderDispatchOnly || assignmentResultOnly || leaderMediatedRouteViolation || workflowChanged) && !taskProjection.changed && !isTerminalTeamTaskStatus(task.Status) { + task.UpdatedAt = now + if err := s.repo.UpdateTask(task); err != nil { + return err + } + } + if member != nil { + member.LastSeenAt = &now + if !passiveMonitorEvent { + applyTeamMemberRuntimeProjection(member, payload, eventType) + } + taskIsActive := task != nil && !isTerminalTeamTaskStatus(task.Status) + if !passiveMonitorEvent && taskIsActive && (leaderDispatchOnly || eventType == "task_received" || eventType == "task_started" || taskProjection.status == models.TeamTaskStatusRunning || taskProjection.status == models.TeamTaskStatusDispatched) { + member.Status = models.TeamMemberStatusBusy + if member.Availability == "" || member.Availability == models.TeamMemberAvailabilityUnknown { + member.Availability = models.TeamMemberAvailabilityBusy + } + member.CurrentTaskID = &task.ID + member.Progress = eventInt(payload, "progress") + } + taskProjectedTerminal := taskProjection.status == models.TeamTaskStatusSucceeded || taskProjection.status == models.TeamTaskStatusFailed + terminalEventWithoutTask := (task == nil || memberTerminalOnly || assignmentResultOnly) && (eventSignalsCompletion || eventSignalsFailure || assignmentResultOnly) + if taskProjectedTerminal || terminalEventWithoutTask { + member.Status = models.TeamMemberStatusIdle + member.CurrentTaskID = nil + if taskProjection.status == models.TeamTaskStatusSucceeded || assignmentResultOnly || (terminalEventWithoutTask && eventSignalsCompletion) { + member.Progress = 100 + if member.Availability != models.TeamMemberAvailabilityBlocked { + member.Availability = models.TeamMemberAvailabilityIdle + member.BlockedReason = nil + } + } else if taskProjection.status == models.TeamTaskStatusFailed || (terminalEventWithoutTask && eventSignalsFailure) { + member.Progress = 0 + if member.Availability == "" || member.Availability == models.TeamMemberAvailabilityUnknown { + member.Availability = models.TeamMemberAvailabilityBlocked + } + if member.BlockedReason == nil { + if errText := eventString(payload, "error_message", "error", "reason", "diagnostic", "lastSummary", "last_summary"); errText != "" { + member.BlockedReason = &errText + } + } + } + } + if task != nil && task.Status == models.TeamTaskStatusSucceeded { + member.Status = models.TeamMemberStatusIdle + member.CurrentTaskID = nil + member.Progress = 100 + member.Availability = models.TeamMemberAvailabilityIdle + member.BlockedReason = nil + } + member.UpdatedAt = now + if err := s.repo.UpdateMember(member); err != nil { + return err + } + } + if isCompletionProposal && !atomicRootCompletionAccepted { + decision := eventString(payload, "completionDecision", "completion_decision") + reason := eventString(payload, "completionDecisionReason", "completion_decision_reason") + if assignmentResultOnly || memberTerminalOnly || eventSignalsFailure { + decision = teamCompletionDecisionAccepted + if eventSignalsFailure { + reason = "failure_recorded" + } else { + reason = "assignment_result_recorded" + } + } + if decision == "" { + decision = teamCompletionDecisionDeferred + } + sourceEventID := eventID + if sourceEventID == "" { + sourceEventID = message.ID + } + var ackOutbox *models.TeamEventOutbox + if eventString(payload, "completionId", "completion_id") != "" { + var err error + ackOutbox, err = completionAcknowledgementOutbox(team, task, member, payload, decision, reason, sourceEventID, now) + if err != nil { + return err + } + if err := s.repo.CreateEventOutbox(ackOutbox); err != nil { + return err + } + } + if err := s.emitCompletionAcknowledgement(team, bus, task, member, payload, decision, reason); err != nil { + return err + } + if ackOutbox != nil && ackOutbox.ID > 0 && bus != nil { + if err := s.repo.MarkEventOutboxDelivered(ackOutbox.ID, time.Now().UTC()); err != nil { + return err + } + } + } + return nil +} + +func isLeaderMediatedRecoverableWarning(team *models.Team, eventType string, payload map[string]interface{}, member *models.TeamMember, task *models.TeamTask) bool { + if team == nil || task == nil || member == nil || payload == nil { + return false + } + if !isLeaderMediatedTeam(team) || isLeaderTeamMember(member) || isTerminalTeamTaskStatus(task.Status) { + return false + } + if isPassiveAssignmentMonitorEvent(eventType, payload) { + return false + } + if eventBool(payload, "rootTaskTerminal", "root_task_terminal") { + return false + } + eventKind := strings.ToLower(strings.TrimSpace(eventString(payload, "eventKind", "event_kind", "kind"))) + if eventKind == "assignment_recovery_exhausted" { + return false + } + if eventBool(payload, "artifactValidationFailed", "artifact_validation_failed") { + return true + } + if eventBool(payload, "nonAuthoritativeCompletion", "non_authoritative_completion") { + return true + } + if strings.EqualFold(eventType, "message_warning") { + return true + } + return eventKind == "completion_validation_warning" +} + +func (s *teamService) createLeaderMediatedRecoveryRequest(team *models.Team, bus *redisBus, task *models.TeamTask, member *models.TeamMember, sourcePayload map[string]interface{}, sourceEvent *models.TeamEvent) error { + if s == nil || team == nil || bus == nil || task == nil || member == nil || sourcePayload == nil || sourceEvent == nil { + return nil + } + leaderKey := "leader" + if leader, err := s.repo.GetMemberByID(task.TargetMemberID); err == nil && leader != nil && strings.TrimSpace(leader.MemberKey) != "" { + leaderKey = leader.MemberKey + } + streamRef := "" + if sourceEvent.RedisStreamID != nil { + streamRef = *sourceEvent.RedisStreamID + } + if streamRef == "" { + streamRef = strconv.FormatInt(sourceEvent.CreatedAt.UnixNano(), 10) + } + eventID := fmt.Sprintf("leader-recovery:%d:%d:%s:%s", team.ID, task.ID, member.MemberKey, streamRef) + exists, err := s.repo.EventExistsByEventID(team.ID, eventID) + if err != nil || exists { + return err + } + rootTaskRef := fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID) + summary := eventString(sourcePayload, "summary", "diagnostic", "error_message", "error", "message", "text") + if summary == "" { + summary = "Recoverable assignment issue detected; Leader should replan, retry, or reassign before surfacing an error to the user." + } + workID := eventString(sourcePayload, "workId", "work_id", "assignmentId", "assignment_id") + if workID == "" { + workID = "member-" + normalizeTeamMemberRouteKey(member.MemberKey) + } + notificationPayload := map[string]interface{}{ + "event": "assignment_recovery_started", + "type": "assignment_recovery_started", + "eventKind": "assignment_recovery_started", + "protocolVersion": 2, + "source": "clawmanager_recovery_controller", + "recoverable": true, + "rootTaskTerminal": false, + "nonAuthoritative": true, + "rootTaskId": rootTaskRef, + "rootMessageId": task.MessageID, + "messageId": task.MessageID, + "assignmentId": workID, + "workId": workID, + "from": "clawmanager", + "memberId": member.MemberKey, + "to": leaderKey, + "target": leaderKey, + "status": models.TeamTaskStatusRunning, + "runtimeStatus": models.TeamTaskStatusRunning, + "availability": models.TeamMemberAvailabilityBusy, + "summary": summary, + "sourceEventId": sourceEvent.ID, + "sourceEventType": sourceEvent.EventType, + "sourcePayload": sourcePayload, + "visibleToChat": true, + "recoveryAction": "leader_replan_or_reissue", + "collaborationStep": map[string]interface{}{ + "type": "progress", + "status": models.TeamTaskStatusRunning, + "actor": "clawmanager", + "target": member.MemberKey, + "rootTaskId": rootTaskRef, + "rootMessageId": task.MessageID, + "workId": workID, + "title": "Recovery started for " + member.MemberKey, + "summary": summary, + "content": summary, + "source": "clawmanager_recovery_controller", + }, + } + payloadJSON, err := marshalOptionalJSON(notificationPayload) + if err != nil { + return err + } + now := time.Now().UTC() + event := &models.TeamEvent{ + TeamID: team.ID, + TaskID: &task.ID, + MemberID: &member.ID, + MessageID: &task.MessageID, + EventID: &eventID, + EventType: "assignment_recovery_started", + PayloadJSON: payloadJSON, + OccurredAt: &now, + CreatedAt: now, + } + if err := s.repo.CreateEvent(event); err != nil { + return err + } + s.dispatchLeaderMediatedRecoveryRequestToInbox(team, bus, task, member, leaderKey, notificationPayload, eventID) + return nil +} + +func (s *teamService) dispatchLeaderMediatedRecoveryRequestToInbox(team *models.Team, bus *redisBus, task *models.TeamTask, member *models.TeamMember, leaderKey string, notificationPayload map[string]interface{}, eventID string) { + if team == nil || bus == nil || task == nil || member == nil || notificationPayload == nil { + return + } + if strings.TrimSpace(leaderKey) == "" { + leaderKey = "leader" + } + summary := eventString(notificationPayload, "summary") + prompt := fmt.Sprintf( + "ClawManager detected a recoverable issue for member %s on root task %s.\n\nIssue:\n%s\n\nFirst try to recover without exposing this as a user-facing failure: inspect existing progress/artifacts, ask the same member to continue if appropriate, reissue the assignment with corrected instructions, or reassign to another member. Only report failure to the user after recovery is exhausted.", + member.MemberKey, + task.MessageID, + summary, + ) + envelope := map[string]interface{}{ + "v": 1, + "messageId": eventID, + "teamId": strconv.Itoa(team.ID), + "from": "clawmanager", + "to": leaderKey, + "replyTo": teamTaskReplyTarget, + "requiresCompletion": false, + "completionTool": teamTaskCompletionTool, + "intent": "assignment_recovery_request", + "taskId": fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID), + "rootTaskId": fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID), + "rootMessageId": task.MessageID, + "workId": eventString(notificationPayload, "workId", "assignmentId"), + "assignmentId": eventString(notificationPayload, "assignmentId", "workId"), + "title": member.MemberKey + " assignment recovery requested", + "prompt": prompt, + "rawPrompt": prompt, + "monitorPolicy": defaultTeamMonitorPolicy(), + "metadata": notificationPayload, + "createdAt": time.Now().UTC().Format(time.RFC3339Nano), + } + applyTeamTaskEnvelopeContext(envelope, task, leaderKey) + envelopeJSON, err := marshalJSON(envelope) + if err != nil { + fmt.Printf("Warning: failed to encode Leader recovery notification for Team %d task %d: %v\n", team.ID, task.ID, err) + return + } + if _, err := bus.XAdd(context.Background(), teamInboxKey(team.ID, leaderKey), map[string]string{ + "payload": envelopeJSON, + "team_id": strconv.Itoa(team.ID), + "task_id": strconv.Itoa(task.ID), + "message_id": eventID, + "member_id": leaderKey, + }); err != nil { + fmt.Printf("Warning: failed to dispatch Leader recovery notification for Team %d task %d: %v\n", team.ID, task.ID, err) + } +} + +func (s *teamService) createLeaderMediatedResultNotification(team *models.Team, bus *redisBus, task *models.TeamTask, member *models.TeamMember, sourcePayload map[string]interface{}, sourceEvent *models.TeamEvent) error { + if s == nil || team == nil || task == nil || member == nil || sourcePayload == nil || sourceEvent == nil { + return nil + } + if !isLeaderMediatedTeam(team) || isLeaderTeamMember(member) { + return nil + } + workID := eventString(sourcePayload, "assignmentId", "assignment_id", "canonicalWorkId", "canonical_work_id", "workId", "work_id") + if workID == "" { + workID = "member-" + normalizeTeamMemberRouteKey(member.MemberKey) + } + revision := eventInt(sourcePayload, "revision", "workRevision", "work_revision") + if revision <= 0 { + revision = 1 + } + source := eventString(sourcePayload, "normalizedResultSource") + if source == "" { + source = "legacy_normalized_reply" + } + summary := eventString(sourcePayload, "summary", "title") + resultMarkdown := eventString(sourcePayload, "resultMarkdown", "result_markdown", "result", "answer", "text", "message", "summary") + if summary == "" { + summary = truncateForSummary(resultMarkdown, 180) + } + if strings.TrimSpace(summary) == "" && strings.TrimSpace(resultMarkdown) == "" { + return nil + } + contentHash := eventString(sourcePayload, "contentHash", "content_hash") + if contentHash == "" { + contentHash = teamResultContentHash(sourcePayload) + } + alreadyConfirmed, err := s.hasLeaderMediatedResultConfirmation(team.ID, task.ID, member.MemberKey, workID, contentHash) + if err != nil || alreadyConfirmed { + return err + } + confirmationSeed := fmt.Sprintf("%d:%d:%s:%s:%d:%s", team.ID, task.ID, normalizeTeamMemberRouteKey(member.MemberKey), workID, revision, contentHash) + confirmationDigest := sha256.Sum256([]byte(confirmationSeed)) + eventID := fmt.Sprintf("member-result-confirmed:%d:%d:%x", team.ID, task.ID, confirmationDigest[:12]) + exists, err := s.repo.EventExistsByEventID(team.ID, eventID) + if err != nil || exists { + return err + } + rootTaskRef := fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID) + sourceEventID := "" + if sourceEvent.EventID != nil { + sourceEventID = strings.TrimSpace(*sourceEvent.EventID) + } + if sourceEventID == "" && sourceEvent.RedisStreamID != nil { + sourceEventID = strings.TrimSpace(*sourceEvent.RedisStreamID) + } + if sourceEventID == "" && sourceEvent.ID > 0 { + sourceEventID = strconv.Itoa(sourceEvent.ID) + } + sourceCompletionID := eventString(sourcePayload, "completionId", "completion_id") + sourceWorkID := eventString(sourcePayload, "workId", "work_id", "assignmentId", "assignment_id") + if sourceWorkID == "" { + sourceWorkID = workID + } + notificationPayload := map[string]interface{}{ + "event": "member_result_confirmed", + "type": "member_result_confirmed", + "protocolVersion": 2, + "source": "clawmanager_assignment_ledger", + "normalizedResultSource": source, + "memberResultConfirmed": true, + "assignmentResultOnly": true, + "rootTaskTerminal": false, + "visibleToChat": false, + "visible_to_chat": false, + "chatPolicy": "hidden", + "sourceEventId": sourceEventID, + "sourceCompletionId": sourceCompletionID, + "sourceWorkId": sourceWorkID, + "contentHash": contentHash, + "rootTaskId": rootTaskRef, + "rootMessageId": task.MessageID, + "messageId": task.MessageID, + "assignmentId": workID, + "workId": workID, + "revision": revision, + "from": member.MemberKey, + "memberId": member.MemberKey, + "to": "leader", + "target": "leader", + "status": models.TeamTaskStatusSucceeded, + "summary": member.MemberKey + " assignment result confirmed", + "collaborationStep": map[string]interface{}{ + "type": "ack", + "status": models.TeamTaskStatusSucceeded, + "actor": member.MemberKey, + "target": "leader", + "rootTaskId": rootTaskRef, + "rootMessageId": task.MessageID, + "workId": workID, + "title": member.MemberKey + " result confirmed", + "summary": member.MemberKey + " assignment result confirmed", + "source": "clawmanager_assignment_ledger", + }, + } + payloadJSON, err := marshalOptionalJSON(notificationPayload) + if err != nil { + return err + } + now := time.Now().UTC() + event := &models.TeamEvent{ + TeamID: team.ID, + TaskID: &task.ID, + MemberID: &member.ID, + MessageID: &task.MessageID, + EventID: &eventID, + EventType: "member_result_confirmed", + PayloadJSON: payloadJSON, + OccurredAt: &now, + CreatedAt: now, + } + resultJSON, err := marshalOptionalJSON(sourcePayload) + if err != nil { + return err + } + workItem := &models.TeamWorkItem{ + TeamID: team.ID, + RootTaskID: task.ID, + WorkID: workID, + AssignmentID: &workID, + CanonicalWorkID: &workID, + Revision: revision, + RequiredForRoot: true, + OwnerMemberID: &member.ID, + Title: member.MemberKey + " delivers result", + Status: models.TeamTaskStatusSucceeded, + ResultJSON: resultJSON, + FinishedAt: &now, + UpdatedAt: now, + } + if items, listErr := s.repo.ListWorkItemsByRootTaskID(task.ID); listErr == nil { + for idx := range items { + candidateID := derefTeamString(items[idx].AssignmentID) + if candidateID == "" { + candidateID = items[idx].WorkID + } + if items[idx].TeamID == team.ID && candidateID == workID && teamMaxInt(items[idx].Revision, 1) == revision { + clone := items[idx] + clone.Status = models.TeamTaskStatusSucceeded + clone.ResultJSON = resultJSON + clone.FinishedAt = &now + clone.UpdatedAt = now + workItem = &clone + break + } + } + } + deliveryPayload := make(map[string]interface{}, len(notificationPayload)+3) + for key, value := range notificationPayload { + deliveryPayload[key] = value + } + deliveryPayload["summary"] = summary + deliveryPayload["resultMarkdown"] = resultMarkdown + deliveryPayload["artifactRefs"] = explicitTeamArtifactReferences(sourcePayload) + deliveryEnvelope, leaderKey := s.buildLeaderMediatedResultNotificationEnvelope(team, task, member, deliveryPayload, eventID) + deliveryJSON, err := marshalJSON(deliveryEnvelope) + if err != nil { + return err + } + outbox := &models.TeamEventOutbox{ + TeamID: team.ID, + SourceEventID: eventID, + Destination: teamInboxKey(team.ID, leaderKey), + MessageID: eventID, + PayloadJSON: deliveryJSON, + Status: "pending", + AvailableAt: now, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.repo.ConfirmWorkItemResult(workItem, event, outbox); err != nil { + return err + } + if outbox.ID > 0 { + if err := s.deliverTeamEventOutbox(team, bus, outbox); err != nil { + _ = s.repo.MarkEventOutboxFailed(outbox.ID, now.Add(teamOutboxRetryDelay(outbox.Attempts)), err.Error()) + return nil + } + if err := s.repo.MarkEventOutboxDelivered(outbox.ID, time.Now().UTC()); err != nil { + return err + } + } + return nil +} + +func (s *teamService) hasLeaderMediatedResultConfirmation(teamID, taskID int, memberKey string, assignmentID string, contentHashes ...string) (bool, error) { + if s == nil || s.repo == nil { + return false, nil + } + events, err := s.repo.ListEventsByTeamID(teamID, 500) + if err != nil { + return false, err + } + normalizedMember := normalizeTeamMemberRouteKey(memberKey) + expectedAssignment := strings.TrimSpace(assignmentID) + expectedHash := "" + if len(contentHashes) > 0 { + expectedHash = strings.TrimSpace(contentHashes[0]) + } + for idx := range events { + event := events[idx] + if event.EventType != "member_result_confirmed" { + continue + } + if event.TaskID == nil || *event.TaskID != taskID { + continue + } + payload := teamEventPayloadMap(event) + from := normalizeTeamMemberRouteKey(eventString(payload, "from", "memberId", "member_id")) + if from == "" || !teamMemberRouteEquivalent(from, normalizedMember) { + continue + } + confirmedAssignmentID := eventString(payload, "assignmentId", "assignment_id", "workId", "work_id", "sourceWorkId", "source_work_id") + if expectedAssignment != "" && confirmedAssignmentID != "" && confirmedAssignmentID != expectedAssignment { + continue + } + confirmedHash := eventString(payload, "contentHash", "content_hash") + if confirmedHash == "" { + confirmedHash = teamResultContentHash(payload) + } + if expectedHash == "" || confirmedHash == "" { + continue + } + if confirmedHash == expectedHash { + return true, nil + } + } + return false, nil +} + +func (s *teamService) buildLeaderMediatedResultNotificationEnvelope(team *models.Team, task *models.TeamTask, member *models.TeamMember, notificationPayload map[string]interface{}, eventID string) (map[string]interface{}, string) { + leaderKey := "leader" + if s != nil && s.repo != nil { + if leader, err := s.repo.GetMemberByID(task.TargetMemberID); err == nil && leader != nil && strings.TrimSpace(leader.MemberKey) != "" { + leaderKey = leader.MemberKey + } + } + resultMarkdown := eventString(notificationPayload, "resultMarkdown", "summary") + prompt := fmt.Sprintf( + "Member %s delivered a confirmed assignment result for root task %s.\n\nResult:\n%s\n\nRecord this result in your synthesis context. If every required assignment has delivered, provide the final user-facing synthesis and call %s for the root task. Do not close the root task if any required assignment is still missing or blocked.", + member.MemberKey, + task.MessageID, + resultMarkdown, + teamTaskCompletionTool, + ) + envelope := map[string]interface{}{ + "v": 1, + "protocolVersion": 2, + "messageId": eventID, + "teamId": strconv.Itoa(team.ID), + "from": "clawmanager", + "to": leaderKey, + "replyTo": teamTaskReplyTarget, + "requiresCompletion": false, + "completionTool": teamTaskCompletionTool, + "intent": "member_result_confirmed", + "taskId": fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID), + "rootTaskId": fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID), + "rootMessageId": task.MessageID, + "workId": eventString(notificationPayload, "workId", "assignmentId"), + "assignmentId": eventString(notificationPayload, "assignmentId", "workId"), + "title": member.MemberKey + " assignment result confirmed", + "prompt": prompt, + "rawPrompt": prompt, + "monitorPolicy": defaultTeamMonitorPolicy(), + "metadata": notificationPayload, + "createdAt": time.Now().UTC().Format(time.RFC3339Nano), + } + applyTeamTaskEnvelopeContext(envelope, task, leaderKey) + return envelope, leaderKey +} + +func truncateForSummary(text string, max int) string { + value := strings.TrimSpace(text) + if value == "" || max <= 0 { + return value + } + runes := []rune(value) + if len(runes) <= max { + return value + } + return string(runes[:max]) + "..." +} + +func (s *teamService) resolveTeamTaskFromEventReferences(teamID int, payload map[string]interface{}) (*models.TeamTask, error) { + if payload == nil { + return nil, nil + } + for _, key := range []string{ + "rootMessageId", "root_message_id", + "completionMessageId", "completion_message_id", + "messageId", "message_id", + "parentMessageId", "parent_message_id", + "inReplyTo", "in_reply_to", + "replyTo", "reply_to", + } { + messageID := eventString(payload, key) + if messageID == "" { + continue + } + if found, err := s.repo.GetTaskByMessageID(teamID, messageID); err != nil { + return nil, err + } else if found != nil && found.TeamID == teamID { + return found, nil + } + } + for _, key := range []string{ + "rootTaskId", "root_task_id", + "completionTaskId", "completion_task_id", + "taskId", "task_id", + "parentTaskId", "parent_task_id", + "currentTaskId", "current_task_id", + "runtimeTaskId", "runtime_task_id", + } { + if taskID := parseClawManagerTeamTaskRef(teamID, eventString(payload, key)); taskID > 0 { + found, err := s.repo.GetTaskByID(taskID) + if err != nil { + return nil, err + } + if found != nil && found.TeamID == teamID { + return found, nil + } + } + } + if step, ok := payload["collaborationStep"].(map[string]interface{}); ok { + return s.resolveTeamTaskFromEventReferences(teamID, step) + } + return nil, nil +} + +func (s *teamService) resolveLeaderMediatedTaskFromWorkItem(teamID int, payload map[string]interface{}, member *models.TeamMember) (*models.TeamTask, error) { + if s == nil || payload == nil || member == nil || isLeaderTeamMember(member) { + return nil, nil + } + status := normalizedTeamTaskEventStatus(payload) + eventType := strings.ToLower(strings.TrimSpace(eventString(payload, "event", "type", "event_type"))) + if !isLeaderMediatedWorkerResultLikeEvent(eventType, status, payload) && !isPassiveAssignmentMonitorEvent(eventType, payload) { + return nil, nil + } + ownerKey := normalizeTeamMemberRouteKey(member.MemberKey) + if ownerKey == "" { + return nil, nil + } + workIDs := map[string]struct{}{ + "member-" + ownerKey: {}, + } + for _, key := range []string{"workId", "work_id", "assignmentId", "assignment_id", "subtaskId", "subtask_id"} { + if value := normalizeTeamMemberRouteKey(eventString(payload, key)); value != "" { + workIDs[value] = struct{}{} + } + } + if step, ok := payload["collaborationStep"].(map[string]interface{}); ok { + for _, key := range []string{"workId", "work_id", "id", "assignmentId", "assignment_id"} { + if value := normalizeTeamMemberRouteKey(eventString(step, key)); value != "" { + workIDs[value] = struct{}{} + } + } + } + items, err := s.repo.ListWorkItemsByTeamID(teamID, 500) + if err != nil { + return nil, err + } + for idx := range items { + item := items[idx] + if item.TeamID != teamID || item.OwnerMemberID == nil || *item.OwnerMemberID != member.ID { + continue + } + normalizedWorkID := normalizeTeamMemberRouteKey(item.WorkID) + _, directWorkMatch := workIDs[normalizedWorkID] + canonicalOwnerMatch := normalizedWorkID == "member-"+ownerKey + if !directWorkMatch && !canonicalOwnerMatch { + continue + } + task, err := s.repo.GetTaskByID(item.RootTaskID) + if err != nil { + return nil, err + } + if task == nil || task.TeamID != teamID { + continue + } + if isPassiveAssignmentMonitorEvent(eventType, payload) && isTerminalTeamTaskStatus(task.Status) { + return nil, nil + } + payload["rootTaskId"] = fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID) + payload["rootMessageId"] = task.MessageID + if eventString(payload, "workId", "work_id") == "" { + payload["workId"] = item.WorkID + } + if eventString(payload, "assignmentId", "assignment_id") == "" { + payload["assignmentId"] = item.WorkID + } + return task, nil + } + return nil, nil +} + +func isLeaderMediatedWorkerResultLikeEvent(eventType, status string, payload map[string]interface{}) bool { + if isTeamTaskCompletionSignal(eventType, status, payload) || eventBool(payload, "assignmentResultOnly", "assignment_result_only") { + return true + } + switch strings.ToLower(strings.TrimSpace(eventType)) { + case "reply", "outbound", "completion", "task_completed": + return teamEventHasBody(payload) + default: + return false + } +} + +func parseClawManagerTeamTaskRef(teamID int, raw string) int { + value := strings.TrimSpace(raw) + if value == "" { + return 0 + } + if matches := regexp.MustCompile(`^team-(\d+)-task-(\d+)$`).FindStringSubmatch(strings.ToLower(value)); len(matches) == 3 { + refTeamID, _ := strconv.Atoi(matches[1]) + taskID, _ := strconv.Atoi(matches[2]) + if refTeamID == teamID { + return taskID + } + return 0 + } + if matches := regexp.MustCompile(`^clawmanager-task-(\d+)$`).FindStringSubmatch(strings.ToLower(value)); len(matches) == 2 { + taskID, _ := strconv.Atoi(matches[1]) + return taskID + } + return 0 +} + +func (s *teamService) activeTaskForMember(teamID int, member *models.TeamMember, requireTarget bool) (*models.TeamTask, error) { + if member == nil || member.CurrentTaskID == nil { + return nil, nil + } + found, err := s.repo.GetTaskByID(*member.CurrentTaskID) + if err != nil { + return nil, err + } + if found == nil || found.TeamID != teamID || isTerminalTeamTaskStatus(found.Status) { + return nil, nil + } + if requireTarget && found.TargetMemberID != member.ID { + return nil, nil + } + return found, nil +} + +func (s *teamService) activeTaskFromPeerContext(teamID int, payload map[string]interface{}, member *models.TeamMember) (*models.TeamTask, error) { + candidates := []*models.TeamMember{member} + for _, key := range []string{"from", "source", "sourceMemberId", "source_member_id", "sender", "senderMemberId", "sender_member_id", "to", "recipient", "target", "targetMemberId", "target_member_id", "memberId", "member_id"} { + memberKey := eventString(payload, key) + if memberKey == "" || memberKey == teamTaskReplyTarget { + continue + } + found, err := s.repo.GetMemberByTeamKey(teamID, memberKey) + if err != nil { + return nil, err + } + if found != nil { + candidates = append(candidates, found) + } + } + seen := map[int]struct{}{} + for _, candidate := range candidates { + if candidate == nil { + continue + } + if _, ok := seen[candidate.ID]; ok { + continue + } + seen[candidate.ID] = struct{}{} + if task, err := s.activeTaskForMember(teamID, candidate, false); err != nil { + return nil, err + } else if task != nil { + return task, nil + } + } + return nil, nil +} + +func (s *teamService) normalizePeerAssistedTeamEvent(team *models.Team, eventType string, payload map[string]interface{}, member *models.TeamMember) (string, *models.TeamMember, error) { + if team == nil || payload == nil || !isPeerCapableTeamEvent(eventType) { + return eventType, member, nil + } + communicationMode := normalizedTeamCommunicationMode(team.CommunicationMode) + if communicationMode != teamCommunicationModePeerAssisted && communicationMode != teamCommunicationModeFullMesh { + return eventType, member, nil + } + sourceKey := eventString(payload, "from", "source", "sourceMemberId", "source_member_id", "sender", "senderMemberId", "sender_member_id") + if sourceKey == "" && member != nil { + sourceKey = member.MemberKey + } + targetKey := eventString(payload, "to", "recipient", "target", "targetMemberId", "target_member_id") + if sourceKey == "" || targetKey == "" || sourceKey == targetKey || sourceKey == teamTaskReplyTarget || targetKey == teamTaskReplyTarget { + return eventType, member, nil + } + + sourceMember, err := s.repo.GetMemberByTeamKey(team.ID, sourceKey) + if err != nil { + return eventType, member, err + } + targetMember, err := s.repo.GetMemberByTeamKey(team.ID, targetKey) + if err != nil { + return eventType, member, err + } + if !isActiveTeamMember(sourceMember) || !isActiveTeamMember(targetMember) { + return eventType, member, nil + } + + if member == nil || member.MemberKey != sourceMember.MemberKey { + member = sourceMember + } + payload["peer"] = true + payload["communicationMode"] = communicationMode + payload["sourceMemberId"] = sourceMember.MemberKey + payload["targetMemberId"] = targetMember.MemberKey + payload["from"] = sourceMember.MemberKey + payload["to"] = targetMember.MemberKey + + rawAction := eventString(payload, "peerAction", "peer_action", "action", "intent", "kind") + action := normalizeTeamPeerAction(rawAction) + if strings.TrimSpace(rawAction) == "" && (eventType == "outbound" || eventType == "task_assigned" || eventType == "team_send") && isTeamLeaderRole(sourceMember.Role) && !isTeamLeaderRole(targetMember.Role) { + action = "handoff" + } + payload["peerAction"] = action + if strings.HasPrefix(eventType, "peer_") { + return eventType, member, nil + } + switch action { + case "review_request", "peer_review": + return "peer_review_request", member, nil + case "handoff": + return "peer_handoff", member, nil + default: + return "peer_request", member, nil + } +} + +func isPeerCapableTeamEvent(eventType string) bool { + switch strings.ToLower(strings.TrimSpace(eventType)) { + case "outbound", "task_assigned", "team_send", "peer_request", "peer_handoff", "peer_review_request", "peer_reply": + return true + default: + return false + } +} + +func normalizeTeamPeerAction(raw string) string { + action := strings.ToLower(strings.TrimSpace(raw)) + action = strings.ReplaceAll(action, "-", "_") + action = strings.ReplaceAll(action, " ", "_") + switch action { + case "handoff", "assign", "assignment": + return "handoff" + case "review", "review_request", "peer_review", "code_review": + return "review_request" + case "artifact", "artifact_request", "file_request": + return "artifact_request" + case "blocker", "blocker_help", "help": + return "blocker_help" + default: + return "ask" + } +} + +func isActiveTeamMember(member *models.TeamMember) bool { + return member != nil && + member.Status != models.TeamMemberStatusDeleted && + member.Status != models.TeamMemberStatusDeleting +} + +func enrichTeamCollaborationStep(team *models.Team, eventType string, payload map[string]interface{}, member *models.TeamMember, task *models.TeamTask) { + if payload == nil { + return + } + if existing, ok := payload["collaborationStep"].(map[string]interface{}); ok && len(existing) > 0 { + normalizeExistingCollaborationStep(existing, team, eventType, payload, member, task) + return + } + + stepType := collaborationStepTypeForEvent(eventType, payload) + if stepType == "" { + return + } + actor := collaborationActorKey(payload, member) + target := eventString(payload, "to", "recipient", "target", "targetMemberId", "target_member_id", "memberId") + if stepType == "assignment" && target == "" { + target = eventString(payload, "assignee", "owner", "targetMember") + } + status := collaborationStepStatusForEvent(eventType, payload) + title := collaborationStepTitle(stepType, actor, target, payload) + summary := eventString(payload, "summary", "resultMarkdown", "result_markdown", "result", "text", "message", "prompt", "instruction", "instructions", "diagnostic", "error", "reason") + fullContent := collaborationStepContentForEvent(stepType, payload, summary) + messageID := eventString(payload, "messageId", "message_id") + rootTaskID := "" + rootMessageID := "" + if task != nil { + rootTaskID = fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID) + rootMessageID = task.MessageID + if messageID == "" { + messageID = task.MessageID + } + } + if rootTaskID == "" { + rootTaskID = eventString(payload, "rootTaskId", "root_task_id", "parentTaskId", "parent_task_id") + } + if rootMessageID == "" { + rootMessageID = eventString(payload, "rootMessageId", "root_message_id", "parentMessageId", "parent_message_id", "inReplyTo", "in_reply_to") + } + workID := collaborationWorkID(stepType, actor, target, messageID, payload) + step := map[string]interface{}{ + "id": workID, + "workId": workID, + "type": stepType, + "status": status, + "title": title, + "summary": summary, + "content": fullContent, + "actor": actor, + "target": target, + "messageId": messageID, + "rootTaskId": rootTaskID, + "rootMessageId": rootMessageID, + "eventType": eventType, + "source": "clawmanager", + } + if progress := eventInt(payload, "progress"); progress > 0 { + step["progress"] = progress + } + if action := eventString(payload, "peerAction", "peer_action", "action", "intent", "kind"); action != "" { + step["action"] = normalizeTeamPeerAction(action) + } + if phase := inferCollaborationPhase(stepType, title, summary, payload); phase != "" { + step["phase"] = phase + } + payload["collaborationStep"] = step + if rootTaskID != "" && eventString(payload, "rootTaskId", "root_task_id") == "" { + payload["rootTaskId"] = rootTaskID + } + if rootMessageID != "" && eventString(payload, "rootMessageId", "root_message_id") == "" { + payload["rootMessageId"] = rootMessageID + } +} + +func (s *teamService) projectTeamWorkItem( + team *models.Team, + task *models.TeamTask, + member *models.TeamMember, + eventType string, + payload map[string]interface{}, + event *models.TeamEvent, +) error { + if s == nil || team == nil || task == nil || payload == nil || event == nil { + return nil + } + if isLeaderControlPlaneSnapshotTask(task, payload) { + return nil + } + step, _ := payload["collaborationStep"].(map[string]interface{}) + stepType := eventString(step, "type") + if stepType == "" { + stepType = collaborationStepTypeForEvent(eventType, payload) + } + if stepType == "" || stepType == "warning" { + return nil + } + if isPassiveAssignmentMonitorEvent(eventType, payload) { + return nil + } + explicitAssignmentID := eventString(payload, "assignmentId", "assignment_id") + explicitSourceWorkID := eventString(payload, "workId", "work_id", "subtaskId", "subtask_id") + explicitWorkID := explicitAssignmentID + if explicitWorkID == "" { + explicitWorkID = explicitSourceWorkID + } + // Transport acknowledgements and unscoped heartbeats are useful in the + // event log, but they are not business work and must not become Kanban + // cards. A progress event is materialized only when the orchestrator gave + // it a stable work identifier. + if stepType == "ack" || (stepType == "progress" && explicitWorkID == "") { + return nil + } + actor := eventString(step, "actor") + target := eventString(step, "target") + if actor == "" { + actor = collaborationActorKey(payload, member) + } + if target == "" { + target = leaderMediatedRouteTarget(payload) + } + if isLeaderMediatedTeam(team) && stepType == "assignment" { + normalizedTarget := normalizeTeamMemberRouteKey(target) + if normalizedTarget == "" || isLeaderRouteTarget(normalizedTarget) { + return nil + } + } + ownerKey := actor + eventKind := strings.ToLower(strings.TrimSpace(eventString(payload, "eventKind", "event_kind", "kind"))) + if eventKind == "assignment_heartbeat" || strings.EqualFold(eventType, "assignment_heartbeat") { + return nil + } + if stepType == "progress" && eventKind == "assignment_check_requested" && target != "" { + ownerKey = target + } + if stepType == "assignment" && target != "" { + ownerKey = target + } + rootCompletion := isLeaderTeamMember(member) && isTeamTaskCompletionSignal(eventType, normalizedTeamTaskEventStatus(payload), payload) + if rootCompletion { + ownerKey = member.MemberKey + stepType = "final_synthesis" + } + ownerKey = normalizeTeamMemberRouteKey(ownerKey) + if ownerKey == "" || ownerKey == "system" || ownerKey == "clawmanager" { + return nil + } + if isLeaderMediatedTeam(team) && stepType == "progress" && isLeaderRouteTarget(ownerKey) { + return nil + } + owner := member + if owner == nil || !teamMemberRouteEquivalent(owner.MemberKey, ownerKey) { + found, err := s.repo.GetMemberByTeamKey(team.ID, ownerKey) + if err != nil { + return err + } + owner = found + } + if eventBool(payload, "assignmentResultOnly", "assignment_result_only") && owner != nil { + items, listErr := s.repo.ListWorkItemsByRootTaskID(task.ID) + if listErr != nil { + return listErr + } + matching := make([]models.TeamWorkItem, 0) + for idx := range items { + candidate := items[idx] + if candidate.OwnerMemberID == nil || *candidate.OwnerMemberID != owner.ID || candidate.SupersededBy != nil || candidate.WorkID == "leader-final-synthesis" { + continue + } + candidateAssignmentID := derefTeamString(candidate.AssignmentID) + if candidateAssignmentID == "" { + candidateAssignmentID = candidate.WorkID + } + if explicitWorkID != "" && (candidateAssignmentID == explicitWorkID || candidate.WorkID == explicitWorkID) { + matching = []models.TeamWorkItem{candidate} + break + } + matching = append(matching, candidate) + } + if len(matching) == 1 { + canonical := derefTeamString(matching[0].AssignmentID) + if canonical == "" { + canonical = matching[0].WorkID + } + if explicitSourceWorkID != "" && explicitSourceWorkID != canonical { + payload["sourceWorkId"] = explicitSourceWorkID + } + explicitAssignmentID = canonical + explicitWorkID = canonical + payload["assignmentId"] = canonical + payload["canonicalWorkId"] = canonical + if matching[0].PhaseID != nil && eventString(payload, "phaseId", "phase_id") == "" { + payload["phaseId"] = *matching[0].PhaseID + } + if matching[0].Revision > 0 && eventInt(payload, "revision") <= 0 { + payload["revision"] = matching[0].Revision + } + } + } + assignmentID := explicitAssignmentID + if assignmentID == "" { + assignmentID = explicitWorkID + } + workID := assignmentID + // The runtime may echo an assignmentId from a prompt or old plugin payload, + // but the backend ledger must be owned by the member that actually produced + // the result. Otherwise one worker can accidentally overwrite another + // worker's Kanban lane and the Leader will wait on the wrong member. + if rootCompletion { + workID = "leader-final-synthesis" + assignmentID = workID + } else if workID == "" { + workID = "member-" + normalizeTeamMemberRouteKey(ownerKey) + assignmentID = workID + } + revision := eventInt(payload, "revision", "workRevision", "work_revision", "artifactRevision", "artifact_revision") + if revision <= 0 { + revision = 1 + } + canonicalWorkID := eventString(payload, "canonicalWorkId", "canonical_work_id") + if canonicalWorkID == "" { + canonicalWorkID = assignmentID + } + if revision > 1 && !strings.HasSuffix(workID, fmt.Sprintf(":r%d", revision)) { + workID = fmt.Sprintf("%s:r%d", workID, revision) + } + status := models.TeamTaskStatusRunning + switch stepType { + case "assignment": + status = models.TeamTaskStatusDispatched + case "ack", "progress", "peer_request", "peer_reply": + status = models.TeamTaskStatusRunning + case "result", "final_synthesis": + status = models.TeamTaskStatusSucceeded + case "blocker": + status = models.TeamTaskStatusFailed + } + now := event.CreatedAt + if now.IsZero() { + now = time.Now().UTC() + } + title := eventString(step, "title") + if title == "" { + title = collaborationStepTitle(stepType, actor, target, payload) + } + requiredForRoot := !eventBool(payload, "optional", "isOptional", "is_optional") + if required, ok := teamEventBoolValue(payload, "required", "requiredForRoot", "required_for_root"); ok { + requiredForRoot = required + } + item := &models.TeamWorkItem{ + TeamID: team.ID, + RootTaskID: task.ID, + WorkID: workID, + Title: title, + Status: status, + Revision: revision, + RequiredForRoot: requiredForRoot, + ReviewRequired: eventBool(payload, "reviewRequired", "review_required"), + UpdatedAt: now, + } + if assignmentID != "" { + item.AssignmentID = &assignmentID + } + if canonicalWorkID != "" { + item.CanonicalWorkID = &canonicalWorkID + } + phaseID := eventString(payload, "phaseId", "phase_id", "phase", "stage") + if phaseID == "" { + phaseID = eventString(step, "phase") + } + if phaseID != "" { + item.PhaseID = &phaseID + } + if validatedRevision := eventInt(payload, "validatedRevision", "validated_revision", "reviewedRevision", "reviewed_revision"); validatedRevision > 0 { + item.ValidatedRevision = &validatedRevision + } + if supersededBy := eventString(payload, "supersededBy", "superseded_by"); supersededBy != "" { + item.SupersededBy = &supersededBy + } + if owner != nil { + item.OwnerMemberID = &owner.ID + } + if status == models.TeamTaskStatusRunning { + item.StartedAt = &now + } + if status == models.TeamTaskStatusSucceeded || status == models.TeamTaskStatusFailed { + item.FinishedAt = &now + if encoded, err := marshalOptionalJSON(payload); err == nil { + item.ResultJSON = encoded + } + if refs := explicitTeamArtifactReferences(payload); len(refs) > 0 { + if encoded, err := json.Marshal(refs); err == nil { + value := string(encoded) + item.ArtifactRefsJSON = &value + } + } + } + dependencies := normalizeContextRefs(step["dependsOn"]) + if len(dependencies) == 0 { + dependencies = normalizeContextRefs(firstTeamValue(payload, "dependsOn", "depends_on")) + } + if len(dependencies) > 0 { + if encoded, err := json.Marshal(dependencies); err == nil { + value := string(encoded) + item.DependsOnJSON = &value + } + } + if assignmentID != "" && revision > 1 { + existingItems, listErr := s.repo.ListWorkItemsByRootTaskID(task.ID) + if listErr != nil { + return listErr + } + for idx := range existingItems { + existing := existingItems[idx] + existingAssignmentID := derefTeamString(existing.AssignmentID) + if existingAssignmentID == "" { + existingAssignmentID = existing.WorkID + } + if existingAssignmentID != assignmentID || teamMaxInt(existing.Revision, 1) >= revision || existing.SupersededBy != nil { + continue + } + supersededBy := workID + existing.SupersededBy = &supersededBy + existing.UpdatedAt = now + if err := s.repo.UpsertWorkItem(&existing); err != nil { + return err + } + } + } + return s.repo.UpsertWorkItem(item) +} + +func normalizeExistingCollaborationStep(step map[string]interface{}, team *models.Team, eventType string, payload map[string]interface{}, member *models.TeamMember, task *models.TeamTask) { + if eventBool(payload, "leaderMediatedRouteViolation", "leader_mediated_route_violation") { + step["type"] = "warning" + step["status"] = "warning" + } + if eventBool(payload, "assignmentResultOnly", "assignment_result_only") { + step["type"] = "result" + step["status"] = models.TeamTaskStatusSucceeded + } + if eventString(step, "type") == "" { + step["type"] = collaborationStepTypeForEvent(eventType, payload) + } + if eventString(step, "status") == "" { + step["status"] = collaborationStepStatusForEvent(eventType, payload) + } + if eventString(step, "actor") == "" { + step["actor"] = collaborationActorKey(payload, member) + } + if eventString(step, "target") == "" { + if target := eventString(payload, "to", "recipient", "target", "targetMemberId", "target_member_id", "memberId"); target != "" { + step["target"] = target + } + } + if task != nil { + if eventString(step, "rootTaskId") == "" { + step["rootTaskId"] = fmt.Sprintf("team-%d-task-%d", task.TeamID, task.ID) + } + if eventString(step, "rootMessageId") == "" { + step["rootMessageId"] = task.MessageID + } + } + if eventString(step, "eventType") == "" { + step["eventType"] = eventType + } + if eventString(step, "content", "detail") == "" { + stepType := eventString(step, "type") + summary := eventString(step, "summary") + if content := collaborationStepContentForEvent(stepType, payload, summary); content != "" { + step["content"] = content + } + } + if eventString(step, "source") == "" { + step["source"] = "clawmanager" + } + if eventString(step, "id", "workId") == "" { + stepType := eventString(step, "type") + actor := eventString(step, "actor") + target := eventString(step, "target") + messageID := eventString(step, "messageId", "message_id") + if messageID == "" { + messageID = eventString(payload, "messageId", "message_id") + } + workID := collaborationWorkID(stepType, actor, target, messageID, payload) + step["id"] = workID + step["workId"] = workID + } + if team != nil && eventString(step, "teamId") == "" { + step["teamId"] = strconv.Itoa(team.ID) + } +} + +func collaborationStepContentForEvent(stepType string, payload map[string]interface{}, fallback string) string { + if stepType == "result" || stepType == "blocker" { + if full := eventString(payload, "resultMarkdown", "result_markdown", "result", "answer", "diagnostic", "error_message", "error", "message", "text"); full != "" { + return full + } + } + if content := eventString(payload, "content", "detail", "text", "message", "resultMarkdown", "result_markdown", "result", "answer"); content != "" { + return content + } + return fallback +} + +func (s *teamService) projectTeamWorkflowLedger(team *models.Team, task *models.TeamTask, member *models.TeamMember, eventType string, payload map[string]interface{}, now time.Time) (bool, error) { + if s == nil || team == nil || task == nil || payload == nil || isTerminalTeamTaskStatus(task.Status) { + return false, nil + } + eventKind := strings.ToLower(strings.TrimSpace(eventString(payload, "eventKind", "event_kind", "kind"))) + step, _ := payload["collaborationStep"].(map[string]interface{}) + stepType := strings.ToLower(strings.TrimSpace(eventString(step, "type"))) + isPlan := eventKind == "leader_plan" && member != nil && isLeaderTeamMember(member) + isAssignment := stepType == "assignment" || eventBool(payload, "leaderDispatchOnly", "leader_dispatch_only") || eventType == "team_send" || eventType == "task_assigned" || eventType == "outbound" + isAssignmentResult := eventBool(payload, "assignmentResultOnly", "assignment_result_only") + if !isPlan && !isAssignment && !isAssignmentResult { + return false, nil + } + + changed := false + planVersion := int64(eventInt(payload, "planVersion", "plan_version")) + if planVersion <= 0 { + planVersion = task.PlanVersion + if planVersion <= 0 { + planVersion = 1 + } + } + if planVersion > task.PlanVersion { + task.PlanVersion = planVersion + changed = true + } + + if isPlan { + phaseValues := firstTeamValue(payload, "phases", "workflowPhases", "workflow_phases") + if plan, ok := payload["workflowPlan"].(map[string]interface{}); ok { + if nested := firstTeamValue(plan, "phases", "workflowPhases", "workflow_phases"); nested != nil { + phaseValues = nested + } + } + if rawPhases, ok := phaseValues.([]interface{}); ok { + for index, rawPhase := range rawPhases { + phaseMap, ok := rawPhase.(map[string]interface{}) + if !ok { + continue + } + phaseID := eventString(phaseMap, "phaseId", "phase_id", "id", "name") + if phaseID == "" { + continue + } + phase := &models.TeamWorkflowPhase{ + TeamID: team.ID, + RootTaskID: task.ID, + PhaseID: phaseID, + PlanVersion: planVersion, + SequenceNo: index, + Status: eventString(phaseMap, "status"), + RequiredForRoot: true, + DecisionRequired: eventBool(phaseMap, "decisionRequiredAfterCompletion", "decisionRequired", "decision_required"), + UpdatedAt: now, + } + if phase.Status == "" { + phase.Status = teamPhaseStatusPlanned + } + if required, exists := teamEventBoolValue(phaseMap, "required", "requiredForRoot", "required_for_root"); exists { + phase.RequiredForRoot = required + } + if dependencies := normalizeContextRefs(firstTeamValue(phaseMap, "dependsOn", "depends_on")); len(dependencies) > 0 { + if encoded, err := json.Marshal(dependencies); err == nil { + value := string(encoded) + phase.DependsOnJSON = &value + } + } + if nextPhaseID := eventString(phaseMap, "nextPhaseId", "next_phase_id"); nextPhaseID != "" { + phase.NextPhaseID = &nextPhaseID + } + if policy := eventString(phaseMap, "completionPolicy", "completion_policy"); policy != "" { + phase.CompletionPolicy = &policy + } + if err := s.repo.UpsertWorkflowPhase(phase); err != nil { + return changed, err + } + if task.CurrentPhaseID == nil && phase.Status != teamPhaseStatusPlanned { + task.CurrentPhaseID = &phaseID + } + changed = true + } + } + workflowState := eventString(payload, "workflowState", "workflow_state") + if workflowState == "" || workflowState == "open" { + workflowState = teamWorkflowStateExecuting + } + if task.WorkflowState != workflowState { + task.WorkflowState = workflowState + changed = true + } + } + + phaseID := eventString(payload, "phaseId", "phase_id", "phase", "stage") + if phaseID == "" { + phaseID = eventString(step, "phase") + } + if isAssignment { + if task.WorkflowState != teamWorkflowStateExecuting && task.WorkflowState != teamWorkflowStateAwaitingPhaseResults { + task.WorkflowState = teamWorkflowStateExecuting + changed = true + } + if phaseID != "" { + task.CurrentPhaseID = &phaseID + phase := &models.TeamWorkflowPhase{ + TeamID: team.ID, + RootTaskID: task.ID, + PhaseID: phaseID, + PlanVersion: planVersion, + Status: teamPhaseStatusAwaitingResults, + RequiredForRoot: true, + UpdatedAt: now, + } + if err := s.repo.UpsertWorkflowPhase(phase); err != nil { + return changed, err + } + changed = true + } + } + if isAssignmentResult && phaseID != "" { + phases, err := s.repo.ListWorkflowPhasesByRootTaskID(task.ID) + if err != nil { + return changed, err + } + items, err := s.repo.ListWorkItemsByRootTaskID(task.ID) + if err != nil { + return changed, err + } + phaseIncomplete := false + for idx := range items { + item := items[idx] + if derefTeamString(item.PhaseID) != phaseID || item.SupersededBy != nil || !(item.RequiredForRoot || item.AssignmentID == nil) { + continue + } + if item.Status != models.TeamTaskStatusSucceeded { + phaseIncomplete = true + break + } + } + for idx := range phases { + phase := phases[idx] + if phase.PhaseID != phaseID || phase.PlanVersion != planVersion { + continue + } + if phaseIncomplete { + phase.Status = teamPhaseStatusAwaitingResults + task.WorkflowState = teamWorkflowStateAwaitingPhaseResults + } else if phase.DecisionRequired { + phase.Status = teamPhaseStatusAwaitingLeaderDecision + task.WorkflowState = teamWorkflowStateAwaitingLeaderDecision + } else { + phase.Status = teamPhaseStatusCompleted + phase.CompletedAt = &now + task.WorkflowState = teamWorkflowStateAwaitingLeaderDecision + } + phase.UpdatedAt = now + if err := s.repo.UpsertWorkflowPhase(&phase); err != nil { + return changed, err + } + changed = true + break + } + } + if changed { + task.LedgerVersion++ + task.UpdatedAt = now + } + return changed, nil +} + +func collaborationStepTypeForEvent(eventType string, payload map[string]interface{}) string { + status := normalizedTeamTaskEventStatus(payload) + eventKind := strings.ToLower(strings.TrimSpace(eventString(payload, "eventKind", "event_kind", "kind"))) + switch eventKind { + case "leader_plan", "worker_plan", "worker_progress", "artifact_changed", "assignment_check_requested", "assignment_check_result", "assignment_heartbeat", "leader_synthesis", "leader_synthesis_reminder", "leader_decision_reminder", "completion_deferred", "assignment_recovery_started", "assignment_reissued", "agent_narrative", "agent_plan", "agent_assignment", "agent_handoff", "agent_progress", "agent_delivery", "agent_review", "agent_synthesis": + return "progress" + case "completion_candidate": + return "progress" + case "completion_validation_warning", "completion_needs_confirmation", "completion_rejected", "assignment_recovery_exhausted": + return "warning" + } + if eventBool(payload, "leaderMediatedRouteViolation", "leader_mediated_route_violation") { + return "warning" + } + if eventBool(payload, "assignmentResultOnly", "assignment_result_only") { + return "result" + } + if eventBool(payload, "leaderDispatchOnly", "leader_dispatch_only") { + return "assignment" + } + if isTeamTaskCompletionSignal(eventType, status, payload) { + return "result" + } + if isNonAuthoritativeDispatchFailure(eventType, payload) { + return "warning" + } + switch eventType { + case "leader_plan", "worker_plan", "worker_progress", "artifact_changed", "assignment_check_requested", "assignment_check_result", "assignment_heartbeat", "leader_synthesis", "leader_synthesis_reminder", "leader_decision_reminder", "completion_candidate", "completion_deferred", "assignment_recovery_started", "assignment_reissued", "agent_narrative", "agent_plan", "agent_assignment", "agent_handoff", "agent_progress", "agent_delivery", "agent_review", "agent_synthesis": + return "progress" + case "completion_validation_warning", "completion_needs_confirmation", "completion_rejected", "assignment_recovery_exhausted": + return "warning" + case "outbound", "task_assigned", "team_send": + return "assignment" + case "peer_handoff": + return "assignment" + case "peer_request", "peer_review_request": + return "peer_request" + case "peer_reply": + return "peer_reply" + case "task_received": + return "ack" + case "task_started", "task_progress", "progress": + return "progress" + case "task_completed", "completion": + return "progress" + case "task_failed", "message_failed", "task_stale": + if !isTeamTaskFailureSignal(eventType, status, payload) && eventType != "task_stale" { + return "warning" + } + return "blocker" + case "message_warning": + return "warning" + case "reply": + if eventBool(payload, "final", "isFinal", "complete", "completed", "taskCompleted") || isFinalCompletionPayload(payload) { + return "result" + } + return "progress" + default: + if eventString(payload, "to", "recipient", "target", "targetMemberId", "target_member_id") != "" { + return "progress" + } + return "" + } +} + +func collaborationStepStatusForEvent(eventType string, payload map[string]interface{}) string { + status := normalizedTeamTaskEventStatus(payload) + eventKind := strings.ToLower(strings.TrimSpace(eventString(payload, "eventKind", "event_kind", "kind"))) + switch eventKind { + case "assignment_check_requested", "assignment_check_result", "assignment_heartbeat", "leader_synthesis_reminder", "assignment_recovery_started", "assignment_reissued": + return models.TeamTaskStatusRunning + case "completion_candidate": + return "result_pending_confirmation" + case "completion_validation_warning", "assignment_recovery_exhausted": + return "warning" + } + if eventBool(payload, "leaderMediatedRouteViolation", "leader_mediated_route_violation") { + return "warning" + } + if eventBool(payload, "assignmentResultOnly", "assignment_result_only") { + return models.TeamTaskStatusSucceeded + } + if eventBool(payload, "leaderDispatchOnly", "leader_dispatch_only") { + return models.TeamTaskStatusDispatched + } + if isTeamTaskCompletionSignal(eventType, status, payload) { + return models.TeamTaskStatusSucceeded + } + if isSuccessfulTeamTaskEventStatus(status) { + switch eventType { + case "task_completed", "completion", "task_failed", "message_failed": + return models.TeamTaskStatusRunning + default: + return models.TeamTaskStatusSucceeded + } + } + if isFailedTeamTaskEventStatus(status) || eventType == "task_failed" || eventType == "message_failed" { + if isNonAuthoritativeDispatchFailure(eventType, payload) { + return "warning" + } + return models.TeamTaskStatusFailed + } + switch eventType { + case "assignment_check_requested", "assignment_check_result", "assignment_heartbeat", "leader_synthesis_reminder", "assignment_recovery_started", "assignment_reissued": + return models.TeamTaskStatusRunning + case "completion_candidate": + return "result_pending_confirmation" + case "completion_validation_warning", "assignment_recovery_exhausted": + return "warning" + case "task_stale": + return models.TeamTaskStatusStale + case "outbound", "task_assigned", "team_send", "peer_request", "peer_handoff", "peer_review_request": + return models.TeamTaskStatusDispatched + case "task_received": + return "acknowledged" + case "task_started", "task_progress", "progress", "reply", "peer_reply": + return models.TeamTaskStatusRunning + case "message_warning": + return "warning" + default: + if status != "" { + return status + } + return "observed" + } +} + +func collaborationActorKey(payload map[string]interface{}, member *models.TeamMember) string { + if actor := eventString(payload, "from", "sourceMemberId", "source_member_id", "sender", "senderMemberId", "sender_member_id", "memberId", "member_id"); actor != "" { + return actor + } + if member != nil { + return member.MemberKey + } + return "system" +} + +func collaborationStepTitle(stepType, actor, target string, payload map[string]interface{}) string { + if title := eventString(payload, "stepTitle", "step_title", "title", "intent"); title != "" && !looksLikeOpaqueRuntimeTaskID(title) { + return title + } + switch strings.ToLower(strings.TrimSpace(eventString(payload, "eventKind", "event_kind", "kind"))) { + case "leader_plan": + return "Leader execution plan" + case "worker_plan": + return actor + " execution plan" + case "worker_progress": + return actor + " updates progress" + case "assignment_check_requested": + if target != "" { + return "ClawManager checks " + target + } + return "ClawManager checks assignment status" + case "assignment_check_result": + return actor + " reports status" + case "assignment_heartbeat": + return actor + " heartbeat" + case "leader_synthesis": + return "Leader synthesizes results" + case "agent_narrative", "agent_plan", "agent_assignment", "agent_handoff", "agent_progress", "agent_delivery", "agent_review", "agent_synthesis": + return actor + " collaboration update" + case "leader_synthesis_reminder": + return "Leader final synthesis requested" + case "completion_candidate": + return actor + " produced candidate result" + case "completion_validation_warning": + return "Completion needs artifact recovery" + case "assignment_recovery_started": + return "Recovery started for " + actor + case "assignment_reissued": + return "Assignment reissued to " + actor + case "assignment_recovery_exhausted": + return "Recovery needs attention" + } + switch stepType { + case "assignment": + if target != "" { + return "Assign to " + target + } + return "Assign subtask" + case "peer_request": + if target != "" { + return actor + " asks " + target + } + return "Peer request" + case "peer_reply": + return actor + " replies" + case "ack": + return actor + " accepted task" + case "progress": + return actor + " updates progress" + case "result": + return actor + " delivers result" + case "blocker": + return actor + " reports blocker" + case "warning": + return actor + " reports warning" + default: + return stepType + } +} + +func collaborationWorkID(stepType, actor, target, messageID string, payload map[string]interface{}) string { + if id := eventString(payload, "workId", "work_id", "stepId", "step_id", "subtaskId", "subtask_id"); id != "" { + return id + } + parts := []string{stepType, actor, target} + if messageID != "" { + parts = append(parts, messageID) + } + if len(parts) == 0 { + return "team-step" + } + return strings.ToLower(strings.Trim(strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' { + return r + } + if r == '-' || r == '_' { + return '-' + } + return '-' + }, strings.Join(parts, "-")), "-")) +} + +func inferCollaborationPhase(stepType, title, summary string, payload map[string]interface{}) string { + if phase := eventString(payload, "phase", "stage"); phase != "" { + return phase + } + text := strings.ToLower(title + " " + summary) + switch { + case strings.Contains(text, "review") || strings.Contains(text, "verify"): + return "verification" + case strings.Contains(text, "design") || strings.Contains(text, "ui") || strings.Contains(text, "prototype"): + return "design" + case strings.Contains(text, "research") || strings.Contains(text, "璋冪爺"): + return "research" + case stepType == "assignment": + return "decomposition" + case stepType == "result": + return "delivery" + default: + return "execution" + } +} + +func isFinalCompletionPayload(payload map[string]interface{}) bool { + status := normalizedTeamTaskEventStatus(payload) + return hasAuthoritativeTeamCompletionPayload("reply", status, payload) +} + +func looksLikeOpaqueRuntimeTaskID(value string) bool { + normalized := strings.TrimSpace(value) + return regexp.MustCompile(`^(task[-_][a-z0-9-]+|team-\d+-task-\d+)$`).MatchString(strings.ToLower(normalized)) +} + +func normalizeFinalReplyTaskEvent(eventType string, payload map[string]interface{}, task *models.TeamTask, _ *models.TeamMember) string { + if task == nil || !strings.EqualFold(strings.TrimSpace(eventType), "reply") { + return eventType + } + hasCompletionTool := hasTeamTaskCompletionToolCall(payload) + if !hasCompletionTool { + return eventType + } + if !eventBool(payload, "final", "isFinal", "complete", "completed", "taskCompleted") { + return eventType + } + if !teamEventHasBody(payload) { + return eventType + } + payload["originalEvent"] = eventType + payload["event"] = "task_completed" + payload["type"] = "task_completed" + payload["status"] = "succeeded" + payload["availability"] = models.TeamMemberAvailabilityIdle + payload["runtimeStatus"] = "succeeded" + if eventString(payload, "resultMarkdown") == "" { + if text := eventString(payload, "text", "result", "summary"); text != "" { + payload["resultMarkdown"] = text + } + } + if eventString(payload, "summary") == "" { + if text := eventString(payload, "text", "resultMarkdown", "result"); text != "" { + payload["summary"] = text + } + } + return "task_completed" +} + +func markLegacyRuntimeCompletionCandidate(eventType string, payload map[string]interface{}, task *models.TeamTask, member *models.TeamMember) string { + if task == nil || member == nil || payload == nil || teamRedisProtocolVersion(payload) >= 2 { + return eventType + } + normalizedEvent := strings.ToLower(strings.TrimSpace(eventType)) + if normalizedEvent != "reply" && normalizedEvent != "message" { + return eventType + } + explicitControlPlaneCompletion := isLeaderControlPlaneSnapshotTask(task, payload) && + hasTeamCompletionResultBody(payload) && + (strings.EqualFold(strings.TrimSpace(eventString(payload, "completionSource", "completion_source")), teamTaskCompletionTool) || + hasTeamTaskCompletionToolCall(payload) || + eventBool(payload, "explicitCompletion", "explicit_completion")) + if member.ID != task.TargetMemberID || (isDispatchOnlyCompletionPayload(payload) && !explicitControlPlaneCompletion) { + return eventType + } + resultText := directTaskCompletionReplyText(payload) + if resultText == "" { + resultText = eventString(payload, "summary") + } + if resultText == "" { + return eventType + } + if !explicitControlPlaneCompletion { + if isInterimOrDelegationReplyText(resultText) || !looksLikeLegacyRuntimeCompletionReport(task, payload, resultText) { + return eventType + } + } + payload["legacyCompletionCandidate"] = true + payload["completionSource"] = "legacy_runtime_reply" + payload["rootTaskTerminal"] = true + payload["final"] = true + payload["status"] = models.TeamTaskStatusSucceeded + payload["availability"] = models.TeamMemberAvailabilityIdle + payload["runtimeStatus"] = models.TeamTaskStatusSucceeded + if eventString(payload, "resultMarkdown", "result_markdown") == "" { + payload["resultMarkdown"] = resultText + } + if eventString(payload, "summary") == "" { + payload["summary"] = compactTeamEventSummary(resultText, 240) + } + if eventString(payload, "originalEvent") == "" { + payload["originalEvent"] = eventType + } + payload["event"] = "task_completed" + payload["type"] = "task_completed" + return "task_completed" +} + +func looksLikeLegacyRuntimeCompletionReport(task *models.TeamTask, payload map[string]interface{}, resultText string) bool { + text := strings.TrimSpace(resultText) + if text == "" { + return false + } + lower := strings.ToLower(text) + compact := strings.ToLower(strings.Join(strings.Fields(text), "")) + hasCanonicalArtifact := regexp.MustCompile(`/team/(artifacts|results|tasks)/[^\s)\]<>{},\\\x60"']+\.[A-Za-z0-9]+`).FindString(text) != "" + hasCompletionWord := strings.Contains(lower, "completed") || + strings.Contains(lower, "complete") || + strings.Contains(lower, "succeeded") || + strings.Contains(lower, "delivered") || + strings.Contains(text, "\u5b8c\u6210") || + strings.Contains(text, "\u5df2\u5b8c\u6210") || + strings.Contains(text, "\u4ea4\u4ed8") + hasBootstrapWord := strings.Contains(lower, "bootstrap") || + strings.Contains(lower, "introduction") || + strings.Contains(text, "\u5f15\u5bfc") || + strings.Contains(text, "\u4ecb\u7ecd") || + strings.Contains(text, "\u56e2\u961f") + if hasCanonicalArtifact && (hasCompletionWord || looksLikeFinalResultText(text) || len([]rune(compact)) >= 80) { + return true + } + if strings.Contains(strings.ToLower(task.MessageID), "bootstrap-introduction") && hasBootstrapWord && hasCompletionWord && len([]rune(compact)) >= 80 { + return true + } + if eventBool(payload, "final", "isFinal", "complete", "completed", "taskCompleted") && looksLikeFinalResultText(text) && len([]rune(compact)) >= 120 { + return true + } + return false +} + +func isImplicitDirectTaskCompletionReply(task *models.TeamTask, member *models.TeamMember, payload map[string]interface{}) bool { + if task == nil || member == nil || payload == nil { + return false + } + if member.ID != task.TargetMemberID { + return false + } + status := normalizedTeamTaskEventStatus(payload) + if isTeamTaskFailureSignal("reply", status, payload) || isTeamTaskRunningSignal("reply", status, payload) { + return false + } + resultText := directTaskCompletionReplyText(payload) + if resultText == "" || isInterimOrDelegationReplyText(resultText) { + return false + } + if eventBool(payload, "final", "isFinal", "complete", "completed", "taskCompleted") { + return true + } + if eventString(payload, "resultMarkdown", "result_markdown", "result", "answer") != "" { + return true + } + switch status { + case "succeeded", "success", "completed", "complete", "done", "finished", "ok": + return true + } + compact := strings.TrimSpace(resultText) + compact = strings.Join(strings.Fields(compact), "") + return len([]rune(compact)) >= 36 && looksLikeSubstantialFinalReply(resultText) +} + +func directTaskCompletionReplyText(payload map[string]interface{}) string { + for _, record := range eventRecordCandidates(payload) { + if text := eventString(record, "resultMarkdown", "result_markdown", "result", "answer", "text", "message", "summary"); text != "" { + return text + } + } + return "" +} + +func compactTeamEventSummary(value string, limit int) string { + text := strings.TrimSpace(strings.Join(strings.Fields(value), " ")) + if text == "" || limit <= 0 { + return text + } + runes := []rune(text) + if len(runes) <= limit { + return text + } + return string(runes[:limit]) + "..." +} + +func isInterimOrDelegationReplyText(text string) bool { + normalized := strings.TrimSpace(text) + if normalized == "" { + return true + } + if looksLikeLeaderDispatchOnlyText(normalized) { + return true + } + if looksLikeFinalResultText(normalized) { + return false + } + lower := strings.ToLower(normalized) + compact := strings.ToLower(strings.Join(strings.Fields(normalized), "")) + if len([]rune(compact)) <= 12 { + return true + } + for _, prefix := range interimReplyPrefixes() { + if strings.HasPrefix(lower, strings.ToLower(prefix)) || strings.HasPrefix(normalized, prefix) { + return true + } + } + if containsAnyTeamTextMarker(normalized, interimReplyMarkers()) { + return true + } + return false +} + +func interimReplyPrefixes() []string { + return []string{ + "\u6536\u5230", + "\u597d\u7684", + "\u597d\uff0c", + "\u597d", + "ok", + "okay", + "\u5904\u7406\u4e2d", + "\u6b63\u5728", + "\u51c6\u5907", + "\u6211\u5c06", + "\u8ba9\u6211", + "\u5148\u770b", + "\u7a0d\u7b49", + "let me", + "now let me", + "i will", + "i'll", + "checking", + "working on", + "good, i have", + "i have the", + "i have all", + "i can see", + } +} + +func interimReplyMarkers() []string { + return []string{ + "\u6b63\u5728\u6574\u7406", + "\u73b0\u5728\u6574\u7406", + "\u7a0d\u540e", + "\u7ee7\u7eed\u7b49\u5f85", + "\u4ecd\u5728\u7b49\u5f85", + "\u5df2\u6536\u5230\u5e76\u5f52\u6863", + "\u5ef6\u8fdf\u9001\u8fbe", + "\u91cd\u590d\u901a\u77e5", + "\u91cd\u590d\u7ed3\u679c", + "\u91cd\u590d\u6295\u9012", + "\u7b49\u5f85\u5b8c\u6210", + "\u7b49\u5f85\u4ed6", + "\u7b49\u5f85\u5979", + "\u7b49\u5f85 designer", + "\u7b49\u5f85 architect", + "\u7b49\u5f85 pm", + "\u7b49\u5f85worker", + "\u7b49\u5f85 worker", + "\u6d3e\u5355", + "\u5df2\u6d3e\u53d1", + "\u6d3e\u53d1\u7ed9", + "\u4e0b\u53d1\u7ed9", + "\u8f6c\u6d3e\u7ed9", + "\u4ea4\u7ed9worker", + "\u4ea4\u7ed9 worker", + "\u8ba9worker", + "\u8ba9 worker", + "\u8bf7worker", + "\u8bf7 worker", + "worker\u5728\u7ebf\u7a7a\u95f2", + "sent to worker", + "assigned to worker", + "waiting for worker", + "still waiting", + "continuing to wait", + "waiting on ", + "duplicate notification", + "duplicate result", + "duplicate delivery", + "delayed notification", + "handoff to worker", + "now let me write", + "then finalize", + "and then finalize", + "write the comprehensive report", + } +} + +func looksLikeSubstantialFinalReply(text string) bool { + normalized := strings.TrimSpace(text) + if normalized == "" { + return false + } + if strings.ContainsAny(normalized, "#*>|`") { + return true + } + if strings.ContainsAny(normalized, ";\n") || strings.Contains(normalized, "\u3002") || strings.Contains(normalized, "\uff1b") || strings.Contains(normalized, "\uff0c") { + return true + } + return containsAnyTeamTextMarker(normalized, []string{ + "\u5b8c\u6210", + "\u603b\u7ed3", + "\u62a5\u544a", + "\u7ed3\u679c", + "completed", + "summary", + }) +} +func hasTeamTaskCompletionToolCall(payload map[string]interface{}) bool { + if payload == nil { + return false + } + if eventString(payload, "toolCallName", "tool_call_name", "calledTool", "called_tool") == teamTaskCompletionTool { + return true + } + for _, key := range []string{"toolCall", "tool_call", "function_call"} { + record, ok := payload[key].(map[string]interface{}) + if !ok || record == nil { + continue + } + if eventString(record, "name", "function", "functionName", "function_name", "tool", "toolName", "tool_name") == teamTaskCompletionTool { + return true + } + } + return false +} + +func eventRecordCandidates(payload map[string]interface{}) []map[string]interface{} { + if payload == nil { + return nil + } + records := []map[string]interface{}{payload} + for _, key := range []string{"sent", "metadata", "data", "envelope", "task", "toolCall", "tool_call", "function_call"} { + if record, ok := payload[key].(map[string]interface{}); ok && record != nil { + records = append(records, record) + } + } + return records +} + +func (s *teamService) enrichOutboundEventFromInbox(teamID int, bus *redisBus, payload map[string]interface{}, messageID string) (map[string]interface{}, error) { + targetMember := eventString(payload, "to", "recipient", "target", "targetMemberId", "target_member_id") + if targetMember == "" { + return payload, nil + } + var lastErr error + for attempt := 0; attempt < 5; attempt++ { + if attempt > 0 { + time.Sleep(100 * time.Millisecond) + } + messages, err := bus.XRevRange(context.Background(), teamInboxKey(teamID, targetMember), 100) + if err != nil { + lastErr = err + continue + } + for _, inboxMessage := range messages { + if !redisStreamMessageMatches(inboxMessage, messageID) { + continue + } + envelope := mergeRedisEventPayload(inboxMessage.Fields) + return mergeMissingEventFields(payload, envelope), nil + } + } + return payload, lastErr +} + +func redisStreamMessageMatches(message redisStreamMessage, messageID string) bool { + if strings.TrimSpace(message.Fields["message_id"]) == messageID { + return true + } + payload := mergeRedisEventPayload(message.Fields) + return eventString(payload, "message_id", "messageId") == messageID +} + +func mergeMissingEventFields(base map[string]interface{}, extra map[string]interface{}) map[string]interface{} { + merged := map[string]interface{}{} + for key, value := range base { + merged[key] = value + } + for key, value := range extra { + if existing, ok := merged[key]; !ok || isEmptyEventValue(existing) { + merged[key] = value + } + } + if metadata, ok := extra["metadata"].(map[string]interface{}); ok { + for key, value := range metadata { + if existing, ok := merged[key]; !ok || isEmptyEventValue(existing) { + merged[key] = value + } + } + } + return merged +} + +func isEmptyEventValue(value interface{}) bool { + if value == nil { + return true + } + if text, ok := value.(string); ok { + return strings.TrimSpace(text) == "" + } + return false +} + +func isOutboundTeamEvent(eventType string) bool { + switch eventType { + case "outbound", "task_assigned": + return true + default: + return false + } +} + +func teamEventHasBody(payload map[string]interface{}) bool { + if eventString(payload, "text", "title", "prompt", "instruction", "instructions", "summary", "resultMarkdown") != "" { + return true + } + for idx, record := range eventRecordCandidates(payload) { + if idx == 0 { + continue + } + if eventString(record, "text", "title", "prompt", "instruction", "instructions", "summary", "resultMarkdown") != "" { + return true + } + } + return false +} + +func (s *teamService) normalizeTeamArtifactReferences(team *models.Team, payload map[string]interface{}) { + if s == nil || team == nil || payload == nil { + return + } + physicalRoot := filepath.ToSlash(filepath.Clean(s.teamRuntimeSharedPathFor(team.UserID, team.ID))) + logicalRoot := strings.TrimRight(strings.TrimSpace(team.SharedMountPath), "/") + if logicalRoot == "" { + logicalRoot = teamSharedMountPath + } + normalizePath := func(raw string) string { + value := strings.TrimSpace(raw) + if value == "" { + return raw + } + value = strings.TrimPrefix(value, "file://") + value = strings.ReplaceAll(value, "\\", "/") + switch { + case strings.HasPrefix(value, logicalRoot+"/"): + return teamSharedMountPath + strings.TrimPrefix(value, logicalRoot) + case strings.HasPrefix(value, teamSharedMountPath+"/"): + return value + case strings.HasPrefix(value, physicalRoot+"/"): + return teamSharedMountPath + strings.TrimPrefix(value, physicalRoot) + case strings.HasPrefix(value, "$CLAWMANAGER_TEAM_SHARED_DIR/"): + return teamSharedMountPath + strings.TrimPrefix(value, "$CLAWMANAGER_TEAM_SHARED_DIR") + case strings.HasPrefix(value, "CLAWMANAGER_TEAM_SHARED_DIR/"): + return teamSharedMountPath + strings.TrimPrefix(value, "CLAWMANAGER_TEAM_SHARED_DIR") + case strings.HasPrefix(value, "team/artifacts/") || + strings.HasPrefix(value, "team/results/") || + strings.HasPrefix(value, "team/tasks/") || + strings.HasPrefix(value, "team/inbox/") || + strings.HasPrefix(value, "team/tmp/") || + strings.HasPrefix(value, "team/status/"): + return "/" + value + default: + return raw + } + } + normalizeText := func(raw string) string { + if strings.TrimSpace(raw) == "" { + return raw + } + // Free-form fields may contain serialized JSON and Markdown. Replacing + // every backslash corrupts escapes such as `\n` into `/n`, after which + // artifact extraction sees JSON syntax as part of the filename. + text := raw + if physicalRoot != "" { + text = strings.ReplaceAll(text, "file://"+physicalRoot+"/", teamSharedMountPath+"/") + text = strings.ReplaceAll(text, physicalRoot+"/", teamSharedMountPath+"/") + } + text = strings.ReplaceAll(text, "$CLAWMANAGER_TEAM_SHARED_DIR/", teamSharedMountPath+"/") + text = regexp.MustCompile(`(^|[\s(\[>])team/(artifacts|results|tasks|inbox|status|tmp)/([^\s)\]<>{},\\\x60"']+)`).ReplaceAllString(text, `${1}/team/${2}/${3}`) + return text + } + var normalizeValue func(value interface{}) interface{} + normalizeValue = func(value interface{}) interface{} { + switch typed := value.(type) { + case string: + normalized := normalizePath(typed) + if normalized != typed { + return normalized + } + return normalizeText(typed) + case []interface{}: + for idx := range typed { + typed[idx] = normalizeValue(typed[idx]) + } + return typed + case map[string]interface{}: + for key, nested := range typed { + typed[key] = normalizeValue(nested) + } + return typed + default: + return value + } + } + for key, value := range payload { + payload[key] = normalizeValue(value) + } +} + +func (s *teamService) missingTeamArtifactReferences(team *models.Team, payload map[string]interface{}) []string { + if s == nil || team == nil || payload == nil { + return nil + } + explicitRefsOnly := teamRedisProtocolVersion(payload) >= 2 + refs := collectTeamArtifactReferences(payload) + invalidRelativeRefs := collectInvalidRelativeTeamArtifactReferences(payload) + if explicitRefsOnly { + refs = explicitTeamArtifactReferences(payload) + invalidRelativeRefs = nil + } + if len(refs) == 0 && len(invalidRelativeRefs) == 0 { + return nil + } + root := filepath.Clean(s.teamRuntimeSharedPathFor(team.UserID, team.ID)) + missing := make([]string, 0) + for _, ref := range invalidRelativeRefs { + missing = append(missing, "invalid relative team artifact path: "+ref) + } + seen := map[string]struct{}{} + for _, ref := range refs { + rel := strings.TrimPrefix(strings.TrimSpace(ref), teamSharedMountPath+"/") + rel = strings.TrimPrefix(rel, "/") + if rel == "" || strings.HasPrefix(rel, "..") { + continue + } + target := filepath.Clean(filepath.Join(root, filepath.FromSlash(rel))) + if target != root && !strings.HasPrefix(target, root+string(filepath.Separator)) { + continue + } + if _, ok := seen[ref]; ok { + continue + } + seen[ref] = struct{}{} + if info, err := os.Stat(target); err != nil { + if os.IsNotExist(err) { + missing = append(missing, ref) + } + } else if !info.Mode().IsRegular() && explicitRefsOnly { + missing = append(missing, ref) + } + } + return missing +} + +func explicitTeamArtifactReferences(payload map[string]interface{}) []string { + if payload == nil { + return nil + } + value, ok := payload["artifactRefs"] + if !ok { + value = payload["artifact_refs"] + } + refs := make([]string, 0) + seen := map[string]struct{}{} + appendRef := func(raw interface{}) { + ref := trimTeamArtifactReferenceToken(fmt.Sprintf("%v", raw)) + if !strings.HasPrefix(ref, teamSharedMountPath+"/") || strings.ContainsAny(ref, "`\"'{}[]") { + return + } + if strings.HasSuffix(ref, "/") { + return + } + if _, exists := seen[ref]; exists { + return + } + seen[ref] = struct{}{} + refs = append(refs, ref) + } + switch typed := value.(type) { + case []interface{}: + for _, item := range typed { + appendRef(item) + } + case []string: + for _, item := range typed { + appendRef(item) + } + } + return refs +} + +func collectTeamArtifactReferences(value interface{}) []string { + refs := make([]string, 0) + // Stop at Markdown delimiters and JSON syntax. Runtime adapters can embed + // the same completion payload as text, Markdown, or serialized JSON; a + // greedy token here turns `/team/file.md` into a nonexistent filename such + // as `/team/file.md`/n\",\"artifactRefs\"...`. + pattern := regexp.MustCompile(`/team/(artifacts|results|tasks|inbox|status|tmp)/[^\s)\]<>{},\\\x60"']+`) + seen := map[string]struct{}{} + var walk func(interface{}) + walk = func(current interface{}) { + switch typed := current.(type) { + case string: + for _, match := range pattern.FindAllString(typed, -1) { + ref := trimTeamArtifactReferenceToken(match) + if ref == "" || !looksLikeTeamArtifactFileReference(ref) { + continue + } + if _, exists := seen[ref]; exists { + continue + } + seen[ref] = struct{}{} + refs = append(refs, ref) + } + case []interface{}: + for _, item := range typed { + walk(item) + } + case map[string]interface{}: + for _, item := range typed { + walk(item) + } + } + } + walk(value) + return refs +} + +func trimTeamArtifactReferenceToken(value string) string { + ref := strings.TrimSpace(value) + if ref == "" { + return "" + } + ref = strings.TrimRight(ref, " \t\r\n.,;:!?。;:,、!?)") + ref = strings.TrimRight(ref, "]}>)】》”’") + return ref +} + +func looksLikeTeamArtifactFileReference(ref string) bool { + ref = strings.TrimSpace(ref) + if ref == "" || strings.HasSuffix(ref, "/") { + return false + } + lastSlash := strings.LastIndex(ref, "/") + name := ref + if lastSlash >= 0 { + name = ref[lastSlash+1:] + } + return strings.Contains(name, ".") +} +func collectInvalidRelativeTeamArtifactReferences(value interface{}) []string { + refs := make([]string, 0) + pattern := regexp.MustCompile(`(^|[\s(\[>])(team/[^\s)\]<>{},\\\x60"']+)`) + seen := map[string]struct{}{} + var walk func(interface{}) + walk = func(current interface{}) { + switch typed := current.(type) { + case string: + for _, match := range pattern.FindAllStringSubmatch(typed, -1) { + if len(match) < 3 { + continue + } + ref := strings.TrimRight(match[2], ".,;:") + if strings.HasPrefix(ref, "team/artifacts/") || + strings.HasPrefix(ref, "team/results/") || + strings.HasPrefix(ref, "team/tasks/") || + strings.HasPrefix(ref, "team/inbox/") || + strings.HasPrefix(ref, "team/tmp/") || + strings.HasPrefix(ref, "team/status/") { + continue + } + if _, exists := seen[ref]; exists { + continue + } + seen[ref] = struct{}{} + refs = append(refs, ref) + } + case []interface{}: + for _, item := range typed { + walk(item) + } + case map[string]interface{}: + for _, item := range typed { + walk(item) + } + } + } + walk(value) + return refs +} +func buildInitialLeaderTaskPayload(teamName string) map[string]interface{} { + normalizedTeamName := strings.TrimSpace(teamName) + if normalizedTeamName == "" { + normalizedTeamName = "current" + } + prompt := fmt.Sprintf("\u8bf7\u4ecb\u7ecdteam %s\u5f53\u524d Redis Team\u6210\u5458\u6784\u6210\uff0c\u5305\u62ec\u5404\u89d2\u8272\u7684\u804c\u8d23\u5206\u5de5\u3001\u8fd0\u884c\u72b6\u6001\u4e0e\u6280\u672f\u80fd\u529b\u8fb9\u754c\u3002\u540c\u65f6\u8bf4\u660e\u56e2\u961f\u5185\u90e8\u7684\u534f\u4f5c\u4e0e\u901a\u4fe1\u673a\u5236(team_send)\uff0c\u4f8b\u5982\u4efb\u52a1\u6d41\u8f6c\u65b9\u5f0f\u3001\u6d88\u606f\u540c\u6b65\u65b9\u5f0f\u3001\u4e0a\u4e0b\u6587\u5171\u4eab\u65b9\u5f0f\u4ee5\u53ca\u53ef\u8c03\u7528\u7684\u65b9\u6cd5\u3001\u5de5\u5177\u4e0e\u64cd\u4f5c\u80fd\u529b\uff0c\u4ee5\u4fbf\u540e\u7eed\u80fd\u591f\u66f4\u9ad8\u6548\u5730\u5f00\u5c55\u56e2\u961f\u5de5\u4f5c", normalizedTeamName) + return map[string]interface{}{ + "intent": initialLeaderTaskIntent, + "title": "\u4ecb\u7ecd\u5f53\u524d Redis Team \u6210\u5458\u4e0e\u534f\u4f5c\u673a\u5236", + "prompt": prompt, + "origin": "system_bootstrap", + "executionMode": "leader_control_plane_snapshot", + "requiresDelegation": false, + "anchorEligible": false, + } +} + +func (s *teamService) markTeamFailed(team *models.Team, cause error) error { + team.Status = models.TeamStatusFailed + team.UpdatedAt = time.Now().UTC() + _ = s.repo.UpdateTeam(team) + return cause +} + +func (s *teamService) rollbackTeamCreation(userID int, team *models.Team, cause error) error { + members, err := s.repo.ListMembersByTeamID(team.ID) + if err != nil { + fmt.Printf("Warning: failed to list Team %d members during create rollback: %v\n", team.ID, err) + } + for idx := range members { + member := members[idx] + if member.InstanceID != nil && *member.InstanceID > 0 { + if err := s.instanceService.Delete(*member.InstanceID); err != nil { + fmt.Printf("Warning: failed to delete Team %d member %s instance %d during create rollback: %v\n", team.ID, member.MemberKey, *member.InstanceID, err) + } + } + member.Status = models.TeamMemberStatusDeleted + member.CurrentTaskID = nil + member.UpdatedAt = time.Now().UTC() + _ = s.repo.UpdateMember(&member) + } + ctx := context.Background() + if strings.TrimSpace(derefTeamString(team.TeamTokenSecretName)) != "" { + if err := s.secretService.DeleteSecret(ctx, userID, derefTeamString(team.TeamTokenSecretName)); err != nil { + fmt.Printf("Warning: failed to delete Team %d secret during create rollback: %v\n", team.ID, err) + } + } + if err := s.configMapService.DeleteConfigMap(ctx, userID, s.teamConfigMapName(team.ID)); err != nil { + fmt.Printf("Warning: failed to delete Team %d configmap during create rollback: %v\n", team.ID, err) + } + if err := s.pvcService.DeleteTeamSharedPVC(ctx, userID, team.ID); err != nil { + fmt.Printf("Warning: failed to delete Team %d shared PVC during create rollback: %v\n", team.ID, err) + } + team.Name = deletedTeamName(team.Name, team.ID) + team.Status = models.TeamStatusDeleted + team.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateTeam(team); err != nil { + fmt.Printf("Warning: failed to mark Team %d deleted during create rollback: %v\n", team.ID, err) + } + return cause +} + +func teamTaskPayloads(tasks []models.TeamTask) []TeamTaskPayload { + result := make([]TeamTaskPayload, 0, len(tasks)) + for _, task := range tasks { + payload, err := teamTaskPayload(task) + if err != nil { + result = append(result, TeamTaskPayload{TeamTask: task}) + continue + } + result = append(result, *payload) + } + return result +} + +func normalizeTeamHistoryLimit(limit, defaultLimit, maxLimit int) int { + if defaultLimit <= 0 { + defaultLimit = 50 + } + if maxLimit <= 0 { + maxLimit = defaultLimit + } + if limit <= 0 { + return defaultLimit + } + if limit > maxLimit { + return maxLimit + } + return limit +} + +func nextTeamTaskBeforeID(tasks []TeamTaskPayload) *int { + if len(tasks) == 0 { + return nil + } + minID := tasks[0].ID + for _, task := range tasks[1:] { + if task.ID < minID { + minID = task.ID + } + } + if minID <= 0 { + return nil + } + next := minID + return &next +} + +func nextTeamEventBeforeID(events []TeamEventPayload) *int { + if len(events) == 0 { + return nil + } + minID := events[0].ID + for _, event := range events[1:] { + if event.ID < minID { + minID = event.ID + } + } + if minID <= 0 { + return nil + } + next := minID + return &next +} + +func teamTaskPayload(task models.TeamTask) (*TeamTaskPayload, error) { + payload := TeamTaskPayload{TeamTask: task} + if strings.TrimSpace(task.PayloadJSON) != "" { + if err := json.Unmarshal([]byte(task.PayloadJSON), &payload.Payload); err != nil { + return nil, err + } + } + if task.ResultJSON != nil && strings.TrimSpace(*task.ResultJSON) != "" { + if err := json.Unmarshal([]byte(*task.ResultJSON), &payload.Result); err != nil { + return nil, err + } + } + return &payload, nil +} + +func teamEventPayloads(events []models.TeamEvent) []TeamEventPayload { + result := make([]TeamEventPayload, 0, len(events)) + for _, event := range events { + payload := TeamEventPayload{TeamEvent: event} + if event.PayloadJSON != nil && strings.TrimSpace(*event.PayloadJSON) != "" { + _ = json.Unmarshal([]byte(*event.PayloadJSON), &payload.Payload) + } + chatPolicy := strings.ToLower(strings.TrimSpace(eventString(payload.Payload, "chatPolicy", "chat_policy"))) + hidden, hiddenDefined := teamEventBoolValue(payload.Payload, "visibleToChat", "visible_to_chat") + eventKind := strings.ToLower(strings.TrimSpace(eventString(payload.Payload, "eventKind", "event_kind", "kind"))) + business := teamChatEventIsBusinessContent(strings.ToLower(strings.TrimSpace(event.EventType)), eventKind, payload.Payload) + if event.EventType == "member_result_confirmed" || (!business && (chatPolicy == "hidden" || (hiddenDefined && !hidden && chatPolicy == ""))) { + continue + } + result = append(result, payload) + } + return result +} + +func teamWorkItemPayloads(items []models.TeamWorkItem) []TeamWorkItemPayload { + result := make([]TeamWorkItemPayload, 0, len(items)) + for _, item := range items { + payload := TeamWorkItemPayload{TeamWorkItem: item} + if item.DependsOnJSON != nil && strings.TrimSpace(*item.DependsOnJSON) != "" { + _ = json.Unmarshal([]byte(*item.DependsOnJSON), &payload.DependsOn) + } + if item.ResultJSON != nil && strings.TrimSpace(*item.ResultJSON) != "" { + _ = json.Unmarshal([]byte(*item.ResultJSON), &payload.Result) + } + if item.ArtifactRefsJSON != nil && strings.TrimSpace(*item.ArtifactRefsJSON) != "" { + _ = json.Unmarshal([]byte(*item.ArtifactRefsJSON), &payload.ArtifactRefs) + } + result = append(result, payload) + } + return result +} + +func mergeRedisEventPayload(fields map[string]string) map[string]interface{} { + payload := map[string]interface{}{} + if raw := strings.TrimSpace(fields["payload"]); raw != "" { + _ = json.Unmarshal([]byte(raw), &payload) + } + for key, value := range fields { + if _, exists := payload[key]; !exists { + payload[key] = value + } + } + return payload +} + +func eventString(payload map[string]interface{}, keys ...string) string { + for _, key := range keys { + value, ok := payload[key] + if !ok || value == nil { + continue + } + switch typed := value.(type) { + case string: + return strings.TrimSpace(typed) + case float64: + return strconv.Itoa(int(typed)) + case int: + return strconv.Itoa(typed) + default: + return strings.TrimSpace(fmt.Sprintf("%v", typed)) + } + } + return "" +} + +func eventBool(payload map[string]interface{}, keys ...string) bool { + for _, key := range keys { + value, ok := payload[key] + if !ok || value == nil { + continue + } + switch typed := value.(type) { + case bool: + return typed + case string: + switch strings.ToLower(strings.TrimSpace(typed)) { + case "1", "true", "yes", "y", "on": + return true + case "0", "false", "no", "n", "off": + return false + } + case float64: + return typed != 0 + case int: + return typed != 0 + default: + text := strings.ToLower(strings.TrimSpace(fmt.Sprintf("%v", typed))) + if text == "true" || text == "yes" || text == "1" { + return true + } + } + } + return false +} + +func applyTeamMemberRuntimeProjection(member *models.TeamMember, payload map[string]interface{}, eventType string) { + if member == nil { + return + } + nonAuthoritativeDispatchWarning := isNonAuthoritativeDispatchWarning(eventType, payload) + status := normalizedTeamTaskEventStatus(payload) + availability := normalizeTeamAvailability(eventString(payload, "availability", "memberAvailability")) + if nonAuthoritativeDispatchWarning && availability == models.TeamMemberAvailabilityBlocked { + availability = "" + } + explicitlyBlocked := availability == models.TeamMemberAvailabilityBlocked + if availability != "" { + member.Availability = availability + } + if member.Availability == "" { + member.Availability = models.TeamMemberAvailabilityUnknown + } + if nonAuthoritativeDispatchWarning { + if member.Availability == models.TeamMemberAvailabilityBlocked || member.Availability == models.TeamMemberAvailabilityUnknown { + member.Availability = models.TeamMemberAvailabilityIdle + } + member.BlockedReason = nil + return + } + if runtimeStatus := eventString(payload, "runtime_status", "runtimeStatus", "runtime", "liveness"); runtimeStatus != "" { + member.RuntimeStatus = &runtimeStatus + } + if runtimeTaskID := eventString(payload, "runtime_task_id", "runtimeTaskId", "current_task_id", "currentTaskId", "taskId"); runtimeTaskID != "" { + member.RuntimeTaskID = &runtimeTaskID + } + if runtimeIntent := eventString(payload, "runtime_intent", "runtimeIntent", "current_intent", "currentIntent", "intent"); runtimeIntent != "" { + member.RuntimeIntent = &runtimeIntent + } + if summary := eventString(payload, "last_summary", "lastSummary", "summary", "diagnostic"); summary != "" && eventType != "assignment_heartbeat" { + member.LastSummary = &summary + } + if reason := eventString(payload, "blocked_reason", "blockedReason", "error_message", "error", "reason"); reason != "" { + if !nonAuthoritativeDispatchWarning { + member.BlockedReason = &reason + } + } + switch eventType { + case "presence", "member_presence", "status", "member_status": + return + case "task_completed": + if !explicitlyBlocked { + member.Availability = models.TeamMemberAvailabilityIdle + member.BlockedReason = nil + } + case "task_failed", "message_failed": + if !isTeamTaskFailureSignal(eventType, status, payload) { + return + } + if member.Availability == "" || member.Availability == models.TeamMemberAvailabilityUnknown || member.Availability == models.TeamMemberAvailabilityBusy { + member.Availability = models.TeamMemberAvailabilityBlocked + } + } +} + +func normalizeTeamAvailability(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "idle", "available", "ready": + return models.TeamMemberAvailabilityIdle + case "busy", "running", "working": + return models.TeamMemberAvailabilityBusy + case "blocked", "error", "failed": + return models.TeamMemberAvailabilityBlocked + case "offline", "unavailable": + return models.TeamMemberAvailabilityOffline + case "unknown": + return models.TeamMemberAvailabilityUnknown + default: + return "" + } +} + +func eventInt(payload map[string]interface{}, keys ...string) int { + for _, key := range keys { + value, ok := payload[key] + if !ok || value == nil { + continue + } + switch typed := value.(type) { + case float64: + return int(typed) + case int: + return typed + case string: + parsed, _ := strconv.Atoi(strings.TrimSpace(typed)) + return parsed + } + } + return 0 +} + +func teamEventBoolValue(payload map[string]interface{}, keys ...string) (bool, bool) { + if payload == nil { + return false, false + } + for _, key := range keys { + value, ok := payload[key] + if !ok || value == nil { + continue + } + switch typed := value.(type) { + case bool: + return typed, true + case float64: + return typed != 0, true + case int: + return typed != 0, true + case string: + normalized := strings.ToLower(strings.TrimSpace(typed)) + if normalized == "" { + continue + } + return normalized == "true" || normalized == "1" || normalized == "yes" || normalized == "on", true + } + } + return false, false +} + +func eventTime(payload map[string]interface{}) *time.Time { + for _, key := range []string{"occurred_at", "occurredAt", "timestamp"} { + raw := eventString(payload, key) + if raw == "" { + continue + } + if parsed, err := time.Parse(time.RFC3339Nano, raw); err == nil { + return &parsed + } + } + now := time.Now().UTC() + return &now +} + +func normalizeContextRefs(value interface{}) []string { + rawItems, ok := value.([]interface{}) + if !ok { + if typed, ok := value.([]string); ok { + return typed + } + return nil + } + refs := make([]string, 0, len(rawItems)) + for _, item := range rawItems { + ref := strings.TrimSpace(fmt.Sprintf("%v", item)) + if ref != "" { + refs = append(refs, ref) + } + } + return refs +} + +func (s *teamService) workspaceProxyInstance(userID, teamID int) (*models.Team, int, error) { + team, err := s.requireOwnedTeam(userID, teamID) + if err != nil { + return nil, 0, err + } + members, err := s.repo.ListMembersByTeamID(teamID) + if err != nil { + return nil, 0, err + } + for _, member := range activeTeamMembers(members) { + if member.InstanceID == nil { + continue + } + instance, err := s.instanceService.GetByID(*member.InstanceID) + if err != nil || instance == nil || instance.UserID != userID { + continue + } + if instance.Status == "running" || member.Status == models.TeamMemberStatusIdle || member.Status == models.TeamMemberStatusBusy { + return team, instance.ID, nil + } + } + for _, member := range activeTeamMembers(members) { + if member.InstanceID != nil { + instance, err := s.instanceService.GetByID(*member.InstanceID) + if err == nil && instance != nil && instance.UserID == userID { + return team, instance.ID, nil + } + } + } + return nil, 0, fmt.Errorf("no available Team member instance for workspace access") +} + +func (s *teamService) execTeamWorkspace(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error { + if s.podService == nil || s.podService.GetClient() == nil || s.podService.GetClient().Clientset == nil { + return fmt.Errorf("k8s client not initialized") + } + pod, err := s.podService.GetPod(ctx, userID, instanceID) + if err != nil { + return fmt.Errorf("failed to get pod: %w", err) + } + req := s.podService.GetClient().Clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("exec") + req.VersionedParams(&corev1.PodExecOptions{ + Container: "desktop", + Command: command, + Stdin: stdin != nil, + Stdout: stdout != nil, + Stderr: stderr != nil, + TTY: false, + }, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(s.podService.GetClient().Config, "POST", req.URL()) + if err != nil { + return fmt.Errorf("failed to initialize exec stream: %w", err) + } + return exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + Tty: false, + }) +} + +func cleanTeamWorkspacePath(raw string) (string, error) { + value := strings.TrimSpace(strings.ReplaceAll(raw, "\\", "/")) + value = strings.TrimPrefix(value, "./") + value = strings.TrimPrefix(value, "/") + if value == "team" { + value = "" + } else if strings.HasPrefix(value, "team/") { + value = strings.TrimPrefix(value, "team/") + } + if value == "" || value == "." { + return "", nil + } + cleaned := posixpath.Clean(value) + if cleaned == "." { + return "", nil + } + for _, segment := range strings.Split(cleaned, "/") { + if segment == "" || segment == "." || segment == ".." { + return "", fmt.Errorf("invalid workspace path") + } + } + return cleaned, nil +} + +func cleanWorkspaceEntryName(raw string) (string, error) { + name := strings.TrimSpace(strings.ReplaceAll(raw, "\\", "/")) + if name == "" || strings.Contains(name, "/") || name == "." || name == ".." { + return "", fmt.Errorf("invalid file or folder name") + } + return name, nil +} + +func joinTeamWorkspacePath(base, child string) string { + if strings.TrimSpace(base) == "" { + return child + } + if strings.TrimSpace(child) == "" { + return base + } + return posixpath.Clean(base + "/" + child) +} + +func teamWorkspaceFullPath(team *models.Team, relPath string) string { + root := strings.TrimRight(strings.TrimSpace(team.SharedMountPath), "/") + if root == "" { + root = teamSharedMountPath + } + if relPath == "" { + return root + } + return root + "/" + relPath +} + +func (s *teamService) resolveTeamWorkspacePath(ctx context.Context, userID, teamID int, cleanPath string) (*models.Team, string, string, error) { + if err := ctx.Err(); err != nil { + return nil, "", "", err + } + team, err := s.requireOwnedTeam(userID, teamID) + if err != nil { + return nil, "", "", err + } + root := filepath.Clean(s.teamRuntimeSharedPathFor(userID, team.ID)) + if root == "." || root == string(filepath.Separator) || strings.TrimSpace(root) == "" { + return nil, "", "", fmt.Errorf("invalid Team workspace root") + } + if err := ensureTeamWorkspaceDirectory(root); err != nil { + return nil, "", "", fmt.Errorf("failed to prepare Team workspace: %w", err) + } + target := root + if cleanPath != "" { + target = filepath.Join(root, filepath.FromSlash(cleanPath)) + } + target = filepath.Clean(target) + if target != root && !strings.HasPrefix(target, root+string(filepath.Separator)) { + return nil, "", "", fmt.Errorf("invalid Team workspace path") + } + return team, root, target, nil +} + +func teamWorkspaceDisplayRoot(team *models.Team, runtimeRoot string) string { + if team != nil { + if value := strings.TrimSpace(team.SharedMountPath); value != "" { + return value + } + } + return filepath.ToSlash(runtimeRoot) +} + +func teamWorkspaceFileEntryFromInfo(parentPath string, info os.FileInfo) TeamWorkspaceFileEntry { + entryType := "file" + size := info.Size() + if info.IsDir() { + entryType = "directory" + size = 0 + } + name := info.Name() + entryPath := joinTeamWorkspacePath(parentPath, name) + modifiedAt := "" + if !info.ModTime().IsZero() { + modifiedAt = info.ModTime().UTC().Format(time.RFC3339) + } + return TeamWorkspaceFileEntry{ + Name: name, + Path: entryPath, + Type: entryType, + Size: size, + ModifiedAt: modifiedAt, + Previewable: entryType == "file" && isPreviewableWorkspaceFile(name), + } +} + +func sortTeamWorkspaceEntries(entries []TeamWorkspaceFileEntry) { + sort.SliceStable(entries, func(i, j int) bool { + if entries[i].Type != entries[j].Type { + return entries[i].Type == "directory" + } + return strings.ToLower(entries[i].Name) < strings.ToLower(entries[j].Name) + }) +} + +func chownTeamWorkspacePath(path string) { + _ = os.Chown(path, teamSharedUID, teamSharedGID) +} + +func ensureTeamWorkspaceDirectory(path string) error { + clean := filepath.Clean(strings.TrimSpace(path)) + if clean == "" || clean == "." || clean == string(filepath.Separator) { + return fmt.Errorf("invalid Team workspace directory") + } + if err := os.MkdirAll(clean, 0o2775); err != nil { + return err + } + _ = os.Chmod(clean, 0o2775) + chownTeamWorkspacePath(clean) + return nil +} + +func zipTeamWorkspaceDirectory(ctx context.Context, root string) ([]byte, error) { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + base := filepath.Base(root) + if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return err + } + info, err := entry.Info() + if err != nil { + return err + } + rel, err := filepath.Rel(filepath.Dir(root), path) + if err != nil { + return err + } + zipName := filepath.ToSlash(rel) + if entry.IsDir() { + if path == root { + return nil + } + _, err := archive.CreateHeader(&zip.FileHeader{ + Name: strings.TrimSuffix(zipName, "/") + "/", + Method: zip.Store, + Modified: info.ModTime(), + }) + return err + } + header, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + if zipName == "." { + zipName = base + } + header.Name = zipName + header.Method = zip.Deflate + writer, err := archive.CreateHeader(header) + if err != nil { + return err + } + file, err := os.Open(path) + if err != nil { + return err + } + _, copyErr := io.Copy(writer, file) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + return closeErr + }); err != nil { + _ = archive.Close() + return nil, fmt.Errorf("failed to download Team workspace folder: %w", err) + } + if err := archive.Close(); err != nil { + return nil, fmt.Errorf("failed to finalize Team workspace folder download: %w", err) + } + return buf.Bytes(), nil +} + +func parseTeamWorkspaceList(parentPath, raw string) []TeamWorkspaceFileEntry { + lines := strings.Split(strings.TrimSpace(raw), "\n") + entries := make([]TeamWorkspaceFileEntry, 0, len(lines)) + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + parts := strings.SplitN(line, "\t", 4) + if len(parts) != 4 { + continue + } + size, _ := strconv.ParseInt(strings.TrimSpace(parts[2]), 10, 64) + mtimeUnix, _ := strconv.ParseInt(strings.TrimSpace(parts[3]), 10, 64) + name := parts[1] + entryPath := joinTeamWorkspacePath(parentPath, name) + modifiedAt := "" + if mtimeUnix > 0 { + modifiedAt = time.Unix(mtimeUnix, 0).UTC().Format(time.RFC3339) + } + entries = append(entries, TeamWorkspaceFileEntry{ + Name: name, + Path: entryPath, + Type: parts[0], + Size: size, + ModifiedAt: modifiedAt, + Previewable: parts[0] == "file" && isPreviewableWorkspaceFile(name), + }) + } + sortTeamWorkspaceEntries(entries) + return entries +} + +func isPreviewableWorkspaceFile(path string) bool { + name := strings.ToLower(strings.TrimSpace(path)) + return strings.HasSuffix(name, ".md") || strings.HasSuffix(name, ".txt") || strings.HasSuffix(name, ".json") +} + +func planTeamMembers(teamName string, members []CreateTeamMemberRequest) ([]plannedTeamMember, error) { + plans := make([]plannedTeamMember, 0, len(members)) + memberKeys := map[string]struct{}{} + leaderCount := 0 + for idx, memberReq := range members { + role := normalizeTeamMemberRole(memberReq.Role, memberReq.IsLeader) + memberKey, err := normalizeTeamMemberKey(memberReq.MemberID, role, idx) + if err != nil { + return nil, err + } + if _, exists := memberKeys[memberKey]; exists { + return nil, fmt.Errorf("duplicate team member id: %s", memberKey) + } + memberKeys[memberKey] = struct{}{} + runtimeType, err := normalizeTeamMemberRuntimeType(memberReq.RuntimeType) + if err != nil { + return nil, err + } + instanceMode, err := normalizeTeamMemberInstanceMode(memberReq.Mode, memberReq.InstanceMode) + if err != nil { + return nil, err + } + + isLeader := memberReq.IsLeader || isTeamLeaderRole(role) + if isLeader { + leaderCount++ + role = "leader" + } + profile := teamMemberProfileFromEnv(memberReq.EnvironmentOverrides) + effectiveRole := role + if !isLeader && strings.TrimSpace(profile.RoleHint) != "" && !isTeamLeaderRole(profile.RoleHint) { + effectiveRole = strings.TrimSpace(profile.RoleHint) + role = effectiveRole + } + displayName := strings.TrimSpace(memberReq.Name) + if displayName == "" { + displayName = fmt.Sprintf("%s-%s", teamName, memberKey) + } + plans = append(plans, plannedTeamMember{ + Request: memberReq, + MemberKey: memberKey, + DisplayName: displayName, + Role: role, + ProfileKey: profile.ProfileKey, + ProfileName: profile.ProfileName, + EffectiveRole: effectiveRole, + RuntimeType: runtimeType, + InstanceMode: instanceMode, + IsLeader: isLeader, + }) + } + if leaderCount != 1 { + return nil, fmt.Errorf("team must include exactly one leader") + } + return plans, nil +} + +func teamMemberInstanceName(teamName string, teamID int, memberKey string) string { + teamPart := normalizeTeamMemberKeyForInstanceName(teamName) + if teamPart == "" { + teamPart = "team" + } + memberPart := normalizeTeamMemberKeyForInstanceName(memberKey) + if memberPart == "" { + memberPart = "member" + } + const maxInstanceNameLength = 50 + idPart := fmt.Sprintf("%d", teamID) + maxMemberLength := maxInstanceNameLength - len(idPart) - len("--t") + if maxMemberLength < 1 { + maxMemberLength = 1 + } + if len(memberPart) > maxMemberLength { + memberPart = strings.Trim(memberPart[:maxMemberLength], "-") + if memberPart == "" { + memberPart = "member" + } + } + suffix := fmt.Sprintf("-%s-%s", idPart, memberPart) + if len(teamPart)+len(suffix) <= maxInstanceNameLength { + return teamPart + suffix + } + maxTeamLength := maxInstanceNameLength - len(suffix) + if maxTeamLength < 1 { + maxTeamLength = 1 + } + return strings.Trim(teamPart[:maxTeamLength], "-") + suffix +} + +func normalizeTeamMemberKeyForInstanceName(value string) string { + normalized := strings.ToLower(strings.TrimSpace(value)) + normalized = strings.ReplaceAll(normalized, "_", "-") + normalized = strings.ReplaceAll(normalized, " ", "-") + normalized = teamMemberInstanceNameInvalidChars.ReplaceAllString(normalized, "") + normalized = teamMemberInstanceNameRepeatedDashs.ReplaceAllString(normalized, "-") + return strings.Trim(normalized, "-") +} + +func normalizeTeamMemberRuntimeType(raw string) (string, error) { + runtimeType := strings.ToLower(strings.TrimSpace(raw)) + if runtimeType == "" { + return "openclaw", nil + } + switch runtimeType { + case "openclaw", "hermes": + return runtimeType, nil + default: + return "", fmt.Errorf("unsupported team member runtime type: %s", raw) + } +} + +func normalizeTeamMemberInstanceMode(rawMode, rawInstanceMode string) (string, error) { + if mode, ok := NormalizeInstanceMode(rawMode); ok { + return mode, nil + } + if strings.TrimSpace(rawMode) != "" { + return "", fmt.Errorf("unsupported team member instance mode: %s", rawMode) + } + if mode, ok := NormalizeInstanceMode(rawInstanceMode); ok { + return mode, nil + } + if strings.TrimSpace(rawInstanceMode) != "" { + return "", fmt.Errorf("unsupported team member instance mode: %s", rawInstanceMode) + } + return InstanceModeLite, nil +} + +func normalizeTeamCommunicationMode(raw string) (string, error) { + mode := strings.ToLower(strings.TrimSpace(raw)) + mode = strings.ReplaceAll(mode, "-", "_") + mode = strings.ReplaceAll(mode, " ", "_") + switch mode { + case "", teamCommunicationModeLeaderMediated, "leader", "leader_only": + return teamCommunicationModeLeaderMediated, nil + case teamCommunicationModePeerAssisted, "peer", "peer_to_peer", "peer_assist": + return teamCommunicationModePeerAssisted, nil + case teamCommunicationModeFullMesh, "mesh": + return teamCommunicationModeFullMesh, nil + default: + return "", fmt.Errorf("unsupported team communication mode: %s", raw) + } +} + +func normalizedTeamCommunicationMode(raw string) string { + mode, err := normalizeTeamCommunicationMode(raw) + if err != nil { + return teamCommunicationModeLeaderMediated + } + return mode +} + +func normalizeTeamMemberRole(raw string, isLeader bool) string { + role := strings.TrimSpace(raw) + if isLeader || isTeamLeaderRole(role) { + return "leader" + } + if role == "" { + return "member" + } + return role +} + +func isTeamLeaderRole(role string) bool { + normalized := strings.ToLower(strings.TrimSpace(role)) + normalized = strings.ReplaceAll(normalized, "_", "-") + normalized = strings.ReplaceAll(normalized, " ", "-") + return normalized == "leader" || normalized == "team-leader" +} + +func findTeamLeader(members []models.TeamMember) *models.TeamMember { + for idx := range members { + if isTeamLeaderRole(members[idx].Role) { + member := members[idx] + return &member + } + } + return nil +} + +func leaderMemberKey(member *models.TeamMember) string { + if member == nil { + return "" + } + return member.MemberKey +} + +type teamRosterConfig struct { + Version int `json:"version"` + TeamID string `json:"teamId"` + LeaderMemberID string `json:"leaderMemberId"` + CommunicationMode string `json:"communicationMode"` + CollaborationPolicy teamRosterCollaborationPolicy `json:"collaborationPolicy"` + SharedDir string `json:"sharedDir"` + Members []teamRosterMember `json:"members"` + Redis teamRosterRedis `json:"redis"` +} + +type teamRosterMember struct { + MemberID string `json:"memberId"` + Role string `json:"role"` + EffectiveRole string `json:"effectiveRole,omitempty"` + ProfileKey string `json:"profileKey,omitempty"` + ProfileName string `json:"profileName,omitempty"` + RuntimeType string `json:"runtimeType"` + InstanceMode string `json:"instanceMode"` + DisplayName string `json:"displayName"` + Description string `json:"description,omitempty"` + IsLeader bool `json:"isLeader"` +} + +type teamRosterRedis struct { + EventsKey string `json:"eventsKey"` + PresenceKey string `json:"presenceKey"` + DLQKey string `json:"dlqKey"` +} + +type teamRosterCollaborationPolicy struct { + Mode string `json:"mode"` + AllowPeerToPeer bool `json:"allowPeerToPeer"` + LeaderFinalizes bool `json:"leaderFinalizes"` + PeerReplyRequired bool `json:"peerReplyRequired"` + AllowedPeerActions []string `json:"allowedPeerActions,omitempty"` +} + +func buildTeamCollaborationPolicy(communicationMode string) teamRosterCollaborationPolicy { + mode := normalizedTeamCommunicationMode(communicationMode) + policy := teamRosterCollaborationPolicy{ + Mode: mode, + LeaderFinalizes: true, + } + switch mode { + case teamCommunicationModePeerAssisted: + policy.AllowPeerToPeer = true + policy.PeerReplyRequired = true + policy.AllowedPeerActions = []string{"ask", "handoff", "review_request", "artifact_request", "blocker_help", "peer_review"} + case teamCommunicationModeFullMesh: + policy.AllowPeerToPeer = true + policy.PeerReplyRequired = true + policy.AllowedPeerActions = []string{"ask", "handoff", "review_request", "artifact_request", "blocker_help", "peer_review", "delegate"} + } + return policy +} + +func buildTeamRosterConfig(team *models.Team, members []plannedTeamMember) (string, error) { + return buildTeamRosterConfigWithSharedDir(team, members, team.SharedMountPath) +} + +func buildTeamRosterConfigWithSharedDir(team *models.Team, members []plannedTeamMember, sharedDir string) (string, error) { + communicationMode := normalizedTeamCommunicationMode(team.CommunicationMode) + sharedDir = strings.TrimSpace(sharedDir) + if sharedDir == "" { + sharedDir = team.SharedMountPath + } + config := teamRosterConfig{ + Version: 1, + TeamID: strconv.Itoa(team.ID), + CommunicationMode: communicationMode, + CollaborationPolicy: buildTeamCollaborationPolicy(communicationMode), + SharedDir: sharedDir, + Members: make([]teamRosterMember, 0, len(members)), + Redis: teamRosterRedis{ + EventsKey: teamEventsKey(team.ID), + PresenceKey: teamPresenceKey(team.ID), + DLQKey: teamDLQKey(team.ID), + }, + } + for _, member := range members { + if member.IsLeader { + config.LeaderMemberID = member.MemberKey + } + config.Members = append(config.Members, teamRosterMember{ + MemberID: member.MemberKey, + Role: member.Role, + EffectiveRole: effectiveTeamMemberRole(member), + ProfileKey: member.ProfileKey, + ProfileName: member.ProfileName, + RuntimeType: member.RuntimeType, + InstanceMode: member.InstanceMode, + DisplayName: member.DisplayName, + Description: plannedTeamMemberDescription(member), + IsLeader: member.IsLeader, + }) + } + if config.LeaderMemberID == "" { + return "", fmt.Errorf("team must include exactly one leader") + } + return marshalJSON(config) +} + +func buildTeamRosterConfigFromMembers(team *models.Team, members []models.TeamMember) (string, error) { + return buildTeamRosterConfigFromMembersWithSharedDir(team, members, team.SharedMountPath) +} + +func buildTeamRosterConfigFromMembersWithSharedDir(team *models.Team, members []models.TeamMember, sharedDir string) (string, error) { + communicationMode := normalizedTeamCommunicationMode(team.CommunicationMode) + sharedDir = strings.TrimSpace(sharedDir) + if sharedDir == "" { + sharedDir = team.SharedMountPath + } + config := teamRosterConfig{ + Version: 1, + TeamID: strconv.Itoa(team.ID), + CommunicationMode: communicationMode, + CollaborationPolicy: buildTeamCollaborationPolicy(communicationMode), + SharedDir: sharedDir, + Members: make([]teamRosterMember, 0, len(members)), + Redis: teamRosterRedis{ + EventsKey: teamEventsKey(team.ID), + PresenceKey: teamPresenceKey(team.ID), + DLQKey: teamDLQKey(team.ID), + }, + } + for _, member := range members { + isLeader := isTeamLeaderRole(member.Role) + runtimeType := strings.TrimSpace(member.RuntimeType) + if runtimeType == "" { + runtimeType = "openclaw" + } + instanceMode := strings.TrimSpace(member.InstanceMode) + if instanceMode == "" { + instanceMode = InstanceModeLite + } + if isLeader { + config.LeaderMemberID = member.MemberKey + } + config.Members = append(config.Members, teamRosterMember{ + MemberID: member.MemberKey, + Role: member.Role, + EffectiveRole: member.Role, + RuntimeType: runtimeType, + InstanceMode: instanceMode, + DisplayName: member.DisplayName, + Description: derefTeamString(member.Description), + IsLeader: isLeader, + }) + } + if config.LeaderMemberID == "" { + return "", fmt.Errorf("team must include exactly one leader") + } + return marshalJSON(config) +} + +func activeTeamMembers(members []models.TeamMember) []models.TeamMember { + active := make([]models.TeamMember, 0, len(members)) + for _, member := range members { + if member.Status == models.TeamMemberStatusDeleted || member.Status == models.TeamMemberStatusDeleting { + continue + } + active = append(active, member) + } + return active +} + +func activeTeams(teams []models.Team) []models.Team { + active := make([]models.Team, 0, len(teams)) + for _, team := range teams { + if team.Status == models.TeamStatusDeleted { + continue + } + active = append(active, team) + } + return active +} + +func deletedTeamName(name string, teamID int) string { + const maxTeamNameLength = 255 + suffix := fmt.Sprintf("__deleted_%d", teamID) + trimmed := strings.TrimSpace(name) + if trimmed == "" { + trimmed = "team" + } + if strings.HasSuffix(trimmed, suffix) { + return trimmed + } + if len(trimmed)+len(suffix) <= maxTeamNameLength { + return trimmed + suffix + } + runes := []rune(trimmed) + maxPrefixLength := maxTeamNameLength - len(suffix) + if len(runes) > maxPrefixLength { + runes = runes[:maxPrefixLength] + } + return string(runes) + suffix +} + +func normalizeTeamMemberKey(raw, role string, index int) (string, error) { + value := strings.ToLower(strings.TrimSpace(raw)) + if value == "" { + value = strings.ToLower(strings.TrimSpace(role)) + } + if value == "" { + value = fmt.Sprintf("member-%d", index+1) + } + value = strings.ReplaceAll(value, "_", "-") + value = strings.ReplaceAll(value, " ", "-") + if !teamMemberKeyPattern.MatchString(value) { + return "", fmt.Errorf("team member id is invalid") + } + return value, nil +} + +func defaultTeamRedisURL() string { + for _, key := range []string{"CLAWMANAGER_TEAM_REDIS_URL", "TEAM_REDIS_URL", "REDIS_URL"} { + if value := strings.TrimSpace(os.Getenv(key)); value != "" { + return value + } + } + if value, ok := defaultTeamRedisServiceURL(); ok { + return value + } + return "" +} + +func defaultTeamRedisServiceURL() (string, bool) { + systemNamespace := strings.TrimSpace(os.Getenv("CLAWMANAGER_SYSTEM_NAMESPACE")) + if systemNamespace == "" { + if client := k8s.GetClient(); client != nil { + systemNamespace = client.GetSystemNamespace() + } else if baseNamespace := strings.TrimSpace(os.Getenv("K8S_NAMESPACE")); baseNamespace != "" { + systemNamespace = fmt.Sprintf("%s-system", baseNamespace) + } + } + if systemNamespace == "" { + return "", false + } + + serviceName := strings.TrimSpace(os.Getenv("CLAWMANAGER_TEAM_REDIS_SERVICE_NAME")) + if serviceName == "" { + serviceName = strings.TrimSpace(os.Getenv("CLAWMANAGER_TEAM_REDIS_SERVICE")) + } + if serviceName == "" { + serviceName = "clawmanager-team-redis" + } + + port := normalizePortValue( + strings.TrimSpace(os.Getenv("CLAWMANAGER_TEAM_REDIS_SERVICE_PORT")), + strings.TrimSpace(os.Getenv("CLAWMANAGER_TEAM_REDIS_PORT")), + ) + if port == "" { + port = "6379" + } + + db := strings.TrimSpace(os.Getenv("CLAWMANAGER_TEAM_REDIS_DB")) + if db == "" { + db = strings.TrimSpace(os.Getenv("TEAM_REDIS_DB")) + } + if db == "" { + db = "0" + } + + return fmt.Sprintf("redis://%s.%s.svc.cluster.local:%s/%s", serviceName, systemNamespace, port, db), true +} + +func teamTaskStaleTimeout() time.Duration { + raw := strings.TrimSpace(os.Getenv("CLAWMANAGER_TEAM_TASK_STALE_SECONDS")) + if raw == "" { + return defaultTeamTaskStaleTimeout + } + seconds, err := strconv.Atoi(raw) + if err != nil { + return defaultTeamTaskStaleTimeout + } + if seconds <= 0 { + return 0 + } + return time.Duration(seconds) * time.Second +} + +func defaultTeamManagerBaseURL() (string, bool) { + if override := strings.TrimSpace(os.Getenv("CLAWMANAGER_TEAM_MANAGER_BASE_URL")); override != "" { + return override, true + } + return defaultAgentControlBaseURL() +} + +func teamInboxKey(teamID int, memberID string) string { + return fmt.Sprintf("claw:team:%d:inbox:%s", teamID, memberID) +} + +func teamEventsKey(teamID int) string { + return fmt.Sprintf("claw:team:%d:events", teamID) +} + +func teamCompletionAckKey(teamID int, completionID, attemptID string) string { + return fmt.Sprintf("claw:team:%d:completion-ack:%s:%s", teamID, normalizeTeamRedisKeyPart(completionID), normalizeTeamRedisKeyPart(attemptID)) +} + +func teamCompletionStateKey(teamID int, completionID string) string { + return fmt.Sprintf("claw:team:%d:completion-state:%s", teamID, normalizeTeamRedisKeyPart(completionID)) +} + +func teamCompletionAckStreamKey(teamID int, memberKey string) string { + return fmt.Sprintf("claw:team:%d:completion-acks:%s", teamID, normalizeTeamRedisKeyPart(memberKey)) +} + +func normalizeTeamRedisKeyPart(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "unknown" + } + return strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == ':' || r == '.' { + return r + } + return '-' + }, value) +} + +func teamPresenceKey(teamID int) string { + return fmt.Sprintf("claw:team:%d:presence", teamID) +} + +func teamDLQKey(teamID int) string { + return fmt.Sprintf("claw:team:%d:dlq", teamID) +} + +func defaultInt(value, fallback int) int { + if value > 0 { + return value + } + return fallback +} + +func defaultFloat(value, fallback float64) float64 { + if value > 0 { + return value + } + return fallback +} + +func derefTeamString(value *string) string { + if value == nil { + return "" + } + return strings.TrimSpace(*value) +} diff --git a/backend/internal/services/team_service_test.go b/backend/internal/services/team_service_test.go new file mode 100644 index 0000000..d8297ff --- /dev/null +++ b/backend/internal/services/team_service_test.go @@ -0,0 +1,5144 @@ +package services + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "clawreef/internal/models" +) + +func TestTeamMemberEnvUsesSecretBackedRedisAndToken(t *testing.T) { + t.Setenv("CLAWMANAGER_TEAM_MANAGER_BASE_URL", "http://manager.example") + + service := &teamService{} + env := service.teamMemberEnv(&models.Team{ + ID: 12, + SharedMountPath: "/team", + }, plannedTeamMember{ + MemberKey: "leader", + Role: "lead", + }) + + if env["CLAWMANAGER_TEAM_ID"] != "12" { + t.Fatalf("expected Team id env, got %q", env["CLAWMANAGER_TEAM_ID"]) + } + if env["CLAWMANAGER_TEAM_MEMBER_ID"] != "leader" { + t.Fatalf("expected member id env, got %q", env["CLAWMANAGER_TEAM_MEMBER_ID"]) + } + if env["CLAWMANAGER_TEAM_ROLE"] != "lead" { + t.Fatalf("expected Team role env, got %q", env["CLAWMANAGER_TEAM_ROLE"]) + } + if env["CLAWMANAGER_TEAM_INBOX_KEY"] != "claw:team:12:inbox:leader" { + t.Fatalf("unexpected inbox key: %q", env["CLAWMANAGER_TEAM_INBOX_KEY"]) + } + if env["CLAWMANAGER_TEAM_EVENTS_KEY"] != "claw:team:12:events" { + t.Fatalf("unexpected events key: %q", env["CLAWMANAGER_TEAM_EVENTS_KEY"]) + } + if env["CLAWMANAGER_TEAM_MANAGER_URL"] != "http://manager.example" { + t.Fatalf("unexpected manager url: %q", env["CLAWMANAGER_TEAM_MANAGER_URL"]) + } + if env["CLAWMANAGER_TEAM_CONFIG_PATH"] != "/etc/clawmanager/team/team.json" { + t.Fatalf("unexpected Team config path: %q", env["CLAWMANAGER_TEAM_CONFIG_PATH"]) + } + if env["CLAWMANAGER_TEAM_SHARED_UID"] != "1000" || env["CLAWMANAGER_TEAM_SHARED_GID"] != "1000" || env["CLAWMANAGER_TEAM_UMASK"] != "0002" { + t.Fatalf("expected Team shared permission env, got %#v", env) + } + if env["PUID"] != "1000" || env["PGID"] != "1000" || env["UMASK"] != "0002" { + t.Fatalf("expected runtime shared permission env, got %#v", env) + } + if env["CLAWMANAGER_TEAM_AUTORUN"] != "true" || env["CLAWMANAGER_TEAM_CONSUMER_GROUP"] != "team-members" { + t.Fatalf("expected Team autorun and consumer group env, got %#v", env) + } + for key := range env { + if strings.Contains(key, "REDIS_URL") || strings.Contains(key, "TOKEN") { + t.Fatalf("sensitive Team env %s must come from Secret, not plain env", key) + } + } +} + +func TestTeamMemberEnvInjectsRoleGuidance(t *testing.T) { + t.Setenv("CLAWMANAGER_TEAM_MANAGER_BASE_URL", "http://manager.example") + description := "Senior Developer: implements scoped changes and reports verification." + + service := &teamService{} + env := service.teamMemberEnv(&models.Team{ + ID: 12, + SharedMountPath: "/team", + }, plannedTeamMember{ + MemberKey: "worker", + DisplayName: "team-worker", + Role: "senior-developer", + Request: CreateTeamMemberRequest{ + Description: &description, + }, + }) + + if env["CLAWMANAGER_TEAM_ROLE"] != "senior-developer" { + t.Fatalf("expected specific Team role env, got %q", env["CLAWMANAGER_TEAM_ROLE"]) + } + if env["CLAWMANAGER_TEAM_MEMBER_DESCRIPTION"] != description { + t.Fatalf("expected description env, got %q", env["CLAWMANAGER_TEAM_MEMBER_DESCRIPTION"]) + } + if !strings.Contains(env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"], "senior-developer") { + t.Fatalf("expected Team system prompt to include role, got %q", env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"]) + } + if env["HERMES_AGENT_HELP_GUIDANCE"] != env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"] { + t.Fatalf("expected Hermes guidance alias to match Team system prompt") + } + for _, expected := range []string{"exact CLAWMANAGER_TEAM_SHARED_DIR", "team/... is invalid", "/team/"} { + if !strings.Contains(env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"], expected) { + t.Fatalf("expected shared workspace guidance %q, got %q", expected, env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"]) + } + } +} + +func TestCleanTeamWorkspacePathTreatsTeamPrefixAsLogicalAlias(t *testing.T) { + cases := map[string]string{ + "/team/results/report.md": "results/report.md", + "team/results/report.md": "results/report.md", + "./team/results/report.md": "results/report.md", + "/team": "", + "team": "", + "results/report.md": "results/report.md", + } + + for raw, expected := range cases { + cleaned, err := cleanTeamWorkspacePath(raw) + if err != nil { + t.Fatalf("cleanTeamWorkspacePath(%q) returned error: %v", raw, err) + } + if cleaned != expected { + t.Fatalf("cleanTeamWorkspacePath(%q) = %q, want %q", raw, cleaned, expected) + } + } +} + +func TestBuildTeamMemberInstanceRequestUsesSharedPermissionDefaults(t *testing.T) { + service := &teamService{} + pvcName := "clawreef-team-7-shared" + secretName := "clawreef-team-7-bus" + req := service.buildTeamMemberInstanceRequest(&models.Team{ + ID: 7, + Name: "Delivery", + SharedPVCName: &pvcName, + SharedMountPath: "/team", + TeamTokenSecretName: &secretName, + }, plannedTeamMember{ + MemberKey: "delivery-lead", + DisplayName: "delivery lead", + Role: "leader", + RuntimeType: "openclaw", + }) + + if req.Team == nil { + t.Fatalf("expected Team instance config") + } + if req.Team.SharedUID != 1000 || req.Team.SharedGID != 1000 || req.Team.SharedUmask != "0002" { + t.Fatalf("unexpected Team shared permission config: %#v", req.Team) + } + if req.Team.ConfigMountPath != "/etc/clawmanager/team" { + t.Fatalf("unexpected Team config mount path: %q", req.Team.ConfigMountPath) + } + if req.Team.Environment["CLAWMANAGER_TEAM_CONFIG_PATH"] != "/etc/clawmanager/team/team.json" { + t.Fatalf("unexpected Team config env: %#v", req.Team.Environment) + } +} + +func TestBuildTeamMemberInstanceRequestMountsHermesSoul(t *testing.T) { + service := &teamService{} + pvcName := "clawreef-team-7-shared" + secretName := "clawreef-team-7-bus" + req := service.buildTeamMemberInstanceRequest(&models.Team{ + ID: 7, + Name: "Delivery", + SharedPVCName: &pvcName, + SharedMountPath: "/team", + TeamTokenSecretName: &secretName, + }, plannedTeamMember{ + MemberKey: "worker", + DisplayName: "team worker", + Role: "senior-developer", + RuntimeType: "hermes", + }) + + if req.Team == nil { + t.Fatalf("expected Team instance config") + } + if req.Team.PersonaConfigKey != "hermes-soul-worker.md" { + t.Fatalf("expected Hermes persona config key, got %q", req.Team.PersonaConfigKey) + } +} + +func TestBuildTeamMemberInstanceRequestSupportsLiteMode(t *testing.T) { + service := &teamService{runtimeWorkspaceRoot: "/workspaces"} + team := &models.Team{ + UserID: 1, + ID: 8, + Name: "Lite Team", + SharedMountPath: "/team", + } + memberPlan := plannedTeamMember{ + Request: CreateTeamMemberRequest{ + Mode: "Lite", + InstanceMode: "Lite", + }, + MemberKey: "lite-worker", + DisplayName: "lite worker", + Role: "developer", + RuntimeType: "openclaw", + InstanceMode: InstanceModeLite, + } + req := service.buildTeamMemberInstanceRequest(team, memberPlan) + + if req.Mode != InstanceModeLite || req.InstanceMode != InstanceModeLite { + t.Fatalf("expected Team member instance request to preserve lite mode, got mode=%q instance_mode=%q", req.Mode, req.InstanceMode) + } + if req.RuntimeType != RuntimeBackendGateway { + t.Fatalf("expected lite Team member to target gateway runtime, got %q", req.RuntimeType) + } + + rosterJSON := `{"teamId":"8","members":[{"memberId":"lite-worker"}]}` + liteReq := service.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, &teamRuntimeSecrets{ + RedisURL: "redis://team-redis:6379/0", + Token: "team_test_token", + }, rosterJSON) + if liteReq.EnvironmentOverrides[teamRedisURLSecretKey] != "redis://team-redis:6379/0" || + liteReq.EnvironmentOverrides[teamTokenSecretKey] != "team_test_token" { + t.Fatalf("expected Lite Team runtime secrets in gateway env overrides, got %#v", liteReq.EnvironmentOverrides) + } + if liteReq.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"] != rosterJSON { + t.Fatalf("Lite Team roster JSON should preserve upstream logical sharedDir contract, got %#v", liteReq.EnvironmentOverrides) + } +} + +func TestBuildTeamMemberInstanceRequestPointsLiteSharedDirAtRuntimeWorkspace(t *testing.T) { + service := &teamService{runtimeWorkspaceRoot: "/workspaces"} + team := &models.Team{ + UserID: 1, + ID: 28, + Name: "Mixed Team", + SharedMountPath: "/team", + } + memberPlan := plannedTeamMember{ + MemberKey: "backend", + DisplayName: "backend", + Role: "developer", + RuntimeType: "openclaw", + InstanceMode: InstanceModeLite, + } + + req := service.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, &teamRuntimeSecrets{ + RedisURL: "redis://team-redis:6379/0", + Token: "team_test_token", + }, `{"sharedDir":"/team"}`) + + wantSharedDir := "/workspaces/teams/user-1/team-28-shared" + if req.EnvironmentOverrides["CLAWMANAGER_TEAM_SHARED_DIR"] != wantSharedDir { + t.Fatalf("expected Lite shared dir %q, got %#v", wantSharedDir, req.EnvironmentOverrides) + } + if req.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"] != `{"sharedDir":"/team"}` { + t.Fatalf("Lite roster JSON should preserve logical /team sharedDir, got %s", req.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"]) + } + if req.Team.SharedMountPath != "/team" { + t.Fatalf("Pro Team mount path should remain /team, got %q", req.Team.SharedMountPath) + } +} + +func TestOpenClawConfigPlanForTeamMemberFiltersOnlyWorkers(t *testing.T) { + originalPlan := &OpenClawConfigPlan{ + Mode: OpenClawConfigPlanModeManual, + ResourceIDs: []int{10, 20}, + } + filteredPlan := &OpenClawConfigPlan{ + Mode: OpenClawConfigPlanModeManual, + ResourceIDs: []int{20}, + } + planner := &teamOpenClawConfigPlannerStub{nextPlan: filteredPlan} + service := &teamService{openClawConfigPlanner: planner} + + leaderPlan, err := service.openClawConfigPlanForTeamMember(7, plannedTeamMember{ + IsLeader: true, + Request: CreateTeamMemberRequest{OpenClawConfigPlan: originalPlan}, + }) + if err != nil { + t.Fatalf("leader plan returned error: %v", err) + } + if leaderPlan != originalPlan { + t.Fatalf("expected leader to keep original OpenClaw plan") + } + if planner.calls != 0 { + t.Fatalf("expected leader plan to skip filtering, got %d calls", planner.calls) + } + + workerPlan, err := service.openClawConfigPlanForTeamMember(7, plannedTeamMember{ + IsLeader: false, + Request: CreateTeamMemberRequest{OpenClawConfigPlan: originalPlan}, + }) + if err != nil { + t.Fatalf("worker plan returned error: %v", err) + } + if workerPlan != filteredPlan { + t.Fatalf("expected worker to use filtered OpenClaw plan") + } + if planner.calls != 1 || planner.userID != 7 || planner.plan != originalPlan { + t.Fatalf("unexpected planner call: %#v", planner) + } +} + +func TestNewRedisBusParsesURLWithoutNetwork(t *testing.T) { + bus, err := newRedisBus("redis://:pass@redis.example:6380/3") + if err != nil { + t.Fatalf("newRedisBus returned error: %v", err) + } + if bus.address != "redis.example:6380" || bus.password != "pass" || bus.db != 3 || bus.useTLS { + t.Fatalf("unexpected redis bus config: %#v", bus) + } +} + +func TestDefaultTeamRedisURLUsesClusterServiceFallback(t *testing.T) { + t.Setenv("CLAWMANAGER_TEAM_REDIS_URL", "") + t.Setenv("TEAM_REDIS_URL", "") + t.Setenv("REDIS_URL", "") + t.Setenv("CLAWMANAGER_SYSTEM_NAMESPACE", "") + t.Setenv("K8S_NAMESPACE", "clawmanager") + t.Setenv("CLAWMANAGER_TEAM_REDIS_SERVICE_NAME", "") + t.Setenv("CLAWMANAGER_TEAM_REDIS_SERVICE", "") + t.Setenv("CLAWMANAGER_TEAM_REDIS_SERVICE_PORT", "") + t.Setenv("CLAWMANAGER_TEAM_REDIS_PORT", "") + t.Setenv("CLAWMANAGER_TEAM_REDIS_DB", "") + t.Setenv("TEAM_REDIS_DB", "") + + got := defaultTeamRedisURL() + want := "redis://clawmanager-team-redis.clawmanager-system.svc.cluster.local:6379/0" + if got != want { + t.Fatalf("expected default Team redis URL %q, got %q", want, got) + } +} + +func TestProjectTeamTaskRuntimeStateUsesExplicitCompletionSignals(t *testing.T) { + now := time.Date(2026, 6, 15, 10, 0, 0, 0, time.UTC) + resultJSON := `{"status":"done","resultMarkdown":"finished"}` + task := &models.TeamTask{Status: models.TeamTaskStatusFailed} + + projection := projectTeamTaskRuntimeState(task, map[string]interface{}{ + "status": "done", + "resultMarkdown": "finished", + "explicitCompletion": true, + }, "completion", &resultJSON, now) + + if !projection.changed || projection.status != models.TeamTaskStatusSucceeded { + t.Fatalf("expected succeeded projection, got %#v", projection) + } + if task.Status != models.TeamTaskStatusSucceeded || task.FinishedAt == nil || task.ResultJSON == nil { + t.Fatalf("expected task to be completed with result, got %#v", task) + } +} + +func TestProjectTeamTaskRuntimeStateDoesNotLetLateFailureOverrideSuccess(t *testing.T) { + now := time.Date(2026, 6, 15, 10, 0, 0, 0, time.UTC) + task := &models.TeamTask{Status: models.TeamTaskStatusSucceeded} + + projection := projectTeamTaskRuntimeState(task, map[string]interface{}{ + "status": "failed", + "error": "late failure", + }, "task_failed", nil, now) + + if projection.changed || task.Status != models.TeamTaskStatusSucceeded { + t.Fatalf("late failure must not override success, projection=%#v task=%#v", projection, task) + } +} + +func TestProjectTeamTaskRuntimeStateDoesNotTreatPlainReplyAsCompletion(t *testing.T) { + now := time.Date(2026, 6, 15, 10, 0, 0, 0, time.UTC) + task := &models.TeamTask{Status: models.TeamTaskStatusRunning} + + projection := projectTeamTaskRuntimeState(task, map[string]interface{}{ + "message": "worker is preparing the result and will report back soon", + }, "reply", nil, now) + + if projection.changed || task.Status != models.TeamTaskStatusRunning || task.FinishedAt != nil { + t.Fatalf("plain reply must not complete task, projection=%#v task=%#v", projection, task) + } +} + +func TestProjectTeamTaskRuntimeStateDoesNotDowngradeTerminalTaskToRunning(t *testing.T) { + now := time.Date(2026, 6, 15, 10, 0, 0, 0, time.UTC) + task := &models.TeamTask{Status: models.TeamTaskStatusSucceeded} + + projection := projectTeamTaskRuntimeState(task, map[string]interface{}{ + "progress": 30, + }, "task_started", nil, now) + + if projection.changed || task.Status != models.TeamTaskStatusSucceeded || task.StartedAt != nil { + t.Fatalf("running signal must not downgrade terminal task, projection=%#v task=%#v", projection, task) + } +} + +func TestPlanTeamMembersRequiresExactlyOneLeader(t *testing.T) { + _, err := planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "worker", Role: "developer"}, + }) + if err == nil || !strings.Contains(err.Error(), "exactly one leader") { + t.Fatalf("expected exactly one leader validation error, got %v", err) + } + + plans, err := planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "lead", Role: "team leader"}, + {MemberID: "worker", Role: "developer"}, + }) + if err != nil { + t.Fatalf("planTeamMembers returned error: %v", err) + } + if len(plans) != 2 || !plans[0].IsLeader || plans[0].Role != "leader" { + t.Fatalf("expected first member to be normalized as leader, got %#v", plans) + } + if plans[1].RuntimeType != "openclaw" { + t.Fatalf("expected default runtime type openclaw, got %#v", plans[1]) + } +} + +func TestPlanTeamMembersSupportsHermesRuntime(t *testing.T) { + plans, err := planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "lead", Role: "leader"}, + {MemberID: "hermes-writer", Role: "writer", RuntimeType: "Hermes", InstanceMode: "Pro"}, + }) + if err != nil { + t.Fatalf("planTeamMembers returned error: %v", err) + } + if plans[1].RuntimeType != "hermes" { + t.Fatalf("expected Hermes runtime to be normalized, got %#v", plans[1]) + } + if plans[1].InstanceMode != InstanceModePro { + t.Fatalf("expected Pro instance mode to be normalized, got %#v", plans[1]) + } + + _, err = planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "lead", Role: "leader"}, + {MemberID: "worker", Role: "developer", RuntimeType: "ubuntu"}, + }) + if err == nil || !strings.Contains(err.Error(), "unsupported team member runtime type") { + t.Fatalf("expected unsupported runtime validation error, got %v", err) + } + + _, err = planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "lead", Role: "leader"}, + {MemberID: "worker", Role: "developer", Mode: "mini"}, + }) + if err == nil || !strings.Contains(err.Error(), "unsupported team member instance mode") { + t.Fatalf("expected unsupported instance mode validation error, got %v", err) + } +} + +func TestTeamMemberInstanceNameUsesTeamIDAndMemberKey(t *testing.T) { + name := teamMemberInstanceName("Software Engineering Team", 42, "code-reviewer") + if name != "software-engineering-team-42-code-reviewer" { + t.Fatalf("unexpected Team member instance name: %q", name) + } + + longName := teamMemberInstanceName("very-long-software-engineering-platform-team", 12345, "extremely-long-code-reviewer-member-key") + if len(longName) > 50 { + t.Fatalf("expected instance name to stay within 50 chars, got %d: %q", len(longName), longName) + } + if !strings.Contains(longName, "-12345-") { + t.Fatalf("expected instance name to include Team ID, got %q", longName) + } +} + +func TestBuildTeamRosterConfigOmitsSecrets(t *testing.T) { + description := "reviews implementation and validates results" + plans, err := planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "leader", Role: "leader"}, + {MemberID: "worker", Role: "developer", Description: &description}, + }) + if err != nil { + t.Fatalf("planTeamMembers returned error: %v", err) + } + roster, err := buildTeamRosterConfig(&models.Team{ + ID: 9, + CommunicationMode: "leader_mediated", + SharedMountPath: "/team", + }, plans) + if err != nil { + t.Fatalf("buildTeamRosterConfig returned error: %v", err) + } + for _, forbidden := range []string{"REDIS_URL", "TOKEN", "OPENAI_API_KEY", "secret"} { + if strings.Contains(roster, forbidden) { + t.Fatalf("roster must not contain sensitive value marker %q: %s", forbidden, roster) + } + } + if !strings.Contains(roster, `"leaderMemberId":"leader"`) || !strings.Contains(roster, `"eventsKey":"claw:team:9:events"`) { + t.Fatalf("roster missing expected leader or redis keys: %s", roster) + } + if !strings.Contains(roster, description) { + t.Fatalf("roster missing member description: %s", roster) + } + if !strings.Contains(roster, `"runtimeType":"openclaw"`) { + t.Fatalf("roster missing member runtime type: %s", roster) + } + if !strings.Contains(roster, `"instanceMode":"lite"`) { + t.Fatalf("roster missing member instance mode: %s", roster) + } + if !strings.Contains(roster, `"communicationMode":"leader_mediated"`) || !strings.Contains(roster, `"allowPeerToPeer":false`) { + t.Fatalf("roster missing leader-mediated collaboration policy: %s", roster) + } +} + +func TestPlanTeamMembersUsesProfileEffectiveRole(t *testing.T) { + profileEnv := map[string]string{ + "CLAWMANAGER_AGENT_PERSONA_JSON": `{"profileKey":"agency.senior-developer","name":"Senior Developer","displayName":"Senior Developer","roleHint":"senior-developer","summary":"Implements scoped engineering tasks.","systemPrompt":"You are a senior implementation specialist."}`, + } + plans, err := planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "leader", Role: "leader"}, + {MemberID: "worker", Role: "developer", EnvironmentOverrides: profileEnv}, + }) + if err != nil { + t.Fatalf("planTeamMembers returned error: %v", err) + } + worker := plans[1] + if worker.Role != "senior-developer" || worker.EffectiveRole != "senior-developer" { + t.Fatalf("expected profile effective role to override generic role, got role=%q effective=%q", worker.Role, worker.EffectiveRole) + } + if worker.ProfileKey != "agency.senior-developer" || worker.ProfileName != "Senior Developer" { + t.Fatalf("expected profile metadata, got key=%q name=%q", worker.ProfileKey, worker.ProfileName) + } + + roster, err := buildTeamRosterConfig(&models.Team{ + ID: 19, + CommunicationMode: teamCommunicationModeLeaderMediated, + SharedMountPath: "/team", + }, plans) + if err != nil { + t.Fatalf("buildTeamRosterConfig returned error: %v", err) + } + for _, expected := range []string{ + `"role":"senior-developer"`, + `"effectiveRole":"senior-developer"`, + `"profileKey":"agency.senior-developer"`, + `"profileName":"Senior Developer"`, + } { + if !strings.Contains(roster, expected) { + t.Fatalf("roster missing %q: %s", expected, roster) + } + } +} + +func TestTeamMemberEnvIncludesPeerAssistedPolicy(t *testing.T) { + description := "Developer: implements scoped tasks." + plan, err := planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "leader", Role: "leader"}, + {MemberID: "worker", Role: "developer", Description: &description}, + }) + if err != nil { + t.Fatalf("planTeamMembers returned error: %v", err) + } + env := (&teamService{}).teamMemberEnv(&models.Team{ + ID: 12, + CommunicationMode: teamCommunicationModePeerAssisted, + SharedMountPath: "/team", + }, plan[1]) + if env["CLAWMANAGER_TEAM_COMMUNICATION_MODE"] != teamCommunicationModePeerAssisted { + t.Fatalf("expected peer_assisted env, got %#v", env) + } + if !strings.Contains(env["CLAWMANAGER_TEAM_COLLABORATION_POLICY_JSON"], `"allowPeerToPeer":true`) { + t.Fatalf("expected peer-to-peer policy env, got %#v", env["CLAWMANAGER_TEAM_COLLABORATION_POLICY_JSON"]) + } + if !strings.Contains(env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"], "Collaboration mode: peer_assisted") { + t.Fatalf("expected peer-assisted guidance in system prompt: %s", env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"]) + } + if !strings.Contains(env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"], "Direct handoff is mandatory") { + t.Fatalf("expected mandatory direct handoff guidance in system prompt: %s", env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"]) + } + if env["GATEWAY_ALLOW_ALL_USERS"] != "true" { + t.Fatalf("expected Hermes teammate messages to be allowed, got %#v", env["GATEWAY_ALLOW_ALL_USERS"]) + } +} + +func TestAppendTeamTaskCompletionInstructionSeparatesCollaborationModes(t *testing.T) { + leaderMediated := appendTeamTaskCompletionInstruction("Do the work.", teamCommunicationModeLeaderMediated, "") + if !strings.Contains(leaderMediated, "Leader-mediated mode") || strings.Contains(leaderMediated, "Worker-direct mode") { + t.Fatalf("leader-mediated completion contract mixed modes: %s", leaderMediated) + } + for _, expected := range []string{ + "strict hub-and-spoke workflow", + "answer self-contained control-plane or simple tasks directly", + "wait for the assigned workers' actual results", + "Do not hand off directly to another Worker", + "Only the Leader may finalize the root task", + } { + if !strings.Contains(leaderMediated, expected) { + t.Fatalf("leader-mediated completion contract missing %q: %s", expected, leaderMediated) + } + } + + peerAssisted := appendTeamTaskCompletionInstruction("Do the work.", teamCommunicationModePeerAssisted, "") + for _, expected := range []string{ + "Worker-direct mode", + "MUST hand off to that exact member", + "required, not optional", + "fallback only", + } { + if !strings.Contains(peerAssisted, expected) { + t.Fatalf("peer-assisted completion contract missing %q: %s", expected, peerAssisted) + } + } + if strings.Contains(peerAssisted, "Leader-mediated mode") { + t.Fatalf("peer-assisted completion contract should not include leader-mediated flow: %s", peerAssisted) + } +} + +func TestTeamMemberEnvKeepsLeaderMediatedFlowIsolated(t *testing.T) { + description := "Developer: implements assigned work and reports to the Leader." + plans, err := planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "leader", Role: "leader"}, + {MemberID: "worker", Role: "developer", Description: &description}, + }) + if err != nil { + t.Fatalf("planTeamMembers returned error: %v", err) + } + team := &models.Team{ + ID: 14, + CommunicationMode: teamCommunicationModeLeaderMediated, + SharedMountPath: "/team", + } + for _, plan := range plans { + env := (&teamService{}).teamMemberEnv(team, plan) + if env["CLAWMANAGER_TEAM_COMMUNICATION_MODE"] != teamCommunicationModeLeaderMediated { + t.Fatalf("expected leader-mediated env for %s: %#v", plan.MemberKey, env) + } + if !strings.Contains(env["CLAWMANAGER_TEAM_COLLABORATION_POLICY_JSON"], `"allowPeerToPeer":false`) { + t.Fatalf("leader-mediated policy allowed peer-to-peer for %s: %s", plan.MemberKey, env["CLAWMANAGER_TEAM_COLLABORATION_POLICY_JSON"]) + } + guidance := env["CLAWMANAGER_TEAM_SYSTEM_PROMPT"] + for _, expected := range []string{"strict hub-and-spoke workflow", "Workers must not hand off directly to other workers", "only the Leader may finalize"} { + if !strings.Contains(guidance, expected) { + t.Fatalf("leader-mediated guidance for %s missing %q: %s", plan.MemberKey, expected, guidance) + } + } + if strings.Contains(guidance, "Direct handoff is mandatory") { + t.Fatalf("leader-mediated guidance leaked worker-direct rules for %s: %s", plan.MemberKey, guidance) + } + } +} + +func TestAppendTeamTaskCompletionInstructionUsesLeaderOnlyBootstrapContract(t *testing.T) { + bootstrap := appendTeamTaskCompletionInstruction( + "Introduce the current Team.", + teamCommunicationModeLeaderMediated, + initialLeaderTaskIntent, + ) + for _, expected := range []string{ + "Bootstrap completion contract", + "assigned only to the Leader", + "Do not delegate it", + "Complete this bootstrap in the current turn", + "metadata.teamConfigJson", + "/team/team.json from the shared workspace", + "optional system fallbacks", + } { + if !strings.Contains(bootstrap, expected) { + t.Fatalf("bootstrap contract missing %q: %s", expected, bootstrap) + } + } + for _, forbidden := range []string{"For multi-member Teams", "workers report deliverables back to the Leader", "Never look for /team/team.json"} { + if strings.Contains(bootstrap, forbidden) { + t.Fatalf("bootstrap contract must not include normal collaboration rule %q: %s", forbidden, bootstrap) + } + } +} + +func TestBuildTeamMemberSoulMarkdownIncludesProfileGuidance(t *testing.T) { + description := "Senior Developer: owns implementation and verification." + member := plannedTeamMember{ + MemberKey: "worker", + DisplayName: "team-worker", + Role: "senior-developer", + RuntimeType: "hermes", + Request: CreateTeamMemberRequest{ + Description: &description, + }, + } + + soul := buildTeamMemberSoulMarkdown(member, teamCommunicationModePeerAssisted) + for _, expected := range []string{ + "# team-worker", + "Member ID: worker", + "Role: senior-developer", + description, + "Collaboration mode: peer_assisted", + "If asked about your role", + "team/... is invalid", + "Report shared artifact links as /team/", + } { + if !strings.Contains(soul, expected) { + t.Fatalf("SOUL.md missing %q: %s", expected, soul) + } + } +} + +func TestWriteLiteTeamMemberIdentityFiles(t *testing.T) { + workspace := t.TempDir() + profileEnv := map[string]string{ + "CLAWMANAGER_AGENT_PERSONA_JSON": `{"profileKey":"agency.senior-developer","name":"Senior Developer","roleHint":"senior-developer","summary":"Implements scoped engineering tasks.","systemPrompt":"You are a senior implementation specialist."}`, + } + plans, err := planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "leader", Role: "leader"}, + {MemberID: "worker", Role: "developer", RuntimeType: "hermes", Mode: InstanceModeLite, InstanceMode: InstanceModeLite, EnvironmentOverrides: profileEnv}, + }) + if err != nil { + t.Fatalf("planTeamMembers returned error: %v", err) + } + member := plans[1] + team := &models.Team{ + UserID: 1, + ID: 31, + CommunicationMode: teamCommunicationModeLeaderMediated, + SharedMountPath: "/team", + } + roster, err := buildTeamRosterConfig(team, plans) + if err != nil { + t.Fatalf("buildTeamRosterConfig returned error: %v", err) + } + instance := &models.Instance{ + Type: "hermes", + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: &workspace, + } + + if err := (&teamService{}).writeLiteTeamMemberIdentityFiles(instance, team, member, roster); err != nil { + t.Fatalf("writeLiteTeamMemberIdentityFiles returned error: %v", err) + } + for _, name := range []string{teamAgentsFileName, teamSoulFileName, teamConfigFileName, filepath.Join(".hermes", teamSoulFileName)} { + if _, err := os.Stat(filepath.Join(workspace, name)); err != nil { + t.Fatalf("expected Lite identity file %s: %v", name, err) + } + } + soulBytes, err := os.ReadFile(filepath.Join(workspace, teamSoulFileName)) + if err != nil { + t.Fatalf("failed to read SOUL.md: %v", err) + } + soul := string(soulBytes) + for _, expected := range []string{ + "Effective role: senior-developer", + "Profile key: agency.senior-developer", + "Profile name: Senior Developer", + "If team.json contains effectiveRole/profileName", + } { + if !strings.Contains(soul, expected) { + t.Fatalf("SOUL.md missing %q: %s", expected, soul) + } + } + agentsBytes, err := os.ReadFile(filepath.Join(workspace, teamAgentsFileName)) + if err != nil { + t.Fatalf("failed to read AGENTS.md: %v", err) + } + if !strings.Contains(string(agentsBytes), "SOUL.md as the member-specific identity") { + t.Fatalf("AGENTS.md missing identity source guidance: %s", string(agentsBytes)) + } + rosterBytes, err := os.ReadFile(filepath.Join(workspace, teamConfigFileName)) + if err != nil { + t.Fatalf("failed to read team.json: %v", err) + } + if !strings.Contains(string(rosterBytes), `"effectiveRole":"senior-developer"`) { + t.Fatalf("team.json missing effective role: %s", string(rosterBytes)) + } +} + +func TestBuildInitialLeaderTaskPayloadDescribesRosterAndTeamSend(t *testing.T) { + payload := buildInitialLeaderTaskPayload("Software Engineering Team") + + if payload["intent"] != initialLeaderTaskIntent { + t.Fatalf("unexpected bootstrap intent: %#v", payload) + } + if payload["title"] == "" { + t.Fatalf("expected bootstrap task title: %#v", payload) + } + if payload["executionMode"] != "leader_control_plane_snapshot" || payload["requiresDelegation"] != false { + t.Fatalf("expected leader-only bootstrap execution metadata: %#v", payload) + } + if payload["anchorEligible"] != false { + t.Fatalf("bootstrap task must not become a user question anchor: %#v", payload) + } + prompt, ok := payload["prompt"].(string) + if !ok { + t.Fatalf("expected prompt string: %#v", payload) + } + for _, expected := range []string{ + "team Software Engineering Team", + "Redis Team\u6210\u5458\u6784\u6210", + "\u8fd0\u884c\u72b6\u6001\u4e0e\u6280\u672f\u80fd\u529b\u8fb9\u754c", + "\u534f\u4f5c\u4e0e\u901a\u4fe1\u673a\u5236(team_send)", + "\u4efb\u52a1\u6d41\u8f6c\u65b9\u5f0f", + "\u6d88\u606f\u540c\u6b65\u65b9\u5f0f", + "\u4e0a\u4e0b\u6587\u5171\u4eab\u65b9\u5f0f", + "\u53ef\u8c03\u7528\u7684\u65b9\u6cd5\u3001\u5de5\u5177\u4e0e\u64cd\u4f5c\u80fd\u529b", + } { + if !strings.Contains(prompt, expected) { + t.Fatalf("bootstrap prompt missing %q: %s", expected, prompt) + } + } +} + +func TestBuildTeamTaskEnvelopeIncludesCompletionContract(t *testing.T) { + task := &models.TeamTask{ID: 67} + payload := map[string]interface{}{ + "intent": "team_bootstrap_introduction", + "title": "Introduce the team", + "prompt": "Generate the team report.", + "teamConfigJson": `{"sharedDir":"/team"}`, + } + + envelope := buildTeamTaskEnvelope(31, "leader", task, "team-31-bootstrap-introduction", payload, nil, time.Unix(123, 0).UTC()) + + if envelope["replyTo"] != "clawmanager" { + t.Fatalf("expected replyTo clawmanager, got %#v", envelope["replyTo"]) + } + if envelope["requiresCompletion"] != true { + t.Fatalf("expected requiresCompletion=true, got %#v", envelope["requiresCompletion"]) + } + if envelope["completionTool"] != "team_complete_task" { + t.Fatalf("expected completion tool team_complete_task, got %#v", envelope["completionTool"]) + } + monitorPolicy, ok := envelope["monitorPolicy"].(map[string]interface{}) + if !ok || monitorPolicy["enabled"] != true || monitorPolicy["visibleToChat"] != true { + t.Fatalf("expected visible monitor policy in envelope, got %#v", envelope["monitorPolicy"]) + } + if monitorPolicy["heartbeatEverySec"] != 30 || monitorPolicy["visibleHeartbeatEverySec"] != 180 { + t.Fatalf("expected 30s internal heartbeat and 180s chat digest policy, got %#v", monitorPolicy) + } + resultSink, ok := envelope["resultSink"].(map[string]interface{}) + if !ok { + t.Fatalf("expected resultSink map, got %#v", envelope["resultSink"]) + } + if resultSink["type"] != "redis_stream" || resultSink["eventsKey"] != "claw:team:31:events" { + t.Fatalf("unexpected resultSink: %#v", resultSink) + } + if resultSink["successEvent"] != "completion_proposed" || resultSink["failureEvent"] != "task_failed" { + t.Fatalf("unexpected resultSink events: %#v", resultSink) + } + if envelope["teamConfigJson"] != `{"sharedDir":"/team"}` { + t.Fatalf("expected teamConfigJson in envelope, got %#v", envelope["teamConfigJson"]) + } + prompt, ok := envelope["prompt"].(string) + if !ok { + t.Fatalf("expected prompt string, got %#v", envelope["prompt"]) + } + for _, expected := range []string{"Generate the team report.", "team_complete_task", "resultMarkdown", "task_completed"} { + if !strings.Contains(prompt, expected) { + t.Fatalf("completion prompt missing %q: %s", expected, prompt) + } + } +} + +func TestBuildTeamTaskEnvelopeCarriesLocaleAndMemberWorkspace(t *testing.T) { + task := &models.TeamTask{ID: 88} + payload := map[string]interface{}{ + "prompt": "请实现一个计算器应用", + "responseLocale": "zh-CN", + "workspaceContract": map[string]interface{}{ + "physicalSharedDir": "/workspaces/teams/user-1/team-54-shared", + "taskRef": "team-54-task-88", + }, + } + envelope := buildTeamTaskEnvelope(54, "ui-designer", task, "root-message", payload, map[string]string{}, time.Unix(123, 0).UTC()) + if envelope["responseLocale"] != "zh-CN" { + t.Fatalf("response locale was not propagated: %#v", envelope) + } + shared, ok := envelope["sharedWorkspace"].(map[string]interface{}) + if !ok || shared["physicalPath"] != "/workspaces/teams/user-1/team-54-shared" || shared["memberArtifactPhysicalRoot"] != "/workspaces/teams/user-1/team-54-shared/artifacts/team-54-task-88/members/ui-designer" { + t.Fatalf("member shared workspace was not resolved: %#v", envelope["sharedWorkspace"]) + } + prompt, _ := envelope["prompt"].(string) + if !strings.Contains(prompt, "use zh-CN") || !strings.Contains(prompt, "team_artifact_write") { + t.Fatalf("runtime prompt is missing locale/artifact guidance: %s", prompt) + } +} + +func TestCompleteInitialLeaderTaskFromSnapshotWritesReportAndCompletion(t *testing.T) { + workspaceRoot := t.TempDir() + team := &models.Team{ + ID: 49, + UserID: 1, + Name: "delivery-team", + CommunicationMode: teamCommunicationModeLeaderMediated, + SharedMountPath: "/team", + } + task := &models.TeamTask{ + ID: 91, + TeamID: team.ID, + TargetMemberID: 700, + MessageID: "team-49-bootstrap-introduction", + Status: models.TeamTaskStatusPending, + CreatedAt: time.Now().UTC(), + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ID: 700, TeamID: team.ID, MemberKey: "delivery-lead", DisplayName: "Delivery Lead", Role: "leader", Status: models.TeamMemberStatusBusy, Availability: models.TeamMemberAvailabilityBusy} + developer := &models.TeamMember{ID: 701, TeamID: team.ID, MemberKey: "developer", DisplayName: "Developer", Role: "developer", RuntimeType: "openclaw", InstanceMode: "lite", Status: models.TeamMemberStatusIdle, Availability: models.TeamMemberAvailabilityIdle} + repo := &teamRepositoryStub{ + membersByKey: map[string]*models.TeamMember{ + "delivery-lead": leader, + "developer": developer, + }, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: workspaceRoot} + payload := map[string]interface{}{ + "intent": initialLeaderTaskIntent, + "workspaceContract": map[string]interface{}{ + "sharedDir": "/team", + "physicalSharedDir": filepath.Join(workspaceRoot, "teams", "user-1", "team-49-shared"), + }, + } + + result, err := service.completeInitialLeaderTaskFromSnapshot(1, team, task, leader, payload) + if err != nil { + t.Fatalf("completeInitialLeaderTaskFromSnapshot returned error: %v", err) + } + if result.Status != models.TeamTaskStatusSucceeded || result.FinishedAt == nil { + t.Fatalf("expected backend bootstrap task to finish, got %#v", result.TeamTask) + } + reportPath := filepath.Join(workspaceRoot, "teams", "user-1", "team-49-shared", "results", "team-49-task-91", "team-introduction.md") + reportBytes, err := os.ReadFile(reportPath) + if err != nil { + t.Fatalf("expected bootstrap report to be written: %v", err) + } + report := string(reportBytes) + if !strings.Contains(report, "Developer") || !strings.Contains(report, "Redis Streams") { + t.Fatalf("bootstrap report missing member or mechanism detail: %s", report) + } + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded || repo.updatedTask.ResultJSON == nil { + t.Fatalf("expected task result to be persisted, got %#v", repo.updatedTask) + } + if repo.updatedMember == nil || repo.updatedMember.MemberKey != "delivery-lead" || repo.updatedMember.Status != models.TeamMemberStatusIdle { + t.Fatalf("expected leader member to become idle, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "task_completed" { + t.Fatalf("expected one backend task_completed event, got %#v", repo.createdEvents) + } + stored := teamEventPayloadMap(repo.createdEvents[0]) + if eventString(stored, "completionSource") != "clawmanager_backend" || eventBool(stored, "backendGenerated") != true { + t.Fatalf("expected backend completion markers, got %#v", stored) + } +} + +func TestProjectTeamEventDoesNotTreatPlainFinalReplyAsTaskCompleted(t *testing.T) { + taskID := 67 + messageID := "team-31-bootstrap-introduction" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + member := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": member}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "reply", + "messageId": messageID, + "memberId": "leader", + "taskId": "team-31-task-67", + "final": true, + "summary": "Team report ready", + "resultMarkdown": "Full report", + "text": "Full report", + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178655-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask != nil { + t.Fatalf("plain final reply must not mark task succeeded, got %#v", repo.updatedTask) + } + if repo.updatedMember == nil || repo.updatedMember.Status == models.TeamMemberStatusIdle && repo.updatedMember.Progress == 100 { + t.Fatalf("plain final reply must not mark member completed, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "reply" { + t.Fatalf("expected stored event type reply, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventKeepsSubstantialDirectTargetReplyNonTerminal(t *testing.T) { + taskID := 68 + messageID := "team-31-bootstrap-introduction" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusDispatched, + UpdatedAt: time.Now().UTC(), + } + member := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": member}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "reply", + "messageId": messageID, + "memberId": "leader", + "taskId": "team-31-task-68", + "summary": "Team introduction ready", + "text": strings.Join([]string{ + "# Team report", + "The team has two members. Leader coordinates planning, handoff, verification, and final synthesis.", + "Worker handles scoped implementation tasks, reports concrete outputs, and keeps changes practical.", + }, "\n"), + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178656-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask != nil && (repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil) { + t.Fatalf("substantial reply without an explicit completion tool must stay non-terminal, got %#v", repo.updatedTask) + } + if repo.updatedMember != nil && repo.updatedMember.Progress == 100 { + t.Fatalf("substantial reply without explicit completion must not complete the member, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "reply" { + t.Fatalf("expected stored event type reply, got %#v", repo.createdEvents) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored event payload: %v", err) + } + step, ok := stored["collaborationStep"].(map[string]interface{}) + if !ok { + t.Fatalf("expected collaboration step in stored event, got %#v", stored) + } + if got := step["summary"]; got != "Team introduction ready" { + t.Fatalf("expected collaboration step summary to stay compact, got %#v", got) + } + if got, _ := step["content"].(string); !strings.Contains(got, "# Team report") { + t.Fatalf("expected collaboration step content to preserve full reply text, got %#v", got) + } +} + +func TestProjectTeamEventDoesNotTreatDelegationReplyAsTaskCompleted(t *testing.T) { + taskID := 69 + messageID := "team-31-task-69" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusDispatched, + UpdatedAt: time.Now().UTC(), + } + member := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": member}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "reply", + "messageId": messageID, + "memberId": "leader", + "taskId": "team-31-task-69", + "final": true, + "text": "Assigned to worker and waiting for worker to finish the report.", + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178657-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask == nil || repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil { + t.Fatalf("delegation reply should touch but not complete task, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "reply" { + t.Fatalf("expected stored event type reply, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventDoesNotTreatProcessOnlyTaskCompletedAsRootCompletion(t *testing.T) { + taskID := 70 + messageID := "team-31-bootstrap-introduction" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + member := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": member}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "task_completed", + "messageId": messageID, + "memberId": "leader", + "taskId": "team-31-task-70", + "status": "succeeded", + "collaborationStep": map[string]interface{}{ + "type": "progress", + "summary": "Good, I have the team configuration.", + "content": "Good, I have the team configuration. Now let me write the comprehensive report to the shared workspace and then finalize.", + }, + "toolCall": map[string]interface{}{ + "name": teamTaskCompletionTool, + }, + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178661-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask == nil || repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil { + t.Fatalf("process-only completion wrapper must not complete task, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "reply" { + t.Fatalf("expected process-only completion wrapper to be stored as reply, got %#v", repo.createdEvents) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored event payload: %v", err) + } + if stored["nonAuthoritativeCompletion"] != true { + t.Fatalf("expected nonAuthoritativeCompletion marker, got %#v", stored) + } +} + +func TestProjectTeamEventLeaderDispatchCompletionDoesNotCloseRootTask(t *testing.T) { + taskID := 169 + messageID := "team-31-task-169" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC().Add(-10 * time.Minute), + } + member := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Role: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": member}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "task_completed", + "messageId": messageID, + "memberId": "leader", + "taskId": "team-31-task-169", + "status": "succeeded", + "summary": "Dispatched to designer.", + "resultMarkdown": strings.Join([]string{ + "[ASSIGNMENT] Self introduction", + "", + "designer, the user wants you to write a short self introduction.", + "", + "Please include your role identity, capability boundary, and working style.", + "After finishing, write the content into the shared workspace and report back to me.", + "", + "Shared directory: $CLAWMANAGER_TEAM_SHARED_DIR/results/team-31-task-169/", + "Canonical path: /team/results/team-31-task-169/intro-designer.md", + }, "\n"), + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178669-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask == nil { + t.Fatalf("expected root task to be touched so stale detection sees active delegation") + } + if repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil || repo.updatedTask.ResultJSON != nil { + t.Fatalf("leader dispatch must not complete root task, got %#v", repo.updatedTask) + } + if repo.updatedMember == nil || repo.updatedMember.Status != models.TeamMemberStatusBusy || repo.updatedMember.Progress == 100 { + t.Fatalf("leader dispatch must keep leader/root task active, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "reply" { + t.Fatalf("expected dispatch stored as reply, got %#v", repo.createdEvents) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored payload: %v", err) + } + if stored["leaderDispatchOnly"] != true || stored["rootTaskTerminal"] != false { + t.Fatalf("expected leader dispatch marker without root terminal, got %#v", stored) + } + step := stored["collaborationStep"].(map[string]interface{}) + if step["type"] != "assignment" || step["status"] != models.TeamTaskStatusDispatched { + t.Fatalf("expected assignment collaboration step, got %#v", step) + } + if got, _ := step["content"].(string); !strings.Contains(got, "designer, the user wants you to write a short self introduction") { + t.Fatalf("expected assignment content preserved, got %#v", got) + } +} + +func TestProjectTeamEventLeaderPlanningDoesNotCreateLeaderAssignmentLane(t *testing.T) { + taskID := 169 + messageID := "team-31-task-169" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Role: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + } + service := &teamService{repo: repo} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "reply", + "memberId": "leader", + "messageId": messageID, + "rootTaskId": messageID, + "status": "dispatched", + "summary": "Planning: decomposing the task into assignments", + "leaderDispatchOnly": true, + "collaborationStep": map[string]interface{}{ + "type": "assignment", + "status": "dispatched", + "actor": "leader", + "target": "leader", + "rootTaskId": messageID, + "content": "Planning: decomposing the task into assignments", + }, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178657-1", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if len(repo.workItems) != 0 { + t.Fatalf("leader planning/dispatch-to-self must not create Kanban work items, got %#v", repo.workItems) + } +} + +func TestProjectTeamEventDoesNotTreatNonTargetReplyAsTaskCompleted(t *testing.T) { + taskID := 70 + messageID := "team-31-task-70" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusDispatched, + UpdatedAt: time.Now().UTC(), + } + worker := &models.TeamMember{ + ID: 121, + TeamID: 31, + MemberKey: "worker", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"worker": worker}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "reply", + "messageId": messageID, + "memberId": "worker", + "taskId": "team-31-task-70", + "summary": "Worker report ready", + "text": "# Worker report\nThis is a detailed result, but it belongs to a member that is not the target of the parent task.", + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178658-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask != nil && (repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil) { + t.Fatalf("non-target reply must not mark task succeeded, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 2 || repo.createdEvents[0].EventType != "reply" || repo.createdEvents[1].EventType != "member_result_confirmed" { + t.Fatalf("expected stored event type reply, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventDowngradesLiteDispatchWrapperFailure(t *testing.T) { + taskID := 71 + messageID := "team-31-task-71" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 121, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + worker := &models.TeamMember{ + ID: 121, + TeamID: 31, + MemberKey: "worker", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"worker": worker}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "message_failed", + "memberId": "worker", + "availability": "blocked", + "reason": "dispatch finished without reply/completion", + "text": "Redis Team task failed", + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178659-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask != nil { + t.Fatalf("lite wrapper dispatch failure must not fail the task, got %#v", repo.updatedTask) + } + if repo.updatedMember == nil || repo.updatedMember.Availability == models.TeamMemberAvailabilityBlocked { + t.Fatalf("lite wrapper dispatch failure must not block the member, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "message_warning" { + t.Fatalf("expected warning event, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventDoesNotTreatSuccessfulFailedWrapperAsCompletion(t *testing.T) { + taskID := 74 + messageID := "team-31-task-74" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 121, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + worker := &models.TeamMember{ + ID: 121, + TeamID: 31, + MemberKey: "worker", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"worker": worker}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "task_failed", + "memberId": "worker", + "messageId": messageID, + "status": "succeeded", + "resultMarkdown": "Delivered correct result.", + "summary": "Finished successfully despite wrapper event type.", + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178660-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask != nil && repo.updatedTask.Status == models.TeamTaskStatusSucceeded { + t.Fatalf("a contradictory task_failed wrapper must not complete the task, got %#v", repo.updatedTask) + } + if repo.updatedMember == nil || repo.updatedMember.Availability == models.TeamMemberAvailabilityBlocked { + t.Fatalf("successful task_failed wrapper must not block member, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 1 { + t.Fatalf("expected one stored event, got %#v", repo.createdEvents) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored payload: %v", err) + } + step, ok := stored["collaborationStep"].(map[string]interface{}) + if !ok || step["type"] == "result" || step["status"] == models.TeamTaskStatusSucceeded { + t.Fatalf("expected contradictory wrapper to remain non-terminal, got %#v", stored) + } +} + +func TestProjectTeamEventAssociatesLeaderPeerHandoffWithRootTask(t *testing.T) { + taskID := 73 + messageID := "team-31-task-73" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Role: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + worker := &models.TeamMember{ + ID: 121, + TeamID: 31, + MemberKey: "worker", + Role: "developer", + Status: models.TeamMemberStatusIdle, + Availability: models.TeamMemberAvailabilityIdle, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "worker": worker}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "outbound", + "from": "leader", + "to": "worker", + "messageId": "worker-task-1", + "title": "Research requirement", + "text": "Please research the user segment and return evidence.", + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModePeerAssisted}, nil, redisStreamMessage{ + ID: "1781171178661-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask != nil && repo.updatedTask.Status == models.TeamTaskStatusSucceeded { + t.Fatalf("leader handoff must not complete root task, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].TaskID == nil || *repo.createdEvents[0].TaskID != taskID { + t.Fatalf("expected handoff event linked to root task, got %#v", repo.createdEvents) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored payload: %v", err) + } + step, ok := stored["collaborationStep"].(map[string]interface{}) + if !ok { + t.Fatalf("expected collaborationStep in payload: %#v", stored) + } + if step["type"] != "assignment" || step["status"] != models.TeamTaskStatusDispatched || step["target"] != "worker" { + t.Fatalf("unexpected collaboration step: %#v", step) + } + if step["rootTaskId"] != "team-31-task-73" || step["rootMessageId"] != messageID { + t.Fatalf("expected root task context, got %#v", step) + } +} + +func TestProjectTeamEventDoesNotCompleteRootTaskFromPeerMemberTerminal(t *testing.T) { + taskID := 75 + messageID := "team-31-task-75" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Role: "leader", + Status: models.TeamMemberStatusBusy, + Availability: models.TeamMemberAvailabilityBusy, + } + workerTaskID := taskID + worker := &models.TeamMember{ + ID: 121, + TeamID: 31, + MemberKey: "worker", + Role: "developer", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &workerTaskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "worker": worker}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "task_completed", + "memberId": "worker", + "messageId": messageID, + "taskId": messageID, + "status": "succeeded", + "summary": "Worker delivery ready", + "resultMarkdown": "Worker completed the assigned research and produced a report for leader synthesis.", + "explicitCompletion": true, + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModePeerAssisted}, nil, redisStreamMessage{ + ID: "1781171178662-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask == nil || repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil { + t.Fatalf("peer member terminal event should touch but not complete root task, got %#v", repo.updatedTask) + } + if repo.updatedMember == nil || repo.updatedMember.MemberKey != "worker" || repo.updatedMember.Status != models.TeamMemberStatusIdle || repo.updatedMember.Progress != 100 { + t.Fatalf("expected peer member to be marked idle and complete, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].TaskID == nil || *repo.createdEvents[0].TaskID != taskID { + t.Fatalf("expected member terminal event linked to root task for visibility, got %#v", repo.createdEvents) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored payload: %v", err) + } + if stored["memberTerminalOnly"] != true || stored["rootTaskTerminal"] != false { + t.Fatalf("expected member terminal marker without root completion, got %#v", stored) + } +} + +func TestProjectTeamEventLeaderMediatedWorkerCompletionDoesNotCloseRootTask(t *testing.T) { + taskID := 76 + messageID := "team-31-task-76" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader"} + worker := &models.TeamMember{ + ID: 121, + TeamID: 31, + MemberKey: "worker", + Role: "developer", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "worker": worker}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "task_completed", + "memberId": "worker", + "messageId": messageID, + "taskId": messageID, + "status": "succeeded", + "summary": "Worker delivery ready for Leader verification", + "resultMarkdown": "The assigned work is complete with evidence and artifact paths.", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178663-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil { + t.Fatalf("leader-mediated worker completion should touch but not close root task, got %#v", repo.updatedTask) + } + if repo.updatedMember == nil || repo.updatedMember.MemberKey != "worker" || repo.updatedMember.Status != models.TeamMemberStatusIdle || repo.updatedMember.Progress != 100 { + t.Fatalf("worker delivery should complete only the worker lane, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 2 || repo.createdEvents[0].TaskID == nil || *repo.createdEvents[0].TaskID != taskID || repo.createdEvents[1].EventType != "member_result_confirmed" { + t.Fatalf("worker delivery must remain linked to the Leader root task, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventLeaderMediatedWorkerReplyToLeaderIsAssignmentResult(t *testing.T) { + taskID := 176 + messageID := "team-31-task-176" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + worker := &models.TeamMember{ + ID: 121, + TeamID: 31, + MemberKey: "designer", + Role: "ui-ux-designer", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"designer": worker}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "outbound", + "memberId": "designer", + "from": "designer", + "to": "leader", + "rootTaskId": messageID, + "messageId": "reply-designer-1", + "text": "1", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178676-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil { + t.Fatalf("worker assignment result must not close Leader root task, got %#v", repo.updatedTask) + } + if repo.updatedMember == nil || repo.updatedMember.MemberKey != "designer" || repo.updatedMember.Status != models.TeamMemberStatusIdle || repo.updatedMember.Progress != 100 { + t.Fatalf("worker reply should complete only the member lane, got %#v", repo.updatedMember) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored payload: %v", err) + } + if stored["assignmentResultOnly"] != true || stored["rootTaskTerminal"] != false { + t.Fatalf("expected assignment-result marker, got %#v", stored) + } + if stored["memberResultConfirmed"] != true || stored["normalizedResultSource"] != "legacy_normalized_reply" { + t.Fatalf("expected normalized member result marker, got %#v", stored) + } + step := stored["collaborationStep"].(map[string]interface{}) + if step["type"] != "result" || step["status"] != models.TeamTaskStatusSucceeded || step["actor"] != "designer" { + t.Fatalf("expected worker result collaboration step, got %#v", step) + } + if len(repo.createdEvents) != 2 || repo.createdEvents[1].EventType != "member_result_confirmed" { + t.Fatalf("expected leader ledger notification after worker result, got %#v", repo.createdEvents) + } + var notification map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[1].PayloadJSON), ¬ification); err != nil { + t.Fatalf("decode leader notification: %v", err) + } + if notification["workId"] != "member-designer" || notification["to"] != "leader" || notification["memberResultConfirmed"] != true { + t.Fatalf("expected structured leader notification, got %#v", notification) + } +} + +func TestProjectTeamEventLeaderMediatedIgnoresGenericWorkerCompletionSummary(t *testing.T) { + taskID := 276 + messageID := "team-31-task-276" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + worker := &models.TeamMember{ + ID: 121, + TeamID: 31, + MemberKey: "designer", + Role: "ui-ux-designer", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"designer": worker}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "task_completed", + "memberId": "designer", + "from": "designer", + "to": "leader", + "taskId": messageID, + "messageId": "msg-worker-assignment", + "status": "succeeded", + "summary": "Redis Team task processing completed", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178678-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if len(repo.createdEvents) != 1 { + t.Fatalf("generic runtime completion summary must not create ledger notification, got %#v", repo.createdEvents) + } + if repo.createdEvents[0].EventType != "reply" { + t.Fatalf("generic runtime completion should be downgraded to non-authoritative reply, got %#v", repo.createdEvents[0]) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored payload: %v", err) + } + if stored["assignmentResultOnly"] == true || stored["memberResultConfirmed"] == true { + t.Fatalf("generic runtime completion must not be a member result, got %#v", stored) + } +} + +func TestProjectTeamEventLeaderMediatedRejectsWorkerSelfRoute(t *testing.T) { + taskID := 177 + messageID := "team-31-task-177" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + architect := &models.TeamMember{ + ID: 122, + TeamID: 31, + MemberKey: "architect", + Role: "architect", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + membersByKey: map[string]*models.TeamMember{"architect": architect}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "outbound", + "memberId": "architect", + "from": "architect", + "to": "architect", + "rootTaskId": messageID, + "messageId": "self-loop", + "text": "42", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178677-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil { + t.Fatalf("invalid worker route must not close root task, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "message_warning" { + t.Fatalf("expected protocol warning event, got %#v", repo.createdEvents) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored payload: %v", err) + } + if stored["leaderMediatedRouteViolation"] != true || stored["nonAuthoritative"] != true { + t.Fatalf("expected route violation marker, got %#v", stored) + } + step := stored["collaborationStep"].(map[string]interface{}) + if step["type"] != "warning" { + t.Fatalf("expected warning collaboration step, got %#v", step) + } +} + +func TestProjectTeamEventLeaderMediatedPrematureLeaderCompletionWaitsForAssignments(t *testing.T) { + taskID := 178 + messageID := "team-31-task-178" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Role: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + assignmentPayload := `{"event":"reply","leaderDispatchOnly":true,"status":"dispatched","rootTaskId":"team-31-task-178","memberId":"leader","collaborationStep":{"type":"assignment","status":"dispatched","actor":"leader","target":"designer","rootTaskId":"team-31-task-178","content":"Please return a number."}}` + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + createdEvents: []models.TeamEvent{{ + TeamID: 31, + TaskID: &taskID, + EventType: "reply", + PayloadJSON: &assignmentPayload, + }}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "task_completed", + "memberId": "leader", + "messageId": messageID, + "taskId": messageID, + "status": "succeeded", + "summary": "Final result ready too early", + "resultMarkdown": "Designer result is pending.", + "explicitCompletion": true, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178678-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil { + t.Fatalf("premature Leader completion must keep root task open, got %#v", repo.updatedTask) + } + stored := map[string]interface{}{} + if err := json.Unmarshal([]byte(*repo.createdEvents[len(repo.createdEvents)-1].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored payload: %v", err) + } + if stored["completionDecision"] != teamCompletionDecisionDeferred || stored["completionDecisionReason"] != "pending_legacy_assignments" || stored["rootTaskTerminal"] != false { + t.Fatalf("expected structured deferred completion, got %#v", stored) + } +} + +func TestProjectTeamEventLeaderMediatedLeaderCompletionAfterAssignmentResultsClosesRoot(t *testing.T) { + taskID := 179 + messageID := "team-31-task-179" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Role: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + assignmentPayload := `{"event":"reply","leaderDispatchOnly":true,"status":"dispatched","rootTaskId":"team-31-task-179","memberId":"leader","collaborationStep":{"type":"assignment","status":"dispatched","actor":"leader","target":"designer","rootTaskId":"team-31-task-179","content":"Please return a number."}}` + resultPayload := `{"event":"outbound","assignmentResultOnly":true,"status":"succeeded","rootTaskId":"team-31-task-179","memberId":"designer","from":"designer","to":"leader","text":"1","collaborationStep":{"type":"result","status":"succeeded","actor":"designer","target":"leader","rootTaskId":"team-31-task-179","content":"1"}}` + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + createdEvents: []models.TeamEvent{ + {TeamID: 31, TaskID: &taskID, EventType: "reply", PayloadJSON: &assignmentPayload}, + {TeamID: 31, TaskID: &taskID, EventType: "outbound", PayloadJSON: &resultPayload}, + }, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "task_completed", + "memberId": "leader", + "messageId": messageID, + "taskId": messageID, + "status": "succeeded", + "summary": "Designer returned 1; final synthesis complete.", + "resultMarkdown": "Final result: designer=1.", + "explicitCompletion": true, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178679-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt == nil { + t.Fatalf("Leader final synthesis after member results should close root task, got %#v", repo.updatedTask) + } + if repo.updatedTask.ResultJSON == nil || !strings.Contains(*repo.updatedTask.ResultJSON, "designer=1") { + t.Fatalf("expected final synthesis result stored, got %#v", repo.updatedTask.ResultJSON) + } + if len(repo.outboxRows) != 1 || !strings.Contains(repo.outboxRows[0].Destination, "completion-acks") || !strings.Contains(repo.outboxRows[0].PayloadJSON, `"decision":"accepted"`) { + t.Fatalf("accepted root completion must atomically persist its acknowledgement: %#v", repo.outboxRows) + } + acceptedPayload := teamEventPayloadMap(repo.createdEvents[len(repo.createdEvents)-1]) + if eventString(acceptedPayload, "chatKind") != "final_delivery" || eventString(acceptedPayload, "displayKey") != "root-final:179" || eventString(acceptedPayload, "resultMarkdown") == "" { + t.Fatalf("accepted completion must expose one full final delivery event: %#v", acceptedPayload) + } +} + +func TestProjectTeamEventLeaderMediatedInterimLeaderCompletionDoesNotCloseAfterMemberResults(t *testing.T) { + taskID := 183 + messageID := "team-31-task-183" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + pm := &models.TeamMember{ID: 121, TeamID: 31, MemberKey: "pm", Role: "product-manager", Status: models.TeamMemberStatusIdle, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityIdle} + designer := &models.TeamMember{ID: 122, TeamID: 31, MemberKey: "designer", Role: "designer", Status: models.TeamMemberStatusIdle, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityIdle} + architect := &models.TeamMember{ID: 123, TeamID: 31, MemberKey: "architect", Role: "architect", Status: models.TeamMemberStatusIdle, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityIdle} + now := time.Now().UTC() + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{ + "leader": leader, + "pm": pm, + "designer": designer, + "architect": architect, + }, + workItems: []models.TeamWorkItem{ + {TeamID: 31, RootTaskID: taskID, WorkID: "member-pm", OwnerMemberID: &pm.ID, Status: models.TeamTaskStatusSucceeded, UpdatedAt: now}, + {TeamID: 31, RootTaskID: taskID, WorkID: "member-designer", OwnerMemberID: &designer.ID, Status: models.TeamTaskStatusSucceeded, UpdatedAt: now}, + {TeamID: 31, RootTaskID: taskID, WorkID: "member-architect", OwnerMemberID: &architect.ID, Status: models.TeamTaskStatusSucceeded, UpdatedAt: now}, + }, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "task_completed", + "memberId": "leader", + "messageId": messageID, + "taskId": messageID, + "status": "succeeded", + "summary": "PM result archived. Still waiting on Designer and Architect.", + "resultMarkdown": "PM result archived. Still waiting on Designer and Architect.", + "explicitCompletion": true, + "rootTaskTerminal": true, + "completionId": "leader-fallback-msg-1", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178682-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil { + t.Fatalf("interim Leader completion must keep root task open even after member results, got %#v", repo.updatedTask) + } + stored := map[string]interface{}{} + if err := json.Unmarshal([]byte(*repo.createdEvents[len(repo.createdEvents)-1].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored payload: %v", err) + } + if stored["completionDecision"] != teamCompletionDecisionNeedsConfirmation || stored["completionDecisionReason"] != "narrative_indicates_remaining_work" || stored["rootTaskTerminal"] != false { + t.Fatalf("expected completion confirmation request, got %#v", stored) + } +} + +func TestEvaluateProtocolV3CompletionAllowsUnusedPlannedPhaseAfterWorkflowSeal(t *testing.T) { + leaderID := 120 + workerID := 121 + task := &models.TeamTask{ + ID: 190, TeamID: 31, TargetMemberID: leaderID, Status: models.TeamTaskStatusRunning, + WorkflowState: teamWorkflowStateAwaitingLeaderDecision, PlanVersion: 2, LedgerVersion: 7, + } + assignmentID := "research-pm" + phaseID := "research" + repo := &teamRepositoryStub{ + workItems: []models.TeamWorkItem{{ + TeamID: 31, RootTaskID: task.ID, WorkID: assignmentID, AssignmentID: &assignmentID, + PhaseID: &phaseID, Revision: 1, RequiredForRoot: true, OwnerMemberID: &workerID, + Status: models.TeamTaskStatusSucceeded, + }}, + workflowPhases: []models.TeamWorkflowPhase{ + {TeamID: 31, RootTaskID: task.ID, PhaseID: "research", PlanVersion: 2, Status: teamPhaseStatusCompleted, RequiredForRoot: true}, + {TeamID: 31, RootTaskID: task.ID, PhaseID: "implementation", PlanVersion: 2, Status: teamPhaseStatusPlanned, RequiredForRoot: true}, + }, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "protocolVersion": 3, "completionId": "completion-190", "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, "rootTaskTerminal": true, "workflowFinal": true, "finalAnswerReady": true, + "planVersion": 2, "ledgerVersion": 7, "status": "succeeded", "summary": "第一阶段完成", + "resultMarkdown": "第一阶段结果已经汇总。", + } + evaluation, err := service.evaluateLeaderRootCompletion( + &models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, task, + &models.TeamMember{ID: leaderID, TeamID: 31, MemberKey: "leader", Role: "leader"}, payload, + ) + if err != nil { + t.Fatal(err) + } + if evaluation.Decision != teamCompletionDecisionAccepted { + t.Fatalf("an explicitly sealed workflow must ignore a planned phase with no dispatched work: %#v", evaluation) + } +} + +func TestReconcileTeamWorkflowLedgerRepairsCompletedAndUnusedPhases(t *testing.T) { + now := time.Now().UTC() + leaderID := 120 + workerID := 121 + assignmentID := "review-calculator" + phaseID := "phase-2" + task := &models.TeamTask{ + ID: 201, TeamID: 31, TargetMemberID: leaderID, Status: models.TeamTaskStatusRunning, + WorkflowState: teamWorkflowStateAwaitingPhaseResults, PlanVersion: 1, LedgerVersion: 6, + CurrentPhaseID: &phaseID, + } + repo := &teamRepositoryStub{ + workItems: []models.TeamWorkItem{{ + TeamID: 31, RootTaskID: task.ID, WorkID: assignmentID, AssignmentID: &assignmentID, + PhaseID: &phaseID, OwnerMemberID: &workerID, RequiredForRoot: true, + Revision: 1, Status: models.TeamTaskStatusSucceeded, + }}, + workflowPhases: []models.TeamWorkflowPhase{ + {TeamID: 31, RootTaskID: task.ID, PhaseID: phaseID, PlanVersion: 1, Status: teamPhaseStatusAwaitingResults, RequiredForRoot: true}, + {TeamID: 31, RootTaskID: task.ID, PhaseID: "phase-3", PlanVersion: 1, Status: teamPhaseStatusPlanned, RequiredForRoot: true}, + }, + } + service := &teamService{repo: repo} + changed, err := service.reconcileTeamWorkflowLedger(task, true, now) + if err != nil || !changed { + t.Fatalf("expected workflow ledger repair, changed=%v err=%v", changed, err) + } + if repo.workflowPhases[0].Status != teamPhaseStatusCompleted || repo.workflowPhases[1].Status != teamPhaseStatusCancelled { + t.Fatalf("expected completed current phase and cancelled unused planned phase, got %#v", repo.workflowPhases) + } + if task.WorkflowState != teamWorkflowStateSynthesizing || task.LedgerVersion != 7 || task.CurrentPhaseID != nil { + t.Fatalf("unexpected reconciled task state: %#v", task) + } +} + +func TestMarkStructuredCompletionDeferredRemainsVisibleInChat(t *testing.T) { + payload := map[string]interface{}{ + "completionId": "completion-201", "resultMarkdown": "# Final delivery\n\nFull report.", + } + markStructuredCompletionDecision("completion_proposed", payload, teamCompletionEvaluation{ + Decision: teamCompletionDecisionDeferred, Reason: "open_workflow_phases", LedgerVersion: 9, + }) + if !eventBool(payload, "visibleToChat") || eventString(payload, "chatPolicy") != "warning" || eventString(payload, "chatKind") != "completion_deferred" { + t.Fatalf("deferred completion must retain visible report and diagnostic: %#v", payload) + } +} + +func TestReconcileDeferredCompletionAcceptsAfterLedgerRepair(t *testing.T) { + now := time.Now().UTC() + taskID := 202 + leaderID := 120 + workerID := 121 + messageID := "team-31-task-202" + assignmentID := "review-calculator" + phaseID := "phase-2" + task := &models.TeamTask{ + ID: taskID, TeamID: 31, TargetMemberID: leaderID, MessageID: messageID, + Status: models.TeamTaskStatusRunning, WorkflowState: teamWorkflowStateAwaitingPhaseResults, + PlanVersion: 1, LedgerVersion: 6, UpdatedAt: now, + } + leader := &models.TeamMember{ID: leaderID, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy} + deferredPayload, err := json.Marshal(map[string]interface{}{ + "protocolVersion": 3, "event": "completion_deferred", "completionId": "completion-202", + "attemptId": "attempt-202", "completionSource": teamTaskCompletionTool, "explicitCompletion": true, + "rootTaskTerminal": false, "workflowFinal": true, "finalAnswerReady": true, + "remainingActions": []string{}, "planVersion": 1, "ledgerVersion": 6, + "summary": "Calculator delivered and verified.", + "resultMarkdown": "# Final delivery\n\nCalculator delivered and verified.", + }) + if err != nil { + t.Fatal(err) + } + eventID := "deferred-202" + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + workItems: []models.TeamWorkItem{{ + TeamID: 31, RootTaskID: taskID, WorkID: assignmentID, AssignmentID: &assignmentID, + PhaseID: &phaseID, OwnerMemberID: &workerID, RequiredForRoot: true, + Revision: 1, Status: models.TeamTaskStatusSucceeded, + }}, + workflowPhases: []models.TeamWorkflowPhase{ + {TeamID: 31, RootTaskID: taskID, PhaseID: phaseID, PlanVersion: 1, Status: teamPhaseStatusAwaitingResults, RequiredForRoot: true}, + {TeamID: 31, RootTaskID: taskID, PhaseID: "phase-3", PlanVersion: 1, Status: teamPhaseStatusPlanned, RequiredForRoot: true}, + }, + createdEvents: []models.TeamEvent{{ + TeamID: 31, TaskID: &taskID, MemberID: &leaderID, EventID: &eventID, + EventType: "completion_deferred", PayloadJSON: stringPtr(string(deferredPayload)), + }}, + } + service := &teamService{repo: repo} + reconciled, err := service.reconcileDeferredTeamCompletion(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, task, leader) + if err != nil || !reconciled || task.Status != models.TeamTaskStatusSucceeded { + t.Fatalf("expected deferred completion to self-heal without another Leader turn: reconciled=%v task=%#v err=%v", reconciled, task, err) + } + if repo.workflowPhases[0].Status != teamPhaseStatusCompleted || repo.workflowPhases[1].Status != teamPhaseStatusCancelled { + t.Fatalf("expected repaired phase ledger before acceptance: %#v", repo.workflowPhases) + } +} + +func TestProjectProtocolV3DeferredCompletionPersistsAcknowledgementOutbox(t *testing.T) { + now := time.Now().UTC() + taskID := 193 + messageID := "team-31-task-193" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + WorkflowState: teamWorkflowStateExecuting, + PlanVersion: 1, + LedgerVersion: 2, + UpdatedAt: now, + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID} + workerID := 121 + assignmentID := "implementation" + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + workItems: []models.TeamWorkItem{{ + TeamID: 31, + RootTaskID: taskID, + WorkID: assignmentID, + AssignmentID: &assignmentID, + OwnerMemberID: &workerID, + Status: models.TeamTaskStatusRunning, + RequiredForRoot: true, + Revision: 1, + UpdatedAt: now, + }}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "protocolVersion": 3, + "event": "completion_proposed", + "eventId": "evt-deferred-193", + "completionId": "completion:31:team-31-task-193:leader", + "attemptId": "attempt-deferred-193", + "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, + "rootTaskTerminal": true, + "memberId": "leader", + "messageId": messageID, + "taskId": messageID, + "rootTaskId": messageID, + "status": "succeeded", + "summary": "最终报告已准备,但实现任务仍在执行。", + "resultMarkdown": "# 最终报告\n\n等待结构化账本允许后提交。", + "workflowFinal": true, + "finalAnswerReady": true, + "remainingActions": []string{}, + "planVersion": 1, + "ledgerVersion": 2, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178993-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "completion_deferred" { + t.Fatalf("pending required assignment must defer root completion, got %#v", repo.createdEvents) + } + if len(repo.outboxRows) != 1 || !strings.Contains(repo.outboxRows[0].PayloadJSON, `"decision":"deferred"`) || !strings.Contains(repo.outboxRows[0].PayloadJSON, assignmentID) { + t.Fatalf("deferred completion acknowledgement must be durable and diagnostic: %#v", repo.outboxRows) + } +} + +func TestEvaluateProtocolV3DynamicPhaseRequiresLeaderDecisionOrWorkflowSeal(t *testing.T) { + leaderID := 120 + workerID := 121 + assignmentID := "collect-worker-input" + phaseID := "collection" + task := &models.TeamTask{ID: 191, TeamID: 31, TargetMemberID: leaderID, Status: models.TeamTaskStatusRunning, WorkflowState: teamWorkflowStateAwaitingLeaderDecision, PlanVersion: 1, LedgerVersion: 4} + repo := &teamRepositoryStub{ + workItems: []models.TeamWorkItem{{TeamID: 31, RootTaskID: task.ID, WorkID: assignmentID, AssignmentID: &assignmentID, PhaseID: &phaseID, Revision: 1, RequiredForRoot: true, OwnerMemberID: &workerID, Status: models.TeamTaskStatusSucceeded}}, + workflowPhases: []models.TeamWorkflowPhase{{TeamID: 31, RootTaskID: task.ID, PhaseID: phaseID, PlanVersion: 1, Status: teamPhaseStatusAwaitingLeaderDecision, RequiredForRoot: true, DecisionRequired: true}}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "protocolVersion": 3, "completionId": "completion-191", "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, "rootTaskTerminal": true, "finalAnswerReady": true, + "planVersion": 1, "ledgerVersion": 4, "status": "succeeded", "summary": "信息已收集", "resultMarkdown": "信息已收集。", + } + leader := &models.TeamMember{ID: leaderID, TeamID: 31, MemberKey: "leader", Role: "leader"} + evaluation, err := service.evaluateLeaderRootCompletion(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, task, leader, payload) + if err != nil { + t.Fatal(err) + } + if evaluation.Decision != teamCompletionDecisionDeferred || evaluation.Reason != "workflow_not_sealed" { + t.Fatalf("dynamic phase must wait for Leader decision: %#v", evaluation) + } + payload["workflowFinal"] = true + evaluation, err = service.evaluateLeaderRootCompletion(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, task, leader, payload) + if err != nil { + t.Fatal(err) + } + if evaluation.Decision != teamCompletionDecisionAccepted { + t.Fatalf("explicitly sealed dynamic workflow should be accepted: %#v", evaluation) + } +} + +func TestEvaluateProtocolV3CompletionDoesNotTreatFeatureWordAsInterim(t *testing.T) { + if isInterimOrDelegationReplyText("计算器最终交付报告:重复 = 操作,所有功能均已验证。") { + t.Fatal("legacy narrative helper must not treat an ordinary feature label as an interim reply") + } + task := &models.TeamTask{ID: 192, TeamID: 31, TargetMemberID: 120, Status: models.TeamTaskStatusRunning, WorkflowState: teamWorkflowStateSynthesizing, PlanVersion: 1, LedgerVersion: 1} + service := &teamService{repo: &teamRepositoryStub{}} + payload := map[string]interface{}{ + "protocolVersion": 3, "completionId": "completion-192", "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, "rootTaskTerminal": true, "workflowFinal": true, "finalAnswerReady": true, + "planVersion": 1, "ledgerVersion": 1, "status": "succeeded", "summary": "计算器最终交付报告", + "resultMarkdown": "功能清单:重复 = 操作。所有功能已验证。", + } + evaluation, err := service.evaluateLeaderRootCompletion( + &models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, task, + &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader"}, payload, + ) + if err != nil { + t.Fatal(err) + } + if evaluation.Decision != teamCompletionDecisionAccepted || len(evaluation.StrongContradictions) != 0 { + t.Fatalf("ordinary feature wording must not veto structured completion: %#v", evaluation) + } +} + +func TestEvaluateProtocolV3RequiresStructuredRiskWaiverForFailedRequiredWork(t *testing.T) { + leaderID := 120 + workerID := 121 + assignmentID := "browser-validation" + optionalAssignmentID := "nice-to-have-benchmark" + phaseID := "validation" + task := &models.TeamTask{ID: 195, TeamID: 31, TargetMemberID: leaderID, Status: models.TeamTaskStatusRunning, WorkflowState: teamWorkflowStateSynthesizing, PlanVersion: 1, LedgerVersion: 5} + repo := &teamRepositoryStub{ + workItems: []models.TeamWorkItem{ + {ID: 1, TeamID: 31, RootTaskID: task.ID, WorkID: assignmentID, AssignmentID: &assignmentID, PhaseID: &phaseID, Revision: 1, RequiredForRoot: true, OwnerMemberID: &workerID, Status: models.TeamTaskStatusFailed}, + {ID: 2, TeamID: 31, RootTaskID: task.ID, WorkID: optionalAssignmentID, AssignmentID: &optionalAssignmentID, PhaseID: &phaseID, Revision: 1, RequiredForRoot: false, OwnerMemberID: &workerID, Status: models.TeamTaskStatusRunning}, + }, + workflowPhases: []models.TeamWorkflowPhase{{TeamID: 31, RootTaskID: task.ID, PhaseID: phaseID, PlanVersion: 1, Status: teamPhaseStatusAwaitingResults, RequiredForRoot: true}}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "protocolVersion": 3, "completionId": "completion-195", "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, "rootTaskTerminal": true, "workflowFinal": true, "finalAnswerReady": true, + "planVersion": 1, "ledgerVersion": 5, "status": "succeeded", "summary": "最终交付报告", + "resultMarkdown": "浏览器验证失败,交付报告明确记录该风险。", + } + leader := &models.TeamMember{ID: leaderID, TeamID: 31, MemberKey: "leader", Role: "leader"} + evaluation, err := service.evaluateLeaderRootCompletion(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, task, leader, payload) + if err != nil { + t.Fatal(err) + } + if evaluation.Decision != teamCompletionDecisionDeferred || !containsTeamString(evaluation.PendingAssignments, assignmentID) { + t.Fatalf("failed required work without a waiver must block completion: %#v", evaluation) + } + payload["waivers"] = []interface{}{map[string]interface{}{"assignmentId": assignmentID, "reason": "测试环境不可用"}} + evaluation, err = service.evaluateLeaderRootCompletion(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, task, leader, payload) + if err != nil { + t.Fatal(err) + } + if evaluation.Decision != teamCompletionDecisionDeferred { + t.Fatalf("waiver without an accepted risk must not close the root task: %#v", evaluation) + } + payload["waivers"] = []interface{}{map[string]interface{}{ + "assignmentId": assignmentID, + "reason": "测试环境不可用", + "risk": "浏览器交互仍未验证,发布前必须人工复核", + }} + evaluation, err = service.evaluateLeaderRootCompletion(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, task, leader, payload) + if err != nil { + t.Fatal(err) + } + if evaluation.Decision != teamCompletionDecisionDeferred || !containsTeamString(evaluation.PendingAssignments, optionalAssignmentID+":skip_reason") { + t.Fatalf("omitted optional work must have a structured skip reason: %#v", evaluation) + } + payload["skippedAssignments"] = []interface{}{map[string]interface{}{"assignmentId": optionalAssignmentID, "reason": "不影响核心交付,留待后续性能专项"}} + evaluation, err = service.evaluateLeaderRootCompletion(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, task, leader, payload) + if err != nil { + t.Fatal(err) + } + if evaluation.Decision != teamCompletionDecisionAccepted || !containsTeamString(evaluation.WaivedAssignments, assignmentID) || !containsTeamString(evaluation.SkippedAssignments, optionalAssignmentID) { + t.Fatalf("complete structured waiver and optional skip record should permit a recorded partial-risk delivery: %#v", evaluation) + } +} + +func TestProjectTeamWorkItemKeepsSequentialAssignmentsForSameMember(t *testing.T) { + team := &models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated} + task := &models.TeamTask{ID: 193, TeamID: 31, TargetMemberID: 120, Status: models.TeamTaskStatusRunning} + developer := &models.TeamMember{ID: 121, TeamID: 31, MemberKey: "developer", Role: "developer"} + repo := &teamRepositoryStub{membersByKey: map[string]*models.TeamMember{"developer": developer}} + service := &teamService{repo: repo} + for index, assignmentID := range []string{"developer-research", "developer-implementation"} { + phaseID := []string{"research", "implementation"}[index] + payload := map[string]interface{}{ + "assignmentId": assignmentID, "workId": assignmentID, "phaseId": phaseID, + "required": true, "revision": 1, + "collaborationStep": map[string]interface{}{ + "type": "assignment", "status": "dispatched", "actor": "leader", "target": "developer", + "workId": assignmentID, "phase": phaseID, "title": assignmentID, + }, + } + if err := service.projectTeamWorkItem(team, task, developer, "team_send", payload, &models.TeamEvent{CreatedAt: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + } + if len(repo.workItems) != 2 || repo.workItems[0].WorkID == repo.workItems[1].WorkID { + t.Fatalf("sequential assignments for one member must remain distinct: %#v", repo.workItems) + } +} + +func TestArtifactChangeInvalidatesMatchingReviewedWorkItem(t *testing.T) { + taskID := 196 + leaderID := 120 + developerID := 121 + validatedRevision := 1 + assignmentID := "developer-implementation" + artifactRefs := `["/team/artifacts/team-31-task-196/members/developer/app.js"]` + task := &models.TeamTask{ID: taskID, TeamID: 31, TargetMemberID: leaderID, MessageID: "team-31-task-196", Status: models.TeamTaskStatusRunning, LedgerVersion: 3, UpdatedAt: time.Now().UTC()} + leader := &models.TeamMember{ID: leaderID, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{task.MessageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + workItems: []models.TeamWorkItem{{ + ID: 1, TeamID: 31, RootTaskID: taskID, WorkID: assignmentID, AssignmentID: &assignmentID, + OwnerMemberID: &developerID, Revision: 1, RequiredForRoot: true, ReviewRequired: true, + ValidatedRevision: &validatedRevision, Status: models.TeamTaskStatusSucceeded, ArtifactRefsJSON: &artifactRefs, + }}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "artifact_changed", "eventKind": "artifact_changed", "eventId": "artifact-change-196", + "memberId": "leader", "messageId": task.MessageID, "taskId": task.MessageID, "rootTaskId": task.MessageID, + "assignmentId": "leader-final-synthesis", "artifactChanged": true, + "artifactRefs": []string{"/team/artifacts/team-31-task-196/members/developer/app.js"}, + "status": "running", "summary": "Leader updated the reviewed artifact.", + }) + if err != nil { + t.Fatal(err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178996-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatal(err) + } + for idx := range repo.workItems { + if repo.workItems[idx].ID == 1 && repo.workItems[idx].ValidatedRevision != nil { + t.Fatalf("modifying a reviewed artifact must invalidate the old review: %#v", repo.workItems[idx]) + } + } + if repo.updatedTask == nil || repo.updatedTask.LedgerVersion != 4 { + t.Fatalf("review invalidation must advance the root ledger: %#v", repo.updatedTask) + } +} + +func TestTeamChatPolicyPreservesMeaningfulCheckProgress(t *testing.T) { + payload := map[string]interface{}{ + "eventKind": "assignment_check_result", "checkId": "monitor:team-31-task-1:work-1:2", + "assignmentId": "work-1", "rootTaskId": "team-31-task-1", "progress": 60, + "summary": "已完成核心模块,正在进行集成验证。", + } + applyTeamChatPolicy("task_progress", payload, nil, &models.TeamMember{MemberKey: "developer"}) + if payload["chatPolicy"] != "visible" || payload["visibleToChat"] != true || payload["chatBusinessKind"] != "worker_progress" { + t.Fatalf("meaningful check progress must remain visible: %#v", payload) + } + quiet := map[string]interface{}{ + "eventKind": "assignment_check_result", "checkId": "monitor:team-31-task-1:work-1:3", + "assignmentId": "work-1", "rootTaskId": "team-31-task-1", "summary": "still running", + } + applyTeamChatPolicy("task_progress", quiet, nil, &models.TeamMember{MemberKey: "developer"}) + if quiet["chatPolicy"] != "digest" || quiet["visibleToChat"] != false { + t.Fatalf("unchanged transport check should be digested: %#v", quiet) + } +} + +func TestTeamChatPolicyBusinessNarrativeOverridesLegacyHiddenPolicy(t *testing.T) { + payload := map[string]interface{}{ + "eventKind": "worker_plan", "chatPolicy": "hidden", "visibleToChat": false, + "memberId": "developer", "phaseId": "implementation", + "text": "I will implement the calculator UI and report the deliverable to the Leader.", + "displayKey": "worker-plan:team-31-task-1:", + } + applyTeamChatPolicy("task_progress", payload, nil, &models.TeamMember{MemberKey: "developer"}) + if payload["chatPolicy"] != "visible" || payload["visibleToChat"] != true || eventString(payload, "displayKey") != "" { + t.Fatalf("business narrative must override stale hidden policy and empty worker display key: %#v", payload) + } +} + +func TestTeamChatPolicyKeepsTransportAcknowledgementHidden(t *testing.T) { + payload := map[string]interface{}{ + "eventKind": "assignment_heartbeat", "summary": "still running", "visibleToChat": true, + } + applyTeamChatPolicy("assignment_heartbeat", payload, nil, &models.TeamMember{MemberKey: "developer"}) + if payload["chatPolicy"] != "digest" || payload["visibleToChat"] != false { + t.Fatalf("heartbeat without business content must remain digest-only: %#v", payload) + } +} + +func TestTeamEventPayloadsFilterOnlyHiddenTransportFacts(t *testing.T) { + hiddenJSON := `{"event":"member_result_confirmed","chatPolicy":"hidden","visibleToChat":false}` + progressJSON := `{"event":"task_progress","eventKind":"worker_progress","chatPolicy":"replaceable","visibleToChat":true,"summary":"正在实现核心模块"}` + digestJSON := `{"event":"assignment_heartbeat","chatPolicy":"digest","visibleToChat":false,"summary":"仍在执行"}` + events := []models.TeamEvent{ + {ID: 1, EventType: "member_result_confirmed", PayloadJSON: &hiddenJSON}, + {ID: 2, EventType: "task_progress", PayloadJSON: &progressJSON}, + {ID: 3, EventType: "assignment_heartbeat", PayloadJSON: &digestJSON}, + } + payloads := teamEventPayloads(events) + if len(payloads) != 2 || payloads[0].ID != 2 || payloads[1].ID != 3 { + t.Fatalf("chat projection must preserve business progress and digests while removing hidden confirmation facts: %#v", payloads) + } +} + +func containsTeamString(values []string, expected string) bool { + for _, value := range values { + if value == expected { + return true + } + } + return false +} + +func TestTeamEventPayloadsRestoreLegacyHiddenBusinessNarrative(t *testing.T) { + workerPlanJSON := `{"event":"task_progress","eventKind":"worker_plan","chatPolicy":"hidden","visibleToChat":false,"text":"Worker plan: implement the core module and report to Leader."}` + transportJSON := `{"event":"task_received","chatPolicy":"hidden","visibleToChat":false,"summary":"task_received"}` + events := []models.TeamEvent{ + {ID: 31, EventType: "task_progress", PayloadJSON: &workerPlanJSON}, + {ID: 32, EventType: "task_received", PayloadJSON: &transportJSON}, + } + payloads := teamEventPayloads(events) + if len(payloads) != 1 || payloads[0].ID != 31 { + t.Fatalf("legacy hidden business narrative must be returned while transport acknowledgement stays hidden: %#v", payloads) + } +} + +func TestProjectTeamEventAssignmentResultUsesActualMemberLane(t *testing.T) { + taskID := 180 + messageID := "team-31-task-180" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + developer := &models.TeamMember{ID: 121, TeamID: 31, MemberKey: "developer", Role: "senior-developer", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + reviewer := &models.TeamMember{ID: 122, TeamID: 31, MemberKey: "reviewer", Role: "qa-engineer", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "developer": developer, "reviewer": reviewer}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "outbound", + "assignmentResultOnly": true, + "assignmentId": "assignment-developer-001", + "memberId": "reviewer", + "from": "reviewer", + "to": "leader", + "rootTaskId": messageID, + "status": "succeeded", + "text": "Reviewer delivered a valid result.", + "collaborationStep": map[string]interface{}{ + "type": "result", + "status": "succeeded", + "actor": "reviewer", + "target": "leader", + "rootTaskId": messageID, + "content": "Reviewer delivered a valid result.", + }, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178680-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if len(repo.workItems) != 1 { + t.Fatalf("expected one work item, got %#v", repo.workItems) + } + if repo.workItems[0].WorkID != "assignment-developer-001" { + t.Fatalf("assignment result must preserve the business assignment ID, got workID %q", repo.workItems[0].WorkID) + } + if repo.workItems[0].OwnerMemberID == nil || *repo.workItems[0].OwnerMemberID != reviewer.ID { + t.Fatalf("expected reviewer owner, got %#v", repo.workItems[0].OwnerMemberID) + } +} + +func TestProjectTeamEventLeaderDispatchAndWorkerResultShareMemberLane(t *testing.T) { + taskID := 181 + messageID := "team-31-task-181" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + developer := &models.TeamMember{ID: 121, TeamID: 31, MemberKey: "developer", Role: "senior-developer", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "developer": developer}, + } + service := &teamService{repo: repo} + assignmentJSON, err := json.Marshal(map[string]interface{}{ + "event": "reply", + "leaderDispatchOnly": true, + "assignmentId": "assignment-developer-001", + "memberId": "leader", + "rootTaskId": messageID, + "status": "dispatched", + "collaborationStep": map[string]interface{}{ + "type": "assignment", + "status": "dispatched", + "actor": "leader", + "target": "developer", + "rootTaskId": messageID, + "content": "Please answer with one flower.", + }, + }) + if err != nil { + t.Fatalf("marshal assignment: %v", err) + } + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178681-0", + Fields: map[string]string{"payload": string(assignmentJSON)}, + }); err != nil { + t.Fatalf("project assignment: %v", err) + } + resultJSON, err := json.Marshal(map[string]interface{}{ + "event": "outbound", + "assignmentResultOnly": true, + "assignmentId": "runtime-random-assignment-id", + "memberId": "developer", + "from": "developer", + "to": "leader", + "rootTaskId": messageID, + "status": "succeeded", + "text": "洋牡丹:受欢迎、魅力四射。", + "collaborationStep": map[string]interface{}{ + "type": "result", + "status": "succeeded", + "actor": "developer", + "target": "leader", + "rootTaskId": messageID, + "content": "洋牡丹:受欢迎、魅力四射。", + }, + }) + if err != nil { + t.Fatalf("marshal result: %v", err) + } + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178681-1", + Fields: map[string]string{"payload": string(resultJSON)}, + }); err != nil { + t.Fatalf("project result: %v", err) + } + if len(repo.workItems) != 1 { + t.Fatalf("dispatch and result should share one member lane, got %#v", repo.workItems) + } + item := repo.workItems[0] + if item.WorkID != "assignment-developer-001" || item.Status != models.TeamTaskStatusSucceeded { + t.Fatalf("expected succeeded canonical developer assignment, got %#v", item) + } + if item.ResultJSON == nil || !strings.Contains(*item.ResultJSON, "洋牡丹") { + t.Fatalf("expected full worker result in lane payload, got %#v", item.ResultJSON) + } + if len(repo.createdEvents) < 3 || repo.createdEvents[len(repo.createdEvents)-1].EventType != "member_result_confirmed" { + t.Fatalf("expected a structured leader notification for worker result, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventLeaderMediatedWorkerProgressAndResultShareMemberLane(t *testing.T) { + taskID := 184 + messageID := "team-31-task-184" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + pm := &models.TeamMember{ID: 121, TeamID: 31, MemberKey: "pm", Role: "product-manager", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "pm": pm}, + } + service := &teamService{repo: repo} + progressJSON, err := json.Marshal(map[string]interface{}{ + "event": "progress", + "memberId": "pm", + "from": "pm", + "to": "leader", + "rootTaskId": messageID, + "workId": "w1-pm-flower", + "status": "running", + "progress": 50, + "text": "Working on PM flower choice.", + "collaborationStep": map[string]interface{}{ + "type": "progress", + "status": "running", + "actor": "pm", + "target": "leader", + "rootTaskId": messageID, + "content": "Working on PM flower choice.", + }, + }) + if err != nil { + t.Fatalf("marshal progress: %v", err) + } + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178683-0", + Fields: map[string]string{"payload": string(progressJSON)}, + }); err != nil { + t.Fatalf("project progress: %v", err) + } + if len(repo.workItems) != 1 || repo.workItems[0].WorkID != "w1-pm-flower" || repo.workItems[0].Status != models.TeamTaskStatusRunning { + t.Fatalf("expected running PM assignment after progress, got %#v", repo.workItems) + } + resultJSON, err := json.Marshal(map[string]interface{}{ + "event": "outbound", + "assignmentResultOnly": true, + "assignmentId": "runtime-random-assignment-id", + "memberId": "pm", + "from": "pm", + "to": "leader", + "rootTaskId": messageID, + "status": "succeeded", + "text": "PM flower: Sunflower.", + "collaborationStep": map[string]interface{}{ + "type": "result", + "status": "succeeded", + "actor": "pm", + "target": "leader", + "rootTaskId": messageID, + "content": "PM flower: Sunflower.", + }, + }) + if err != nil { + t.Fatalf("marshal result: %v", err) + } + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178683-1", + Fields: map[string]string{"payload": string(resultJSON)}, + }); err != nil { + t.Fatalf("project result: %v", err) + } + if len(repo.workItems) != 1 { + t.Fatalf("progress and result should share one member lane, got %#v", repo.workItems) + } + item := repo.workItems[0] + if item.WorkID != "w1-pm-flower" || item.Status != models.TeamTaskStatusSucceeded { + t.Fatalf("expected succeeded PM assignment, got %#v", item) + } + if item.ResultJSON == nil || !strings.Contains(*item.ResultJSON, "Sunflower") { + t.Fatalf("expected worker result stored in pm lane, got %#v", item.ResultJSON) + } +} + +func TestProjectTeamEventLeaderMediatedDuplicateWorkerResultDoesNotNotifyTwice(t *testing.T) { + taskID := 185 + messageID := "team-31-task-185" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + pm := &models.TeamMember{ID: 121, TeamID: 31, MemberKey: "pm", Role: "product-manager", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + confirmedPayload := `{"event":"member_result_confirmed","from":"pm","memberId":"pm","rootTaskId":"team-31-task-185","text":"PM flower: Sunflower."}` + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "pm": pm}, + createdEvents: []models.TeamEvent{{ + TeamID: 31, + TaskID: &taskID, + EventType: "member_result_confirmed", + PayloadJSON: &confirmedPayload, + }}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "outbound", + "assignmentResultOnly": true, + "memberId": "pm", + "from": "pm", + "to": "leader", + "rootTaskId": messageID, + "status": "succeeded", + "text": "PM flower: Sunflower.", + "collaborationStep": map[string]interface{}{ + "type": "result", + "status": "succeeded", + "actor": "pm", + "target": "leader", + "rootTaskId": messageID, + "content": "PM flower: Sunflower.", + }, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178684-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("project duplicate result: %v", err) + } + confirmations := 0 + for _, event := range repo.createdEvents { + if event.EventType == "member_result_confirmed" { + confirmations++ + } + } + if confirmations != 1 { + t.Fatalf("duplicate worker result must not create another leader notification, got %d events: %#v", confirmations, repo.createdEvents) + } +} + +func TestProjectTeamEventLeaderMediatedChangedWorkerResultNotifiesAgain(t *testing.T) { + taskID := 186 + messageID := "team-31-task-186" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + reviewer := &models.TeamMember{ID: 121, TeamID: 31, MemberKey: "reviewer", Role: "qa-engineer", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + confirmedPayload := `{"event":"member_result_confirmed","from":"reviewer","memberId":"reviewer","rootTaskId":"team-31-task-186","assignmentId":"review-current","workId":"review-current","text":"Reviewer verdict: FAIL"}` + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "reviewer": reviewer}, + createdEvents: []models.TeamEvent{{ + TeamID: 31, + TaskID: &taskID, + EventType: "member_result_confirmed", + PayloadJSON: &confirmedPayload, + }}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "outbound", + "assignmentResultOnly": true, + "assignmentId": "review-current", + "memberId": "reviewer", + "from": "reviewer", + "to": "leader", + "rootTaskId": messageID, + "status": "succeeded", + "text": "Reviewer verdict: PASS", + "collaborationStep": map[string]interface{}{ + "type": "result", + "status": "succeeded", + "actor": "reviewer", + "target": "leader", + "rootTaskId": messageID, + "content": "Reviewer verdict: PASS", + }, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178685-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("project changed result: %v", err) + } + confirmations := 0 + for _, event := range repo.createdEvents { + if event.EventType == "member_result_confirmed" { + confirmations++ + } + } + if confirmations != 2 { + t.Fatalf("changed worker result must create a fresh leader notification, got %d events: %#v", confirmations, repo.createdEvents) + } + if len(repo.outboxRows) != 1 { + t.Fatalf("changed worker result must be delivered to leader once, got outbox %#v", repo.outboxRows) + } + if len(repo.workItems) != 1 || repo.workItems[0].ResultJSON == nil || !strings.Contains(*repo.workItems[0].ResultJSON, "PASS") { + t.Fatalf("changed worker result must refresh current work item, got %#v", repo.workItems) + } +} + +func TestProjectTeamEventLeaderDispatchRefreshesTerminalCurrentLane(t *testing.T) { + taskID := 187 + messageID := "team-31-task-187" + now := time.Now().UTC() + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: now.Add(-time.Minute), + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + reviewer := &models.TeamMember{ID: 121, TeamID: 31, MemberKey: "reviewer", Role: "qa-engineer", Status: models.TeamMemberStatusIdle, Availability: models.TeamMemberAvailabilityIdle} + oldResult := `{"summary":"Reviewer verdict: FAIL"}` + oldFinishedAt := now.Add(-2 * time.Minute) + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "reviewer": reviewer}, + workItems: []models.TeamWorkItem{{ + TeamID: 31, + RootTaskID: taskID, + WorkID: "review-current", + AssignmentID: stringPtr("review-current"), + OwnerMemberID: &reviewer.ID, + Title: "reviewer delivers result", + Status: models.TeamTaskStatusFailed, + ResultJSON: &oldResult, + FinishedAt: &oldFinishedAt, + UpdatedAt: now.Add(-2 * time.Minute), + }}, + } + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "reply", + "leaderDispatchOnly": true, + "assignmentId": "review-current", + "memberId": "leader", + "from": "leader", + "to": "reviewer", + "rootTaskId": messageID, + "status": "dispatched", + "text": "Please re-check after the calculator fix.", + "collaborationStep": map[string]interface{}{ + "type": "assignment", + "status": "dispatched", + "actor": "leader", + "target": "reviewer", + "rootTaskId": messageID, + "content": "Please re-check after the calculator fix.", + }, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + service := &teamService{repo: repo} + if err := service.projectTeamEvent(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178686-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("project redispatch: %v", err) + } + if len(repo.workItems) != 1 { + t.Fatalf("redispatch should refresh one current lane, got %#v", repo.workItems) + } + item := repo.workItems[0] + if item.Status != models.TeamTaskStatusDispatched || item.ResultJSON != nil || item.FinishedAt != nil { + t.Fatalf("redispatch must reopen current lane and clear old terminal result, got %#v", item) + } +} + +func TestLeaderMediatedRootCompletionReadyCoalescesSameOwnerWorkItems(t *testing.T) { + taskID := 182 + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: "team-31-task-182", + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + developer := &models.TeamMember{ID: 121, TeamID: 31, MemberKey: "developer", Role: "senior-developer", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + repo := &teamRepositoryStub{ + workItems: []models.TeamWorkItem{ + {TeamID: 31, RootTaskID: taskID, WorkID: "assignment-developer-legacy", OwnerMemberID: &developer.ID, Status: models.TeamTaskStatusDispatched, UpdatedAt: time.Now().UTC()}, + {TeamID: 31, RootTaskID: taskID, WorkID: "member-developer", OwnerMemberID: &developer.ID, Status: models.TeamTaskStatusSucceeded, UpdatedAt: time.Now().UTC()}, + }, + } + service := &teamService{repo: repo} + ready, err := service.leaderMediatedRootCompletionReady(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, task, leader) + if err != nil { + t.Fatalf("leaderMediatedRootCompletionReady returned error: %v", err) + } + if !ready { + t.Fatalf("same-owner legacy dispatch card must not block root completion after member result") + } +} + +func TestProjectTeamEventAssociatesCurrentTaskReplyWithoutCompletingIt(t *testing.T) { + taskID := 72 + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 121, + MessageID: "team-31-task-72", + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + worker := &models.TeamMember{ + ID: 121, + TeamID: 31, + MemberKey: "worker", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + membersByKey: map[string]*models.TeamMember{"worker": worker}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "reply", + "memberId": "worker", + "summary": "Weather report ready", + "text": strings.Join([]string{ + "# Melbourne weather", + "Current conditions: partly cloudy, 16C, south wind 27 km/h, humidity 82%.", + "Forecast: showers today, cooler tomorrow, rain on Saturday.", + }, "\n"), + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178660-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask != nil && (repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil) { + t.Fatalf("current task reply must be associated but remain non-terminal, got %#v", repo.updatedTask) + } + if repo.updatedMember != nil && repo.updatedMember.Progress == 100 { + t.Fatalf("reply without explicit completion must not complete worker state, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "reply" || repo.createdEvents[0].TaskID == nil || *repo.createdEvents[0].TaskID != taskID { + t.Fatalf("expected reply event linked to current task, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventTreatsExplicitCompletionToolReplyAsTaskCompleted(t *testing.T) { + taskID := 67 + messageID := "team-31-bootstrap-introduction" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + member := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": member}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "reply", + "messageId": messageID, + "memberId": "leader", + "taskId": "team-31-task-67", + "final": true, + "summary": "Team report ready", + "resultMarkdown": "Full report", + "text": "Full report", + "toolCall": map[string]interface{}{ + "name": teamTaskCompletionTool, + }, + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178655-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded { + t.Fatalf("expected explicit completion tool reply to mark task succeeded, got %#v", repo.updatedTask) + } + if repo.updatedTask.ResultJSON == nil || !strings.Contains(*repo.updatedTask.ResultJSON, "Full report") { + t.Fatalf("expected reply payload to become task result, got %#v", repo.updatedTask.ResultJSON) + } + if repo.updatedMember == nil || repo.updatedMember.Status != models.TeamMemberStatusIdle || repo.updatedMember.Availability != models.TeamMemberAvailabilityIdle { + t.Fatalf("expected member to become idle, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "task_completed" { + t.Fatalf("expected stored event type task_completed, got %#v", repo.createdEvents) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored event payload: %v", err) + } + step, ok := stored["collaborationStep"].(map[string]interface{}) + if !ok { + t.Fatalf("expected collaboration step in stored event, got %#v", stored) + } + if got := step["summary"]; got != "Team report ready" { + t.Fatalf("expected collaboration step summary to stay compact, got %#v", got) + } + if got := step["content"]; got != "Full report" { + t.Fatalf("expected collaboration step content to preserve full resultMarkdown, got %#v", got) + } +} + +func TestProjectTeamEventCompletionReportMentioningDispatchStillClosesTask(t *testing.T) { + taskID := 69 + messageID := "team-31-bootstrap-introduction" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + member := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": member}, + } + service := &teamService{repo: repo} + report := strings.Join([]string{ + "# Team 31 introduction", + "", + "## Collaboration", + "Leader uses team_send for task dispatch to each member, then waits for worker evidence before final synthesis.", + "PM, Designer, and Architect each receive scoped assignments and return concrete artifacts through the shared workspace.", + "Task dispatch is part of this completed report, not a dispatch-only acknowledgement.", + }, "\n") + payload := map[string]interface{}{ + "event": "task_completed", + "messageId": messageID, + "memberId": "leader", + "taskId": "team-31-task-69", + "status": "succeeded", + "summary": "Completed team introduction.", + "resultMarkdown": report, + "explicitCompletion": true, + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178657-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded { + t.Fatalf("expected completion report mentioning dispatch to close task, got %#v", repo.updatedTask) + } + if repo.updatedTask.ResultJSON == nil || !strings.Contains(*repo.updatedTask.ResultJSON, "Task dispatch is part of this completed report") { + t.Fatalf("expected full report to be stored on task result, got %#v", repo.updatedTask.ResultJSON) + } + var stored map[string]interface{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode stored event payload: %v", err) + } + step := stored["collaborationStep"].(map[string]interface{}) + if got, _ := step["content"].(string); !strings.Contains(got, "Leader uses team_send") { + t.Fatalf("expected collaboration step content to preserve report, got %#v", got) + } +} + +func TestProjectTeamEventProtocolV2BootstrapCompletionMentioningTeamSendClosesTask(t *testing.T) { + teamID := 40 + taskID := 69 + messageID := "team-40-bootstrap-introduction" + workspaceRoot := t.TempDir() + resultDir := filepath.Join(workspaceRoot, "teams", "user-1", "team-40-shared", "results", "team-40-task-69") + if err := os.MkdirAll(resultDir, 0o775); err != nil { + t.Fatalf("create result dir: %v", err) + } + for _, name := range []string{"team-introduction.md", "result.md"} { + if err := os.WriteFile(filepath.Join(resultDir, name), []byte("team introduction"), 0o664); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + task := &models.TeamTask{ + ID: taskID, + TeamID: teamID, + TargetMemberID: 145, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ + ID: 145, + TeamID: teamID, + MemberKey: "leader", + Role: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: workspaceRoot} + report := strings.Join([]string{ + "# Product Discovery Team 23 - Bootstrap complete", + "", + "## Collaboration", + "- Mode: Leader Mediated; all user work enters through the Leader.", + "- Communication layer: Redis Streams with team_send, events, inbox, presence, and DLQ.", + "- Worker pipeline: Worker completes, writes result, calls team_complete_task, then reports back.", + "", + "Detailed report: /team/results/team-40-task-69/team-introduction.md", + }, "\n") + payloadJSON, err := json.Marshal(map[string]interface{}{ + "v": 1, + "protocolVersion": 2, + "eventId": "evt_team40_completion", + "event": "task_completed", + "type": "task_completed", + "memberId": "leader", + "messageId": messageID, + "taskId": "team-40-task-69", + "rootTaskId": "team-40-task-69", + "rootMessageId": messageID, + "status": "succeeded", + "runtimeStatus": "succeeded", + "availability": "idle", + "completionId": "completion:40:team-40-task-69:leader", + "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, + "summary": "Bootstrap report completed.", + "result": report, + "resultMarkdown": report, + "artifactRefs": []interface{}{"/team/results/team-40-task-69/team-introduction.md", "/team/results/team-40-task-69/result.md"}, + "sourceMessageId": messageID, + "completionMessageId": "completion:40:team-40-task-69:leader", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1783407850912-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt == nil { + t.Fatalf("protocol v2 completion report should close the root task, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "task_completed" { + t.Fatalf("expected stored task_completed event, got %#v", repo.createdEvents) + } + stored := teamEventPayloadMap(repo.createdEvents[0]) + if eventBool(stored, "leaderDispatchOnly") { + t.Fatalf("explicit completion must not be downgraded to leader dispatch, got %#v", stored) + } + step, ok := stored["collaborationStep"].(map[string]interface{}) + if !ok || step["type"] != "result" || step["status"] != models.TeamTaskStatusSucceeded { + t.Fatalf("expected result collaboration step for explicit completion, got %#v", step) + } +} + +func TestActiveTeamMembersFiltersDeletedMembers(t *testing.T) { + members := activeTeamMembers([]models.TeamMember{ + {MemberKey: "leader", Status: models.TeamMemberStatusIdle}, + {MemberKey: "old", Status: models.TeamMemberStatusDeleted}, + {MemberKey: "gone", Status: models.TeamMemberStatusDeleting}, + }) + if len(members) != 1 || members[0].MemberKey != "leader" { + t.Fatalf("unexpected active members: %#v", members) + } +} + +func TestDeletedTeamNameReleasesUniqueName(t *testing.T) { + name := deletedTeamName("DeepResearch", 42) + if name != "DeepResearch__deleted_42" { + t.Fatalf("unexpected deleted Team name: %q", name) + } + if again := deletedTeamName(name, 42); again != name { + t.Fatalf("deleted Team name should be idempotent, got %q", again) + } +} + +func TestTeamTaskStaleTimeoutUsesEnvironment(t *testing.T) { + t.Setenv("CLAWMANAGER_TEAM_TASK_STALE_SECONDS", "60") + if got := teamTaskStaleTimeout(); got != time.Minute { + t.Fatalf("expected one minute stale timeout, got %s", got) + } + + t.Setenv("CLAWMANAGER_TEAM_TASK_STALE_SECONDS", "0") + if got := teamTaskStaleTimeout(); got != 0 { + t.Fatalf("expected disabled stale timeout, got %s", got) + } +} + +func TestApplyTeamMemberRuntimeProjectionSetsBlockedAvailability(t *testing.T) { + member := &models.TeamMember{Availability: models.TeamMemberAvailabilityBusy} + payload := map[string]interface{}{ + "availability": "blocked", + "lastSummary": "Task failed: LLM request failed: network connection error.", + "currentTaskId": "task_cb1062da-dff2-46ff-836f-86490583d944", + "currentIntent": "weather_query_beijing", + } + + applyTeamMemberRuntimeProjection(member, payload, "status") + + if member.Availability != models.TeamMemberAvailabilityBlocked { + t.Fatalf("expected blocked availability, got %q", member.Availability) + } + if member.LastSummary == nil || !strings.Contains(*member.LastSummary, "LLM request failed") { + t.Fatalf("expected last summary projection, got %#v", member.LastSummary) + } + if member.RuntimeTaskID == nil || *member.RuntimeTaskID != "task_cb1062da-dff2-46ff-836f-86490583d944" { + t.Fatalf("expected runtime task id projection, got %#v", member.RuntimeTaskID) + } +} + +func TestApplyTeamMemberRuntimeProjectionClearsStaleBlockedOnCompletion(t *testing.T) { + reason := "previous task failed" + member := &models.TeamMember{ + Availability: models.TeamMemberAvailabilityBlocked, + BlockedReason: &reason, + } + payload := map[string]interface{}{ + "lastSummary": "Redis Team task processing completed", + "currentTaskId": "task_001", + } + + applyTeamMemberRuntimeProjection(member, payload, "task_completed") + + if member.Availability != models.TeamMemberAvailabilityIdle { + t.Fatalf("expected idle availability after task completion, got %q", member.Availability) + } + if member.BlockedReason != nil { + t.Fatalf("expected stale blocked reason to be cleared, got %#v", *member.BlockedReason) + } +} + +func TestMergeMissingEventFieldsEnrichesOutboundPayload(t *testing.T) { + base := map[string]interface{}{ + "event": "outbound", + "messageId": "msg_123", + "from": "leader", + "to": "worker", + } + extra := map[string]interface{}{ + "messageId": "msg_123", + "title": "Check date", + "text": "Check today's date and send the result back.", + "metadata": map[string]interface{}{ + "prompt": "metadata prompt should also be available", + }, + } + + merged := mergeMissingEventFields(base, extra) + + if merged["messageId"] != "msg_123" || merged["from"] != "leader" || merged["to"] != "worker" { + t.Fatalf("base fields should be preserved, got %#v", merged) + } + if merged["title"] != "Check date" || merged["text"] == "" { + t.Fatalf("expected outbound payload to be enriched with title/text, got %#v", merged) + } + if merged["prompt"] != "metadata prompt should also be available" { + t.Fatalf("expected metadata prompt to be merged, got %#v", merged) + } + if !teamEventHasBody(merged) { + t.Fatalf("expected enriched event to have displayable body: %#v", merged) + } +} + +func TestCollectTeamArtifactReferencesStopsAtMarkdownAndJSONDelimiters(t *testing.T) { + payload := map[string]interface{}{ + "resultMarkdown": "Report: `/team/results/team-22-task-40/team-introduction-report.md`.", + "raw": `{"resultMarkdown":"/team/results/team-22-task-40/team-introduction-report.md\\n","artifactRefs":["/team/results/team-22-task-40/result.md"]}`, + "directory": "Detailed report directory: `/team/results/team-22-task-40/`.", + "artifactRefs": []interface{}{ + "/team/results/team-22-task-40/team-introduction-report.md", + "/team/results/team-22-task-40/result.md", + }, + } + + got := collectTeamArtifactReferences(payload) + want := map[string]bool{ + "/team/results/team-22-task-40/team-introduction-report.md": true, + "/team/results/team-22-task-40/result.md": true, + } + if len(got) != len(want) { + t.Fatalf("artifact refs = %#v, want exactly %#v", got, want) + } + for _, ref := range got { + if !want[ref] { + t.Fatalf("unexpected malformed artifact ref %q in %#v", ref, got) + } + } +} + +func TestProjectTeamEventTreatsLegacyBootstrapReportAsCompletion(t *testing.T) { + teamID := 31 + taskID := 67 + messageID := "team-31-bootstrap-introduction" + workspaceRoot := t.TempDir() + resultDir := filepath.Join(workspaceRoot, "teams", "user-1", "team-31-shared", "results", "team-31-task-67") + if err := os.MkdirAll(resultDir, 0o775); err != nil { + t.Fatalf("create result dir: %v", err) + } + if err := os.WriteFile(filepath.Join(resultDir, "team-introduction.md"), []byte("team introduction"), 0o664); err != nil { + t.Fatalf("write result: %v", err) + } + task := &models.TeamTask{ + ID: taskID, + TeamID: teamID, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ + ID: 120, + TeamID: teamID, + MemberKey: "leader", + Role: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: workspaceRoot} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "reply", + "memberId": "leader", + "messageId": messageID, + "text": "Team 31 bootstrap introduction completed. The Team roster, runtime status, collaboration mode, " + + "capability boundaries, Redis Streams team_send mechanism, shared workspace rules, and available methods " + + "have been delivered. Detailed report: /team/results/team-31-task-67/team-introduction.md", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178692-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt == nil { + t.Fatalf("legacy bootstrap report should close the root task, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "task_completed" { + t.Fatalf("expected one normalized task_completed event, got %#v", repo.createdEvents) + } + payload := teamEventPayloadMap(repo.createdEvents[0]) + if !eventBool(payload, "legacyCompletionCandidate") || eventString(payload, "completionSource") != "legacy_runtime_reply" { + t.Fatalf("expected legacy completion markers, got %#v", payload) + } +} + +func TestProjectTeamEventTreatsServerBootstrapToolReplyAsCompletion(t *testing.T) { + teamID := 30 + taskID := 55 + messageID := "team-30-bootstrap-introduction" + workspaceRoot := t.TempDir() + resultDir := filepath.Join(workspaceRoot, "teams", "user-1", "team-30-shared", "results", "team-30-task-55") + if err := os.MkdirAll(resultDir, 0o775); err != nil { + t.Fatalf("create result dir: %v", err) + } + if err := os.WriteFile(filepath.Join(resultDir, "team-introduction.md"), []byte("team introduction"), 0o664); err != nil { + t.Fatalf("write result: %v", err) + } + task := &models.TeamTask{ID: taskID, TeamID: teamID, TargetMemberID: 120, MessageID: messageID, Status: models.TeamTaskStatusRunning, UpdatedAt: time.Now().UTC()} + leader := &models.TeamMember{ID: 120, TeamID: teamID, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + designer := &models.TeamMember{ID: 121, TeamID: teamID, MemberKey: "ui-designer", Role: "ui-ux-designer", Status: models.TeamMemberStatusIdle, Availability: models.TeamMemberAvailabilityIdle} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "ui-designer": designer}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: workspaceRoot} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "reply", + "memberId": "leader", + "messageId": messageID, + "completionSource": teamTaskCompletionTool, + "status": "dispatched", + "summary": "Bootstrap 团队介绍任务完成。已编制 product-discovery-team (Team 30) 完整团队画像,包含 4 名成员的角色职责、运行时状态、" + + "技术能力边界,以及 leader_mediated 协作模式下的任务流转、消息同步、上下文共享与可调用方法说明。详细报告已落盘至 /team/results/team-30-task-55/team-introduction.md。", + "resultMarkdown": "# Product Discovery Team (team-30) — Bootstrap 团队介绍\n\n" + + "## 团队架构\n\n**协作模式**:`leader_mediated`。Leader 是唯一入口,Worker 之间禁止直连。\n\n" + + "| 成员 | 角色 | 运行时 |\n|------|------|--------|\n| discovery-lead | Leader | OpenClaw Pro |\n| ui-designer | UI Designer | Hermes Pro |\n\n" + + "详细报告:/team/results/team-30-task-55/team-introduction.md", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178693-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt == nil { + t.Fatalf("server bootstrap tool reply should close the root task, got %#v", repo.updatedTask) + } + if len(repo.workItems) != 0 { + t.Fatalf("server bootstrap tool reply must not create synthetic assignments, got %#v", repo.workItems) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "task_completed" { + t.Fatalf("expected one normalized task_completed event, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventDoesNotCreateBootstrapAssignmentWorkItem(t *testing.T) { + teamID := 30 + taskID := 55 + messageID := "team-30-bootstrap-introduction" + task := &models.TeamTask{ID: taskID, TeamID: teamID, TargetMemberID: 120, MessageID: messageID, Status: models.TeamTaskStatusRunning, UpdatedAt: time.Now().UTC()} + leader := &models.TeamMember{ID: 120, TeamID: teamID, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + designer := &models.TeamMember{ID: 121, TeamID: teamID, MemberKey: "ui-designer", Role: "ui-ux-designer", Status: models.TeamMemberStatusIdle, Availability: models.TeamMemberAvailabilityIdle} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "ui-designer": designer}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: t.TempDir()} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "reply", + "memberId": "leader", + "messageId": messageID, + "text": "任务分派:请 ui-designer 补充团队能力边界说明。", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178694-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if len(repo.workItems) != 0 { + t.Fatalf("bootstrap/control-plane replies must not create work items, got %#v", repo.workItems) + } +} + +func TestProjectTeamEventBootstrapCompletionIgnoresStaleSyntheticWorkItem(t *testing.T) { + teamID := 30 + taskID := 55 + messageID := "team-30-bootstrap-introduction" + workspaceRoot := t.TempDir() + resultDir := filepath.Join(workspaceRoot, "teams", "user-1", "team-30-shared", "results", "team-30-task-55") + if err := os.MkdirAll(resultDir, 0o775); err != nil { + t.Fatalf("create result dir: %v", err) + } + if err := os.WriteFile(filepath.Join(resultDir, "team-introduction.md"), []byte("full report"), 0o664); err != nil { + t.Fatalf("write result: %v", err) + } + task := &models.TeamTask{ID: taskID, TeamID: teamID, TargetMemberID: 120, MessageID: messageID, Status: models.TeamTaskStatusRunning, UpdatedAt: time.Now().UTC()} + leader := &models.TeamMember{ID: 120, TeamID: teamID, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + designer := &models.TeamMember{ID: 121, TeamID: teamID, MemberKey: "ui-designer", Role: "ui-ux-designer", Status: models.TeamMemberStatusIdle, Availability: models.TeamMemberAvailabilityIdle} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader, "ui-designer": designer}, + workItems: []models.TeamWorkItem{{ + TeamID: teamID, + RootTaskID: taskID, + WorkID: "member-ui-designer", + OwnerMemberID: &designer.ID, + Title: "Assign to ui-designer", + Status: models.TeamTaskStatusDispatched, + UpdatedAt: time.Now().UTC(), + }}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: workspaceRoot} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "task_completed", + "memberId": "leader", + "messageId": messageID, + "status": "succeeded", + "summary": "Bootstrap complete.", + "resultMarkdown": "Bootstrap 完成。详见 /team/results/team-30-task-55/team-introduction.md", + "artifactRefs": []string{"/team/results/team-30-task-55/team-introduction.md"}, + "explicitCompletion": true, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178695-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded { + t.Fatalf("bootstrap completion must ignore stale synthetic work items, got %#v", repo.updatedTask) + } +} + +func TestProjectTeamEventDoesNotTreatLegacyAckAsCompletion(t *testing.T) { + teamID := 31 + taskID := 68 + messageID := "team-31-bootstrap-introduction" + task := &models.TeamTask{ID: taskID, TeamID: teamID, TargetMemberID: 120, MessageID: messageID, Status: models.TeamTaskStatusRunning, UpdatedAt: time.Now().UTC()} + leader := &models.TeamMember{ID: 120, TeamID: teamID, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: t.TempDir()} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "reply", + "memberId": "leader", + "messageId": messageID, + "text": "Redis Team task completed", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178693-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask != nil { + t.Fatalf("ack-only legacy reply must not close the root task, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "reply" { + t.Fatalf("expected ack to stay as reply, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventAcceptsExistingMarkdownArtifactReference(t *testing.T) { + teamID := 22 + taskID := 40 + messageID := "team-22-bootstrap-introduction" + workspaceRoot := t.TempDir() + resultDir := filepath.Join(workspaceRoot, "teams", "user-1", "team-22-shared", "results", "team-22-task-40") + if err := os.MkdirAll(resultDir, 0o775); err != nil { + t.Fatalf("create result dir: %v", err) + } + if err := os.WriteFile(filepath.Join(resultDir, "team-introduction-report.md"), []byte("full report"), 0o664); err != nil { + t.Fatalf("write result: %v", err) + } + task := &models.TeamTask{ID: taskID, TeamID: teamID, TargetMemberID: 120, MessageID: messageID, Status: models.TeamTaskStatusRunning, UpdatedAt: time.Now().UTC()} + leader := &models.TeamMember{ID: 120, TeamID: teamID, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: workspaceRoot} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "task_completed", + "memberId": "leader", + "messageId": messageID, + "status": "succeeded", + "summary": "Team introduction complete.", + "resultMarkdown": "Full report: `/team/results/team-22-task-40/team-introduction-report.md`.", + "artifactRefs": []string{"/team/results/team-22-task-40/team-introduction-report.md"}, + "explicitCompletion": true, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178688-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded { + t.Fatalf("existing Markdown artifact should allow completion, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "task_completed" { + t.Fatalf("expected task_completed event, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventResolvesProtocolV2CompletionByMessageID(t *testing.T) { + teamID := 22 + taskID := 52 + messageID := "team-22-bootstrap-introduction" + workspaceRoot := t.TempDir() + resultDir := filepath.Join(workspaceRoot, "teams", "user-1", "team-22-shared", "results", "team-22-task-52") + if err := os.MkdirAll(resultDir, 0o775); err != nil { + t.Fatalf("create result dir: %v", err) + } + if err := os.WriteFile(filepath.Join(resultDir, "result.md"), []byte("full report"), 0o664); err != nil { + t.Fatalf("write result: %v", err) + } + task := &models.TeamTask{ + ID: taskID, + TeamID: teamID, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + leader := &models.TeamMember{ + ID: 120, + TeamID: teamID, + MemberKey: "leader", + Role: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: workspaceRoot} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "protocolVersion": 2, + "event": "task_completed", + "eventId": "evt-team-22-task-52", + "completionId": "completion:22:team-22-task-52:leader", + "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, + "memberId": "leader", + "messageId": messageID, + "completionMessageId": messageID, + // Some runtimes preserve their own internal task identifier here. The + // root task must still resolve through the message identifiers above. + "taskId": "runtime-task-52", + "rootTaskId": "runtime-task-52", + "status": "succeeded", + "summary": "Team introduction complete.", + "resultMarkdown": "Full report: /team/results/team-22-task-52/result.md", + "artifactRefs": []string{"/team/results/team-22-task-52/result.md"}, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178690-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt == nil { + t.Fatalf("protocol v2 completion should resolve and close the root task, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "task_completed" { + t.Fatalf("expected accepted task_completed event, got %#v", repo.createdEvents) + } +} + +func TestProjectTeamEventMissingArtifactKeepsFinalAnswerInWarning(t *testing.T) { + teamID := 22 + taskID := 40 + messageID := "team-22-bootstrap-introduction" + task := &models.TeamTask{ID: taskID, TeamID: teamID, TargetMemberID: 120, MessageID: messageID, Status: models.TeamTaskStatusRunning, UpdatedAt: time.Now().UTC()} + leader := &models.TeamMember{ID: 120, TeamID: teamID, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: t.TempDir()} + finalBody := "# Delivery Team\n\nDetailed final answer." + payloadJSON, err := json.Marshal(map[string]interface{}{ + "protocolVersion": 3, + "event": "completion_proposed", + "eventId": "evt-missing-artifact", + "completionId": "completion:22:team-22-task-40:leader", + "attemptId": "attempt-missing-artifact", + "completionSource": teamTaskCompletionTool, + "memberId": "leader", + "messageId": messageID, + "taskId": "team-22-task-40", + "rootTaskId": "team-22-task-40", + "status": "succeeded", + "summary": "Team introduction complete.", + "resultMarkdown": finalBody, + "artifactRefs": []string{"/team/results/team-22-task-40/missing.md"}, + "explicitCompletion": true, + "rootTaskTerminal": true, + "workflowFinal": true, + "finalAnswerReady": true, + "remainingActions": []string{}, + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171178689-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if repo.updatedTask == nil || repo.updatedTask.Status == models.TeamTaskStatusSucceeded || repo.updatedTask.FinishedAt != nil { + t.Fatalf("missing artifact must reopen an incorrectly terminal task, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "message_warning" { + t.Fatalf("expected artifact warning event, got %#v", repo.createdEvents) + } + stored := map[string]interface{}{} + if err := json.Unmarshal([]byte(*repo.createdEvents[0].PayloadJSON), &stored); err != nil { + t.Fatalf("decode warning: %v", err) + } + if stored["resultMarkdown"] != finalBody || stored["summary"] != "Team introduction complete." { + t.Fatalf("artifact warning must preserve final answer, got %#v", stored) + } + if stored["artifactValidationMessage"] == "" { + t.Fatalf("expected separate artifact validation diagnostic, got %#v", stored) + } + if stored["completionDecision"] != teamCompletionDecisionRejected || stored["completionDecisionReason"] != "missing_artifacts" { + t.Fatalf("missing artifact must never acknowledge completion as accepted, got %#v", stored) + } + if len(repo.outboxRows) != 1 || !strings.Contains(repo.outboxRows[0].PayloadJSON, `"decision":"rejected"`) { + t.Fatalf("missing artifact rejection must be delivered reliably, got %#v", repo.outboxRows) + } +} + +func TestNormalizeTeamArtifactReferencesDoesNotCorruptSerializedJSON(t *testing.T) { + service := &teamService{runtimeWorkspaceRoot: "/workspaces"} + payload := map[string]interface{}{ + "raw": `{"resultMarkdown":"line one\n/team/results/team-22-task-40/result.md","artifactRefs":["/team/results/team-22-task-40/result.md"]}`, + } + service.normalizeTeamArtifactReferences(&models.Team{ID: 22, UserID: 1, SharedMountPath: "/team"}, payload) + got, _ := payload["raw"].(string) + if strings.Contains(got, "line one/n") || !strings.Contains(got, `line one\n`) { + t.Fatalf("serialized JSON escapes were corrupted: %q", got) + } +} + +func TestTeamTaskCompletionSignalsRemainBackwardCompatibleAcrossProtocolVersions(t *testing.T) { + v1 := map[string]interface{}{ + "status": "succeeded", + "resultMarkdown": "Legacy runtime final answer", + "explicitCompletion": true, + } + if !isTeamTaskCompletionSignal("task_completed", "succeeded", v1) { + t.Fatal("expected explicit legacy v1 task_completed event to remain supported") + } + plainV1 := cloneStringInterfaceMap(v1) + delete(plainV1, "explicitCompletion") + if isTeamTaskCompletionSignal("task_completed", "succeeded", plainV1) { + t.Fatal("legacy task_completed without an explicit completion marker must remain non-terminal") + } + + v2 := map[string]interface{}{ + "protocolVersion": 2, + "eventId": "evt-70", + "status": "succeeded", + "completionId": "completion:31:task-70:leader", + "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, + "taskId": "team-31-task-70", + "rootTaskId": "team-31-task-70", + "memberId": "leader", + "summary": "Protocol v2 final answer", + "resultMarkdown": "Protocol v2 final answer", + "artifactRefs": []interface{}{"/team/results/team-31-task-70/result.md"}, + } + if isTeamTaskCompletionSignal("reply", "succeeded", v2) { + t.Fatal("expected a v2 reply to remain non-terminal") + } + if !isTeamTaskCompletionSignal("task_completed", "succeeded", v2) { + t.Fatal("expected an explicit v2 task_completed event to be terminal") + } + v2WithoutArtifacts := cloneStringInterfaceMap(v2) + delete(v2WithoutArtifacts, "artifactRefs") + if !isTeamTaskCompletionSignal("task_completed", "succeeded", v2WithoutArtifacts) { + t.Fatal("expected explicit v2 completion with resultMarkdown to be terminal even without artifactRefs") + } + + missingBody := cloneStringInterfaceMap(v2) + delete(missingBody, "resultMarkdown") + if isTeamTaskCompletionSignal("task_completed", "succeeded", missingBody) { + t.Fatal("expected v2 completion without a result body to remain open") + } + + automatic := cloneStringInterfaceMap(v2) + automatic["explicitCompletion"] = false + automatic["completionSource"] = "runtime_processing" + if isTeamTaskCompletionSignal("task_completed", "succeeded", automatic) { + t.Fatal("expected automatic runtime success to remain non-terminal") + } +} + +func TestTeamTaskFailureSignalsRemainBackwardCompatibleAcrossProtocolVersions(t *testing.T) { + if !isTeamTaskFailureSignal("task_failed", "failed", map[string]interface{}{}) { + t.Fatal("expected legacy task_failed event to remain supported") + } + + v2RuntimeFailure := map[string]interface{}{ + "protocolVersion": 2, + "eventId": "evt-71", + "status": "failed", + "completionId": "completion:31:task-71:worker", + "completionSource": "runtime_error", + "taskId": "team-31-task-71", + "rootTaskId": "team-31-task-71", + "memberId": "worker", + "summary": "runtime failed", + "artifactRefs": []interface{}{"/team/results/team-31-task-71/result.md"}, + } + if !isTeamTaskFailureSignal("task_failed", "failed", v2RuntimeFailure) { + t.Fatal("expected structured v2 runtime failure to be terminal") + } + if isTeamTaskFailureSignal("reply", "failed", v2RuntimeFailure) { + t.Fatal("expected failed v2 reply to remain non-terminal") + } +} + +func TestAcceptedTeamCompletionIDOnlyDeduplicatesAcceptedTerminalEvents(t *testing.T) { + completionID := "completion:31:task-72:leader" + terminalPayload, err := json.Marshal(map[string]interface{}{ + "protocolVersion": 2, + "eventId": "evt-72", + "status": "succeeded", + "completionId": completionID, + "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, + "taskId": "team-31-task-72", + "rootTaskId": "team-31-task-72", + "memberId": "leader", + "summary": "Final answer", + "resultMarkdown": "Final answer", + "artifactRefs": []interface{}{"/team/results/team-31-task-72/result.md"}, + }) + if err != nil { + t.Fatalf("marshal terminal payload: %v", err) + } + warningPayload, err := json.Marshal(map[string]interface{}{ + "protocolVersion": 2, + "status": "blocked", + "completionId": completionID, + "completionSource": teamTaskCompletionTool, + }) + if err != nil { + t.Fatalf("marshal warning payload: %v", err) + } + + repo := &teamRepositoryStub{createdEvents: []models.TeamEvent{{ + TeamID: 31, + EventType: "message_warning", + PayloadJSON: stringPtr(string(warningPayload)), + }}} + service := &teamService{repo: repo} + duplicate, err := service.hasAcceptedTeamCompletionID(31, completionID) + if err != nil { + t.Fatalf("check warning completion ID: %v", err) + } + if duplicate { + t.Fatal("expected rejected artifact completion to remain retryable") + } + + repo.createdEvents = append(repo.createdEvents, models.TeamEvent{ + TeamID: 31, + EventType: "task_completed", + PayloadJSON: stringPtr(string(terminalPayload)), + }) + duplicate, err = service.hasAcceptedTeamCompletionID(31, completionID) + if err != nil { + t.Fatalf("check terminal completion ID: %v", err) + } + if !duplicate { + t.Fatal("expected accepted terminal completion ID to be deduplicated") + } +} + +func cloneStringInterfaceMap(source map[string]interface{}) map[string]interface{} { + clone := make(map[string]interface{}, len(source)) + for key, value := range source { + clone[key] = value + } + return clone +} + +func stringPtr(value string) *string { return &value } + +func TestTaskHasRecentActivityUsesBusinessWorkItems(t *testing.T) { + now := time.Now().UTC() + task := &models.TeamTask{ + ID: 88, + TeamID: 42, + MessageID: "team-42-task-root", + Status: models.TeamTaskStatusRunning, + UpdatedAt: now.Add(-time.Hour), + } + repo := &teamRepositoryStub{ + workItems: []models.TeamWorkItem{{ + TeamID: 42, + RootTaskID: 88, + WorkID: "collect-paper", + Status: models.TeamTaskStatusRunning, + UpdatedAt: now.Add(-time.Minute), + }}, + } + service := &teamService{repo: repo} + active, err := service.taskHasRecentActivity(&models.Team{ID: 42}, task, now.Add(-5*time.Minute)) + if err != nil { + t.Fatalf("taskHasRecentActivity returned error: %v", err) + } + if !active { + t.Fatal("expected a recently updated running work item to keep the root task active") + } +} + +func TestLeaderMediatedWorkerProgressIsNotConfirmedAsResult(t *testing.T) { + teamID := 46 + taskID := 79 + messageID := "team-46-task-1783559040281180893" + task := &models.TeamTask{ID: taskID, TeamID: teamID, TargetMemberID: 169, MessageID: messageID, Status: models.TeamTaskStatusRunning, UpdatedAt: time.Now().UTC()} + leader := &models.TeamMember{ID: 169, TeamID: teamID, MemberKey: "delivery-lead", Role: "leader", Status: models.TeamMemberStatusBusy, Availability: models.TeamMemberAvailabilityBusy} + developer := &models.TeamMember{ID: 170, TeamID: teamID, MemberKey: "developer", Role: "developer", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByID: map[int]*models.TeamMember{169: leader, 170: developer}, + membersByKey: map[string]*models.TeamMember{"delivery-lead": leader, "leader": leader, "developer": developer}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: t.TempDir()} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "reply", + "memberId": "developer", + "messageId": "msg-progress-1", + "rootTaskId": "team-46-task-79", + "rootMessageId": messageID, + "status": "running", + "text": "RAG Papers Task - Progress Update: Starting paper search and crawl. I found candidates and am now processing all PDFs.", + }) + if err != nil { + t.Fatal(err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1783559160000-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if len(repo.workItems) != 0 { + t.Fatalf("progress-only worker reply must not create a completed member result, got %#v", repo.workItems) + } + for _, event := range repo.createdEvents { + if event.EventType == "member_result_confirmed" { + t.Fatalf("progress-only worker reply must not emit result confirmation: %#v", event) + } + } +} + +func TestLeaderMediatedMonitorBlockerDoesNotCloseRootTask(t *testing.T) { + teamID := 46 + taskID := 79 + messageID := "team-46-task-1783559040281180893" + task := &models.TeamTask{ID: taskID, TeamID: teamID, TargetMemberID: 169, MessageID: messageID, Status: models.TeamTaskStatusRunning, UpdatedAt: time.Now().UTC()} + leader := &models.TeamMember{ID: 169, TeamID: teamID, MemberKey: "delivery-lead", Role: "leader", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByID: map[int]*models.TeamMember{169: leader}, + membersByKey: map[string]*models.TeamMember{"delivery-lead": leader, "leader": leader}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: t.TempDir()} + monitorSummary := "Monitoring agent for dev-papers-001 completed. Developer unresponsive after 3 consecutive status checks. Zero paper files found, no summary created. Blocker report sent to leader for investigation." + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "task_failed", + "memberId": "delivery-lead", + "messageId": messageID, + "rootTaskId": "team-46-task-79", + "rootMessageId": messageID, + "status": "failed", + "summary": monitorSummary, + }) + if err != nil { + t.Fatal(err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1783560082000-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask != nil && repo.updatedTask.Status == models.TeamTaskStatusFailed { + t.Fatalf("monitor blocker candidate must not close root task, got %#v", repo.updatedTask) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "message_warning" { + t.Fatalf("expected monitor blocker to be stored as warning, got %#v", repo.createdEvents) + } + stored := teamEventPayloadMap(repo.createdEvents[0]) + if !eventBool(stored, "monitorBlockerCandidate") || eventString(stored, "status") != "attention_required" { + t.Fatalf("expected monitor blocker candidate markers, got %#v", stored) + } +} + +func TestLeaderMediatedMemberResultReopensNonAuthoritativeMonitorFailure(t *testing.T) { + teamID := 46 + taskID := 79 + messageID := "team-46-task-1783559040281180893" + finishedAt := time.Now().UTC().Add(-5 * time.Minute) + monitorSummary := "Monitoring agent for dev-papers-001 completed. Developer unresponsive after 3 consecutive status checks. Zero paper files found, no summary created." + task := &models.TeamTask{ID: taskID, TeamID: teamID, TargetMemberID: 169, MessageID: messageID, Status: models.TeamTaskStatusFailed, FinishedAt: &finishedAt, ErrorMessage: &monitorSummary, UpdatedAt: time.Now().UTC()} + leader := &models.TeamMember{ID: 169, TeamID: teamID, MemberKey: "delivery-lead", Role: "leader", Status: models.TeamMemberStatusBusy, Availability: models.TeamMemberAvailabilityBusy} + developer := &models.TeamMember{ID: 170, TeamID: teamID, MemberKey: "developer", Role: "developer", Status: models.TeamMemberStatusBusy, CurrentTaskID: &taskID, Availability: models.TeamMemberAvailabilityBusy} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByID: map[int]*models.TeamMember{169: leader, 170: developer}, + membersByKey: map[string]*models.TeamMember{"delivery-lead": leader, "leader": leader, "developer": developer}, + } + service := &teamService{repo: repo, runtimeWorkspaceRoot: t.TempDir()} + resultText := "Final Delivery Confirmation - dev-papers-001 COMPLETE\n\nFiles are now at /team/artifacts/team-46-task-79/members/developer/dev-papers-001/report.md." + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "reply", + "memberId": "developer", + "messageId": "msg-final-1", + "rootTaskId": "team-46-task-79", + "rootMessageId": messageID, + "status": "succeeded", + "summary": "Developer delivered the paper report.", + "resultMarkdown": resultText, + }) + if err != nil { + t.Fatal(err) + } + + if err := service.projectTeamEvent(&models.Team{ID: teamID, UserID: 1, SharedMountPath: "/team", CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1783560514000-0", Fields: map[string]string{"payload": string(payloadJSON)}, + }); err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusRunning || repo.updatedTask.FinishedAt != nil || repo.updatedTask.ErrorMessage != nil { + t.Fatalf("member result should reopen non-authoritative monitor failure, got %#v", repo.updatedTask) + } + if len(repo.workItems) != 1 || repo.workItems[0].Status != models.TeamTaskStatusSucceeded || repo.workItems[0].WorkID != "member-developer" { + t.Fatalf("expected developer result work item, got %#v", repo.workItems) + } + if len(repo.createdEvents) < 2 || repo.createdEvents[len(repo.createdEvents)-1].EventType != "member_result_confirmed" { + t.Fatalf("expected member result confirmation event, got %#v", repo.createdEvents) + } +} + +func TestAssignmentMonitorEnvelopeIsNonTerminalAndAddressedToWorker(t *testing.T) { + now := time.Date(2026, 7, 9, 10, 30, 0, 0, time.UTC) + team := &models.Team{ID: 46} + task := &models.TeamTask{ID: 78, TeamID: 46, MessageID: "team-46-task-1783559040281180893"} + item := &models.TeamWorkItem{ID: 9001, WorkID: "dev-papers-001", Title: "Assign to developer", UpdatedAt: now.Add(-4 * time.Minute)} + owner := &models.TeamMember{ID: 301, TeamID: 46, MemberKey: "developer"} + + envelope, messageID := buildAssignmentStatusCheckEnvelope(team, task, item, owner, now) + if messageID != "monitor:team-46-task-78:dev-papers-001:9908850" { + t.Fatalf("unexpected monitor message id %q", messageID) + } + if envelope["intent"] != "assignment_status_check" || envelope["to"] != "developer" || envelope["from"] != "clawmanager-monitor" { + t.Fatalf("unexpected monitor routing: %#v", envelope) + } + if envelope["requiresCompletion"] != false || envelope["rootTaskId"] != "team-46-task-78" || envelope["workId"] != "dev-papers-001" { + t.Fatalf("monitor envelope should be non-terminal and assignment scoped: %#v", envelope) + } + if envelope["checkId"] != messageID || envelope["checkSequence"] != int64(9908850) || envelope["requestedAt"] == "" { + t.Fatalf("monitor envelope must carry stable check identity: %#v", envelope) + } + monitorPolicy, ok := envelope["monitorPolicy"].(map[string]interface{}) + if !ok || monitorPolicy["enabled"] != true || monitorPolicy["visibleToChat"] != true { + t.Fatalf("expected visible monitor policy on monitor envelope, got %#v", envelope["monitorPolicy"]) + } + if monitorPolicy["heartbeatEverySec"] != 30 || monitorPolicy["visibleHeartbeatEverySec"] != 180 { + t.Fatalf("expected monitor envelope to separate internal heartbeat and chat digest cadence, got %#v", monitorPolicy) + } + prompt, _ := envelope["prompt"].(string) + if !strings.Contains(prompt, "call team_update_progress") || !strings.Contains(prompt, "assignment_check_result") || strings.Contains(prompt, "call team_complete_task") { + t.Fatalf("monitor prompt should request progress without forcing completion: %s", prompt) + } + metadata, ok := envelope["metadata"].(map[string]interface{}) + if !ok || metadata["monitor"] != true || metadata["monitorType"] != "assignment_status_check" || metadata["eventKind"] != "assignment_check_requested" || metadata["visibleToChat"] != false { + t.Fatalf("unexpected monitor metadata: %#v", envelope["metadata"]) + } +} + +func TestAssignmentMonitorEligibilityAndThrottle(t *testing.T) { + cutoff := time.Date(2026, 7, 9, 10, 30, 0, 0, time.UTC) + ownerID := 301 + if !shouldMonitorTeamWorkItem(models.TeamWorkItem{OwnerMemberID: &ownerID, WorkID: "dev-papers-001", Status: models.TeamTaskStatusRunning, UpdatedAt: cutoff.Add(-time.Second)}, cutoff) { + t.Fatal("expected stale running worker item to be monitored") + } + if shouldMonitorTeamWorkItem(models.TeamWorkItem{OwnerMemberID: &ownerID, WorkID: "dev-papers-001", Status: models.TeamTaskStatusRunning, UpdatedAt: cutoff.Add(time.Second)}, cutoff) { + t.Fatal("fresh worker item should not be monitored") + } + if shouldMonitorTeamWorkItem(models.TeamWorkItem{OwnerMemberID: &ownerID, WorkID: "dev-papers-001", Status: models.TeamTaskStatusSucceeded, UpdatedAt: cutoff.Add(-time.Hour)}, cutoff) { + t.Fatal("terminal worker item should not be monitored") + } + if shouldMonitorTeamWorkItem(models.TeamWorkItem{WorkID: "dev-papers-001", Status: models.TeamTaskStatusRunning, UpdatedAt: cutoff.Add(-time.Hour)}, cutoff) { + t.Fatal("unowned worker item should not be monitored") + } + + service := &teamService{} + if !service.claimAssignmentMonitorSlot("46:78:dev-papers-001:301", cutoff) { + t.Fatal("expected first monitor slot claim to succeed") + } + if service.claimAssignmentMonitorSlot("46:78:dev-papers-001:301", cutoff.Add(teamAssignmentMonitorEvery-time.Second)) { + t.Fatal("expected monitor slot to be throttled inside interval") + } + if !service.claimAssignmentMonitorSlot("46:78:dev-papers-001:301", cutoff.Add(teamAssignmentMonitorEvery+time.Second)) { + t.Fatal("expected monitor slot to reopen after interval") + } +} + +func TestLeaderMediatedRecoverableWarningClassification(t *testing.T) { + team := &models.Team{ID: 47, CommunicationMode: teamCommunicationModeLeaderMediated} + task := &models.TeamTask{ID: 82, TeamID: 47, Status: models.TeamTaskStatusRunning} + member := &models.TeamMember{ID: 12, TeamID: 47, MemberKey: "developer", Role: "developer"} + if !isLeaderMediatedRecoverableWarning(team, "message_warning", map[string]interface{}{ + "artifactValidationFailed": true, + "rootTaskTerminal": false, + }, member, task) { + t.Fatal("artifact validation warnings should start recovery") + } + if !isLeaderMediatedRecoverableWarning(team, "message_warning", map[string]interface{}{ + "nonAuthoritative": true, + "rootTaskTerminal": false, + "eventKind": "completion_validation_warning", + "assignmentResult": "candidate", + "assignmentId": "member-developer", + "assignment_result": "candidate", + }, member, task) { + t.Fatal("non-authoritative worker warnings should start recovery") + } + if isLeaderMediatedRecoverableWarning(team, "message_warning", map[string]interface{}{ + "eventKind": "assignment_recovery_exhausted", + "rootTaskTerminal": false, + }, member, task) { + t.Fatal("recovery exhausted should not start another automatic recovery") + } + if isLeaderMediatedRecoverableWarning(team, "assignment_heartbeat", map[string]interface{}{ + "eventKind": "assignment_heartbeat", + "nonAuthoritative": true, + "rootTaskTerminal": false, + }, member, task) { + t.Fatal("heartbeat must not start recovery") + } + if isLeaderMediatedRecoverableWarning(team, "task_progress", map[string]interface{}{ + "eventKind": "assignment_check_result", + "nonAuthoritative": true, + "rootTaskTerminal": false, + }, member, task) { + t.Fatal("passive assignment check result must not start recovery") + } +} + +func TestProjectTeamEventDropsHeartbeatAfterTerminalTask(t *testing.T) { + taskID := 91 + messageID := "team-49-bootstrap-introduction" + finishedAt := time.Now().UTC().Add(-time.Minute) + task := &models.TeamTask{ + ID: taskID, + TeamID: 49, + TargetMemberID: 700, + MessageID: messageID, + Status: models.TeamTaskStatusSucceeded, + FinishedAt: &finishedAt, + UpdatedAt: finishedAt, + } + leader := &models.TeamMember{ + ID: 700, + TeamID: 49, + MemberKey: "delivery-lead", + Role: "leader", + Status: models.TeamMemberStatusIdle, + Availability: models.TeamMemberAvailabilityIdle, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"delivery-lead": leader}, + } + service := &teamService{repo: repo} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "assignment_heartbeat", + "eventKind": "assignment_heartbeat", + "memberId": "delivery-lead", + "messageId": messageID, + "taskId": "team-49-task-91", + "heartbeatSeq": 12, + "visibleToChat": true, + "summary": "Agent turn is still running", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 49, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1781171179000-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if len(repo.createdEvents) != 0 { + t.Fatalf("terminal heartbeat must not be stored as a chat event, got %#v", repo.createdEvents) + } + if repo.updatedTask != nil || repo.updatedMember != nil { + t.Fatalf("terminal heartbeat must not mutate task/member state, task=%#v member=%#v", repo.updatedTask, repo.updatedMember) + } +} + +func TestProjectTeamEventDropsHeartbeatAfterTerminalWorkItem(t *testing.T) { + rootTaskID := 95 + memberID := 701 + task := &models.TeamTask{ + ID: rootTaskID, + TeamID: 52, + TargetMemberID: 700, + MessageID: "team-52-task-1783601111", + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC().Add(-time.Minute), + } + developer := &models.TeamMember{ + ID: memberID, + TeamID: 52, + MemberKey: "developer", + Role: "developer", + Status: models.TeamMemberStatusIdle, + Availability: models.TeamMemberAvailabilityIdle, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{rootTaskID: task}, + membersByID: map[int]*models.TeamMember{memberID: developer}, + membersByKey: map[string]*models.TeamMember{"developer": developer}, + workItems: []models.TeamWorkItem{{ + TeamID: 52, + RootTaskID: rootTaskID, + WorkID: "member-developer", + OwnerMemberID: &memberID, + Title: "developer delivers result", + Status: models.TeamTaskStatusSucceeded, + UpdatedAt: time.Now().UTC().Add(-time.Minute), + }}, + } + service := &teamService{repo: repo} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "assignment_heartbeat", + "eventKind": "assignment_heartbeat", + "memberId": "developer", + "from": "developer", + "taskId": "team-52-task-95", + "rootTaskId": "team-52-task-95", + "workId": "member-developer", + "assignmentId": "member-developer", + "status": models.TeamTaskStatusRunning, + "nonAuthoritative": true, + "summary": "Agent turn is still running", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 52, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1783601200000-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if len(repo.createdEvents) != 0 { + t.Fatalf("post-terminal work item heartbeat must not be stored, got %#v", repo.createdEvents) + } + if repo.updatedTask != nil || repo.updatedMember != nil { + t.Fatalf("post-terminal work item heartbeat must not mutate task/member state, task=%#v member=%#v", repo.updatedTask, repo.updatedMember) + } +} + +func TestProjectTeamEventMapsGeneratedWorkerCompletionToExistingWorkItem(t *testing.T) { + rootTaskID := 87 + rootMessageID := "team-50-task-1783587467953008490" + ownerID := 183 + task := &models.TeamTask{ + ID: rootTaskID, + TeamID: 50, + TargetMemberID: 181, + MessageID: rootMessageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC().Add(-time.Minute), + } + reviewer := &models.TeamMember{ + ID: ownerID, + TeamID: 50, + MemberKey: "reviewer", + Role: "reviewer", + Status: models.TeamMemberStatusBusy, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{rootTaskID: task}, + membersByID: map[int]*models.TeamMember{ownerID: reviewer}, + membersByKey: map[string]*models.TeamMember{"reviewer": reviewer}, + workItems: []models.TeamWorkItem{{ + TeamID: 50, + RootTaskID: rootTaskID, + WorkID: "member-reviewer", + OwnerMemberID: &ownerID, + Title: "Assign to reviewer", + Status: models.TeamTaskStatusDispatched, + UpdatedAt: time.Now().UTC().Add(-2 * time.Minute), + }}, + } + service := &teamService{repo: repo} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "event": "task_completed", + "protocolVersion": 2, + "completionId": "completion:50:task_01d9:reviewer", + "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, + "assignmentResultOnly": true, + "memberId": "reviewer", + "from": "reviewer", + "to": "leader", + "taskId": "task_01d9ca72-9721-44ea-a389-d43bdf573521", + "rootTaskId": "task_01d9ca72-9721-44ea-a389-d43bdf573521", + "workId": "chat-app-review", + "status": models.TeamTaskStatusSucceeded, + "summary": "Review complete. Verdict: PASS.", + "resultMarkdown": "Review complete. Verdict: PASS.", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 50, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1783589347769-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + if len(repo.createdEvents) == 0 || repo.createdEvents[0].TaskID == nil || *repo.createdEvents[0].TaskID != rootTaskID { + t.Fatalf("expected generated runtime completion to attach to root task, got %#v", repo.createdEvents) + } + if len(repo.createdEvents) < 2 { + t.Fatalf("expected generated runtime completion to create a leader result notification, got %#v", repo.createdEvents) + } + notification := repo.createdEvents[len(repo.createdEvents)-1] + if notification.EventType != "member_result_confirmed" || notification.TaskID == nil || *notification.TaskID != rootTaskID { + t.Fatalf("expected member result confirmation on root task, got %#v", notification) + } + notificationPayload := teamEventPayloadMap(notification) + if eventString(notificationPayload, "rootTaskId") != "team-50-task-87" || + eventString(notificationPayload, "rootMessageId") != rootMessageID || + eventString(notificationPayload, "workId") != "member-reviewer" { + t.Fatalf("member result notification must preserve root context, got %#v", notificationPayload) + } + if eventBool(notificationPayload, "visibleToChat", "visible_to_chat") || + eventString(notificationPayload, "sourceCompletionId") != "completion:50:task_01d9:reviewer" || + eventString(notificationPayload, "sourceWorkId") != "chat-app-review" || + eventString(notificationPayload, "resultMarkdown", "result", "text") != "" { + t.Fatalf("member result confirmation must be hidden and linked to its source: %#v", notificationPayload) + } + if len(repo.outboxRows) != 1 || repo.outboxRows[0].MessageID == "" || repo.outboxRows[0].SourceEventID == "" { + t.Fatalf("member result confirmation must atomically create a delivery outbox row, got %#v", repo.outboxRows) + } + if len(repo.workItems) != 1 || repo.workItems[0].WorkID != "member-reviewer" || repo.workItems[0].Status != models.TeamTaskStatusSucceeded { + t.Fatalf("expected reviewer work item to close, got %#v", repo.workItems) + } + if repo.updatedTask == nil || repo.updatedTask.ID != rootTaskID || repo.updatedTask.Status != models.TeamTaskStatusRunning { + t.Fatalf("worker completion must touch but not close root task, got %#v", repo.updatedTask) + } + if repo.updatedMember == nil || repo.updatedMember.MemberKey != "reviewer" || repo.updatedMember.Status != models.TeamMemberStatusIdle { + t.Fatalf("expected reviewer member to become idle, got %#v", repo.updatedMember) + } +} + +func TestTerminalMemberPresenceCannotRestoreRunningState(t *testing.T) { + teamID := 51 + taskID := 91 + memberID := 220 + messageID := "team-51-task-91-root" + task := &models.TeamTask{ID: taskID, TeamID: teamID, TargetMemberID: 219, MessageID: messageID, Status: models.TeamTaskStatusRunning, UpdatedAt: time.Now().UTC()} + member := &models.TeamMember{ID: memberID, TeamID: teamID, MemberKey: "pm", Role: "product-manager", Status: models.TeamMemberStatusIdle, Availability: models.TeamMemberAvailabilityIdle, Progress: 100} + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByID: map[int]*models.TeamMember{memberID: member}, + membersByKey: map[string]*models.TeamMember{"pm": member}, + workItems: []models.TeamWorkItem{{ + TeamID: teamID, + RootTaskID: taskID, + WorkID: "member-pm", + OwnerMemberID: &memberID, + Status: models.TeamTaskStatusSucceeded, + }}, + } + service := &teamService{repo: repo} + payload, _ := json.Marshal(map[string]interface{}{ + "event": "presence", + "memberId": "pm", + "taskId": "team-51-task-91", + "rootTaskId": "team-51-task-91", + "availability": "busy", + "runtimeStatus": "running", + }) + if err := service.projectTeamEvent(&models.Team{ID: teamID, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, redisStreamMessage{ + ID: "1783593000000-0", Fields: map[string]string{"payload": string(payload)}, + }); err != nil { + t.Fatal(err) + } + if repo.updatedMember == nil || repo.updatedMember.Status != models.TeamMemberStatusIdle || repo.updatedMember.Availability != models.TeamMemberAvailabilityIdle || derefTeamString(repo.updatedMember.RuntimeStatus) != models.TeamTaskStatusSucceeded { + t.Fatalf("terminal worker presence restored running state: %#v", repo.updatedMember) + } +} + +func TestThreeWorkerResultsEachCreateOneConfirmationAndOutbox(t *testing.T) { + team := &models.Team{ID: 52, CommunicationMode: teamCommunicationModeLeaderMediated} + task := &models.TeamTask{ID: 93, TeamID: 52, TargetMemberID: 230, MessageID: "team-52-root", Status: models.TeamTaskStatusRunning} + leader := &models.TeamMember{ID: 230, TeamID: 52, MemberKey: "delivery-lead", Role: "leader"} + members := []*models.TeamMember{ + {ID: 231, TeamID: 52, MemberKey: "pm", Role: "product-manager"}, + {ID: 232, TeamID: 52, MemberKey: "designer", Role: "ui-designer"}, + {ID: 233, TeamID: 52, MemberKey: "architect", Role: "solution-architect"}, + } + repo := &teamRepositoryStub{ + membersByID: map[int]*models.TeamMember{230: leader, 231: members[0], 232: members[1], 233: members[2]}, + membersByKey: map[string]*models.TeamMember{"delivery-lead": leader, "pm": members[0], "designer": members[1], "architect": members[2]}, + } + for _, member := range members { + ownerID := member.ID + repo.workItems = append(repo.workItems, models.TeamWorkItem{ + TeamID: team.ID, + RootTaskID: task.ID, + WorkID: "member-" + member.MemberKey, + OwnerMemberID: &ownerID, + Status: models.TeamTaskStatusSucceeded, + }) + } + service := &teamService{repo: repo} + var wg sync.WaitGroup + errs := make(chan error, len(members)) + for index, member := range members { + index, member := index, member + wg.Add(1) + go func() { + defer wg.Done() + completionID := fmt.Sprintf("completion:52:93:%s", member.MemberKey) + payload := map[string]interface{}{ + "completionId": completionID, + "workId": "assignment-" + member.MemberKey, + "summary": fmt.Sprintf("%s 已完成任务", member.MemberKey), + "resultMarkdown": fmt.Sprintf("%s 已完成任务并提交产物。", member.MemberKey), + } + sourceEventID := fmt.Sprintf("evt-worker-%d", index) + errs <- service.createLeaderMediatedResultNotification(team, nil, task, member, payload, &models.TeamEvent{EventID: &sourceEventID, CreatedAt: time.Now().UTC()}) + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + confirmations := 0 + seenMembers := map[string]bool{} + for _, event := range repo.createdEvents { + if event.EventType != "member_result_confirmed" { + continue + } + confirmations++ + payload := teamEventPayloadMap(event) + seenMembers[eventString(payload, "memberId")] = true + } + if confirmations != 3 || len(seenMembers) != 3 || len(repo.outboxRows) != 3 || !seenMembers["pm"] { + t.Fatalf("three-worker confirmation reconciliation failed: events=%#v outbox=%#v", repo.createdEvents, repo.outboxRows) + } +} + +func TestLeaderSynthesisReminderCreatedWhenWorkersDone(t *testing.T) { + now := time.Now().UTC() + rootTaskID := 89 + leaderID := 201 + developerID := 202 + reviewerID := 203 + rootMessageID := "team-51-task-1783600000000" + task := &models.TeamTask{ + ID: rootTaskID, + TeamID: 51, + TargetMemberID: leaderID, + MessageID: rootMessageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: now.Add(-5 * time.Minute), + } + leader := &models.TeamMember{ID: leaderID, TeamID: 51, MemberKey: "delivery-lead", Role: "leader", Status: models.TeamMemberStatusBusy, Availability: models.TeamMemberAvailabilityBusy} + developer := &models.TeamMember{ID: developerID, TeamID: 51, MemberKey: "developer", Role: "developer", Status: models.TeamMemberStatusIdle, Availability: models.TeamMemberAvailabilityIdle} + reviewer := &models.TeamMember{ID: reviewerID, TeamID: 51, MemberKey: "reviewer", Role: "reviewer", Status: models.TeamMemberStatusIdle, Availability: models.TeamMemberAvailabilityIdle} + developerResult := `{"summary":"Build complete","resultMarkdown":"Developer delivered the chat app."}` + reviewerResult := `{"summary":"PASS","resultMarkdown":"Reviewer verdict: PASS."}` + items := []models.TeamWorkItem{ + {TeamID: 51, RootTaskID: rootTaskID, WorkID: "member-developer", OwnerMemberID: &developerID, Title: "Assign to developer", Status: models.TeamTaskStatusSucceeded, ResultJSON: &developerResult, UpdatedAt: now.Add(-4 * time.Minute)}, + {TeamID: 51, RootTaskID: rootTaskID, WorkID: "member-reviewer", OwnerMemberID: &reviewerID, Title: "Assign to reviewer", Status: models.TeamTaskStatusSucceeded, ResultJSON: &reviewerResult, UpdatedAt: now.Add(-4 * time.Minute)}, + } + membersByID := map[int]*models.TeamMember{ + leaderID: leader, + developerID: developer, + reviewerID: reviewer, + } + ready, resultItems := leaderMediatedRootNeedsSynthesisReminder(task, items, membersByID) + if !ready || len(resultItems) != 2 { + t.Fatalf("expected root to need leader synthesis after worker results, ready=%v items=%#v", ready, resultItems) + } + repo := &teamRepositoryStub{ + teamsByID: map[int]*models.Team{51: &models.Team{ID: 51, CommunicationMode: teamCommunicationModeLeaderMediated}}, + tasksByID: map[int]*models.TeamTask{rootTaskID: task}, + membersByID: map[int]*models.TeamMember{leaderID: leader, developerID: developer, reviewerID: reviewer}, + membersByKey: map[string]*models.TeamMember{"delivery-lead": leader, "developer": developer, "reviewer": reviewer}, + workItems: items, + } + service := &teamService{repo: repo} + if err := service.createLeaderSynthesisReminder(&models.Team{ID: 51, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, task, leader, resultItems, now); err != nil { + t.Fatalf("createLeaderSynthesisReminder returned error: %v", err) + } + if len(repo.createdEvents) != 1 { + t.Fatalf("expected one leader synthesis reminder event, got %#v", repo.createdEvents) + } + event := repo.createdEvents[0] + if event.EventType != "leader_synthesis_reminder" || event.TaskID == nil || *event.TaskID != rootTaskID || event.MemberID == nil || *event.MemberID != leaderID { + t.Fatalf("unexpected leader synthesis reminder event: %#v", event) + } + payload := teamEventPayloadMap(event) + if eventString(payload, "rootTaskId") != "team-51-task-89" || + eventString(payload, "rootMessageId") != rootMessageID || + eventString(payload, "workId") != "leader-final-synthesis" { + t.Fatalf("leader synthesis reminder must preserve root context, got %#v", payload) + } + memberResults, _ := payload["memberResults"].([]interface{}) + if len(memberResults) != 2 { + t.Fatalf("expected reminder to carry member result summaries, got %#v", payload["memberResults"]) + } +} + +func TestReconcileDeferredCompletionNeverReusesReportFromOlderPlan(t *testing.T) { + now := time.Now().UTC() + taskID := 194 + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: "team-31-task-194", + Status: models.TeamTaskStatusRunning, + WorkflowState: teamWorkflowStateAwaitingLeaderDecision, + PlanVersion: 2, + LedgerVersion: 7, + UpdatedAt: now, + } + leader := &models.TeamMember{ID: 120, TeamID: 31, MemberKey: "leader", Role: "leader", Status: models.TeamMemberStatusBusy} + payloadJSON, err := json.Marshal(map[string]interface{}{ + "protocolVersion": 3, + "event": "completion_deferred", + "completionId": "completion:31:team-31-task-194:leader", + "completionSource": teamTaskCompletionTool, + "explicitCompletion": true, + "rootTaskTerminal": false, + "workflowFinal": true, + "finalAnswerReady": true, + "remainingActions": []string{}, + "planVersion": 1, + "ledgerVersion": 3, + "resultMarkdown": "# 第一阶段报告\n\n这份报告不包含第二阶段产物。", + }) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + eventID := "old-plan-deferred" + repo := &teamRepositoryStub{ + tasksByID: map[int]*models.TeamTask{taskID: task}, + membersByKey: map[string]*models.TeamMember{"leader": leader}, + createdEvents: []models.TeamEvent{{ + TeamID: task.TeamID, TaskID: &taskID, MemberID: &leader.ID, EventID: &eventID, + EventType: "completion_deferred", PayloadJSON: stringPtr(string(payloadJSON)), + }}, + } + service := &teamService{repo: repo} + reconciled, err := service.reconcileDeferredTeamCompletion(&models.Team{ID: 31, CommunicationMode: teamCommunicationModeLeaderMediated}, nil, task, leader) + if err != nil { + t.Fatalf("reconcileDeferredTeamCompletion returned error: %v", err) + } + if reconciled || task.Status == models.TeamTaskStatusSucceeded || len(repo.createdEvents) != 1 { + t.Fatalf("an older plan report must not be auto-accepted after plan advancement: reconciled=%v task=%#v events=%#v", reconciled, task, repo.createdEvents) + } +} + +type teamRepositoryStub struct { + mu sync.Mutex + teamsByID map[int]*models.Team + membersByID map[int]*models.TeamMember + membersByKey map[string]*models.TeamMember + tasksByID map[int]*models.TeamTask + tasksByMessageID map[string]*models.TeamTask + createdEvents []models.TeamEvent + workItems []models.TeamWorkItem + workflowPhases []models.TeamWorkflowPhase + outboxRows []models.TeamEventOutbox + updatedTask *models.TeamTask + updatedMember *models.TeamMember + updatedTeam *models.Team +} + +type teamOpenClawConfigPlannerStub struct { + calls int + userID int + plan *OpenClawConfigPlan + nextPlan *OpenClawConfigPlan + err error +} + +func (s *teamOpenClawConfigPlannerStub) PlanWithoutTeamMemberLeaderOnlyChannels(userID int, plan *OpenClawConfigPlan) (*OpenClawConfigPlan, error) { + s.calls++ + s.userID = userID + s.plan = plan + return s.nextPlan, s.err +} + +func (s *teamRepositoryStub) CreateTeam(team *models.Team) error { return nil } +func (s *teamRepositoryStub) UpdateTeam(team *models.Team) error { + clone := *team + s.updatedTeam = &clone + return nil +} +func (s *teamRepositoryStub) GetTeamByID(id int) (*models.Team, error) { + if s.teamsByID != nil { + return s.teamsByID[id], nil + } + return nil, nil +} +func (s *teamRepositoryStub) GetTeamByUserIDAndName(userID int, name string) (*models.Team, error) { + return nil, nil +} +func (s *teamRepositoryStub) ExistsByUserIDAndName(userID int, name string) (bool, error) { + return false, nil +} +func (s *teamRepositoryStub) ListTeamsByUserID(userID int, offset, limit int) ([]models.Team, error) { + return nil, nil +} +func (s *teamRepositoryStub) ListActiveTeams() ([]models.Team, error) { return nil, nil } +func (s *teamRepositoryStub) CountTeamsByUserID(userID int) (int, error) { + return 0, nil +} +func (s *teamRepositoryStub) CreateMember(member *models.TeamMember) error { return nil } +func (s *teamRepositoryStub) UpdateMember(member *models.TeamMember) error { + clone := *member + s.updatedMember = &clone + return nil +} +func (s *teamRepositoryStub) GetMemberByID(id int) (*models.TeamMember, error) { + if s.membersByID != nil { + return s.membersByID[id], nil + } + return nil, nil +} +func (s *teamRepositoryStub) GetMemberByTeamKey(teamID int, memberKey string) (*models.TeamMember, error) { + if s.membersByKey == nil { + return nil, nil + } + member := s.membersByKey[memberKey] + if member == nil || member.TeamID != teamID { + return nil, nil + } + return member, nil +} +func (s *teamRepositoryStub) ListMembersByTeamID(teamID int) ([]models.TeamMember, error) { + result := make([]models.TeamMember, 0) + seen := map[int]bool{} + for _, member := range s.membersByKey { + if member != nil && member.TeamID == teamID && !seen[member.ID] { + result = append(result, *member) + seen[member.ID] = true + } + } + for _, member := range s.membersByID { + if member != nil && member.TeamID == teamID && !seen[member.ID] { + result = append(result, *member) + seen[member.ID] = true + } + } + return result, nil +} +func (s *teamRepositoryStub) CreateTask(task *models.TeamTask) error { return nil } +func (s *teamRepositoryStub) UpdateTask(task *models.TeamTask) error { + clone := *task + s.updatedTask = &clone + return nil +} +func (s *teamRepositoryStub) GetTaskByID(id int) (*models.TeamTask, error) { + if s.tasksByID != nil { + return s.tasksByID[id], nil + } + return nil, nil +} +func (s *teamRepositoryStub) GetTaskByMessageID(teamID int, messageID string) (*models.TeamTask, error) { + if s.tasksByMessageID == nil { + return nil, nil + } + task := s.tasksByMessageID[messageID] + if task == nil || task.TeamID != teamID { + return nil, nil + } + return task, nil +} +func (s *teamRepositoryStub) ListTasksByTeamID(teamID int, limit int) ([]models.TeamTask, error) { + return nil, nil +} +func (s *teamRepositoryStub) ListTasksBeforeID(teamID, beforeID, limit int) ([]models.TeamTask, error) { + return nil, nil +} +func (s *teamRepositoryStub) ListStaleCandidateTasks(cutoff time.Time, limit int) ([]models.TeamTask, error) { + return nil, nil +} +func (s *teamRepositoryStub) CreateEvent(event *models.TeamEvent) error { + s.mu.Lock() + defer s.mu.Unlock() + clone := *event + s.createdEvents = append(s.createdEvents, clone) + return nil +} +func (s *teamRepositoryStub) EventExistsByStreamID(teamID int, streamID string) (bool, error) { + return false, nil +} +func (s *teamRepositoryStub) EventExistsByEventID(teamID int, eventID string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + for _, event := range s.createdEvents { + if event.TeamID == teamID && event.EventID != nil && *event.EventID == eventID { + return true, nil + } + } + return false, nil +} +func (s *teamRepositoryStub) EventExistsByCompletionID(teamID int, completionID string) (bool, error) { + for _, event := range s.createdEvents { + if event.TeamID == teamID { + payload := teamEventPayloadMap(event) + if eventString(payload, "completionId", "completion_id") == completionID && + (isTeamTaskCompletionSignal(event.EventType, normalizedTeamTaskEventStatus(payload), payload) || + isTeamTaskFailureSignal(event.EventType, normalizedTeamTaskEventStatus(payload), payload)) { + return true, nil + } + } + } + return false, nil +} +func (s *teamRepositoryStub) ListEventsByTeamID(teamID int, limit int) ([]models.TeamEvent, error) { + s.mu.Lock() + defer s.mu.Unlock() + events := make([]models.TeamEvent, 0, len(s.createdEvents)) + for _, event := range s.createdEvents { + if event.TeamID == teamID { + events = append(events, event) + } + } + if limit > 0 && len(events) > limit { + events = events[:limit] + } + return events, nil +} +func (s *teamRepositoryStub) ListEventsBeforeID(teamID, beforeID, limit int) ([]models.TeamEvent, error) { + return nil, nil +} +func (s *teamRepositoryStub) UpsertWorkItem(item *models.TeamWorkItem) error { + s.mu.Lock() + defer s.mu.Unlock() + for idx := range s.workItems { + if s.workItems[idx].TeamID == item.TeamID && s.workItems[idx].RootTaskID == item.RootTaskID && s.workItems[idx].WorkID == item.WorkID { + existing := s.workItems[idx] + newRevision := item.Revision > existing.Revision + existingTerminal := existing.Status == models.TeamTaskStatusSucceeded || existing.Status == models.TeamTaskStatusFailed || existing.Status == models.TeamTaskStatusStale + itemTerminal := item.Status == models.TeamTaskStatusSucceeded || item.Status == models.TeamTaskStatusFailed || item.Status == models.TeamTaskStatusStale + reopeningCurrent := !newRevision && existingTerminal && !itemTerminal && + !item.UpdatedAt.IsZero() && + (existing.UpdatedAt.IsZero() || item.UpdatedAt.After(existing.UpdatedAt)) + if !newRevision && !reopeningCurrent { + if item.ResultJSON == nil { + item.ResultJSON = existing.ResultJSON + } + if item.FinishedAt == nil { + item.FinishedAt = existing.FinishedAt + } + } + s.workItems[idx] = *item + return nil + } + } + s.workItems = append(s.workItems, *item) + return nil +} +func (s *teamRepositoryStub) InvalidateWorkItemReview(workItemID int, updatedAt time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + for idx := range s.workItems { + if s.workItems[idx].ID == workItemID && s.workItems[idx].ReviewRequired && s.workItems[idx].ValidatedRevision != nil { + s.workItems[idx].ValidatedRevision = nil + s.workItems[idx].UpdatedAt = updatedAt + } + } + return nil +} +func (s *teamRepositoryStub) ListWorkItemsByRootTaskID(rootTaskID int) ([]models.TeamWorkItem, error) { + s.mu.Lock() + defer s.mu.Unlock() + result := make([]models.TeamWorkItem, 0) + for _, item := range s.workItems { + if item.RootTaskID == rootTaskID { + result = append(result, item) + } + } + return result, nil +} +func (s *teamRepositoryStub) ListWorkItemsByTeamID(teamID int, limit int) ([]models.TeamWorkItem, error) { + result := make([]models.TeamWorkItem, 0) + for _, item := range s.workItems { + if item.TeamID == teamID { + result = append(result, item) + } + } + return result, nil +} +func (s *teamRepositoryStub) UpsertWorkflowPhase(phase *models.TeamWorkflowPhase) error { + s.mu.Lock() + defer s.mu.Unlock() + for idx := range s.workflowPhases { + current := s.workflowPhases[idx] + if current.RootTaskID == phase.RootTaskID && current.PhaseID == phase.PhaseID && current.PlanVersion == phase.PlanVersion { + s.workflowPhases[idx] = *phase + return nil + } + } + clone := *phase + clone.ID = len(s.workflowPhases) + 1 + phase.ID = clone.ID + s.workflowPhases = append(s.workflowPhases, clone) + return nil +} +func (s *teamRepositoryStub) ListWorkflowPhasesByRootTaskID(rootTaskID int) ([]models.TeamWorkflowPhase, error) { + s.mu.Lock() + defer s.mu.Unlock() + result := make([]models.TeamWorkflowPhase, 0) + for _, phase := range s.workflowPhases { + if phase.RootTaskID == rootTaskID { + result = append(result, phase) + } + } + return result, nil +} +func (s *teamRepositoryStub) AcceptRootCompletion(task *models.TeamTask, expectedLedgerVersion int64, event *models.TeamEvent, outbox *models.TeamEventOutbox) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + current := s.tasksByID[task.ID] + if current != nil && current != task && (current.LedgerVersion != expectedLedgerVersion || current.AcceptedCompletionID != nil || isTerminalTeamTaskStatus(current.Status)) { + return false, nil + } + cloneTask := *task + if s.tasksByID == nil { + s.tasksByID = map[int]*models.TeamTask{} + } + s.tasksByID[task.ID] = &cloneTask + s.updatedTask = &cloneTask + cloneEvent := *event + cloneEvent.ID = len(s.createdEvents) + 1 + event.ID = cloneEvent.ID + s.createdEvents = append(s.createdEvents, cloneEvent) + cloneOutbox := *outbox + cloneOutbox.ID = len(s.outboxRows) + 1 + outbox.ID = cloneOutbox.ID + s.outboxRows = append(s.outboxRows, cloneOutbox) + return true, nil +} +func (s *teamRepositoryStub) ConfirmWorkItemResult(item *models.TeamWorkItem, event *models.TeamEvent, outbox *models.TeamEventOutbox) error { + s.mu.Lock() + defer s.mu.Unlock() + updated := false + for idx := range s.workItems { + if s.workItems[idx].TeamID == item.TeamID && s.workItems[idx].RootTaskID == item.RootTaskID && s.workItems[idx].WorkID == item.WorkID { + s.workItems[idx] = *item + updated = true + break + } + } + if !updated { + s.workItems = append(s.workItems, *item) + } + eventExists := false + for idx := range s.createdEvents { + current := s.createdEvents[idx] + if current.TeamID == event.TeamID && current.EventID != nil && *current.EventID == derefTeamString(event.EventID) { + eventExists = true + break + } + } + if !eventExists { + s.createdEvents = append(s.createdEvents, *event) + } + for idx := range s.outboxRows { + if s.outboxRows[idx].TeamID == outbox.TeamID && s.outboxRows[idx].Destination == outbox.Destination && s.outboxRows[idx].MessageID == outbox.MessageID { + return nil + } + } + clone := *outbox + clone.ID = len(s.outboxRows) + 1 + outbox.ID = clone.ID + s.outboxRows = append(s.outboxRows, clone) + return nil +} +func (s *teamRepositoryStub) ListPendingEventOutbox(now time.Time, limit int) ([]models.TeamEventOutbox, error) { + result := make([]models.TeamEventOutbox, 0) + for _, row := range s.outboxRows { + if row.Status == "pending" && !row.AvailableAt.After(now) { + result = append(result, row) + } + } + if limit > 0 && len(result) > limit { + result = result[:limit] + } + return result, nil +} +func (s *teamRepositoryStub) CreateEventOutbox(outbox *models.TeamEventOutbox) error { + if outbox == nil { + return nil + } + for idx := range s.outboxRows { + if s.outboxRows[idx].TeamID == outbox.TeamID && s.outboxRows[idx].MessageID == outbox.MessageID { + return nil + } + } + clone := *outbox + clone.ID = len(s.outboxRows) + 1 + outbox.ID = clone.ID + s.outboxRows = append(s.outboxRows, clone) + return nil +} +func (s *teamRepositoryStub) MarkEventOutboxDelivered(id int, deliveredAt time.Time) error { + for idx := range s.outboxRows { + if s.outboxRows[idx].ID == id { + s.outboxRows[idx].Status = "delivered" + s.outboxRows[idx].DeliveredAt = &deliveredAt + } + } + return nil +} +func (s *teamRepositoryStub) MarkEventOutboxFailed(id int, availableAt time.Time, cause string) error { + for idx := range s.outboxRows { + if s.outboxRows[idx].ID == id { + s.outboxRows[idx].Status = "pending" + s.outboxRows[idx].Attempts++ + s.outboxRows[idx].AvailableAt = availableAt + s.outboxRows[idx].LastError = &cause + } + } + return nil +} diff --git a/backend/internal/services/user_service.go b/backend/internal/services/user_service.go new file mode 100644 index 0000000..e80a8b8 --- /dev/null +++ b/backend/internal/services/user_service.go @@ -0,0 +1,210 @@ +package services + +import ( + "errors" + "fmt" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/utils" +) + +// UserService defines the interface for user operations +type UserService interface { + CreateUser(username, email, password, role string) (*models.User, error) + GetUserByID(id int) (*models.User, error) + GetUserByUsername(username string) (*models.User, error) + ListUsers(offset, limit int) ([]models.User, error) + CountUsers() (int, error) + UpdateUser(user *models.User) error + DeleteUser(id int) error + UpdateUserRole(id int, role string) error + CreateDefaultQuota(userID int) error +} + +func defaultPasswordForRole(role string) string { + return DefaultPasswordForRole(role) +} + +// userService implements UserService +type userService struct { + userRepo repository.UserRepository + quotaRepo repository.QuotaRepository +} + +// NewUserService creates a new user service +func NewUserService(userRepo repository.UserRepository, quotaRepo repository.QuotaRepository) UserService { + return &userService{ + userRepo: userRepo, + quotaRepo: quotaRepo, + } +} + +// CreateUser creates a new user (admin only) +func (s *userService) CreateUser(username, email, password, role string) (*models.User, error) { + if password == "" { + password = defaultPasswordForRole(role) + } + + // Check if username already exists + existingUser, err := s.userRepo.GetByUsername(username) + if err != nil { + return nil, fmt.Errorf("failed to check username: %w", err) + } + if existingUser != nil { + return nil, errors.New("username already exists") + } + + // Check if email already exists + existingUser, err = s.userRepo.GetByEmail(email) + if err != nil { + return nil, fmt.Errorf("failed to check email: %w", err) + } + if existingUser != nil { + return nil, errors.New("email already exists") + } + + // Hash password + passwordHash, err := utils.HashPassword(password) + if err != nil { + return nil, fmt.Errorf("failed to hash password: %w", err) + } + + // Create user + user := &models.User{ + Username: username, + Email: email, + PasswordHash: passwordHash, + Role: role, + IsActive: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + if err := s.userRepo.Create(user); err != nil { + return nil, fmt.Errorf("failed to create user: %w", err) + } + + // Create default quota for user + if _, err := s.quotaRepo.CreateDefaultQuota(user.ID); err != nil { + return nil, fmt.Errorf("failed to create default quota: %w", err) + } + + return user, nil +} + +// GetUserByID gets a user by ID +func (s *userService) GetUserByID(id int) (*models.User, error) { + user, err := s.userRepo.GetByID(id) + if err != nil { + return nil, fmt.Errorf("failed to get user: %w", err) + } + if user == nil { + return nil, errors.New("user not found") + } + return user, nil +} + +// GetUserByUsername gets a user by username +func (s *userService) GetUserByUsername(username string) (*models.User, error) { + user, err := s.userRepo.GetByUsername(username) + if err != nil { + return nil, fmt.Errorf("failed to get user: %w", err) + } + if user == nil { + return nil, errors.New("user not found") + } + return user, nil +} + +// ListUsers lists all users with pagination +func (s *userService) ListUsers(offset, limit int) ([]models.User, error) { + users, err := s.userRepo.List(offset, limit) + if err != nil { + return nil, fmt.Errorf("failed to list users: %w", err) + } + return users, nil +} + +// CountUsers counts all users +func (s *userService) CountUsers() (int, error) { + count, err := s.userRepo.Count() + if err != nil { + return 0, fmt.Errorf("failed to count users: %w", err) + } + return count, nil +} + +// UpdateUser updates a user +func (s *userService) UpdateUser(user *models.User) error { + existingUser, err := s.userRepo.GetByID(user.ID) + if err != nil { + return fmt.Errorf("failed to get user: %w", err) + } + if existingUser == nil { + return errors.New("user not found") + } + + // Update allowed fields + if user.Email != "" { + existingUser.Email = user.Email + } + existingUser.IsActive = user.IsActive + existingUser.UpdatedAt = time.Now() + + if err := s.userRepo.Update(existingUser); err != nil { + return fmt.Errorf("failed to update user: %w", err) + } + + return nil +} + +// DeleteUser deletes a user +func (s *userService) DeleteUser(id int) error { + user, err := s.userRepo.GetByID(id) + if err != nil { + return fmt.Errorf("failed to get user: %w", err) + } + if user == nil { + return errors.New("user not found") + } + + // Delete user's quota first + if err := s.quotaRepo.DeleteByUserID(id); err != nil { + return fmt.Errorf("failed to delete user quota: %w", err) + } + + // Delete user + if err := s.userRepo.Delete(id); err != nil { + return fmt.Errorf("failed to delete user: %w", err) + } + + return nil +} + +// UpdateUserRole updates a user's role +func (s *userService) UpdateUserRole(id int, role string) error { + user, err := s.userRepo.GetByID(id) + if err != nil { + return fmt.Errorf("failed to get user: %w", err) + } + if user == nil { + return errors.New("user not found") + } + + user.Role = role + user.UpdatedAt = time.Now() + + if err := s.userRepo.Update(user); err != nil { + return fmt.Errorf("failed to update user role: %w", err) + } + + return nil +} + +// CreateDefaultQuota creates default quota for a user +func (s *userService) CreateDefaultQuota(userID int) error { + _, err := s.quotaRepo.CreateDefaultQuota(userID) + return err +} diff --git a/backend/internal/services/websocket_service.go b/backend/internal/services/websocket_service.go new file mode 100644 index 0000000..5f13491 --- /dev/null +++ b/backend/internal/services/websocket_service.go @@ -0,0 +1,424 @@ +package services + +import ( + "context" + "encoding/json" + "log" + "net/http" + "strings" + "sync" + "time" + + "clawreef/internal/models" + + "github.com/gorilla/websocket" +) + +var upgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true + }, + ReadBufferSize: 1024, + WriteBufferSize: 1024, +} + +// Client represents a WebSocket client +type Client struct { + ID int + UserID int + Role string + Topic WebSocketTopic + Conn *websocket.Conn + Send chan []byte + hub *Hub +} + +type WebSocketTopic string + +const ( + WebSocketTopicUser WebSocketTopic = "user" + WebSocketTopicRuntimeAdmin WebSocketTopic = "runtime_admin" +) + +// Hub maintains the set of active clients and broadcasts messages +type Hub struct { + clients map[*Client]bool + broadcast chan *Message + register chan *Client + unregister chan *Client + mu sync.RWMutex + stop chan struct{} +} + +// Message represents a WebSocket message +type Message struct { + Type string `json:"type"` + UserID int `json:"user_id,omitempty"` + InstanceID int `json:"instance_id,omitempty"` + Data interface{} `json:"data"` + Timestamp time.Time `json:"timestamp"` +} + +// InstanceStatusUpdate represents an instance status update +type InstanceStatusUpdate struct { + InstanceID int `json:"instance_id"` + Status string `json:"status"` + PodName string `json:"pod_name,omitempty"` + PodIP string `json:"pod_ip,omitempty"` + UpdatedAt string `json:"updated_at"` +} + +var ( + hub *Hub + hubOnce sync.Once +) + +// GetHub returns the global hub instance +func GetHub() *Hub { + hubOnce.Do(func() { + hub = NewHub() + go hub.Run() + }) + return hub +} + +// NewHub creates a new Hub instance +func NewHub() *Hub { + return &Hub{ + clients: make(map[*Client]bool), + broadcast: make(chan *Message), + register: make(chan *Client), + unregister: make(chan *Client), + stop: make(chan struct{}), + } +} + +// Run starts the hub's main loop +func (h *Hub) Run() { + for { + select { + case <-h.stop: + h.mu.Lock() + for client := range h.clients { + close(client.Send) + delete(h.clients, client) + } + h.mu.Unlock() + log.Println("WebSocket hub stopped") + return + + case client := <-h.register: + h.mu.Lock() + h.clients[client] = true + h.mu.Unlock() + log.Printf("WebSocket client registered: user=%d", client.UserID) + + case client := <-h.unregister: + h.mu.Lock() + if _, ok := h.clients[client]; ok { + delete(h.clients, client) + close(client.Send) + } + h.mu.Unlock() + log.Printf("WebSocket client unregistered: user=%d", client.UserID) + + case message := <-h.broadcast: + h.mu.RLock() + clients := make([]*Client, 0, len(h.clients)) + for client := range h.clients { + // Filter by user ID if specified + if message.UserID == 0 || client.UserID == message.UserID { + clients = append(clients, client) + } + } + h.mu.RUnlock() + + for _, client := range clients { + select { + case client.Send <- mustEncode(message): + default: + // Client's send channel is full, close it + h.mu.Lock() + close(client.Send) + delete(h.clients, client) + h.mu.Unlock() + } + } + } + } +} + +// mustEncode encodes a message to JSON, panics on error +func mustEncode(msg *Message) []byte { + data, err := json.Marshal(msg) + if err != nil { + log.Printf("Failed to encode message: %v", err) + return []byte(`{"type":"error","data":"encoding error"}`) + } + return data +} + +// BroadcastInstanceStatus broadcasts instance status update to relevant clients +func (h *Hub) BroadcastInstanceStatus(userID int, instance *models.Instance) { + update := InstanceStatusUpdate{ + InstanceID: instance.ID, + Status: instance.Status, + UpdatedAt: instance.UpdatedAt.Format(time.RFC3339), + } + + if instance.PodName != nil { + update.PodName = *instance.PodName + } + if instance.PodIP != nil { + update.PodIP = *instance.PodIP + } + + msg := &Message{ + Type: "instance_status", + UserID: userID, + Data: update, + Timestamp: time.Now(), + } + h.broadcast <- msg +} + +// BroadcastToAll broadcasts a message to all connected clients +func (h *Hub) BroadcastToAll(msgType string, data interface{}) { + msg := &Message{ + Type: msgType, + Data: data, + Timestamp: time.Now(), + } + h.broadcast <- msg +} + +func (h *Hub) BroadcastRuntimeAdmin(msgType string, data interface{}) { + msg := &Message{ + Type: msgType, + Data: data, + Timestamp: time.Now(), + } + h.broadcastRuntimeAdminMessage(msg) +} + +func (h *Hub) broadcastRuntimeAdminMessage(msg *Message) { + h.mu.RLock() + clients := make([]*Client, 0, len(h.clients)) + for client := range h.clients { + if client.Role == "admin" && client.Topic == WebSocketTopicRuntimeAdmin { + clients = append(clients, client) + } + } + h.mu.RUnlock() + + encoded := mustEncode(msg) + for _, client := range clients { + select { + case client.Send <- encoded: + default: + h.mu.Lock() + if _, ok := h.clients[client]; ok { + close(client.Send) + delete(h.clients, client) + } + h.mu.Unlock() + } + } +} + +// ServeWS handles WebSocket connections +func ServeWS(hub *Hub, w http.ResponseWriter, r *http.Request, userID int) { + ServeWSWithOptions(hub, w, r, userID, "", WebSocketTopicUser) +} + +func ServeWSWithOptions(hub *Hub, w http.ResponseWriter, r *http.Request, userID int, role string, topic WebSocketTopic) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Printf("Failed to upgrade WebSocket: %v", err) + return + } + if topic == "" { + topic = WebSocketTopicUser + } + + client := &Client{ + UserID: userID, + Role: strings.TrimSpace(role), + Topic: topic, + Conn: conn, + Send: make(chan []byte, 256), + hub: hub, + } + + client.hub.register <- client + + // Start goroutines for reading and writing + go client.writePump() + go client.readPump() +} + +func StartRuntimeAdminEventBridge(ctx context.Context, events RuntimeEventService, hub *Hub) { + if events == nil || hub == nil { + return + } + go func() { + lastID := "$" + for { + select { + case <-ctx.Done(): + return + default: + } + + messages, err := events.Read(ctx, lastID, 5*time.Second) + if err != nil { + if ctx.Err() != nil { + return + } + log.Printf("runtime admin event bridge read failed: %v", err) + sleepOrDone(ctx, time.Second) + continue + } + if len(messages) == 0 { + sleepOrDone(ctx, 200*time.Millisecond) + continue + } + for _, message := range messages { + if strings.TrimSpace(message.ID) != "" { + lastID = message.ID + } + if msg, ok := runtimeEventWebSocketMessage(message); ok { + hub.broadcastRuntimeAdminMessage(msg) + } + } + } + }() +} + +func runtimeEventWebSocketMessage(message redisStreamMessage) (*Message, bool) { + eventType := strings.TrimSpace(message.Fields["type"]) + if eventType == "" { + var event RuntimeEvent + if raw := strings.TrimSpace(message.Fields["event"]); raw != "" && json.Unmarshal([]byte(raw), &event) == nil { + eventType = strings.TrimSpace(event.Type) + if eventType == "" { + return nil, false + } + return &Message{ + Type: eventType, + Data: json.RawMessage(event.Payload), + Timestamp: event.CreatedAt, + }, true + } + return nil, false + } + + createdAt := time.Now().UTC() + if rawCreatedAt := strings.TrimSpace(message.Fields["created_at"]); rawCreatedAt != "" { + if parsed, err := time.Parse(time.RFC3339Nano, rawCreatedAt); err == nil { + createdAt = parsed + } + } + var data interface{} = map[string]any{} + if rawPayload := strings.TrimSpace(message.Fields["payload"]); rawPayload != "" { + if json.Valid([]byte(rawPayload)) { + data = json.RawMessage(rawPayload) + } else { + data = rawPayload + } + } + return &Message{ + Type: eventType, + Data: data, + Timestamp: createdAt, + }, true +} + +func sleepOrDone(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +// readPump pumps messages from the websocket connection to the hub +func (c *Client) readPump() { + defer func() { + c.hub.unregister <- c + c.Conn.Close() + }() + + c.Conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + c.Conn.SetPongHandler(func(string) error { + c.Conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + return nil + }) + + for { + _, _, err := c.Conn.ReadMessage() + if err != nil { + if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) { + log.Printf("WebSocket error: %v", err) + } + break + } + // Process incoming messages if needed + } +} + +// writePump pumps messages from the hub to the websocket connection +func (c *Client) writePump() { + ticker := time.NewTicker(54 * time.Second) + defer func() { + ticker.Stop() + c.Conn.Close() + }() + + for { + select { + case message, ok := <-c.Send: + c.Conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + if !ok { + c.Conn.WriteMessage(websocket.CloseMessage, []byte{}) + return + } + + w, err := c.Conn.NextWriter(websocket.TextMessage) + if err != nil { + return + } + w.Write(message) + + // Add queued messages to the current websocket message + n := len(c.Send) + for i := 0; i < n; i++ { + w.Write([]byte{'\n'}) + w.Write(<-c.Send) + } + + if err := w.Close(); err != nil { + return + } + + case <-ticker.C: + c.Conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + if err := c.Conn.WriteMessage(websocket.PingMessage, nil); err != nil { + return + } + } + } +} + +// GetClientCount returns the number of connected clients +func (h *Hub) GetClientCount() int { + h.mu.RLock() + defer h.mu.RUnlock() + return len(h.clients) +} + +// Stop gracefully shuts down the hub, closing all client connections. +func (h *Hub) Stop() { + close(h.stop) +} diff --git a/backend/internal/services/websocket_service_test.go b/backend/internal/services/websocket_service_test.go new file mode 100644 index 0000000..06b088c --- /dev/null +++ b/backend/internal/services/websocket_service_test.go @@ -0,0 +1,143 @@ +package services + +import ( + "encoding/json" + "sync" + "testing" + "time" +) + +func TestGetHubReturnsSameInstance(t *testing.T) { + // Reset the package-level singleton so the test is self-contained. + hubOnce = sync.Once{} + hub = nil + + h1 := GetHub() + h2 := GetHub() + if h1 != h2 { + t.Fatal("GetHub() returned different instances") + } + + // Cleanup: stop the hub so the goroutine doesn't leak. + h1.Stop() +} + +func TestGetHubConcurrentAccess(t *testing.T) { + hubOnce = sync.Once{} + hub = nil + + const goroutines = 50 + results := make(chan *Hub, goroutines) + + var wg sync.WaitGroup + wg.Add(goroutines) + for i := 0; i < goroutines; i++ { + go func() { + defer wg.Done() + results <- GetHub() + }() + } + wg.Wait() + close(results) + + var first *Hub + for h := range results { + if first == nil { + first = h + } else if h != first { + t.Fatal("GetHub() returned different instances under concurrent access") + } + } + + first.Stop() +} + +func TestHubStopClosesClients(t *testing.T) { + h := NewHub() + go h.Run() + + // Register a fake client with a buffered Send channel. + client := &Client{ + UserID: 1, + Send: make(chan []byte, 8), + hub: h, + } + h.register <- client + + // Give the hub a moment to process the registration. + time.Sleep(20 * time.Millisecond) + + if h.GetClientCount() != 1 { + t.Fatalf("expected 1 client, got %d", h.GetClientCount()) + } + + h.Stop() + + // Give the hub a moment to process the stop. + time.Sleep(20 * time.Millisecond) + + if h.GetClientCount() != 0 { + t.Fatalf("expected 0 clients after Stop, got %d", h.GetClientCount()) + } + + // The client's Send channel should be closed. + _, ok := <-client.Send + if ok { + t.Fatal("expected client.Send to be closed after hub Stop") + } +} + +func TestHubBroadcastRuntimeAdminFiltersToAdminRuntimeClients(t *testing.T) { + h := NewHub() + adminRuntime := &Client{ + UserID: 1, + Role: "admin", + Topic: WebSocketTopicRuntimeAdmin, + Send: make(chan []byte, 1), + hub: h, + } + adminUserTopic := &Client{ + UserID: 2, + Role: "admin", + Topic: WebSocketTopicUser, + Send: make(chan []byte, 1), + hub: h, + } + normalRuntime := &Client{ + UserID: 3, + Role: "user", + Topic: WebSocketTopicRuntimeAdmin, + Send: make(chan []byte, 1), + hub: h, + } + h.clients[adminRuntime] = true + h.clients[adminUserTopic] = true + h.clients[normalRuntime] = true + + h.BroadcastRuntimeAdmin("runtime_pod_metrics", map[string]any{"pod_id": int64(9)}) + + select { + case raw := <-adminRuntime.Send: + var msg Message + if err := json.Unmarshal(raw, &msg); err != nil { + t.Fatalf("runtime admin message is not json: %v", err) + } + if msg.Type != "runtime_pod_metrics" { + t.Fatalf("message type = %q, want runtime_pod_metrics", msg.Type) + } + default: + t.Fatalf("admin runtime client did not receive runtime admin message") + } + + select { + case raw := <-adminUserTopic.Send: + t.Fatalf("admin user-topic client received runtime admin message: %s", string(raw)) + default: + } + + select { + case raw := <-normalRuntime.Send: + t.Fatalf("normal runtime client received runtime admin message: %s", string(raw)) + default: + } +} diff --git a/backend/internal/services/workspace_file_service.go b/backend/internal/services/workspace_file_service.go new file mode 100644 index 0000000..4df972d --- /dev/null +++ b/backend/internal/services/workspace_file_service.go @@ -0,0 +1,652 @@ +package services + +import ( + "context" + "errors" + "fmt" + "io" + "mime" + "net/url" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + "unicode" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +const ( + WorkspaceUploadMaxBytesEnv = "WORKSPACE_UPLOAD_MAX_BYTES" + defaultWorkspaceUploadMax = int64(524288000) + textPreviewMaxBytes = int64(1024 * 1024) + imagePreviewMaxBytes = int64(10 * 1024 * 1024) +) + +var ( + ErrWorkspaceDirectoryExpected = errors.New("workspace directory is required") + ErrWorkspaceFileExpected = errors.New("workspace file is required") + ErrWorkspacePreviewTooLarge = errors.New("workspace preview exceeds maximum size") + ErrWorkspaceUploadTooLarge = errors.New("workspace upload exceeds maximum size") + ErrWorkspaceRootOperation = errors.New("workspace root cannot be modified") + ErrWorkspaceFileNameInvalid = errors.New("workspace filename is invalid") + ErrWorkspaceEntryExists = errors.New("workspace entry already exists") +) + +type WorkspaceFileScope struct { + InstanceID int + UserID int + WorkspacePath string +} + +type WorkspaceEntry struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` + ModifiedAt time.Time `json:"modified_at"` + Previewable bool `json:"previewable"` + Downloadable bool `json:"downloadable"` +} + +type WorkspacePreview struct { + Kind string `json:"kind"` + ContentType string `json:"content_type"` + Text string `json:"text,omitempty"` + PreviewURL string `json:"preview_url,omitempty"` + DownloadURL string `json:"download_url,omitempty"` +} + +type WorkspaceFileService interface { + List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) + Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) + OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) + OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) + Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error) + Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error) + Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error) + Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error +} + +type workspaceFileService struct { + auditRepo repository.WorkspaceFileAuditRepository +} + +func NewWorkspaceFileService(auditRepo repository.WorkspaceFileAuditRepository) WorkspaceFileService { + return &workspaceFileService{auditRepo: auditRepo} +} + +func WorkspaceUploadMaxBytes() int64 { + raw := strings.TrimSpace(os.Getenv(WorkspaceUploadMaxBytesEnv)) + if raw == "" { + return defaultWorkspaceUploadMax + } + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil || value <= 0 { + return defaultWorkspaceUploadMax + } + return value +} + +func (s *workspaceFileService) List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false) + if err != nil { + return nil, err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return nil, err + } + defer root.Close() + + dir, err := root.Open(workspaceRootName(resolved.RelativePath)) + if err != nil { + return nil, err + } + defer dir.Close() + info, err := dir.Stat() + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, ErrWorkspaceDirectoryExpected + } + + items, err := dir.ReadDir(-1) + if err != nil { + return nil, err + } + entries := make([]WorkspaceEntry, 0, len(items)) + for _, item := range items { + if isWorkspaceTransientEntry(item.Name()) { + continue + } + itemInfo, infoErr := item.Info() + if infoErr != nil { + return nil, infoErr + } + childRelative := joinWorkspaceRelative(resolved.RelativePath, item.Name()) + entries = append(entries, buildWorkspaceEntry(childRelative, itemInfo)) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].IsDir != entries[j].IsDir { + return entries[i].IsDir + } + left := strings.ToLower(entries[i].Name) + right := strings.ToLower(entries[j].Name) + if left == right { + return entries[i].Name < entries[j].Name + } + return left < right + }) + return entries, nil +} + +func (s *workspaceFileService) Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) { + resolved, file, info, err := s.openExistingFile(ctx, scope, relativePath) + if err != nil { + return nil, err + } + defer file.Close() + + kind, contentType, previewable, maxBytes := workspacePreviewKind(info.Name(), info.Size()) + downloadURL := workspaceDownloadURL(scope.InstanceID, resolved.RelativePath) + if !previewable { + return &WorkspacePreview{ + Kind: "binary", + ContentType: "application/octet-stream", + DownloadURL: downloadURL, + }, nil + } + if maxBytes > 0 && info.Size() > maxBytes { + return nil, ErrWorkspacePreviewTooLarge + } + if kind == "text" { + data, err := io.ReadAll(io.LimitReader(file, textPreviewMaxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > textPreviewMaxBytes { + return nil, ErrWorkspacePreviewTooLarge + } + return &WorkspacePreview{ + Kind: "text", + ContentType: contentType, + Text: string(data), + DownloadURL: downloadURL, + }, nil + } + + return &WorkspacePreview{ + Kind: kind, + ContentType: contentType, + PreviewURL: workspacePreviewURL(scope.InstanceID, resolved.RelativePath), + DownloadURL: downloadURL, + }, nil +} + +func (s *workspaceFileService) OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + file, info, contentType, err := s.openPreviewableFile(ctx, scope, relativePath) + if err != nil { + return nil, "", 0, err + } + return file, contentType, info.Size(), nil +} + +func (s *workspaceFileService) openPreviewableFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, os.FileInfo, string, error) { + _, file, info, err := s.openExistingFile(ctx, scope, relativePath) + if err != nil { + return nil, nil, "", err + } + _, contentType, previewable, maxBytes := workspacePreviewKind(info.Name(), info.Size()) + if !previewable { + file.Close() + return nil, nil, "", ErrWorkspaceFileExpected + } + if maxBytes > 0 && info.Size() > maxBytes { + file.Close() + return nil, nil, "", ErrWorkspacePreviewTooLarge + } + return file, info, contentType, nil +} + +func (s *workspaceFileService) OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + resolved, file, info, err := s.openExistingFile(ctx, scope, relativePath) + if err != nil { + return nil, "", 0, err + } + if err := s.recordAudit(ctx, scope, "download", resolved.RelativePath, info.Size()); err != nil { + file.Close() + return nil, "", 0, err + } + return file, info.Name(), info.Size(), nil +} + +func (s *workspaceFileService) Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + maxBytes := WorkspaceUploadMaxBytes() + if size > maxBytes { + return nil, ErrWorkspaceUploadTooLarge + } + cleanName, err := sanitizeWorkspaceFilename(filename) + if err != nil { + return nil, err + } + dir, err := ResolveWorkspacePath(scope.WorkspacePath, relativeDir, false) + if err != nil { + return nil, err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return nil, err + } + defer root.Close() + dirInfo, err := root.Stat(workspaceRootName(dir.RelativePath)) + if err != nil { + return nil, err + } + if !dirInfo.IsDir() { + return nil, ErrWorkspaceDirectoryExpected + } + + targetRelative := joinWorkspaceRelative(dir.RelativePath, cleanName) + target, err := ResolveWorkspacePath(scope.WorkspacePath, targetRelative, true) + if err != nil { + return nil, err + } + if err := ensureWorkspaceRuntimeOwnership(scope, dir.RelativePath); err != nil { + return nil, err + } + targetName := workspaceRootName(target.RelativePath) + if targetInfo, statErr := root.Stat(targetName); statErr == nil && targetInfo.IsDir() { + return nil, ErrWorkspaceFileExpected + } else if statErr != nil && !os.IsNotExist(statErr) { + return nil, statErr + } + + tempRelative := joinWorkspaceRelative(dir.RelativePath, fmt.Sprintf(".%s.tmp-%d", cleanName, time.Now().UnixNano())) + tempName := workspaceRootName(tempRelative) + file, err := root.OpenFile(tempName, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0640) + if err != nil { + return nil, err + } + written, copyErr := io.Copy(file, io.LimitReader(reader, maxBytes+1)) + closeErr := file.Close() + if copyErr != nil { + _ = root.Remove(tempName) + return nil, copyErr + } + if closeErr != nil { + _ = root.Remove(tempName) + return nil, closeErr + } + if written > maxBytes { + _ = root.Remove(tempName) + return nil, ErrWorkspaceUploadTooLarge + } + if err := applyWorkspaceRuntimeOwnership(scope, tempRelative, 0640); err != nil { + _ = root.Remove(tempName) + return nil, err + } + if err := s.recordAudit(ctx, scope, "upload", target.RelativePath, written); err != nil { + _ = root.Remove(tempName) + return nil, err + } + if err := root.Rename(tempName, targetName); err != nil { + _ = root.Remove(tempName) + return nil, err + } + info, err := root.Stat(targetName) + if err != nil { + return nil, err + } + entry := buildWorkspaceEntry(target.RelativePath, info) + return &entry, nil +} + +func (s *workspaceFileService) Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error) { + if isWorkspaceRootRelative(relativePath) { + return nil, ErrWorkspaceRootOperation + } + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, true) + if err != nil { + return nil, err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return nil, err + } + defer root.Close() + resolvedName := workspaceRootName(resolved.RelativePath) + if _, err := root.Lstat(resolvedName); err == nil { + return nil, ErrWorkspaceEntryExists + } else if !os.IsNotExist(err) { + return nil, err + } + if err := s.recordAudit(ctx, scope, "mkdir", resolved.RelativePath, 0); err != nil { + return nil, err + } + if err := root.MkdirAll(resolvedName, 0750); err != nil { + return nil, err + } + if err := ensureWorkspaceRuntimeOwnership(scope, resolved.RelativePath); err != nil { + _ = root.RemoveAll(resolvedName) + return nil, err + } + info, err := root.Stat(resolvedName) + if err != nil { + return nil, err + } + entry := buildWorkspaceEntry(resolved.RelativePath, info) + return &entry, nil +} + +func (s *workspaceFileService) Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error) { + if isWorkspaceRootRelative(oldPath) || isWorkspaceRootRelative(newPath) { + return nil, ErrWorkspaceRootOperation + } + oldResolved, err := ResolveWorkspacePath(scope.WorkspacePath, oldPath, false) + if err != nil { + return nil, err + } + newResolved, err := ResolveWorkspacePath(scope.WorkspacePath, newPath, true) + if err != nil { + return nil, err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return nil, err + } + defer root.Close() + oldName := workspaceRootName(oldResolved.RelativePath) + newName := workspaceRootName(newResolved.RelativePath) + if _, err := root.Lstat(newName); err == nil { + return nil, ErrWorkspaceEntryExists + } else if !os.IsNotExist(err) { + return nil, err + } + if _, err := root.Lstat(oldName); err != nil { + return nil, err + } + auditPath := oldResolved.RelativePath + " -> " + newResolved.RelativePath + if err := s.recordAudit(ctx, scope, "rename", auditPath, 0); err != nil { + return nil, err + } + if err := root.Rename(oldName, newName); err != nil { + return nil, err + } + if err := ensureWorkspaceRuntimeOwnership(scope, newResolved.RelativePath); err != nil { + return nil, err + } + info, err := root.Stat(newName) + if err != nil { + return nil, err + } + entry := buildWorkspaceEntry(newResolved.RelativePath, info) + return &entry, nil +} + +func (s *workspaceFileService) Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error { + if isWorkspaceRootRelative(relativePath) { + return ErrWorkspaceRootOperation + } + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false) + if err != nil { + return err + } + if err := s.recordAudit(ctx, scope, "delete", resolved.RelativePath, 0); err != nil { + return err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return err + } + defer root.Close() + return root.RemoveAll(workspaceRootName(resolved.RelativePath)) +} + +func (s *workspaceFileService) openExistingFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*ResolvedWorkspacePath, *os.File, os.FileInfo, error) { + if err := ctx.Err(); err != nil { + return nil, nil, nil, err + } + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false) + if err != nil { + return nil, nil, nil, err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return nil, nil, nil, err + } + defer root.Close() + file, err := root.Open(workspaceRootName(resolved.RelativePath)) + if err != nil { + return nil, nil, nil, err + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, nil, nil, err + } + if info.IsDir() { + file.Close() + return nil, nil, nil, ErrWorkspaceFileExpected + } + return resolved, file, info, nil +} + +func (s *workspaceFileService) recordAudit(ctx context.Context, scope WorkspaceFileScope, action, relativePath string, bytes int64) error { + if s.auditRepo == nil { + return nil + } + if err := ctx.Err(); err != nil { + return err + } + return s.auditRepo.Create(ctx, &models.WorkspaceFileAudit{ + InstanceID: scope.InstanceID, + UserID: scope.UserID, + Action: action, + RelativePath: relativePath, + Bytes: bytes, + CreatedAt: time.Now().UTC(), + }) +} + +func buildWorkspaceEntry(relativePath string, info os.FileInfo) WorkspaceEntry { + isDir := info.IsDir() + return WorkspaceEntry{ + Name: info.Name(), + Path: filepath.ToSlash(relativePath), + IsDir: isDir, + Size: info.Size(), + ModifiedAt: info.ModTime().UTC(), + Previewable: workspaceEntryPreviewable(info.Name(), info.Size(), isDir), + Downloadable: !isDir, + } +} + +func isWorkspaceTransientEntry(name string) bool { + return name == ".tmp" || strings.HasPrefix(name, ".tmp-skill-") +} + +func workspaceEntryPreviewable(name string, size int64, isDir bool) bool { + if isDir { + return false + } + kind, _, previewable, maxBytes := workspacePreviewKind(name, size) + if !previewable { + return false + } + return kind == "pdf" || maxBytes <= 0 || size <= maxBytes +} + +func workspacePreviewKind(name string, size int64) (string, string, bool, int64) { + ext := strings.ToLower(filepath.Ext(name)) + switch ext { + case ".txt", ".md", ".json", ".yaml", ".yml", ".log", ".py", ".js", ".ts", ".go", ".sh": + return "text", "text/plain; charset=utf-8", true, textPreviewMaxBytes + case ".png", ".jpg", ".jpeg", ".gif", ".webp": + contentType := mime.TypeByExtension(ext) + if contentType == "" { + contentType = "application/octet-stream" + } + return "image", contentType, true, imagePreviewMaxBytes + case ".pdf": + return "pdf", "application/pdf", true, 0 + default: + return "binary", "application/octet-stream", false, 0 + } +} + +func sanitizeWorkspaceFilename(name string) (string, error) { + name = strings.TrimSpace(name) + if name == "" || name == "." || name == ".." { + return "", ErrWorkspaceFileNameInvalid + } + if strings.ContainsAny(name, `/\`) || strings.Contains(name, "\x00") { + return "", ErrWorkspaceFileNameInvalid + } + cleaned := strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + switch r { + case ':', '*', '?', '"', '<', '>', '|': + return '-' + default: + return r + } + }, name) + cleaned = strings.TrimSpace(cleaned) + if cleaned == "" || cleaned == "." || cleaned == ".." { + return "", ErrWorkspaceFileNameInvalid + } + return cleaned, nil +} + +func joinWorkspaceRelative(base, name string) string { + base = filepath.ToSlash(strings.TrimSpace(base)) + name = filepath.ToSlash(strings.TrimSpace(name)) + if base == "" { + return path.Clean(name) + } + return path.Clean(base + "/" + name) +} + +func ensureWorkspaceRuntimeOwnership(scope WorkspaceFileScope, relativePath string) error { + if _, _, ok := runtimeWorkspaceOwner(scope); !ok { + return nil + } + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return err + } + if relative == "" { + return nil + } + + parts := strings.Split(relative, "/") + for idx := range parts { + current := strings.Join(parts[:idx+1], "/") + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, current, false) + if err != nil { + return err + } + info, err := os.Stat(resolved.RealPath) + if err != nil { + return err + } + mode := os.FileMode(0640) + if info.IsDir() { + mode = 0750 + } + if err := applyWorkspaceRuntimeOwnershipToPath(scope, resolved.RealPath, mode); err != nil { + return err + } + } + return nil +} + +func applyWorkspaceRuntimeOwnership(scope WorkspaceFileScope, relativePath string, mode os.FileMode) error { + if _, _, ok := runtimeWorkspaceOwner(scope); !ok { + return nil + } + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false) + if err != nil { + return err + } + return applyWorkspaceRuntimeOwnershipToPath(scope, resolved.RealPath, mode) +} + +func applyWorkspaceRuntimeOwnershipToPath(scope WorkspaceFileScope, targetPath string, mode os.FileMode) error { + uid, gid, ok := runtimeWorkspaceOwner(scope) + if !ok { + return nil + } + if err := os.Chown(targetPath, uid, gid); err != nil { + return err + } + if err := os.Chmod(targetPath, mode); err != nil { + return err + } + return nil +} + +func runtimeWorkspaceOwner(scope WorkspaceFileScope) (int, int, bool) { + if scope.InstanceID <= 0 || scope.UserID <= 0 { + return 0, 0, false + } + cleaned := filepath.ToSlash(filepath.Clean(strings.TrimSpace(scope.WorkspacePath))) + parts := strings.Split(strings.Trim(cleaned, "/"), "/") + if len(parts) < 3 { + return 0, 0, false + } + instancePart := fmt.Sprintf("instance-%d", scope.InstanceID) + userPart := fmt.Sprintf("user-%d", scope.UserID) + runtimeType := parts[len(parts)-3] + if parts[len(parts)-1] != instancePart || parts[len(parts)-2] != userPart { + return 0, 0, false + } + if runtimeType != RuntimeTypeOpenClaw && runtimeType != RuntimeTypeHermes { + return 0, 0, false + } + linuxID := RuntimeLinuxID(scope.InstanceID) + return linuxID, linuxID, true +} + +func isWorkspaceRootRelative(relativePath string) bool { + cleaned, err := cleanWorkspaceRelativePath(relativePath) + return err == nil && cleaned == "" +} + +func openWorkspaceRoot(workspacePath string) (*os.Root, error) { + resolved, err := ResolveWorkspacePath(workspacePath, "", false) + if err != nil { + return nil, err + } + return os.OpenRoot(resolved.Root) +} + +func workspaceRootName(relativePath string) string { + relativePath = filepath.ToSlash(strings.TrimSpace(relativePath)) + if relativePath == "" { + return "." + } + return filepath.FromSlash(relativePath) +} + +func workspaceDownloadURL(instanceID int, relativePath string) string { + return fmt.Sprintf("/api/v1/instances/%d/workspace/download?path=%s", instanceID, url.QueryEscape(filepath.ToSlash(relativePath))) +} + +func workspacePreviewURL(instanceID int, relativePath string) string { + return fmt.Sprintf("/api/v1/instances/%d/workspace/preview?path=%s&raw=1", instanceID, url.QueryEscape(filepath.ToSlash(relativePath))) +} diff --git a/backend/internal/services/workspace_file_service_test.go b/backend/internal/services/workspace_file_service_test.go new file mode 100644 index 0000000..60d7334 --- /dev/null +++ b/backend/internal/services/workspace_file_service_test.go @@ -0,0 +1,324 @@ +package services + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "clawreef/internal/models" +) + +type recordingWorkspaceFileAuditRepo struct { + items []models.WorkspaceFileAudit + err error +} + +func (r *recordingWorkspaceFileAuditRepo) Create(ctx context.Context, audit *models.WorkspaceFileAudit) error { + if r.err != nil { + return r.err + } + r.items = append(r.items, *audit) + return nil +} + +func workspaceTestScope(root string) WorkspaceFileScope { + return WorkspaceFileScope{ + InstanceID: 12, + UserID: 34, + WorkspacePath: root, + } +} + +func TestWorkspaceFileServiceListSortsDirectoriesFirstAndMarksCapabilities(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "docs"), 0750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "alpha.log"), []byte("hello"), 0640); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "binary.dat"), []byte{0, 1, 2}, 0640); err != nil { + t.Fatal(err) + } + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + entries, err := service.List(context.Background(), workspaceTestScope(root), "") + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.Name) + } + if strings.Join(names, ",") != "docs,alpha.log,binary.dat" { + t.Fatalf("entry order = %v, want dirs first then names", names) + } + if entries[1].Path != "alpha.log" || !entries[1].Previewable || !entries[1].Downloadable { + t.Fatalf("alpha.log capabilities = %#v, want previewable downloadable file", entries[1]) + } + if entries[2].Previewable || !entries[2].Downloadable { + t.Fatalf("binary.dat capabilities = %#v, want download-only file", entries[2]) + } +} + +func TestWorkspaceFileServiceListHidesTransientSkillTempDirs(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, ".tmp"), 0750); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(root, ".tmp-skill-paper-ranker-123"), 0750); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(root, "paper-ranker"), 0750); err != nil { + t.Fatal(err) + } + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + entries, err := service.List(context.Background(), workspaceTestScope(root), "") + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + if len(entries) != 1 || entries[0].Name != "paper-ranker" { + t.Fatalf("entries = %#v, want only paper-ranker", entries) + } +} +func TestWorkspaceFileServicePreviewTextAndDoesNotAudit(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "notes.md"), []byte("# hello"), 0640); err != nil { + t.Fatal(err) + } + auditRepo := &recordingWorkspaceFileAuditRepo{} + + service := NewWorkspaceFileService(auditRepo) + preview, err := service.Preview(context.Background(), workspaceTestScope(root), "notes.md") + if err != nil { + t.Fatalf("Preview returned error: %v", err) + } + + if preview.Kind != "text" || preview.Text != "# hello" { + t.Fatalf("preview = %#v, want text preview", preview) + } + if len(auditRepo.items) != 0 { + t.Fatalf("preview audited %d events, want none", len(auditRepo.items)) + } +} + +func TestWorkspaceFileServiceTreatsSVGAsDownloadOnly(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "icon.svg"), []byte(``), 0640); err != nil { + t.Fatal(err) + } + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + entries, err := service.List(context.Background(), workspaceTestScope(root), "") + if err != nil { + t.Fatalf("List returned error: %v", err) + } + if len(entries) != 1 || entries[0].Previewable { + t.Fatalf("svg entry = %#v, want download-only", entries) + } + preview, err := service.Preview(context.Background(), workspaceTestScope(root), "icon.svg") + if err != nil { + t.Fatalf("Preview returned error: %v", err) + } + if preview.Kind != "binary" || preview.PreviewURL != "" { + t.Fatalf("svg preview = %#v, want binary without preview url", preview) + } +} + +func TestWorkspaceFileServiceRejectsLargeTextPreview(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "large.log"), []byte(strings.Repeat("a", 1024*1024+1)), 0640); err != nil { + t.Fatal(err) + } + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + _, err := service.Preview(context.Background(), workspaceTestScope(root), "large.log") + if !errors.Is(err, ErrWorkspacePreviewTooLarge) { + t.Fatalf("Preview error = %v, want ErrWorkspacePreviewTooLarge", err) + } +} + +func TestWorkspaceFileServiceDownloadAuditsSuccessfulFileOpen(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "report.txt"), []byte("report"), 0640); err != nil { + t.Fatal(err) + } + auditRepo := &recordingWorkspaceFileAuditRepo{} + + service := NewWorkspaceFileService(auditRepo) + file, filename, size, err := service.OpenDownload(context.Background(), workspaceTestScope(root), "report.txt") + if err != nil { + t.Fatalf("OpenDownload returned error: %v", err) + } + file.Close() + + if filename != "report.txt" || size != int64(len("report")) { + t.Fatalf("download metadata filename=%q size=%d", filename, size) + } + if len(auditRepo.items) != 1 || auditRepo.items[0].Action != "download" || auditRepo.items[0].RelativePath != "report.txt" { + t.Fatalf("audit items = %#v, want download audit", auditRepo.items) + } +} + +func TestWorkspaceFileServiceUploadRejectsEscapingSymlinkParent(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { + t.Skipf("symlink creation is not available: %v", err) + } + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + _, err := service.Upload(context.Background(), workspaceTestScope(root), "linked", "owned.txt", strings.NewReader("owned"), int64(len("owned"))) + if !errors.Is(err, ErrWorkspacePathEscape) { + t.Fatalf("Upload error = %v, want ErrWorkspacePathEscape", err) + } + if _, statErr := os.Stat(filepath.Join(outside, "owned.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("outside file stat error = %v, want not exist", statErr) + } +} + +func TestWorkspaceFileServiceUploadEnforcesMaxBytesBeforeWriting(t *testing.T) { + t.Setenv(WorkspaceUploadMaxBytesEnv, "3") + root := t.TempDir() + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + _, err := service.Upload(context.Background(), workspaceTestScope(root), "", "big.txt", strings.NewReader("abcd"), int64(len("abcd"))) + if !errors.Is(err, ErrWorkspaceUploadTooLarge) { + t.Fatalf("Upload error = %v, want ErrWorkspaceUploadTooLarge", err) + } + if _, statErr := os.Stat(filepath.Join(root, "big.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("uploaded file stat error = %v, want not exist", statErr) + } +} + +func TestWorkspaceFileServiceRuntimeWorkspaceOwner(t *testing.T) { + scope := WorkspaceFileScope{ + InstanceID: 74, + UserID: 1, + WorkspacePath: "/workspaces/openclaw/user-1/instance-74", + } + uid, gid, ok := runtimeWorkspaceOwner(scope) + if !ok { + t.Fatal("runtimeWorkspaceOwner ok = false, want true") + } + if uid != RuntimeLinuxID(74) || gid != RuntimeLinuxID(74) { + t.Fatalf("runtime owner uid:gid = %d:%d, want %d:%d", uid, gid, RuntimeLinuxID(74), RuntimeLinuxID(74)) + } + + scope.WorkspacePath = "/tmp/workspaces/openclaw/user-1/instance-74" + if _, _, ok := runtimeWorkspaceOwner(scope); !ok { + t.Fatal("runtimeWorkspaceOwner should support configurable workspace roots") + } + + scope.WorkspacePath = "/workspaces/openclaw/user-2/instance-74" + if _, _, ok := runtimeWorkspaceOwner(scope); ok { + t.Fatal("runtimeWorkspaceOwner ok = true for mismatched user") + } + + scope.WorkspacePath = "/var/lib/clawmanager/user-1/instance-74" + if _, _, ok := runtimeWorkspaceOwner(scope); ok { + t.Fatal("runtimeWorkspaceOwner ok = true for non-runtime workspace") + } +} + +func TestWorkspaceFileServiceUploadAuditFailureRemovesFile(t *testing.T) { + root := t.TempDir() + auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")} + service := NewWorkspaceFileService(auditRepo) + + _, err := service.Upload(context.Background(), workspaceTestScope(root), "", "created.txt", strings.NewReader("body"), int64(len("body"))) + if err == nil || !strings.Contains(err.Error(), "audit unavailable") { + t.Fatalf("Upload error = %v, want audit failure", err) + } + if _, statErr := os.Stat(filepath.Join(root, "created.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("created file stat error = %v, want not exist", statErr) + } +} + +func TestWorkspaceFileServiceMkdirAuditFailureRemovesDirectory(t *testing.T) { + root := t.TempDir() + auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")} + service := NewWorkspaceFileService(auditRepo) + + _, err := service.Mkdir(context.Background(), workspaceTestScope(root), "docs") + if err == nil || !strings.Contains(err.Error(), "audit unavailable") { + t.Fatalf("Mkdir error = %v, want audit failure", err) + } + if _, statErr := os.Stat(filepath.Join(root, "docs")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("created directory stat error = %v, want not exist", statErr) + } +} + +func TestWorkspaceFileServiceRenameAuditFailureRollsBack(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "old.txt"), []byte("body"), 0640); err != nil { + t.Fatal(err) + } + auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")} + service := NewWorkspaceFileService(auditRepo) + + _, err := service.Rename(context.Background(), workspaceTestScope(root), "old.txt", "new.txt") + if err == nil || !strings.Contains(err.Error(), "audit unavailable") { + t.Fatalf("Rename error = %v, want audit failure", err) + } + if _, statErr := os.Stat(filepath.Join(root, "old.txt")); statErr != nil { + t.Fatalf("old file stat error = %v, want rollback", statErr) + } + if _, statErr := os.Stat(filepath.Join(root, "new.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("new file stat error = %v, want not exist", statErr) + } +} + +func TestWorkspaceFileServiceDeleteAuditFailureKeepsFile(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "keep.txt"), []byte("body"), 0640); err != nil { + t.Fatal(err) + } + auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")} + service := NewWorkspaceFileService(auditRepo) + + err := service.Delete(context.Background(), workspaceTestScope(root), "keep.txt") + if err == nil || !strings.Contains(err.Error(), "audit unavailable") { + t.Fatalf("Delete error = %v, want audit failure", err) + } + if _, statErr := os.Stat(filepath.Join(root, "keep.txt")); statErr != nil { + t.Fatalf("kept file stat error = %v", statErr) + } +} + +func TestWorkspaceFileServiceMutationsAuditAndProtectRoot(t *testing.T) { + root := t.TempDir() + auditRepo := &recordingWorkspaceFileAuditRepo{} + service := NewWorkspaceFileService(auditRepo) + scope := workspaceTestScope(root) + + if err := service.Delete(context.Background(), scope, ""); !errors.Is(err, ErrWorkspaceRootOperation) { + t.Fatalf("Delete root error = %v, want ErrWorkspaceRootOperation", err) + } + if _, err := service.Rename(context.Background(), scope, "", "renamed"); !errors.Is(err, ErrWorkspaceRootOperation) { + t.Fatalf("Rename root error = %v, want ErrWorkspaceRootOperation", err) + } + if _, err := service.Mkdir(context.Background(), scope, "docs"); err != nil { + t.Fatalf("Mkdir returned error: %v", err) + } + if _, err := service.Rename(context.Background(), scope, "docs", "notes"); err != nil { + t.Fatalf("Rename returned error: %v", err) + } + if err := service.Delete(context.Background(), scope, "notes"); err != nil { + t.Fatalf("Delete returned error: %v", err) + } + + actions := make([]string, 0, len(auditRepo.items)) + for _, item := range auditRepo.items { + actions = append(actions, item.Action) + } + if strings.Join(actions, ",") != "mkdir,rename,delete" { + t.Fatalf("audit actions = %v, want mkdir,rename,delete", actions) + } +} diff --git a/backend/internal/services/workspace_path_guard.go b/backend/internal/services/workspace_path_guard.go new file mode 100644 index 0000000..c8b1e31 --- /dev/null +++ b/backend/internal/services/workspace_path_guard.go @@ -0,0 +1,202 @@ +package services + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "runtime" + "strings" +) + +var ( + ErrWorkspacePathInvalid = errors.New("workspace path is invalid") + ErrWorkspacePathEscape = errors.New("workspace path escapes workspace") + ErrWorkspacePathNotFound = errors.New("workspace entry not found") +) + +type ResolvedWorkspacePath struct { + Root string + Path string + RealPath string + RelativePath string +} + +func ResolveWorkspacePath(workspaceRoot, relativePath string, allowMissingLeaf bool) (*ResolvedWorkspacePath, error) { + root := strings.TrimSpace(workspaceRoot) + if root == "" { + return nil, fmt.Errorf("%w: workspace root is required", ErrWorkspacePathInvalid) + } + if strings.Contains(root, "\x00") { + return nil, fmt.Errorf("%w: workspace root contains null byte", ErrWorkspacePathInvalid) + } + + rootAbs, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err) + } + rootReal, err := filepath.EvalSymlinks(rootAbs) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("%w: workspace root", ErrWorkspacePathNotFound) + } + return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err) + } + rootReal = filepath.Clean(rootReal) + rootInfo, err := os.Stat(rootReal) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("%w: workspace root", ErrWorkspacePathNotFound) + } + return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err) + } + if !rootInfo.IsDir() { + return nil, fmt.Errorf("%w: workspace root is not a directory", ErrWorkspacePathInvalid) + } + + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return nil, err + } + + targetPath := rootReal + if relative != "" { + targetPath = filepath.Join(rootReal, filepath.FromSlash(relative)) + } + targetPath = filepath.Clean(targetPath) + if !isWorkspaceSubpath(rootReal, targetPath) { + return nil, fmt.Errorf("%w: %s", ErrWorkspacePathEscape, relativePath) + } + + realPath, err := resolveWorkspaceRealPath(targetPath, allowMissingLeaf) + if err != nil { + return nil, err + } + realPath = filepath.Clean(realPath) + if !isWorkspaceSubpath(rootReal, realPath) { + return nil, fmt.Errorf("%w: %s", ErrWorkspacePathEscape, relativePath) + } + + return &ResolvedWorkspacePath{ + Root: rootReal, + Path: targetPath, + RealPath: realPath, + RelativePath: relative, + }, nil +} + +func cleanWorkspaceRelativePath(relativePath string) (string, error) { + raw := strings.TrimSpace(relativePath) + if raw == "" || raw == "." { + return "", nil + } + if strings.Contains(raw, "\x00") { + return "", fmt.Errorf("%w: path contains null byte", ErrWorkspacePathInvalid) + } + if filepath.IsAbs(raw) || filepath.VolumeName(raw) != "" { + return "", fmt.Errorf("%w: absolute paths are not allowed", ErrWorkspacePathEscape) + } + + normalized := strings.ReplaceAll(raw, "\\", "/") + if path.IsAbs(normalized) || isWindowsDrivePath(normalized) { + return "", fmt.Errorf("%w: absolute paths are not allowed", ErrWorkspacePathEscape) + } + + parts := strings.Split(normalized, "/") + cleaned := make([]string, 0, len(parts)) + for _, part := range parts { + switch part { + case "", ".": + continue + case "..": + return "", fmt.Errorf("%w: traversal is not allowed", ErrWorkspacePathEscape) + default: + cleaned = append(cleaned, part) + } + } + if len(cleaned) == 0 { + return "", nil + } + return path.Clean(strings.Join(cleaned, "/")), nil +} + +func isWindowsDrivePath(value string) bool { + if len(value) < 2 || value[1] != ':' { + return false + } + first := value[0] + return (first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z') +} + +func resolveWorkspaceRealPath(targetPath string, allowMissingLeaf bool) (string, error) { + realPath, err := filepath.EvalSymlinks(targetPath) + if err == nil { + return realPath, nil + } + if !os.IsNotExist(err) { + return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err) + } + if !allowMissingLeaf { + return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, targetPath) + } + + existingPath := targetPath + missingParts := []string{} + for { + info, statErr := os.Lstat(existingPath) + if statErr == nil { + if len(missingParts) > 0 { + info, statErr = os.Stat(existingPath) + if statErr != nil { + if os.IsNotExist(statErr) { + return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, existingPath) + } + return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, statErr) + } + if !info.IsDir() { + return "", fmt.Errorf("%w: parent is not a directory", ErrWorkspacePathInvalid) + } + } + parentReal, evalErr := filepath.EvalSymlinks(existingPath) + if evalErr != nil { + if os.IsNotExist(evalErr) { + return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, existingPath) + } + return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, evalErr) + } + parts := append([]string{parentReal}, missingParts...) + return filepath.Join(parts...), nil + } + if !os.IsNotExist(statErr) { + return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, statErr) + } + + parent := filepath.Dir(existingPath) + if parent == existingPath { + return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, targetPath) + } + missingParts = append([]string{filepath.Base(existingPath)}, missingParts...) + existingPath = parent + } +} + +func isWorkspaceSubpath(root, target string) bool { + root = filepath.Clean(root) + target = filepath.Clean(target) + if sameWorkspacePath(root, target) { + return true + } + relative, err := filepath.Rel(root, target) + if err != nil || relative == "." || filepath.IsAbs(relative) { + return false + } + return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + +func sameWorkspacePath(left, right string) bool { + if runtime.GOOS == "windows" { + return strings.EqualFold(left, right) + } + return left == right +} diff --git a/backend/internal/services/workspace_path_guard_test.go b/backend/internal/services/workspace_path_guard_test.go new file mode 100644 index 0000000..1ca2f13 --- /dev/null +++ b/backend/internal/services/workspace_path_guard_test.go @@ -0,0 +1,76 @@ +package services + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestResolveWorkspacePathRejectsTraversalAndAbsolutePaths(t *testing.T) { + root := t.TempDir() + + for _, relativePath := range []string{ + "../secret.txt", + "docs/../../secret.txt", + "/etc/passwd", + `C:\Windows\System32\drivers\etc\hosts`, + "C:/Windows/System32/drivers/etc/hosts", + } { + t.Run(relativePath, func(t *testing.T) { + _, err := ResolveWorkspacePath(root, relativePath, false) + if !errors.Is(err, ErrWorkspacePathEscape) && !errors.Is(err, ErrWorkspacePathInvalid) { + t.Fatalf("ResolveWorkspacePath(%q) error = %v, want path safety error", relativePath, err) + } + }) + } +} + +func TestResolveWorkspacePathAllowsMissingChildInsideWorkspace(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "docs"), 0750); err != nil { + t.Fatal(err) + } + + resolved, err := ResolveWorkspacePath(root, "docs/new.txt", true) + if err != nil { + t.Fatalf("ResolveWorkspacePath returned error: %v", err) + } + + if resolved.RelativePath != "docs/new.txt" { + t.Fatalf("relative path = %q, want docs/new.txt", resolved.RelativePath) + } + if !strings.HasSuffix(filepath.ToSlash(resolved.Path), "/docs/new.txt") { + t.Fatalf("target path = %q, want path under docs/new.txt", resolved.Path) + } +} + +func TestResolveWorkspacePathRejectsExistingSymlinkEscape(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("secret"), 0640); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(outside, "secret.txt"), filepath.Join(root, "leak.txt")); err != nil { + t.Skipf("symlink creation is not available: %v", err) + } + + _, err := ResolveWorkspacePath(root, "leak.txt", false) + if !errors.Is(err, ErrWorkspacePathEscape) { + t.Fatalf("ResolveWorkspacePath through symlink error = %v, want ErrWorkspacePathEscape", err) + } +} + +func TestResolveWorkspacePathRejectsMissingTargetUnderEscapingSymlinkParent(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { + t.Skipf("symlink creation is not available: %v", err) + } + + _, err := ResolveWorkspacePath(root, "linked/new.txt", true) + if !errors.Is(err, ErrWorkspacePathEscape) { + t.Fatalf("ResolveWorkspacePath through symlink parent error = %v, want ErrWorkspacePathEscape", err) + } +} diff --git a/backend/internal/utils/jwt.go b/backend/internal/utils/jwt.go new file mode 100644 index 0000000..5d435f9 --- /dev/null +++ b/backend/internal/utils/jwt.go @@ -0,0 +1,52 @@ +package utils + +import ( + "errors" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +// TokenClaims represents JWT token claims +type TokenClaims struct { + UserID int `json:"user_id"` + TokenType string `json:"token_type"` + jwt.RegisteredClaims +} + +// GenerateToken generates a new JWT token +func GenerateToken(claims TokenClaims, secret string, expiry time.Duration) (string, error) { + now := time.Now() + claims.RegisteredClaims = jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(now.Add(expiry)), + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + tokenString, err := token.SignedString([]byte(secret)) + if err != nil { + return "", err + } + + return tokenString, nil +} + +// ValidateToken validates a JWT token and returns its claims +func ValidateToken(tokenString, secret string) (*TokenClaims, error) { + token, err := jwt.ParseWithClaims(tokenString, &TokenClaims{}, func(token *jwt.Token) (interface{}, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, errors.New("unexpected signing method") + } + return []byte(secret), nil + }) + + if err != nil { + return nil, err + } + + if claims, ok := token.Claims.(*TokenClaims); ok && token.Valid { + return claims, nil + } + + return nil, errors.New("invalid token claims") +} diff --git a/backend/internal/utils/password.go b/backend/internal/utils/password.go new file mode 100644 index 0000000..0956593 --- /dev/null +++ b/backend/internal/utils/password.go @@ -0,0 +1,20 @@ +package utils + +import ( + "golang.org/x/crypto/bcrypt" +) + +// HashPassword hashes a password using bcrypt +func HashPassword(password string) (string, error) { + bytes, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return "", err + } + return string(bytes), nil +} + +// VerifyPassword verifies a password against its hash +func VerifyPassword(password, hash string) bool { + err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) + return err == nil +} diff --git a/backend/internal/utils/response.go b/backend/internal/utils/response.go new file mode 100644 index 0000000..8ba0b9d --- /dev/null +++ b/backend/internal/utils/response.go @@ -0,0 +1,116 @@ +package utils + +import ( + "errors" + "log" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/go-playground/validator/v10" +) + +// Success sends a successful response +func Success(c *gin.Context, status int, message string, data interface{}) { + c.JSON(status, gin.H{ + "success": true, + "message": message, + "data": data, + }) +} + +// Error sends an error response +func Error(c *gin.Context, status int, message string) { + c.JSON(status, gin.H{ + "success": false, + "error": message, + }) +} + +// HandleError handles different types of errors and sends appropriate responses +func HandleError(c *gin.Context, err error) { + // Log the actual error for debugging + log.Printf("[ERROR] %v", err) + + // Handle validation errors + if validationErrors, ok := err.(validator.ValidationErrors); ok { + Error(c, http.StatusBadRequest, formatValidationErrors(validationErrors)) + return + } + + // Handle known errors + errStr := err.Error() + if strings.HasPrefix(errStr, "provider discovery failed:") || strings.HasPrefix(errStr, "failed to call provider discovery endpoint:") || strings.HasPrefix(errStr, "failed to decode provider discovery response:") { + Error(c, http.StatusBadGateway, errStr) + return + } + if strings.HasPrefix(errStr, "failed to get secret ") || strings.HasPrefix(errStr, "secret key ") || strings.HasPrefix(errStr, "secret value is empty") { + Error(c, http.StatusBadGateway, errStr) + return + } + if strings.HasPrefix(errStr, "missing required openclaw config dependency:") || strings.HasPrefix(errStr, "required openclaw config dependency is disabled:") { + Error(c, http.StatusBadRequest, errStr) + return + } + if strings.HasPrefix(errStr, "duplicate team member id:") || strings.HasPrefix(errStr, "redis url is invalid:") || strings.HasPrefix(errStr, "redis db index is invalid:") || strings.HasPrefix(errStr, "instance limit reached:") || strings.HasPrefix(errStr, "CPU cores exceed quota:") || strings.HasPrefix(errStr, "memory exceed quota:") || strings.HasPrefix(errStr, "storage exceed quota:") || strings.HasPrefix(errStr, "GPU count exceed quota:") { + Error(c, http.StatusBadRequest, errStr) + return + } + + switch errStr { + case "username already exists", "email already exists", "instance name already exists", "team name already exists", "openclaw config resource key already exists", "team task message id already exists": + Error(c, http.StatusConflict, errStr) + case "display name already exists": + Error(c, http.StatusConflict, errStr) + case "unsupported instance type", "instance name is required", "image is required", "display name is required", "provider type is required", "base URL is required", "provider model name is required", "input price must be non-negative", "output price must be non-negative", "base URL is invalid", "automatic model discovery for azure-openai is not supported yet", "provider discovery is not supported", "model is required", "messages are required", "streaming is not supported yet", "provider type is not supported yet", "trace id is required", "event type is required", "message is required", "risk hit record is incomplete", "rule id is required", "rule display name is required", "rule pattern is required", "rule pattern is invalid", "risk severity is invalid", "risk action is invalid", "sample text is required", "secret ref format is invalid", "secret namespace is required in secret ref", "invalid openclaw resource type", "invalid openclaw config plan mode", "openclaw config resource name is required", "openclaw config resource key is invalid", "openclaw config schemaVersion is required", "openclaw config kind does not match resource type", "openclaw config format is required", "openclaw config content is required", "openclaw config content must be valid JSON", "openclaw config config payload is required", "openclaw config dependency type is invalid", "openclaw config dependency key is required", "openclaw config dependency is invalid", "openclaw config bundle name is required", "openclaw config bundle must include at least one resource", "openclaw config bundle resource id is required", "openclaw config bundle contains duplicate resources", "openclaw config bundle is required", "openclaw config bundle is disabled", "openclaw config bundle is empty", "at least one openclaw config resource must be selected", "openclaw config resource id is invalid", "openclaw config resource is disabled", "openclaw config bundle contains a disabled resource", "openclaw bootstrap payload is too large", "agent bootstrap token is required", "agent id is required", "unsupported agent protocol version", "invalid agent bootstrap token", "invalid instance command type", "invalid instance command finish status", "team name is required", "team must include at least one member", "team must include exactly one leader", "team redis url is required", "redis url scheme must be redis or rediss", "team member id is invalid", "target member id is required", "task payload is required", "team leader cannot be deleted before assigning a new leader": + Error(c, http.StatusBadRequest, errStr) + case "model is not active or does not exist", "openclaw config resource not found", "openclaw config bundle not found", "openclaw injection snapshot not found", "instance command not found", "instance config revision not found", "team not found", "team member not found": + Error(c, http.StatusNotFound, errStr) + case "risk rule not found": + Error(c, http.StatusNotFound, errStr) + case "sensitive content requires an active secure model", "request was blocked by risk policy": + Error(c, http.StatusForbidden, errStr) + case "invalid username or password", "account is disabled", "invalid or expired agent session token": + Error(c, http.StatusUnauthorized, errStr) + case "agent registration is only supported for openclaw instances", "agent registration is only supported for openclaw or hermes instances", "agent id does not match session", "access denied": + Error(c, http.StatusForbidden, errStr) + case "current password is incorrect": + Error(c, http.StatusBadRequest, errStr) + case "user not found", "model not found": + Error(c, http.StatusNotFound, errStr) + default: + // For development, show actual error; for production, hide details + Error(c, http.StatusInternalServerError, errStr) + } +} + +// ValidationError handles validation errors from gin binding +func ValidationError(c *gin.Context, err error) { + var ve validator.ValidationErrors + if errors.As(err, &ve) { + Error(c, http.StatusBadRequest, formatValidationErrors(ve)) + return + } + Error(c, http.StatusBadRequest, err.Error()) +} + +func formatValidationErrors(errs validator.ValidationErrors) string { + var messages []string + for _, err := range errs { + switch err.Tag() { + case "required": + messages = append(messages, err.Field()+" is required") + case "min": + messages = append(messages, err.Field()+" must be at least "+err.Param()+" characters") + case "max": + messages = append(messages, err.Field()+" must be at most "+err.Param()+" characters") + case "email": + messages = append(messages, err.Field()+" must be a valid email") + case "alphanum": + messages = append(messages, err.Field()+" must be alphanumeric") + default: + messages = append(messages, err.Field()+" is invalid") + } + } + return messages[0] +} diff --git a/backend/test_sync.sh b/backend/test_sync.sh new file mode 100644 index 0000000..bd611b8 --- /dev/null +++ b/backend/test_sync.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Get token first +echo "Getting auth token..." +TOKEN=$(curl -s -X POST http://localhost:9001/api/v1/auth/login \ + -H "Content-Type: application/json" \ + -d '{"username":"test","password":"test123"}' | grep -o '"access_token":"[^"]*"' | cut -d'"' -f4) + +if [ -z "$TOKEN" ]; then + echo "Failed to get token" + exit 1 +fi + +echo "Token: $TOKEN" +echo "" + +# Force sync instance 0 +echo "Force syncing instance 0..." +curl -s -X POST http://localhost:9001/api/v1/instances/0/sync \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" | jq . diff --git a/deployments/container/start.sh b/deployments/container/start.sh new file mode 100644 index 0000000..6c9645e --- /dev/null +++ b/deployments/container/start.sh @@ -0,0 +1,127 @@ +#!/bin/sh +set -eu + +TLS_DIR="/etc/nginx/tls" +TLS_SOURCE_DIR="${TLS_SOURCE_DIR:-/var/run/clawreef-tls}" +TLS_SHARED_DIR="${TLS_SHARED_DIR:-/workspaces/.clawmanager/tls}" +TLS_CN="${TLS_CN:-clawmanager.local}" +TLS_SUBJECT_ALT_NAME="${TLS_SUBJECT_ALT_NAME:-DNS:clawmanager.local,DNS:localhost,IP:127.0.0.1}" +TLS_CERT="${TLS_DIR}/tls.crt" +TLS_KEY="${TLS_DIR}/tls.key" +SOURCE_CERT="${TLS_SOURCE_DIR}/tls.crt" +SOURCE_KEY="${TLS_SOURCE_DIR}/tls.key" +SHARED_CERT="${TLS_SHARED_DIR}/tls.crt" +SHARED_KEY="${TLS_SHARED_DIR}/tls.key" +SHARED_LOCK_DIR="${TLS_SHARED_DIR}.lock" + +generate_self_signed_cert() { + cert_path="$1" + key_path="$2" + openssl req \ + -x509 \ + -nodes \ + -days 365 \ + -newkey rsa:2048 \ + -subj "/CN=${TLS_CN}" \ + -addext "subjectAltName=${TLS_SUBJECT_ALT_NAME}" \ + -keyout "${key_path}" \ + -out "${cert_path}" || \ + openssl req \ + -x509 \ + -nodes \ + -days 365 \ + -newkey rsa:2048 \ + -subj "/CN=${TLS_CN}" \ + -keyout "${key_path}" \ + -out "${cert_path}" +} + +copy_cert_pair() { + cp "$1" "${TLS_CERT}" + cp "$2" "${TLS_KEY}" + chmod 0644 "${TLS_CERT}" + chmod 0600 "${TLS_KEY}" +} + +ensure_shared_cert() { + if [ -f "${SHARED_CERT}" ] && [ -f "${SHARED_KEY}" ]; then + return 0 + fi + + mkdir -p "${TLS_SHARED_DIR}" 2>/dev/null || return 1 + if mkdir "${SHARED_LOCK_DIR}" 2>/dev/null; then + if [ ! -f "${SHARED_CERT}" ] || [ ! -f "${SHARED_KEY}" ]; then + tmp_cert="${SHARED_CERT}.$$" + tmp_key="${SHARED_KEY}.$$" + generate_self_signed_cert "${tmp_cert}" "${tmp_key}" + chmod 0644 "${tmp_cert}" + chmod 0600 "${tmp_key}" + mv "${tmp_cert}" "${SHARED_CERT}" + mv "${tmp_key}" "${SHARED_KEY}" + fi + rmdir "${SHARED_LOCK_DIR}" 2>/dev/null || true + return 0 + fi + + for _ in $(seq 1 60); do + if [ -f "${SHARED_CERT}" ] && [ -f "${SHARED_KEY}" ]; then + return 0 + fi + sleep 1 + done + return 1 +} + +mkdir -p "${TLS_DIR}" + +if [ -f "${SOURCE_CERT}" ] && [ -f "${SOURCE_KEY}" ]; then + copy_cert_pair "${SOURCE_CERT}" "${SOURCE_KEY}" +elif ensure_shared_cert; then + copy_cert_pair "${SHARED_CERT}" "${SHARED_KEY}" +elif [ ! -f "${TLS_CERT}" ] || [ ! -f "${TLS_KEY}" ]; then + echo "TLS certificate not found, generating a self-signed certificate for bootstrap use." + generate_self_signed_cert "${TLS_CERT}" "${TLS_KEY}" +fi + +export SERVER_ADDRESS="${SERVER_ADDRESS:-:9001}" +export SERVER_MODE="${SERVER_MODE:-release}" +export CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB="${CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB:-500}" + +case "${CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB}" in + ''|*[!0-9]*|0) + echo "Invalid CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB=${CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB}; falling back to 500 MiB." + export CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB="500" + ;; +esac + +sed -i "s/client_max_body_size [0-9][0-9]*m;/client_max_body_size ${CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB}m;/" /etc/nginx/nginx.conf + +# Resolve the cluster DNS server for nginx so the desktop location can resolve +# per-instance Service FQDNs at request time. Prefer the first nameserver from +# /etc/resolv.conf, falling back to the common in-cluster DNS ClusterIP. +DNS_RESOLVER="$(awk '/^nameserver/ {print $2; exit}' /etc/resolv.conf 2>/dev/null || true)" +if [ -z "${DNS_RESOLVER}" ]; then + DNS_RESOLVER="10.96.0.10" +fi +sed -i "s/__DNS_RESOLVER__/${DNS_RESOLVER}/g" /etc/nginx/nginx.conf + +/usr/local/bin/clawreef-server & +backend_pid=$! + +nginx -g 'daemon off;' & +nginx_pid=$! + +shutdown() { + kill "${backend_pid}" 2>/dev/null || true + kill "${nginx_pid}" 2>/dev/null || true + wait "${backend_pid}" 2>/dev/null || true + wait "${nginx_pid}" 2>/dev/null || true +} + +trap shutdown INT TERM + +while kill -0 "${backend_pid}" 2>/dev/null && kill -0 "${nginx_pid}" 2>/dev/null; do + sleep 2 +done + +shutdown diff --git a/deployments/hermes-runtime/Dockerfile.tui-dist b/deployments/hermes-runtime/Dockerfile.tui-dist new file mode 100644 index 0000000..ae45802 --- /dev/null +++ b/deployments/hermes-runtime/Dockerfile.tui-dist @@ -0,0 +1,9 @@ +ARG HERMES_RUNTIME_BASE_IMAGE=172.16.1.12:5010/hermes:team-lite-envfix-20260610123402 +FROM ${HERMES_RUNTIME_BASE_IMAGE} + +USER root + +RUN cd /usr/local/lib/hermes-agent/ui-tui \ + && npm run build \ + && test -f dist/entry.js \ + && chmod -R a+rX dist diff --git a/deployments/k3s/cluster/README.md b/deployments/k3s/cluster/README.md new file mode 100644 index 0000000..f1a9290 --- /dev/null +++ b/deployments/k3s/cluster/README.md @@ -0,0 +1,408 @@ +# ClawManager K3s Cluster Manifest + +This directory contains the all-in-one K3s manifest for a test or small cluster +deployment: + +- `clawmanager.yaml`: Longhorn v1.12.0 plus ClawManager workloads. +- `uninstall.yaml`: Longhorn v1.12.0 uninstall job. +- StorageClasses `longhorn` and `longhorn-rwx`. + +The manifest includes Longhorn. Follow the install and uninstall order below; +do not delete the whole manifest as the first uninstall step. + +## Install + +### 1. Enter This Directory + +```sh +cd deployments/k3s/cluster +``` + +Check that `kubectl` points to the target cluster: + +```sh +kubectl config current-context +kubectl get nodes -o wide +``` + +Expected result: every node that may run Longhorn or ClawManager workloads is +`Ready`. + +If a node is not `Ready`, fix the Kubernetes node first. Longhorn will not be +stable on an unhealthy cluster. + +### 2. Check For Old Installation State + +```sh +kubectl get ns longhorn-system clawmanager-system clawmanager-user-1 --ignore-not-found +kubectl get sc longhorn longhorn-rwx --ignore-not-found +kubectl get crd | grep longhorn.io || true +``` + +Expected result for a fresh install: no `longhorn-system`, +`clawmanager-system`, `clawmanager-user-1`, Longhorn StorageClass, or Longhorn +CRD remains. + +If old Longhorn or ClawManager resources exist, run the uninstall flow in this +README before installing again. + +### 3. Check Image Availability + +The manifest uses the images listed in `clawmanager.yaml`. Make sure every node +can pull from the referenced registry before installing: + +```sh +grep -n 'image:' clawmanager.yaml | head -n 40 +``` + +If this is an offline or private-registry deployment, load or push the images to +the registry used by the manifest before continuing. + +If pods later show `ImagePullBackOff`, locate the failing image with: + +```sh +kubectl get pods -A | grep ImagePullBackOff || true +kubectl describe pod -n +``` + +Then verify registry access from the node that runs the pod. + +### 4. Prepare Every Node + +Longhorn uses iSCSI for block volumes and NFS for RWX volumes. Install the +required packages on every node that may run workloads. + +Ubuntu or Debian: + +```sh +apt-get update +apt-get install -y open-iscsi nfs-common cryptsetup dmsetup +systemctl enable --now iscsid || systemctl enable --now open-iscsi +``` + +RHEL, CentOS, Rocky, or AlmaLinux: + +```sh +yum install -y iscsi-initiator-utils nfs-utils cryptsetup device-mapper +systemctl enable --now iscsid +``` + +Check on every node: + +```sh +command -v iscsiadm +command -v mount.nfs +command -v mount.nfs4 +systemctl is-active iscsid || systemctl is-active open-iscsi +``` + +If `mount.nfs` or `mount.nfs4` is missing, pods that mount +`clawmanager-workspaces` can stay in `ContainerCreating` with an event like: + +```text +bad option; you might need a /sbin/mount. helper program +``` + +### 5. Check `multipathd` + +Longhorn volumes can be blocked by `multipathd` if Longhorn devices are not +blacklisted. + +Check on every node: + +```sh +systemctl is-active multipathd || true +multipath -t 2>/dev/null | grep -F 'devnode "^sd[a-z0-9]+"' || true +``` + +Expected result: either `multipathd` is not active, or the blacklist contains: + +```text +devnode "^sd[a-z0-9]+" +``` + +If `multipathd` is active and the blacklist is missing, either disable it if +the node does not need multipath: + +```sh +systemctl disable --now multipathd +``` + +Or add the Longhorn blacklist to `/etc/multipath.conf`, then restart +`multipathd`: + +```conf +blacklist { + devnode "^sd[a-z0-9]+" +} +``` + +```sh +systemctl restart multipathd +multipath -t | grep -F 'devnode "^sd[a-z0-9]+"' +``` + +Common symptom when this check is skipped: + +```text +Waiting for volume share to be available +is apparently in use by the system +``` + +### 6. Check Pod Network Health + +Longhorn manager, CSI, and share-manager pods require cross-node Pod networking. +First check for duplicate Pod IPs: + +```sh +kubectl get pods -A \ + -o custom-columns=IP:.status.podIP,NS:.metadata.namespace,NAME:.metadata.name,NODE:.spec.nodeName \ + --no-headers \ +| awk '$1 != "" { pods[$1] = pods[$1] "\n" $0; count[$1]++ } END { for (ip in count) if (count[ip] > 1) print pods[ip] }' +``` + +Expected result: no output. + +If this command prints any IP, fix the CNI before installing. For Calico, common +checks are: + +```sh +kubectl -n calico-system get pods -o wide +kubectl get blockaffinities.crd.projectcalico.org 2>/dev/null || true +kubectl get ippools.crd.projectcalico.org 2>/dev/null || true +``` + +If a single DaemonSet or Deployment pod has a stale or wrong IP, deleting that +pod and letting its controller recreate it is often enough: + +```sh +kubectl -n delete pod +``` + +Do not continue until the duplicate IP check has no output. + +### 7. Match Longhorn Replica Count To Storage Nodes + +The manifest defaults Longhorn StorageClasses to `numberOfReplicas: "3"`. +Use this default only when at least three Longhorn storage nodes are available. + +Check the current value: + +```sh +grep -n 'numberOfReplicas:' clawmanager.yaml +``` + +For a two-node test cluster, change both StorageClass values to `2` before +installing: + +```sh +sed -i.bak 's/numberOfReplicas: "3"/numberOfReplicas: "2"/g' clawmanager.yaml +grep -n 'numberOfReplicas:' clawmanager.yaml +``` + +If this is not adjusted, volumes may still work but will stay `degraded` with +replica scheduling errors. + +### 8. Apply The Manifest + +```sh +kubectl apply -f clawmanager.yaml +``` + +### 9. Check Longhorn + +Wait for Longhorn manager and CSI components: + +```sh +kubectl -n longhorn-system rollout status daemonset/longhorn-manager --timeout=10m +kubectl -n longhorn-system rollout status deployment/longhorn-driver-deployer --timeout=10m +kubectl -n longhorn-system rollout status daemonset/longhorn-csi-plugin --timeout=10m +kubectl -n longhorn-system get deploy,ds,pod -o wide +kubectl get csidriver driver.longhorn.io +kubectl get sc longhorn longhorn-rwx +``` + +Expected result: + +- `longhorn-manager` is ready on every node. +- `longhorn-driver-deployer` is `1/1`. +- `longhorn-csi-plugin` is ready on every node. +- `driver.longhorn.io`, `longhorn`, and `longhorn-rwx` exist. + +If `longhorn-driver-deployer` is `CrashLoopBackOff`, inspect it: + +```sh +kubectl -n longhorn-system logs -l app=longhorn-driver-deployer --all-containers --tail=200 +kubectl -n longhorn-system describe pod -l app=longhorn-driver-deployer +``` + +If the log contains `connect: connection refused` for +`http://longhorn-backend:9500/v1`, check duplicate Pod IPs and cross-node Pod +networking again. If the log mentions `MountPropagation`, also check that the +Longhorn manager pod has `mountPropagation: Bidirectional` on +`/var/lib/longhorn/`. + +### 10. Check ClawManager + +```sh +kubectl -n clawmanager-system get pvc +kubectl -n clawmanager-system rollout status deployment/clawmanager-app --timeout=15m +kubectl -n clawmanager-system rollout status deployment/openclaw-runtime --timeout=15m +kubectl -n clawmanager-system rollout status deployment/hermes-runtime --timeout=15m +kubectl -n clawmanager-system get deploy,pod,pvc -o wide +``` + +Expected result: + +- All PVCs are `Bound`. +- `clawmanager-app`, `openclaw-runtime`, and `hermes-runtime` are available. +- MySQL, Redis, MinIO, and skill scanner pods are running. + +If PVCs stay `Pending`, check CSI and StorageClass: + +```sh +kubectl get sc longhorn longhorn-rwx +kubectl -n longhorn-system get pods | grep csi +kubectl -n clawmanager-system describe pvc +``` + +If pods stay `ContainerCreating`, check recent events: + +```sh +kubectl -n clawmanager-system get events --sort-by=.lastTimestamp | tail -n 80 +``` + +### 11. Check Longhorn Volumes + +```sh +kubectl -n longhorn-system get volumes.longhorn.io \ + -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness,REPLICAS:.spec.numberOfReplicas,SHARESTATE:.status.shareState,ENDPOINT:.status.shareEndpoint +``` + +Expected result: volumes are `healthy`. The workspace RWX volume should also +show `SHARESTATE` as `running` and an NFS endpoint. + +If volumes are `degraded`, check whether `numberOfReplicas` is greater than the +number of available storage nodes. + +If the RWX volume is stuck at `shareState: starting`, inspect the share-manager: + +```sh +kubectl -n longhorn-system get pods | grep share-manager +kubectl -n longhorn-system logs --tail=200 +kubectl -n longhorn-system describe pod +``` + +Common causes are missing NFS client packages, `multipathd` interference, or +cross-node Pod network problems. + +## Uninstall + +Use this order for a disposable test deployment. The Longhorn uninstall job must +run before deleting the all-in-one manifest. + +### 1. Delete ClawManager Workloads First + +```sh +kubectl delete namespace clawmanager-user-1 --ignore-not-found +kubectl delete namespace clawmanager-system --ignore-not-found +``` + +Check: + +```sh +kubectl get ns clawmanager-user-1 clawmanager-system --ignore-not-found +kubectl get pvc -A | grep clawmanager || true +``` + +If a namespace stays `Terminating`, inspect the remaining resources: + +```sh +kubectl get all,pvc -n clawmanager-system 2>/dev/null || true +kubectl get events -n clawmanager-system --sort-by=.lastTimestamp 2>/dev/null | tail -n 80 || true +``` + +### 2. Enable Longhorn Deletion + +```sh +kubectl -n longhorn-system patch settings.longhorn.io deleting-confirmation-flag \ + --type=merge \ + -p '{"value":"true"}' +``` + +Check: + +```sh +kubectl -n longhorn-system get settings.longhorn.io deleting-confirmation-flag -o yaml +``` + +Expected result: `value: "true"`. + +### 3. Run The Longhorn Uninstall Job + +```sh +kubectl create -f uninstall.yaml +kubectl wait --for=condition=complete job/longhorn-uninstall -n longhorn-system --timeout=10m +``` + +Check: + +```sh +kubectl -n longhorn-system logs job/longhorn-uninstall --tail=200 +``` + +If the job already exists from a previous attempt: + +```sh +kubectl delete -f uninstall.yaml --ignore-not-found +kubectl create -f uninstall.yaml +kubectl wait --for=condition=complete job/longhorn-uninstall -n longhorn-system --timeout=10m +``` + +### 4. Delete The All-In-One Manifest + +```sh +kubectl delete -f clawmanager.yaml --ignore-not-found +kubectl delete -f uninstall.yaml --ignore-not-found +``` + +Final check: + +```sh +kubectl get ns longhorn-system clawmanager-system clawmanager-user-1 --ignore-not-found +kubectl get sc longhorn longhorn-rwx --ignore-not-found +kubectl get crd | grep longhorn.io || true +``` + +Expected result: no Longhorn or ClawManager resources remain. + +## Recovery For A Stuck Uninstall + +If `kubectl delete -f clawmanager.yaml` was run first and the uninstall is now +stuck, check whether the Longhorn webhook service is gone: + +```sh +kubectl -n longhorn-system get svc longhorn-admission-webhook --ignore-not-found +kubectl get validatingwebhookconfiguration longhorn-webhook-validator --ignore-not-found +kubectl get mutatingwebhookconfiguration longhorn-webhook-mutator --ignore-not-found +``` + +If the webhook service is gone but webhook configurations remain, remove the +stale webhook registrations: + +```sh +kubectl delete validatingwebhookconfiguration longhorn-webhook-validator --ignore-not-found +kubectl delete mutatingwebhookconfiguration longhorn-webhook-mutator --ignore-not-found +``` + +If Longhorn CRDs still cannot be deleted after that, use this only when you +intend to wipe Longhorn state from the cluster: + +```sh +NAMESPACE=longhorn-system +for crd in $(kubectl get crd -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep longhorn.io); do + kubectl -n "${NAMESPACE}" get "${crd}" -o yaml | sed 's/- longhorn.io//g' | kubectl apply -f - + kubectl -n "${NAMESPACE}" delete "${crd}" --all --ignore-not-found + kubectl delete "crd/${crd}" --ignore-not-found +done +``` diff --git a/deployments/k3s/cluster/clawmanager.yaml b/deployments/k3s/cluster/clawmanager.yaml new file mode 100644 index 0000000..d556a5d --- /dev/null +++ b/deployments/k3s/cluster/clawmanager.yaml @@ -0,0 +1,6496 @@ +# ClawManager cluster deployment bundle. +# Includes Longhorn v1.12.0 from: +# https://raw.githubusercontent.com/longhorn/longhorn/v1.12.0/deploy/longhorn.yaml +# +# Apply this file on a clean Kubernetes cluster: +# kubectl apply -f +# +# Notes: +# - Longhorn RWX volumes require NFSv4 client support on all workload nodes. +# Install nfs-common on Ubuntu/Debian or nfs-utils on RHEL-compatible systems. +# - ClawManager uses StorageClass longhorn for RWO data and longhorn-rwx for workspace RWX data. +--- +# Builtin: "helm template" does not respect --create-namespace +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system +--- +# Source: longhorn/templates/priorityclass.yaml +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: "longhorn-critical" + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" +globalDefault: false +preemptionPolicy: PreemptLowerPriority +value: 1000000000 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-ui-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-support-bundle + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +--- +# Source: longhorn/templates/default-resource.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-resource + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +data: + default-resource.yaml: |- +--- +# Source: longhorn/templates/default-setting.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-setting + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +data: + default-setting.yaml: |- + priority-class: "longhorn-critical" + disable-revision-counter: "{\"v1\":\"true\"}" +--- +# Source: longhorn/templates/storageclass.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-storageclass + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +data: + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: longhorn + annotations: + storageclass.kubernetes.io/is-default-class: "true" + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: "Delete" + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "3" + staleReplicaTimeout: "30" + fromBackup: "" + fsType: "ext4" + dataLocality: "disabled" + unmapMarkSnapChainRemoved: "ignored" + disableRevisionCounter: "true" + dataEngine: "v1" + backupTargetName: "default" +--- +# Source: longhorn/templates/crds.yaml +# Generated crds.yaml from github.com/longhorn/longhorn-manager/k8s/pkg/apis and the crds.yaml will be copied to longhorn/longhorn chart/templates and cannot be directly used by kubectl apply. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backingimagedatasources.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageDataSource + listKind: BackingImageDataSourceList + plural: backingimagedatasources + shortNames: + - lhbids + singular: backingimagedatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system generated UUID of the provisioned backing image file + jsonPath: .spec.uuid + name: UUID + type: string + - description: The current state of the pod used to provision the backing image + file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size + jsonPath: .status.size + name: Size + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image + data source object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackingImageDataSourceSpec defines the desired state of the + Longhorn backing image data source + properties: + checksum: + type: string + diskPath: + type: string + diskUUID: + type: string + fileTransferred: + type: boolean + nodeID: + type: string + parameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + - clone + type: string + uuid: + type: string + type: object + status: + description: BackingImageDataSourceStatus defines the observed state of + the Longhorn backing image data source + properties: + checksum: + type: string + currentState: + type: string + ip: + type: string + message: + type: string + ownerID: + type: string + progress: + type: integer + runningParameters: + additionalProperties: + type: string + nullable: true + type: object + size: + format: int64 + type: integer + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backingimagemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageManager + listKind: BackingImageManagerList + plural: backingimagemanagers + shortNames: + - lhbim + singular: backingimagemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager + object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackingImageManagerSpec defines the desired state of the + Longhorn backing image manager + properties: + backingImages: + additionalProperties: + type: string + type: object + diskPath: + type: string + diskUUID: + type: string + image: + type: string + nodeID: + type: string + type: object + status: + description: BackingImageManagerStatus defines the observed state of the + Longhorn backing image manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImageFileMap: + additionalProperties: + properties: + currentChecksum: + type: string + message: + type: string + name: + type: string + progress: + type: integer + realSize: + format: int64 + type: integer + senderManagerAddress: + type: string + sendingReference: + type: integer + size: + format: int64 + type: integer + state: + type: string + uuid: + type: string + virtualSize: + format: int64 + type: integer + type: object + nullable: true + type: object + currentState: + type: string + ip: + type: string + ownerID: + type: string + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backingimages.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImage + listKind: BackingImageList + plural: backingimages + shortNames: + - lhbi + singular: backingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system generated UUID + jsonPath: .status.uuid + name: UUID + type: string + - description: The source of the backing image file data + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size in each disk + jsonPath: .status.size + name: Size + type: string + - description: The virtual size of the image (may be larger than file size) + jsonPath: .status.virtualSize + name: VirtualSize + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackingImageSpec defines the desired state of the Longhorn + backing image + properties: + checksum: + type: string + dataEngine: + default: v1 + enum: + - v1 + - v2 + type: string + diskFileSpecMap: + additionalProperties: + properties: + dataEngine: + enum: + - v1 + - v2 + type: string + evictionRequested: + type: boolean + type: object + type: object + diskSelector: + items: + type: string + type: array + disks: + additionalProperties: + type: string + description: Deprecated. We are now using DiskFileSpecMap to assign + different spec to the file on different disks. + type: object + minNumberOfCopies: + type: integer + nodeSelector: + items: + type: string + type: array + secret: + type: string + secretNamespace: + type: string + sourceParameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + - clone + type: string + type: object + status: + description: BackingImageStatus defines the observed state of the Longhorn + backing image status + properties: + checksum: + type: string + diskFileStatusMap: + additionalProperties: + properties: + dataEngine: + enum: + - v1 + - v2 + type: string + lastStateTransitionTime: + type: string + message: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + diskLastRefAtMap: + additionalProperties: + type: string + nullable: true + type: object + ownerID: + type: string + realSize: + description: Real size of image in bytes, which may be smaller than + the size when the file is a sparse file. Will be zero until known + (e.g. while a backing image is uploading) + format: int64 + type: integer + size: + format: int64 + type: integer + uuid: + type: string + v2FirstCopyDisk: + type: string + v2FirstCopyStatus: + description: It is pending -> in-progress -> ready/failed + type: string + virtualSize: + description: Virtual size of image in bytes, which may be larger than + physical size. Will be zero until known (e.g. while a backing image + is uploading) + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backupbackingimages.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupBackingImage + listKind: BackupBackingImageList + plural: backupbackingimages + shortNames: + - lhbbi + singular: backupbackingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .status.backingImage + name: BackingImage + type: string + - description: The backing image size + jsonPath: .status.size + name: Size + type: string + - description: The backing image backup upload finished time + jsonPath: .status.backupCreatedAt + name: BackupCreatedAt + type: string + - description: The backing image backup state + jsonPath: .status.state + name: State + type: string + - description: The last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupBackingImage is where Longhorn stores backing image backup + object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackupBackingImageSpec defines the desired state of the Longhorn + backing image backup + properties: + backingImage: + description: The backing image name. + type: string + backupTargetName: + description: The backup target name. + nullable: true + type: string + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + type: object + syncRequestedAt: + description: The time to request run sync the remote backing image + backup. + format: date-time + nullable: true + type: string + userCreated: + description: Is this CR created by user through API or UI. + type: boolean + required: + - backingImage + - userCreated + type: object + status: + description: BackupBackingImageStatus defines the observed state of the + Longhorn backing image backup + properties: + backingImage: + description: The backing image name. + type: string + backupCreatedAt: + description: The backing image backup upload finished time. + type: string + checksum: + description: The checksum of the backing image. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the backing image backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backing image backup was synced + with the remote backup target. + format: date-time + nullable: true + type: string + managerAddress: + description: The address of the backing image manager that runs backing + image backup. + type: string + messages: + additionalProperties: + type: string + description: The error messages when listing or inspecting backing + image backup. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to + reconcile this CR. + type: string + progress: + description: The backing image backup progress. + type: integer + secret: + description: Record the secret if this backup backing image is encrypted + type: string + secretNamespace: + description: Record the secret namespace if this backup backing image + is encrypted + type: string + size: + description: The backing image size. + format: int64 + type: integer + state: + description: |- + The backing image backup creation state. + Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The backing image backup URL. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backups.longhorn.io +spec: + group: longhorn.io + names: + kind: Backup + listKind: BackupList + plural: backups + shortNames: + - lhb + singular: backup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup target name + jsonPath: .status.backupTargetName + name: BackupTarget + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackupSpec defines the desired state of the Longhorn backup + properties: + backupBlockSize: + description: The backup block size. 0 means the legacy default size + 2MiB, and -1 indicate the block size is invalid. + enum: + - "-1" + - "2097152" + - "16777216" + format: int64 + type: string + backupMode: + description: |- + The backup mode of this backup. + Can be "full" or "incremental" + enum: + - full + - incremental + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + type: object + snapshotName: + description: The snapshot name. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupStatus defines the observed state of the Longhorn backup + properties: + backupCreatedAt: + description: The snapshot backup upload finished time. + type: string + backupTargetName: + description: The backup target name. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the snapshot backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backup was synced with the remote + backup target. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when calling longhorn engine on listing + or inspecting backups. + nullable: true + type: object + newlyUploadDataSize: + description: Size in bytes of newly uploaded data + type: string + ownerID: + description: The node ID on which the controller is responsible to + reconcile this backup CR. + type: string + progress: + description: The snapshot backup progress. + type: integer + reUploadedDataSize: + description: Size in bytes of reuploaded data + type: string + replicaAddress: + description: The address of the replica that runs snapshot backup. + type: string + size: + description: The snapshot size. + type: string + snapshotCreatedAt: + description: The snapshot creation time. + type: string + snapshotName: + description: The snapshot name. + type: string + state: + description: |- + The backup creation state. + Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The snapshot backup URL. + type: string + volumeBackingImageName: + description: The volume's backing image name. + type: string + volumeCreated: + description: The volume creation time. + type: string + volumeName: + description: The volume name. + type: string + volumeSize: + description: The volume size. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backuptargets.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupTarget + listKind: BackupTargetList + plural: backuptargets + shortNames: + - lhbt + singular: backuptarget + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackupTargetSpec defines the desired state of the Longhorn + backup target + properties: + backupTargetURL: + description: The backup target URL. + type: string + credentialSecret: + description: The backup target credential secret. + type: string + pollInterval: + description: The interval that the cluster needs to run sync with + the backup target. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup target. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupTargetStatus defines the observed state of the Longhorn + backup target + properties: + available: + description: Available indicates if the remote backup target is available + or not. + type: boolean + conditions: + description: Records the reason on why the backup target is unavailable. + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + lastSyncedAt: + description: The last time that the controller synced with the remote + backup target. + format: date-time + nullable: true + type: string + ownerID: + description: The node ID on which the controller is responsible to + reconcile this backup target CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backupvolumes.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupVolume + listKind: BackupVolumeList + plural: backupvolumes + shortNames: + - lhbv + singular: backupvolume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target name + jsonPath: .spec.backupTargetName + name: BackupTarget + type: string + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackupVolumeSpec defines the desired state of the Longhorn + backup volume + properties: + backupTargetName: + description: The backup target name that the backup volume was synced. + nullable: true + type: string + syncRequestedAt: + description: The time to request run sync the remote backup volume. + format: date-time + nullable: true + type: string + volumeName: + description: The volume name that the backup volume was used to backup. + type: string + type: object + status: + description: BackupVolumeStatus defines the observed state of the Longhorn + backup volume + properties: + backingImageChecksum: + description: the backing image checksum. + type: string + backingImageName: + description: The backing image name. + type: string + createdAt: + description: The backup volume creation time. + type: string + dataStored: + description: The backup volume block count. + type: string + labels: + additionalProperties: + type: string + description: The backup volume labels. + nullable: true + type: object + lastBackupAt: + description: The latest volume backup time. + type: string + lastBackupName: + description: The latest volume backup name. + type: string + lastModificationTime: + description: The backup volume config last modification time. + format: date-time + nullable: true + type: string + lastSyncedAt: + description: The last time that the backup volume was synced into + the cluster. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when call longhorn engine on list + or inspect backup volumes. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to + reconcile this backup volume CR. + type: string + size: + description: The backup volume size. + type: string + storageClassName: + description: the storage class name of pv/pvc binding with the volume. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: enginefrontends.longhorn.io +spec: + group: longhorn.io + names: + kind: EngineFrontend + listKind: EngineFrontendList + plural: enginefrontends + shortNames: + - lhef + singular: enginefrontend + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The data engine of the engine frontend + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the engine frontend + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine frontend is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine frontend + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineFrontend is where Longhorn stores engine frontend object + for v2 data engine initiator. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EngineFrontendSpec defines the desired state of the Longhorn + engine frontend (v2 initiator) + properties: + active: + type: boolean + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + disableFrontend: + type: boolean + engineName: + description: EngineName is the name of the v2 engine target (required + for EngineFrontend instance creation) + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - ublk + - "" + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + salvageRequested: + type: boolean + size: + description: |- + Size is the desired size of the frontend device in bytes, as requested + by the volume owner. The EngineFrontend controller drives the frontend + device toward this size independently of the engine's target size. + format: int64 + type: string + targetIP: + description: TargetIP is the IP address of the v2 engine target + type: string + targetPort: + description: TargetPort is the port of the v2 engine target + type: integer + ublkNumberOfQueue: + description: ublkNumberOfQueue controls the number of queues for ublk + frontend. + type: integer + ublkQueueDepth: + description: ublkQueueDepth controls the depth of each queue for ublk + frontend. + type: integer + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineFrontendStatus defines the observed state of the Longhorn + engine frontend + properties: + activePath: + description: ActivePath is the currently active frontend path address. + type: string + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentSize: + description: |- + CurrentSize is the current size of the frontend device in bytes, as + observed from the data plane. It is 0 while the engine frontend is not + running. + format: int64 + type: string + currentState: + type: string + endpoint: + description: Endpoint is the initiator endpoint (e.g., /dev/longhorn/vol-name) + type: string + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + paths: + description: Paths describes the currently known frontend multipath + state. + items: + properties: + anaState: + type: string + engineName: + type: string + nguid: + type: string + nqn: + type: string + targetIP: + type: string + targetPort: + type: integer + type: object + type: array + port: + type: integer + preferredPath: + description: PreferredPath is the preferred frontend path address. + type: string + salvageExecuted: + type: boolean + started: + type: boolean + starting: + type: boolean + storageIP: + type: string + switchoverPhase: + description: SwitchoverPhase is the last completed switchover phase + reported by the data plane. + type: string + targetIP: + description: TargetIP is the currently connected IP address of the + v2 engine target + type: string + targetPort: + description: TargetPort is the currently connected port of the v2 + engine target + type: integer + ublkID: + format: int32 + type: integer + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: engineimages.longhorn.io +spec: + group: longhorn.io + names: + kind: EngineImage + listKind: EngineImageList + plural: engineimages + shortNames: + - lhei + singular: engineimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Compatibility of the engine image + jsonPath: .status.incompatible + name: Incompatible + type: boolean + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EngineImageSpec defines the desired state of the Longhorn + engine image + properties: + image: + minLength: 1 + type: string + required: + - image + type: object + status: + description: EngineImageStatus defines the observed state of the Longhorn + engine image + properties: + buildDate: + type: string + cliAPIMinVersion: + type: integer + cliAPIVersion: + type: integer + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + controllerAPIMinVersion: + type: integer + controllerAPIVersion: + type: integer + dataFormatMinVersion: + type: integer + dataFormatVersion: + type: integer + gitCommit: + type: string + incompatible: + type: boolean + noRefSince: + type: string + nodeDeploymentMap: + additionalProperties: + type: boolean + nullable: true + type: object + ownerID: + type: string + refCount: + type: integer + state: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: engines.longhorn.io +spec: + group: longhorn.io + names: + kind: Engine + listKind: EngineList + plural: engines + shortNames: + - lhe + singular: engine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The data engine of the engine + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EngineSpec defines the desired state of the Longhorn engine + properties: + active: + type: boolean + backupVolume: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + disableFrontend: + type: boolean + frontend: + enum: + - blockdev + - iscsi + - nvmf + - ublk + - "" + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + rebuildConcurrentSyncLimit: + description: |- + RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run + concurrently during a single replica rebuild. + It is determined by the global setting or the volume spec field with the same name. + maximum: 5 + minimum: 0 + type: integer + replicaAddressMap: + additionalProperties: + type: string + type: object + requestedBackupRestore: + type: string + requestedDataSource: + type: string + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + ublkNumberOfQueue: + description: ublkNumberOfQueue controls the number of queues for ublk + frontend. + type: integer + ublkQueueDepth: + description: ublkQueueDepth controls the depth of each queue for ublk + frontend. + type: integer + unmapMarkSnapChainRemovedEnabled: + type: boolean + upgradedReplicaAddressMap: + additionalProperties: + type: string + type: object + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineStatus defines the observed state of the Longhorn engine + properties: + backupStatus: + additionalProperties: + properties: + backupURL: + type: string + error: + type: string + progress: + type: integer + replicaAddress: + type: string + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + cloneStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isCloning: + type: boolean + progress: + type: integer + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentReplicaAddressMap: + additionalProperties: + type: string + nullable: true + type: object + currentSize: + format: int64 + type: string + currentState: + type: string + endpoint: + type: string + instanceManagerName: + type: string + ip: + type: string + isExpanding: + type: boolean + lastExpansionError: + type: string + lastExpansionFailedAt: + type: string + lastRestoredBackup: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + purgeStatus: + additionalProperties: + properties: + error: + type: string + isPurging: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + rebuildConcurrentSyncLimit: + description: |- + RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run + concurrently during a single replica rebuild. + It is determined by the global setting or the volume spec field with the same name. + minimum: 0 + type: integer + rebuildStatus: + additionalProperties: + properties: + appliedRebuildingMBps: + format: int64 + type: integer + error: + type: string + fromReplicaAddress: + description: Deprecated. We are now using FromReplicaAddressList + to list all source replicas. + type: string + fromReplicaAddressList: + items: + type: string + type: array + isRebuilding: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + replicaModeMap: + additionalProperties: + type: string + nullable: true + type: object + replicaTransitionTimeMap: + additionalProperties: + type: string + description: |- + ReplicaTransitionTimeMap records the time a replica in ReplicaModeMap transitions from one mode to another (or + from not being in the ReplicaModeMap to being in it). This information is sometimes required by other controllers + (e.g. the volume controller uses it to determine the correct value for replica.Spec.lastHealthyAt). + type: object + restoreStatus: + additionalProperties: + properties: + backupURL: + type: string + currentRestoringBackup: + type: string + error: + type: string + filename: + type: string + isRestoring: + type: boolean + lastRestored: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + salvageExecuted: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + snapshots: + additionalProperties: + properties: + children: + additionalProperties: + type: boolean + nullable: true + type: object + created: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + name: + type: string + parent: + type: string + removed: + type: boolean + size: + type: string + usercreated: + type: boolean + type: object + nullable: true + type: object + snapshotsError: + type: string + started: + type: boolean + starting: + type: boolean + storageIP: + type: string + ublkID: + format: int32 + type: integer + unmapMarkSnapChainRemovedEnabled: + type: boolean + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: instancemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: InstanceManager + listKind: InstanceManagerList + plural: instancemanagers + shortNames: + - lhim + singular: instancemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The data engine of the instance manager + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: InstanceManagerSpec defines the desired state of the Longhorn + instance manager + properties: + dataEngine: + type: string + dataEngineSpec: + properties: + v2: + properties: + cpuMask: + type: string + type: object + type: object + image: + type: string + nodeID: + type: string + type: + enum: + - aio + - engine + - replica + type: string + type: object + status: + description: InstanceManagerStatus defines the observed state of the Longhorn + instance manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImages: + additionalProperties: + properties: + currentChecksum: + type: string + diskUUID: + type: string + message: + type: string + name: + type: string + progress: + type: integer + size: + format: int64 + type: integer + state: + type: string + uuid: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentState: + type: string + dataEngineStatus: + properties: + v2: + properties: + cpuMask: + type: string + interruptModeEnabled: + description: |- + InterruptModeEnabled indicates whether the V2 data engine is running in + interrupt mode (true) or polling mode (false). Set by Longhorn manager; + read-only to users. + enum: + - "" + - "true" + - "false" + type: string + type: object + type: object + instanceEngineFrontends: + additionalProperties: + properties: + spec: + properties: + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + activePath: + type: string + conditions: + additionalProperties: + type: boolean + nullable: true + type: object + endpoint: + type: string + errorMsg: + type: string + frontend: + type: string + listen: + type: string + paths: + items: + properties: + anaState: + type: string + engineName: + type: string + nguid: + type: string + nqn: + type: string + targetIP: + type: string + targetPort: + type: integer + type: object + type: array + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + preferredPath: + type: string + resourceVersion: + format: int64 + type: integer + state: + type: string + targetPortEnd: + format: int32 + type: integer + targetPortStart: + format: int32 + type: integer + type: + type: string + ublkID: + format: int32 + type: integer + uuid: + type: string + type: object + type: object + nullable: true + type: object + instanceEngines: + additionalProperties: + properties: + spec: + properties: + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + activePath: + type: string + conditions: + additionalProperties: + type: boolean + nullable: true + type: object + endpoint: + type: string + errorMsg: + type: string + frontend: + type: string + listen: + type: string + paths: + items: + properties: + anaState: + type: string + engineName: + type: string + nguid: + type: string + nqn: + type: string + targetIP: + type: string + targetPort: + type: integer + type: object + type: array + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + preferredPath: + type: string + resourceVersion: + format: int64 + type: integer + state: + type: string + targetPortEnd: + format: int32 + type: integer + targetPortStart: + format: int32 + type: integer + type: + type: string + ublkID: + format: int32 + type: integer + uuid: + type: string + type: object + type: object + nullable: true + type: object + instanceReplicas: + additionalProperties: + properties: + spec: + properties: + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + activePath: + type: string + conditions: + additionalProperties: + type: boolean + nullable: true + type: object + endpoint: + type: string + errorMsg: + type: string + frontend: + type: string + listen: + type: string + paths: + items: + properties: + anaState: + type: string + engineName: + type: string + nguid: + type: string + nqn: + type: string + targetIP: + type: string + targetPort: + type: integer + type: object + type: array + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + preferredPath: + type: string + resourceVersion: + format: int64 + type: integer + state: + type: string + targetPortEnd: + format: int32 + type: integer + targetPortStart: + format: int32 + type: integer + type: + type: string + ublkID: + format: int32 + type: integer + uuid: + type: string + type: object + type: object + nullable: true + type: object + ip: + type: string + ownerID: + type: string + proxyApiMinVersion: + type: integer + proxyApiVersion: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: nodes.longhorn.io +spec: + group: longhorn.io + names: + kind: Node + listKind: NodeList + plural: nodes + shortNames: + - lhn + singular: node + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for + the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeSpec defines the desired state of the Longhorn node + properties: + allowScheduling: + type: boolean + disks: + additionalProperties: + properties: + allowScheduling: + type: boolean + diskDriver: + enum: + - "" + - auto + - aio + - nvme + type: string + diskType: + enum: + - filesystem + - block + type: string + evictionRequested: + type: boolean + path: + type: string + storageReserved: + format: int64 + type: integer + tags: + items: + type: string + type: array + type: object + type: object + evictionRequested: + type: boolean + instanceManagerCPURequest: + type: integer + name: + type: string + tags: + items: + type: string + type: array + type: object + status: + description: NodeStatus defines the observed state of the Longhorn node + properties: + autoEvicting: + type: boolean + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskDriver: + type: string + diskName: + type: string + diskPath: + type: string + diskType: + type: string + diskUUID: + type: string + filesystemType: + type: string + healthData: + additionalProperties: + properties: + attributes: + items: + properties: + id: + type: integer + name: + type: string + rawString: + type: string + rawValue: + format: int64 + type: integer + threshold: + type: integer + value: + type: integer + whenFailed: + type: string + worst: + type: integer + type: object + type: array + capacity: + format: int64 + type: integer + diskName: + type: string + diskType: + type: string + firmwareVersion: + type: string + healthStatus: + enum: + - FAILED + - PASSED + - UNKNOWN + - WARNING + type: string + modelName: + type: string + serialNumber: + type: string + source: + enum: + - SMART + - SPDK + type: string + temperature: + type: integer + type: object + type: object + healthDataLastCollectedAt: + format: date-time + type: string + instanceManagerName: + type: string + scheduledBackingImage: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + scheduledReplica: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + storageAvailable: + format: int64 + type: integer + storageMaximum: + format: int64 + type: integer + storageScheduled: + format: int64 + type: integer + type: object + nullable: true + type: object + region: + type: string + snapshotCheckStatus: + properties: + lastPeriodicCheckedAt: + format: date-time + type: string + type: object + zone: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: orphans.longhorn.io +spec: + group: longhorn.io + names: + kind: Orphan + listKind: OrphanList + plural: orphans + shortNames: + - lho + singular: orphan + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The type of the orphan + jsonPath: .spec.orphanType + name: Type + type: string + - description: The node that the orphan is on + jsonPath: .spec.nodeID + name: Node + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Orphan is where Longhorn stores orphan object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: OrphanSpec defines the desired state of the Longhorn orphaned + data + properties: + dataEngine: + description: |- + The type of data engine for instance orphan. + Can be "v1", "v2". + enum: + - v1 + - v2 + type: string + nodeID: + description: The node ID on which the controller is responsible to + reconcile this orphan CR. + type: string + orphanType: + description: |- + The type of the orphaned data. + Can be "replica". + type: string + parameters: + additionalProperties: + type: string + description: The parameters of the orphaned data + type: object + type: object + status: + description: OrphanStatus defines the observed state of the Longhorn orphaned + data + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: recurringjobs.longhorn.io +spec: + group: longhorn.io + names: + kind: RecurringJob + listKind: RecurringJobList + plural: recurringjobs + shortNames: + - lhrj + singular: recurringjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be + added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", + "snapshot-delete", "backup", "backup-force-create", "filesystem-trim" or "system-backup" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: RecurringJobSpec defines the desired state of the Longhorn + recurring job + properties: + concurrency: + description: The concurrency of taking the snapshot/backup. + type: integer + cron: + description: The cron setting. + type: string + groups: + description: The recurring job group. + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: The label of the snapshot/backup. + type: object + name: + description: The recurring job name. + type: string + parameters: + additionalProperties: + type: string + description: |- + The parameters of the snapshot/backup. + Support parameters: "full-backup-interval", "volume-backup-policy". + type: object + retain: + description: The retain count of the snapshot/backup. + type: integer + task: + description: |- + The recurring job task. + Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create", "filesystem-trim" or "system-backup". + enum: + - snapshot + - snapshot-force-create + - snapshot-cleanup + - snapshot-delete + - backup + - backup-force-create + - filesystem-trim + - system-backup + type: string + type: object + status: + description: RecurringJobStatus defines the observed state of the Longhorn + recurring job + properties: + executionCount: + description: The number of jobs that have been triggered. + type: integer + ownerID: + description: The owner ID which is responsible to reconcile this recurring + job CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: replicas.longhorn.io +spec: + group: longhorn.io + names: + kind: Replica + listKind: ReplicaList + plural: replicas + shortNames: + - lhr + singular: replica + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The data engine of the replica + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ReplicaSpec defines the desired state of the Longhorn replica + properties: + active: + type: boolean + backingImage: + type: string + dataDirectoryName: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + diskID: + type: string + diskPath: + type: string + engineName: + type: string + evictionRequested: + type: boolean + failedAt: + description: |- + FailedAt is set when a running replica fails or when a running engine is unable to use a replica for any reason. + FailedAt indicates the time the failure occurred. When FailedAt is set, a replica is likely to have useful + (though possibly stale) data. A replica with FailedAt set must be rebuilt from a non-failed replica (or it can + be used in a salvage if all replicas are failed). FailedAt is cleared before a rebuild or salvage. FailedAt may + be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume + controller acknowledges the change. + type: string + hardNodeAffinity: + type: string + healthyAt: + description: |- + HealthyAt is set the first time a replica becomes read/write in an engine after creation or rebuild. HealthyAt + indicates the time the last successful rebuild occurred. When HealthyAt is set, a replica is likely to have + useful (though possibly stale) data. HealthyAt is cleared before a rebuild. HealthyAt may be later than the + corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller + acknowledges the change. + type: string + image: + type: string + lastFailedAt: + description: |- + LastFailedAt is always set at the same time as FailedAt. Unlike FailedAt, LastFailedAt is never cleared. + LastFailedAt is not a reliable indicator of the state of a replica's data. For example, a replica with + LastFailedAt may already be healthy and in use again. However, because it is never cleared, it can be compared to + LastHealthyAt to help prevent dangerous replica deletion in some corner cases. LastFailedAt may be later than the + corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller + acknowledges the change. + type: string + lastHealthyAt: + description: |- + LastHealthyAt is set every time a replica becomes read/write in an engine. Unlike HealthyAt, LastHealthyAt is + never cleared. LastHealthyAt is not a reliable indicator of the state of a replica's data. For example, a + replica with LastHealthyAt set may be in the middle of a rebuild. However, because it is never cleared, it can be + compared to LastFailedAt to help prevent dangerous replica deletion in some corner cases. LastHealthyAt may be + later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume + controller acknowledges the change. + type: string + logRequested: + type: boolean + migrationEngineName: + description: |- + MigrationEngineName is indicating the migrating engine which current connected to this replica. This is only + used for live migration of v2 data engine + type: string + nodeID: + type: string + rebuildRetryCount: + type: integer + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkDiskChainRemovedEnabled: + type: boolean + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: ReplicaStatus defines the observed state of the Longhorn + replica + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentState: + type: string + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + salvageExecuted: + type: boolean + started: + type: boolean + starting: + type: boolean + storageIP: + type: string + ublkID: + format: int32 + type: integer + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: settings.longhorn.io +spec: + group: longhorn.io + names: + kind: Setting + listKind: SettingList + plural: settings + shortNames: + - lhs + singular: setting + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - description: The setting is applied + jsonPath: .status.applied + name: Applied + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + description: The status of the setting. + properties: + applied: + description: The setting is applied. + type: boolean + required: + - applied + type: object + value: + description: |- + The value of the setting. + - It can be a non-JSON formatted string that is applied to all the applicable data engines listed in the setting definition. + - It can be a JSON formatted string that contains values for applicable data engines listed in the setting definition's Default. + type: string + required: + - value + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: sharemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: ShareManager + listKind: ShareManagerList + plural: sharemanagers + shortNames: + - lhsm + singular: sharemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - description: The current image of the share manager + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ShareManagerSpec defines the desired state of the Longhorn + share manager + properties: + image: + description: Share manager image used for creating a share manager + pod + type: string + type: object + status: + description: ShareManagerStatus defines the observed state of the Longhorn + share manager + properties: + currentImage: + description: The image currently used by the share manager pod + type: string + endpoint: + description: NFS endpoint that can access the mounted filesystem of + the volume + type: string + ownerID: + description: The node ID on which the controller is responsible to + reconcile this share manager resource + type: string + state: + description: The state of the share manager resource + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: snapshots.longhorn.io +spec: + group: longhorn.io + names: + kind: Snapshot + listKind: SnapshotList + plural: snapshots + shortNames: + - lhsnap + singular: snapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The volume that this snapshot belongs to + jsonPath: .spec.volume + name: Volume + type: string + - description: Timestamp when the point-in-time snapshot was taken + jsonPath: .status.creationTime + name: CreationTime + type: string + - description: Indicates if the snapshot is ready to be used to restore/backup + a volume + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the minimum size of volume required to rehydrate from + this snapshot + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The actual size of the snapshot + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Snapshot is the Schema for the snapshots API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SnapshotSpec defines the desired state of Longhorn Snapshot + properties: + createSnapshot: + description: require creating a new snapshot + type: boolean + labels: + additionalProperties: + type: string + description: The labels of snapshot + nullable: true + type: object + volume: + description: |- + the volume that this snapshot belongs to. + This field is immutable after creation. + type: string + required: + - volume + type: object + status: + description: SnapshotStatus defines the observed state of Longhorn Snapshot + properties: + checksum: + type: string + checksumCalculatedAt: + description: |- + ChecksumCalculatedAt is the RFC3339 timestamp indicating when the checksum + for this snapshot was last calculated or updated. + type: string + children: + additionalProperties: + type: boolean + nullable: true + type: object + creationTime: + type: string + error: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + markRemoved: + type: boolean + ownerID: + type: string + parent: + type: string + readyToUse: + type: boolean + requestedTime: + type: string + restoreSize: + format: int64 + type: integer + size: + format: int64 + type: integer + userCreated: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: supportbundles.longhorn.io +spec: + group: longhorn.io + names: + kind: SupportBundle + listKind: SupportBundleList + plural: supportbundles + shortNames: + - lhbundle + singular: supportbundle + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the support bundle + jsonPath: .status.state + name: State + type: string + - description: The issue URL + jsonPath: .spec.issueURL + name: Issue + type: string + - description: A brief description of the issue + jsonPath: .spec.description + name: Description + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SupportBundle is where Longhorn stores support bundle object + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SupportBundleSpec defines the desired state of the Longhorn + SupportBundle + properties: + description: + description: A brief description of the issue + type: string + issueURL: + description: The issue URL + nullable: true + type: string + nodeID: + description: The preferred responsible controller node ID. + type: string + required: + - description + type: object + status: + description: SupportBundleStatus defines the observed state of the Longhorn + SupportBundle + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + type: array + filename: + type: string + filesize: + format: int64 + type: integer + image: + description: The support bundle manager image + type: string + managerIP: + description: The support bundle manager IP + type: string + ownerID: + description: The current responsible controller node ID + type: string + progress: + type: integer + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: systembackups.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemBackup + listKind: SystemBackupList + plural: systembackups + shortNames: + - lhsb + singular: systembackup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system backup Longhorn version + jsonPath: .status.version + name: Version + type: string + - description: The system backup state + jsonPath: .status.state + name: State + type: string + - description: The system backup creation time + jsonPath: .status.createdAt + name: Created + type: string + - description: The last time that the system backup was synced into the cluster + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemBackup is where Longhorn stores system backup object + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SystemBackupSpec defines the desired state of the Longhorn + SystemBackup + properties: + volumeBackupPolicy: + description: |- + The create volume backup policy + Can be "if-not-present", "always" or "disabled" + nullable: true + type: string + type: object + status: + description: SystemBackupStatus defines the observed state of the Longhorn + SystemBackup + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + createdAt: + description: The system backup creation time. + format: date-time + type: string + gitCommit: + description: The saved Longhorn manager git commit. + nullable: true + type: string + lastSyncedAt: + description: The last time that the system backup was synced into + the cluster. + format: date-time + nullable: true + type: string + managerImage: + description: The saved manager image. + type: string + ownerID: + description: The node ID of the responsible controller to reconcile + this SystemBackup. + type: string + state: + description: The system backup state. + type: string + version: + description: The saved Longhorn version. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: systemrestores.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemRestore + listKind: SystemRestoreList + plural: systemrestores + shortNames: + - lhsr + singular: systemrestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system restore state + jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemRestore is where Longhorn stores system restore object + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SystemRestoreSpec defines the desired state of the Longhorn + SystemRestore + properties: + systemBackup: + description: The system backup name in the object store. + type: string + required: + - systemBackup + type: object + status: + description: SystemRestoreStatus defines the observed state of the Longhorn + SystemRestore + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + description: The node ID of the responsible controller to reconcile + this SystemRestore. + type: string + sourceURL: + description: The source system backup URL. + type: string + state: + description: The system restore state. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: volumeattachments.longhorn.io +spec: + group: longhorn.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + shortNames: + - lhva + singular: volumeattachment + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: VolumeAttachment stores attachment information of a Longhorn + volume + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: VolumeAttachmentSpec defines the desired state of Longhorn + VolumeAttachment + properties: + attachmentTickets: + additionalProperties: + properties: + generation: + description: |- + A sequence number representing a specific generation of the desired state. + Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate + different attachments of the same volume. + type: string + nodeID: + description: The node that this attachment is requesting + type: string + parameters: + additionalProperties: + type: string + description: Optional additional parameter for this attachment + type: object + type: + type: string + type: object + type: object + volume: + description: The name of Longhorn volume of this VolumeAttachment + type: string + required: + - volume + type: object + status: + description: VolumeAttachmentStatus defines the observed state of Longhorn + VolumeAttachment + properties: + attachmentTicketStatuses: + additionalProperties: + properties: + conditions: + description: Record any error when trying to fulfill this attachment + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + generation: + description: |- + A sequence number representing a specific generation of the desired state. + Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate + different attachments of the same volume. + type: string + satisfied: + description: Indicate whether this attachment ticket has been + satisfied + type: boolean + required: + - conditions + - satisfied + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: volumes.longhorn.io +spec: + group: longhorn.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + shortNames: + - lhv + singular: volume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The data engine of the volume + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - description: The engine switchover state + jsonPath: .status.switchoverState + name: Switchover + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: VolumeSpec defines the desired state of the Longhorn volume + properties: + Standby: + type: boolean + accessMode: + enum: + - rwo + - rwop + - rwx + type: string + backingImage: + type: string + x-kubernetes-validations: + - message: BackingImage is immutable + rule: self == oldSelf + backupBlockSize: + description: BackupBlockSize indicate the block size to create backups. + The block size is immutable. + enum: + - "2097152" + - "16777216" + format: int64 + type: string + backupCompressionMethod: + enum: + - none + - lz4 + - gzip + type: string + backupTargetName: + description: The backup target name that the volume will be backed + up to or is synced. + type: string + cloneMode: + enum: + - "" + - full-copy + - linked-clone + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + dataLocality: + enum: + - disabled + - best-effort + - strict-local + type: string + dataSource: + type: string + disableFrontend: + type: boolean + diskSelector: + items: + type: string + type: array + encrypted: + type: boolean + x-kubernetes-validations: + - message: Encrypted is immutable + rule: self == oldSelf + engineNodeID: + description: |- + engineNodeID defines the node where the backend engine (target) runs. + If empty, falls back to NodeID. + type: string + freezeFilesystemForSnapshot: + description: Setting that freezes the filesystem on the root partition + before a snapshot is created. + enum: + - ignored + - enabled + - disabled + type: string + fromBackup: + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - ublk + - "" + type: string + image: + type: string + lastAttachedBy: + type: string + migratable: + type: boolean + migrationNodeID: + type: string + nodeID: + description: nodeID defines the node where the volume is attached + (where the frontend initiator runs). + type: string + nodeSelector: + items: + type: string + type: array + numberOfReplicas: + type: integer + offlineRebuilding: + description: |- + Specifies whether Longhorn should rebuild replicas while the detached volume is degraded. + - ignored: Use the global setting for offline replica rebuilding. + - enabled: Enable offline rebuilding for this volume, regardless of the global setting. + - disabled: Disable offline rebuilding for this volume, regardless of the global setting + enum: + - ignored + - disabled + - enabled + type: string + rebuildConcurrentSyncLimit: + description: |- + RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run + concurrently during a single replica rebuild. + When set to 0, it means following the global setting. + maximum: 5 + minimum: 0 + type: integer + replicaAutoBalance: + enum: + - ignored + - disabled + - least-effort + - best-effort + type: string + replicaDiskSoftAntiAffinity: + description: Replica disk soft anti affinity of the volume. Set enabled + to allow replicas to be scheduled in the same disk. + enum: + - ignored + - enabled + - disabled + type: string + replicaRebuildingBandwidthLimit: + description: ReplicaRebuildingBandwidthLimit controls the maximum + write bandwidth (in megabytes per second) allowed on the destination + replica during the rebuilding process. Set this value to 0 to disable + bandwidth limiting. + format: int64 + minimum: 0 + type: integer + replicaSoftAntiAffinity: + description: Replica soft anti affinity of the volume. Set enabled + to allow replicas to be scheduled on the same node. + enum: + - ignored + - enabled + - disabled + type: string + replicaZoneSoftAntiAffinity: + description: Replica zone soft anti affinity of the volume. Set enabled + to allow replicas to be scheduled in the same zone. + enum: + - ignored + - enabled + - disabled + type: string + restoreVolumeRecurringJob: + enum: + - ignored + - enabled + - disabled + type: string + revisionCounterDisabled: + type: boolean + size: + format: int64 + type: string + snapshotDataIntegrity: + enum: + - ignored + - disabled + - enabled + - fast-check + type: string + snapshotHashingRequestedAt: + description: |- + SnapshotHashingRequestedAt is the RFC3339 timestamp (e.g., "2026-03-16T10:30:00Z") when an on-demand snapshot checksum calculation is requested. + When this value is set and is later than LastOnDemandSnapshotHashingCompleteAt, the system will calculate checksums + for all user snapshots. + + If SnapshotHashingRequestedAt differs from LastOnDemandSnapshotHashingCompleteAt, it indicates that a hashing request + is still in progress, and a new request will be rejected. + type: string + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + staleReplicaTimeout: + type: integer + ublkNumberOfQueue: + description: ublkNumberOfQueue controls the number of queues for ublk + frontend. + type: integer + ublkQueueDepth: + description: ublkQueueDepth controls the depth of each queue for ublk + frontend. + type: integer + unmapMarkSnapChainRemoved: + enum: + - ignored + - disabled + - enabled + type: string + type: object + status: + description: VolumeStatus defines the observed state of the Longhorn volume + properties: + actualSize: + format: int64 + type: integer + cloneStatus: + properties: + attemptCount: + type: integer + nextAllowedAttemptAt: + type: string + snapshot: + type: string + sourceVolume: + type: string + state: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentEngineNodeID: + description: the node that the engine (target) is currently running + on. + type: string + currentImage: + type: string + currentMigrationNodeID: + description: the node that this volume is currently migrating to + type: string + currentNodeID: + type: string + expansionRequired: + type: boolean + frontendDisabled: + type: boolean + isStandby: + type: boolean + kubernetesStatus: + properties: + lastPVCRefAt: + type: string + lastPodRefAt: + type: string + namespace: + description: determine if PVC/Namespace is history or not + type: string + pvName: + type: string + pvStatus: + type: string + pvcName: + type: string + workloadsStatus: + description: determine if Pod/Workload is history or not + items: + properties: + podName: + type: string + podStatus: + type: string + workloadName: + type: string + workloadType: + type: string + type: object + nullable: true + type: array + type: object + lastAutoSalvagedAt: + type: string + lastBackup: + type: string + lastBackupAt: + type: string + lastDegradedAt: + type: string + lastOnDemandSnapshotHashingCompleteAt: + description: |- + LastOnDemandSnapshotHashingCompleteAt is the RFC3339 timestamp (e.g., "2026-03-16T10:30:00Z") when the + most recent on-demand snapshot checksum calculation completed. + When this value matches SnapshotHashingRequestedAt, the requested on-demand checksum calculation is considered complete. + type: string + ownerID: + type: string + remountRequestedAt: + type: string + restoreInitiated: + type: boolean + restoreRequired: + type: boolean + robustness: + type: string + shareEndpoint: + type: string + shareState: + type: string + state: + type: string + switchoverState: + description: |- + SwitchoverState describes the current progress of a v2 engine live switchover. + Empty when no switchover is in progress. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-role + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "delete", "deletecollection"] +- apiGroups: [""] + resources: ["secrets", "services", "endpoints", "configmaps", "serviceaccounts", "pods/log"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events", "persistentvolumes", "persistentvolumeclaims", "persistentvolumeclaims/status", "nodes"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["apps"] + resources: ["daemonsets", "statefulsets", "deployments", "replicasets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list", "watch"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "volumeattributesclasses", "csinodes", "csidrivers", "csistoragecapacities"] + verbs: ["*"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["*"] +- apiGroups: ["longhorn.io"] + resources: ["volumes", "volumes/status", "enginefrontends", "enginefrontends/status", "engines", "engines/status", "replicas", "replicas/status", "settings", "settings/status", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch"] +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] +- apiGroups: ["apiregistration.k8s.io"] + resources: ["apiservices"] + verbs: ["list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "create", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "list", "watch"] +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterrolebindings", "clusterroles"] + verbs: ["*"] +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-bind + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: longhorn-system +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-support-bundle + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: longhorn-support-bundle + namespace: longhorn-system +--- +# Source: longhorn/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhorn + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +rules: +- apiGroups: [""] + resources: ["pods", "pods/log", "events", "secrets", "services", "endpoints", "configmaps", "serviceaccounts", "persistentvolumeclaims", "persistentvolumeclaims/status"] + verbs: ["*"] +- apiGroups: ["apps"] + resources: ["daemonsets", "deployments", "statefulsets", "replicasets"] + verbs: ["*"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["*"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["*"] +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["*"] +--- +# Source: longhorn/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhorn + namespace: longhorn-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhorn +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: longhorn-system +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-manager + name: longhorn-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: manager + port: 9500 + targetPort: manager +--- +# Source: longhorn/templates/deployment-ui.yaml +kind: Service +apiVersion: v1 +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-ui + name: longhorn-frontend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-ui + ports: + - name: http + port: 80 + targetPort: http + nodePort: null +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-admission-webhook + name: longhorn-admission-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + longhorn.io/admission-webhook: longhorn-admission-webhook + ports: + - name: admission-webhook + port: 9502 + targetPort: admission-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-recovery-backend + name: longhorn-recovery-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + longhorn.io/recovery-backend: longhorn-recovery-backend + ports: + - name: recovery-backend + port: 9503 + targetPort: recov-backend +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-manager + name: longhorn-manager + namespace: longhorn-system +spec: + selector: + matchLabels: + app: longhorn-manager + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-manager + spec: + containers: + - name: longhorn-manager + image: docker.io/longhornio/longhorn-manager:v1.12.0 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - longhorn-manager + - -d + - daemon + - --engine-image + - "docker.io/longhornio/longhorn-engine:v1.12.0" + - --instance-manager-image + - "docker.io/longhornio/longhorn-instance-manager:v1.12.0" + - --share-manager-image + - "docker.io/longhornio/longhorn-share-manager:v1.12.0" + - --backing-image-manager-image + - "docker.io/longhornio/backing-image-manager:v1.12.0" + - --support-bundle-manager-image + - "docker.io/longhornio/support-bundle-kit:v0.0.86" + - --manager-image + - "docker.io/longhornio/longhorn-manager:v1.12.0" + - --service-account + - longhorn-service-account + - --upgrade-version-check + ports: + - containerPort: 9500 + name: manager + - containerPort: 9502 + name: admission-wh + - containerPort: 9503 + name: recov-backend + readinessProbe: + httpGet: + path: /v1/healthz + port: 9502 + scheme: HTTPS + volumeMounts: + - name: boot + mountPath: /host/boot/ + readOnly: true + - name: dev + mountPath: /host/dev/ + - name: proc + mountPath: /host/proc/ + readOnly: true + - name: etc + mountPath: /host/etc/ + readOnly: true + - name: longhorn + mountPath: /var/lib/longhorn/ + mountPropagation: Bidirectional + - name: longhorn-grpc-tls + mountPath: /tls-files/ + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: LONGHORN_DISTRO + value: "longhorn" + + - name: pre-pull-share-manager-image + imagePullPolicy: IfNotPresent + image: docker.io/longhornio/longhorn-share-manager:v1.12.0 + command: ["sh", "-c", "echo share-manager image pulled && sleep infinity"] + volumes: + - name: boot + hostPath: + path: /boot/ + - name: dev + hostPath: + path: /dev/ + - name: proc + hostPath: + path: /proc/ + - name: etc + hostPath: + path: /etc/ + - name: longhorn + hostPath: + path: /var/lib/longhorn/ + - name: longhorn-grpc-tls + secret: + secretName: longhorn-grpc-tls + optional: true + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + updateStrategy: + rollingUpdate: + maxUnavailable: 100% +--- +# Source: longhorn/templates/deployment-driver.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhorn-driver-deployer + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +spec: + replicas: 1 + selector: + matchLabels: + app: longhorn-driver-deployer + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-driver-deployer + spec: + initContainers: + - name: wait-longhorn-manager + image: docker.io/longhornio/longhorn-manager:v1.12.0 + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] + containers: + - name: longhorn-driver-deployer + image: docker.io/longhornio/longhorn-manager:v1.12.0 + imagePullPolicy: IfNotPresent + command: + - longhorn-manager + - -d + - deploy-driver + - --manager-image + - "docker.io/longhornio/longhorn-manager:v1.12.0" + - --manager-url + - http://longhorn-backend:9500/v1 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: CSI_ATTACHER_IMAGE + value: "docker.io/longhornio/csi-attacher:v4.12.0" + - name: CSI_PROVISIONER_IMAGE + value: "docker.io/longhornio/csi-provisioner:v5.3.0-20260514" + - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE + value: "docker.io/longhornio/csi-node-driver-registrar:v2.17.0" + - name: CSI_RESIZER_IMAGE + value: "docker.io/longhornio/csi-resizer:v2.1.0-20260514" + - name: CSI_SNAPSHOTTER_IMAGE + value: "docker.io/longhornio/csi-snapshotter:v8.5.0-20260514" + - name: CSI_LIVENESS_PROBE_IMAGE + value: "docker.io/longhornio/livenessprobe:v2.19.0" + + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + securityContext: + runAsUser: 0 +--- +# Source: longhorn/templates/deployment-ui.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-ui + name: longhorn-ui + namespace: longhorn-system +spec: + replicas: 2 + selector: + matchLabels: + app: longhorn-ui + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-ui + spec: + serviceAccountName: longhorn-ui-service-account + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-ui + topologyKey: kubernetes.io/hostname + weight: 1 + containers: + - name: longhorn-ui + image: docker.io/longhornio/longhorn-ui:v1.12.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: nginx-cache + mountPath: /var/cache/nginx/ + - name: nginx-config + mountPath: /var/config/nginx/ + - name: var-run + mountPath: /var/run/ + ports: + - containerPort: 8000 + name: http + env: + - name: LONGHORN_MANAGER_IP + value: "http://longhorn-backend:9500" + - name: LONGHORN_UI_PORT + value: "8000" + + volumes: + - emptyDir: {} + name: nginx-cache + - emptyDir: {} + name: nginx-config + - emptyDir: {} + name: var-run + priorityClassName: "longhorn-critical" +--- +# Source: longhorn/templates/validate-psp-install.yaml +# +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: longhorn-rwx + annotations: + storageclass.kubernetes.io/is-default-class: "false" +provisioner: driver.longhorn.io +allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + numberOfReplicas: "3" + staleReplicaTimeout: "30" + fromBackup: "" + fsType: "ext4" +--- +# ClawManager cluster install profile. +# Official validation target: Longhorn CSI example. +# Defaults use longhorn for RWO data and longhorn-rwx for RWX workspaces. +# These StorageClass names are examples and may be replaced by any compatible CSI classes. +apiVersion: v1 +kind: Namespace +metadata: + name: clawmanager-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: clawmanager-secrets + namespace: clawmanager-system +type: Opaque +stringData: + mysql-root-password: root123 + mysql-password: clawreef123 + jwt-secret: change-me-in-production + runtime-agent-control-token: change-me-runtime-control-token + runtime-agent-report-token: change-me-runtime-report-token + openclaw-gateway-token: change-me-openclaw-gateway-token + minio-root-user: minioadmin + minio-root-password: minioadmin123 + minio-access-key: minioadmin + minio-secret-key: minioadmin123 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: clawmanager-mysql-init + namespace: clawmanager-system +data: + 001_init_schema.sql: | + CREATE DATABASE IF NOT EXISTS clawmanager CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + USE clawmanager; + + CREATE TABLE IF NOT EXISTS users ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(255) UNIQUE NOT NULL, + email VARCHAR(320) UNIQUE NOT NULL, + password_hash VARCHAR(255) NOT NULL, + role ENUM('admin', 'user') DEFAULT 'user', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + last_login TIMESTAMP, + INDEX idx_username (username), + INDEX idx_role (role) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instances ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT, + type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu', + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + status ENUM('creating', 'running', 'stopped', 'error', 'deleting') DEFAULT 'creating', + cpu_cores DECIMAL(10,2) NOT NULL, + memory_gb INT NOT NULL, + disk_gb INT NOT NULL, + gpu_enabled BOOLEAN DEFAULT FALSE, + gpu_type VARCHAR(100), + gpu_count INT DEFAULT 0, + os_type VARCHAR(50) NOT NULL, + os_version VARCHAR(50) NOT NULL, + image_registry VARCHAR(255), + image_tag VARCHAR(100), + storage_class VARCHAR(50) DEFAULT 'standard', + mount_path VARCHAR(255) DEFAULT '/data', + pod_name VARCHAR(255), + pod_namespace VARCHAR(255), + pod_ip VARCHAR(45), + access_url VARCHAR(500), + access_token VARCHAR(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + started_at TIMESTAMP, + stopped_at TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id), + INDEX idx_status (status), + INDEX idx_type (type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS persistent_volumes ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + pvc_name VARCHAR(255) UNIQUE NOT NULL, + pvc_namespace VARCHAR(255) NOT NULL, + storage_size_gb INT NOT NULL, + storage_class VARCHAR(50), + mount_path VARCHAR(255), + status ENUM('pending', 'bound', 'released', 'failed') DEFAULT 'pending', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + UNIQUE KEY uk_pvc_name_namespace (pvc_name, pvc_namespace) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backups ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + backup_name VARCHAR(255) NOT NULL, + backup_size_gb INT, + backup_path VARCHAR(500), + status ENUM('creating', 'completed', 'failed', 'deleted') DEFAULT 'creating', + backup_type ENUM('manual', 'scheduled') DEFAULT 'manual', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP, + expires_at TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backup_schedules ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + schedule_name VARCHAR(255), + cron_expression VARCHAR(100) NOT NULL, + retention_days INT DEFAULT 30, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS user_quotas ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL UNIQUE, + max_instances INT DEFAULT 10, + max_cpu_cores DECIMAL(10,2) DEFAULT 40, + max_memory_gb INT DEFAULT 100, + max_storage_gb INT DEFAULT 500, + max_gpu_count INT DEFAULT 2, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_usage ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + cpu_usage_percent DECIMAL(5,2), + memory_usage_gb DECIMAL(10,2), + disk_usage_gb DECIMAL(10,2), + gpu_usage_percent DECIMAL(5,2), + uptime_seconds INT, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_recorded (instance_id, recorded_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS audit_logs ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT, + action VARCHAR(100) NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_id INT, + details JSON, + ip_address VARCHAR(45), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL, + INDEX idx_user_id (user_id), + INDEX idx_action (action), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + INSERT INTO users (username, email, password_hash, role, is_active) + SELECT 'admin', 'admin@clawmanager.local', '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi', 'admin', TRUE + WHERE NOT EXISTS (SELECT 1 FROM users WHERE username = 'admin'); + + INSERT INTO user_quotas (user_id, max_instances, max_cpu_cores, max_memory_gb, max_storage_gb, max_gpu_count) + SELECT id, 100, 200, 1000, 5000, 10 FROM users + WHERE username = 'admin' + AND NOT EXISTS (SELECT 1 FROM user_quotas WHERE user_id = users.id); + 002_add_webtop_instance_type.sql: | + USE clawmanager; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 003_add_system_image_settings.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS system_image_settings ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_type VARCHAR(50) NOT NULL, + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + display_name VARCHAR(255) NOT NULL, + image VARCHAR(500) NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_instance_type (instance_type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 004_fix_seeded_admin_password.sql: | + USE clawmanager; + UPDATE users + SET password_hash = '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi' + WHERE username = 'admin' + AND password_hash = '$2a$10$N9qo8uLOickgx2ZMRZoMy.MqrzL9wGC3qD3Q.ZHqQH6t3q7l1L5uG'; + 005_update_openclaw_default_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + 006_add_openclaw_config_center.sql: | + USE clawmanager; + SET @openclaw_snapshot_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'openclaw_config_snapshot_id' + ); + SET @openclaw_snapshot_column_sql = IF( + @openclaw_snapshot_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN openclaw_config_snapshot_id INT NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE openclaw_snapshot_column_stmt FROM @openclaw_snapshot_column_sql; + EXECUTE openclaw_snapshot_column_stmt; + DEALLOCATE PREPARE openclaw_snapshot_column_stmt; + + CREATE TABLE IF NOT EXISTS openclaw_config_resources ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_key VARCHAR(100) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + tags_json LONGTEXT NOT NULL, + content_json LONGTEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_resource_key (user_id, resource_type, resource_key), + INDEX idx_openclaw_resource_user_type (user_id, resource_type), + INDEX idx_openclaw_resource_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundles ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_openclaw_bundle_user (user_id), + INDEX idx_openclaw_bundle_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_items ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + resource_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (resource_id) REFERENCES openclaw_config_resources(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_resource (bundle_id, resource_id), + INDEX idx_openclaw_bundle_item_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_injection_snapshots ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NULL, + user_id INT NOT NULL, + mode VARCHAR(20) NOT NULL, + bundle_id INT NULL, + selected_resource_ids_json LONGTEXT NOT NULL, + resolved_resources_json LONGTEXT NOT NULL, + rendered_manifest_json LONGTEXT NOT NULL, + rendered_env_json LONGTEXT NOT NULL, + secret_name VARCHAR(255) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + activated_at TIMESTAMP NULL, + INDEX idx_openclaw_snapshot_user_created (user_id, created_at), + INDEX idx_openclaw_snapshot_instance (instance_id), + INDEX idx_openclaw_snapshot_bundle (bundle_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 007_add_instance_agent_control_plane.sql: | + USE clawmanager; + SET @instance_agent_bootstrap_token_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'agent_bootstrap_token' + ); + SET @instance_agent_bootstrap_token_column_sql = IF( + @instance_agent_bootstrap_token_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN agent_bootstrap_token VARCHAR(255) NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE instance_agent_bootstrap_token_column_stmt FROM @instance_agent_bootstrap_token_column_sql; + EXECUTE instance_agent_bootstrap_token_column_stmt; + DEALLOCATE PREPARE instance_agent_bootstrap_token_column_stmt; + + CREATE TABLE IF NOT EXISTS instance_agents ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NOT NULL, + agent_version VARCHAR(50) NOT NULL, + protocol_version VARCHAR(50) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'online', + capabilities_json LONGTEXT NOT NULL, + host_info_json LONGTEXT NULL, + session_token VARCHAR(255) NULL, + session_expires_at TIMESTAMP NULL, + last_heartbeat_at TIMESTAMP NULL, + last_reported_at TIMESTAMP NULL, + last_seen_ip VARCHAR(45) NULL, + registered_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_agents_instance (instance_id), + UNIQUE KEY uk_instance_agents_session_token (session_token), + INDEX idx_instance_agents_agent_id (agent_id), + INDEX idx_instance_agents_status (status), + INDEX idx_instance_agents_last_heartbeat (last_heartbeat_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_runtime_status ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + infra_status VARCHAR(30) NOT NULL DEFAULT 'creating', + agent_status VARCHAR(30) NOT NULL DEFAULT 'offline', + openclaw_status VARCHAR(30) NOT NULL DEFAULT 'unknown', + openclaw_pid INT NULL, + openclaw_version VARCHAR(100) NULL, + current_config_revision_id INT NULL, + desired_config_revision_id INT NULL, + summary_json LONGTEXT NULL, + system_info_json LONGTEXT NULL, + health_json LONGTEXT NULL, + last_reported_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_runtime_status_instance (instance_id), + INDEX idx_instance_runtime_status_agent_status (agent_status), + INDEX idx_instance_runtime_status_openclaw_status (openclaw_status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_desired_state ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + desired_power_state VARCHAR(30) NOT NULL DEFAULT 'running', + desired_config_revision_id INT NULL, + desired_runtime_action VARCHAR(50) NULL, + updated_by INT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_desired_state_instance (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_commands ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NULL, + command_type VARCHAR(50) NOT NULL, + payload_json LONGTEXT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + idempotency_key VARCHAR(255) NOT NULL, + issued_by INT NULL, + issued_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + dispatched_at TIMESTAMP NULL, + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + timeout_seconds INT NOT NULL DEFAULT 300, + result_json LONGTEXT NULL, + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (issued_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_commands_idempotency (instance_id, idempotency_key), + INDEX idx_instance_commands_instance_status (instance_id, status), + INDEX idx_instance_commands_agent_status (agent_id, status), + INDEX idx_instance_commands_issued_at (issued_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_config_revisions ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + source_snapshot_id INT NULL, + source_bundle_id INT NULL, + revision_no INT NOT NULL, + content_json LONGTEXT NOT NULL, + checksum VARCHAR(255) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'published', + published_by INT NULL, + published_at TIMESTAMP NULL, + activated_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (published_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_config_revision_unique (instance_id, revision_no), + INDEX idx_instance_config_revision_instance (instance_id, revision_no), + INDEX idx_instance_config_revision_status (status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 008_add_skill_management.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS skill_blobs ( + id INT AUTO_INCREMENT PRIMARY KEY, + content_hash VARCHAR(128) NOT NULL, + archive_hash VARCHAR(128) NOT NULL, + object_key VARCHAR(512) NOT NULL, + file_name VARCHAR(255) NOT NULL, + media_type VARCHAR(100) NOT NULL DEFAULT 'application/gzip', + size_bytes BIGINT NOT NULL DEFAULT 0, + scan_status VARCHAR(30) NOT NULL DEFAULT 'pending', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_skill_blobs_content_hash (content_hash), + INDEX idx_skill_blobs_scan_status (scan_status), + INDEX idx_skill_blobs_risk_level (risk_level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + skill_key VARCHAR(120) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + current_version_id INT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + status VARCHAR(30) NOT NULL DEFAULT 'active', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_skills_user_key (user_id, skill_key), + INDEX idx_skills_user_status (user_id, status), + INDEX idx_skills_risk_level (risk_level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skill_versions ( + id INT AUTO_INCREMENT PRIMARY KEY, + skill_id INT NOT NULL, + blob_id INT NOT NULL, + version_no INT NOT NULL, + manifest_json LONGTEXT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE RESTRICT, + UNIQUE KEY uk_skill_versions_skill_version (skill_id, version_no), + UNIQUE KEY uk_skill_versions_skill_blob (skill_id, blob_id), + INDEX idx_skill_versions_skill_id (skill_id, version_no) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + skill_id INT NOT NULL, + skill_version_id INT NULL, + source_type VARCHAR(40) NOT NULL DEFAULT 'discovered_in_instance', + install_path VARCHAR(1024) NULL, + observed_hash VARCHAR(128) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'active', + last_seen_at TIMESTAMP NULL, + removed_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (skill_version_id) REFERENCES skill_versions(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_skills_instance_skill (instance_id, skill_id), + INDEX idx_instance_skills_instance (instance_id, status), + INDEX idx_instance_skills_skill (skill_id, status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skill_scan_results ( + id INT AUTO_INCREMENT PRIMARY KEY, + blob_id INT NOT NULL, + engine VARCHAR(60) NOT NULL, + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + status VARCHAR(30) NOT NULL DEFAULT 'completed', + summary TEXT NULL, + findings_json LONGTEXT NULL, + scanned_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE CASCADE, + INDEX idx_skill_scan_results_blob (blob_id, scanned_at), + INDEX idx_skill_scan_results_risk (risk_level, scanned_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 009_cpu_cores_decimal.sql: | + USE clawmanager; + ALTER TABLE instances MODIFY COLUMN cpu_cores DECIMAL(10,2) NOT NULL; + ALTER TABLE user_quotas MODIFY COLUMN max_cpu_cores DECIMAL(10,2) DEFAULT 40; + 011_add_hermes_instance_type.sql: | + USE clawmanager; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 012_update_agents_runtime_default_images.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' + WHERE instance_type = 'hermes' + AND image IN ( + 'registry.example.com/hermes-webtop:latest', + 'lscr.io/linuxserver/webtop:ubuntu-xfce' + ); + 013_normalize_agents_runtime_image_names.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/openclaw:latest'; + + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/hermes:latest'; + 019_add_instance_runtime_type.sql: | + USE clawmanager; + SET @instance_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'runtime_type' + ); + SET @instance_runtime_type_column_sql = IF( + @instance_runtime_type_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER type', + 'SELECT 1' + ); + PREPARE instance_runtime_type_column_stmt FROM @instance_runtime_type_column_sql; + EXECUTE instance_runtime_type_column_stmt; + DEALLOCATE PREPARE instance_runtime_type_column_stmt; + 020_add_system_image_runtime_type.sql: | + USE clawmanager; + SET @system_image_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'runtime_type' + ); + SET @system_image_runtime_type_column_sql = IF( + @system_image_runtime_type_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER instance_type', + 'SELECT 1' + ); + PREPARE system_image_runtime_type_column_stmt FROM @system_image_runtime_type_column_sql; + EXECUTE system_image_runtime_type_column_stmt; + DEALLOCATE PREPARE system_image_runtime_type_column_stmt; + 021_add_openclaw_shell_runtime_image.sql: | + USE clawmanager; + SET @system_image_is_enabled_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'is_enabled' + ); + SET @system_image_is_enabled_column_sql = IF( + @system_image_is_enabled_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN is_enabled BOOLEAN NOT NULL DEFAULT TRUE', + 'SELECT 1' + ); + PREPARE system_image_is_enabled_column_stmt FROM @system_image_is_enabled_column_sql; + EXECUTE system_image_is_enabled_column_stmt; + DEALLOCATE PREPARE system_image_is_enabled_column_stmt; + + SET @system_image_unique_instance_type_index_name = ( + SELECT INDEX_NAME + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'instance_type' + AND NON_UNIQUE = 0 + AND INDEX_NAME <> 'PRIMARY' + LIMIT 1 + ); + SET @system_image_unique_instance_type_index_sql = IF( + @system_image_unique_instance_type_index_name IS NOT NULL, + CONCAT( + 'ALTER TABLE system_image_settings DROP INDEX `', + REPLACE(@system_image_unique_instance_type_index_name, '`', '``'), + '`' + ), + 'SELECT 1' + ); + PREPARE system_image_unique_instance_type_index_stmt FROM @system_image_unique_instance_type_index_sql; + EXECUTE system_image_unique_instance_type_index_stmt; + DEALLOCATE PREPARE system_image_unique_instance_type_index_stmt; + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'desktop', + 'OpenClaw Desktop', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'desktop' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'shell', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + 022_add_openclaw_config_bundle_skills.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + skill_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id), + INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 023_normalize_runtime_config_mounts.sql: | + USE clawmanager; + UPDATE instances + SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP + WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); + 031_add_hermes_lite_runtime_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + 'registry.example.com/your-custom-shell-image:latest' + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'hermes', + 'shell', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND is_enabled = FALSE + ); +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: longhorn +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: longhorn +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clawmanager-workspaces + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Gi + storageClassName: longhorn-rwx +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: mysql + template: + metadata: + labels: + app: mysql + spec: + containers: + - name: mysql + image: mysql:8.4.8 + imagePullPolicy: IfNotPresent + args: + - "--innodb-use-native-aio=0" + ports: + - containerPort: 3306 + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-root-password + - name: MYSQL_DATABASE + value: clawmanager + - name: MYSQL_USER + value: clawmanager + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + - name: mysql-init + mountPath: /docker-entrypoint-initdb.d + readinessProbe: + exec: + command: ["sh", "-c", "mysqladmin ping -h 127.0.0.1 -uroot -p$MYSQL_ROOT_PASSWORD"] + initialDelaySeconds: 20 + periodSeconds: 10 + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: mysql-data + - name: mysql-init + configMap: + name: clawmanager-mysql-init +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: clawmanager-system +spec: + selector: + app: mysql + ports: + - name: mysql + port: 3306 + targetPort: 3306 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio:latest + imagePullPolicy: IfNotPresent + args: + - server + - /data + - --console-address + - ":9001" + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-root-password + ports: + - name: api + containerPort: 9000 + - name: console + containerPort: 9001 + volumeMounts: + - name: minio-data + mountPath: /data + readinessProbe: + tcpSocket: + port: api + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: minio-data + persistentVolumeClaim: + claimName: minio-data +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: clawmanager-system +spec: + selector: + app: minio + ports: + - name: api + port: 9000 + targetPort: api + - name: console + port: 9001 + targetPort: console +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-team-redis + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: clawmanager-team-redis + template: + metadata: + labels: + app: clawmanager-team-redis + spec: + containers: + - name: redis + image: redis:7-alpine + imagePullPolicy: IfNotPresent + args: + - redis-server + - --appendonly + - "yes" + ports: + - name: redis + containerPort: 6379 + readinessProbe: + tcpSocket: + port: redis + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: redis-data + mountPath: /data + volumes: + - name: redis-data + persistentVolumeClaim: + claimName: redis-data +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-redis + namespace: clawmanager-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-team-redis + namespace: clawmanager-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: clawmanager-runtime-manager +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - apiGroups: + - apps + resources: + - deployments + - deployments/scale + verbs: + - get + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager-runtime-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clawmanager-runtime-manager +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager-app-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: skill-scanner + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: skill-scanner + template: + metadata: + labels: + app: skill-scanner + spec: + containers: + - name: skill-scanner + image: ghcr.io/yuan-lab-llm/skill-scanner:latest + imagePullPolicy: IfNotPresent + command: + - /opt/skill-scanner-venv/bin/skill-scanner-api + - --host + - 0.0.0.0 + - --port + - "8000" + env: + - name: SKILL_SCANNER_LLM_API_KEY + value: "" + - name: SKILL_SCANNER_LLM_MODEL + value: "" + - name: SKILL_SCANNER_LLM_BASE_URL + value: "" + - name: SKILL_SCANNER_META_LLM_API_KEY + value: "" + - name: SKILL_SCANNER_META_LLM_MODEL + value: "" + - name: SKILL_SCANNER_META_LLM_BASE_URL + value: "" + ports: + - name: http + containerPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: skill-scanner + namespace: clawmanager-system +spec: + selector: + app: skill-scanner + ports: + - name: http + port: 8000 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-app + namespace: clawmanager-system +spec: + replicas: 3 + selector: + matchLabels: + app: clawmanager-app + template: + metadata: + labels: + app: clawmanager-app + spec: + serviceAccountName: clawmanager-app + containers: + - name: clawmanager-app + image: ghcr.io/yuan-lab-llm/clawmanager:latest + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 8443 + - name: proxy + containerPort: 9001 + env: + - name: SERVER_ADDRESS + value: ":9001" + - name: SERVER_MODE + value: "release" + - name: DB_HOST + value: "mysql" + - name: DB_PORT + value: "3306" + - name: DB_USER + value: "clawmanager" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + - name: DB_NAME + value: "clawmanager" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: jwt-secret + - name: OBJECT_STORAGE_ENDPOINT + value: "minio.clawmanager-system.svc.cluster.local:9000" + - name: OBJECT_STORAGE_REGION + value: "" + - name: OBJECT_STORAGE_ACCESS_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-access-key + - name: OBJECT_STORAGE_SECRET_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-secret-key + - name: OBJECT_STORAGE_BUCKET + value: "clawmanager-skills" + - name: OBJECT_STORAGE_USE_SSL + value: "false" + - name: OBJECT_STORAGE_BASE_PATH + value: "skills" + - name: OBJECT_STORAGE_FORCE_PATH_STYLE + value: "true" + - name: K8S_MODE + value: "incluster" + - name: K8S_NAMESPACE + value: "clawmanager" + - name: K8S_STORAGE_CLASS + value: "longhorn" + - name: CLAWMANAGER_STORAGE_PROFILE + value: "cluster" + - name: K8S_HOSTPATH_FALLBACK_ENABLED + value: "false" + - name: K8S_PVC_BIND_TIMEOUT + value: "2m" + - name: K8S_CONTROL_PLANE_STORAGE_CLASS + value: "longhorn" + - name: K8S_INSTANCE_STORAGE_CLASS + value: "longhorn" + - name: K8S_WORKSPACE_STORAGE_CLASS + value: "longhorn-rwx" + - name: K8S_WORKSPACE_ACCESS_MODE + value: "ReadWriteMany" + - name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB + value: "500" + - name: PLATFORM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: TEAM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_TEAM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_TEAM_MANAGER_BASE_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: RUNTIME_NAMESPACE + value: "clawmanager-system" + - name: RUNTIME_WORKSPACE_ROOT + value: "/workspaces" + - name: RUNTIME_WORKSPACE_PVC_CLAIM + value: "clawmanager-workspaces" + - name: RUNTIME_MAX_GATEWAYS_PER_POD + value: "100" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_SCHEDULER_ENABLED + value: "true" + - name: RUNTIME_HEARTBEAT_TIMEOUT + value: "10s" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: SKILL_SCANNER_ENABLED + value: "true" + - name: SKILL_SCANNER_BASE_URL + value: "http://skill-scanner.clawmanager-system.svc.cluster.local:8000" + - name: SKILL_SCANNER_TIMEOUT_SECONDS + value: "120" + - name: SKILL_SCANNER_NAMESPACE + value: "clawmanager-system" + - name: SKILL_SCANNER_DEPLOYMENT + value: "skill-scanner" + volumeMounts: + - name: tls-cert + mountPath: /var/run/clawreef-tls + readOnly: true + - name: workspaces + mountPath: /workspaces + readinessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 30 + periodSeconds: 20 + volumes: + - name: tls-cert + secret: + secretName: clawmanager-tls + optional: true + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openclaw-runtime + namespace: clawmanager-system + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw +spec: + replicas: 1 + selector: + matchLabels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + template: + metadata: + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + imagePullPolicy: Always + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: openclaw + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: openclaw-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_COMMAND + value: "/usr/local/bin/openclaw gateway run --allow-unconfigured --auth token --bind lan --force" + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-runtime + namespace: clawmanager-system + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes +spec: + replicas: 1 + selector: + matchLabels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + template: + metadata: + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + imagePullPolicy: IfNotPresent + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: hermes + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: hermes-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + - name: CLAWMANAGER_AGENT_PORT + value: "19090" + - name: CLAWMANAGER_GATEWAY_PORT_START + value: "20000" + - name: CLAWMANAGER_GATEWAY_PORT_END + value: "20099" + - name: HERMES_TUI_DIR + value: /usr/local/lib/hermes-agent/ui-tui + - name: CLAWMANAGER_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: CLAWMANAGER_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-frontend + namespace: clawmanager-system +spec: + type: NodePort + selector: + app: clawmanager-app + ports: + - name: https + port: 443 + targetPort: https + nodePort: 30443 +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-gateway + namespace: clawmanager-system +spec: + type: ClusterIP + selector: + app: clawmanager-app + ports: + - name: api + port: 9001 + targetPort: 9001 +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-egress-proxy + namespace: clawmanager-system +spec: + type: ClusterIP + selector: + app: clawmanager-app + ports: + - name: proxy + port: 3128 + targetPort: 9001 diff --git a/deployments/k3s/cluster/uninstall.yaml b/deployments/k3s/cluster/uninstall.yaml new file mode 100644 index 0000000..e87daa5 --- /dev/null +++ b/deployments/k3s/cluster/uninstall.yaml @@ -0,0 +1,148 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-uninstall-service-account + namespace: longhorn-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-uninstall-role +rules: + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" + - apiGroups: + - "" + resources: + - pods + - persistentvolumes + - persistentvolumeclaims + - nodes + - configmaps + - secrets + - services + - endpoints + verbs: + - "*" + - apiGroups: + - apps + resources: + - daemonsets + - statefulsets + - deployments + verbs: + - "*" + - apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - "*" + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - "*" + - apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - watch + - list + - apiGroups: + - storage.k8s.io + resources: + - csidrivers + - storageclasses + - volumeattachments + verbs: + - "*" + - apiGroups: + - longhorn.io + resources: + - backingimagedatasources + - backingimagemanagers + - backingimages + - backupbackingimages + - backups + - backuptargets + - backupvolumes + - enginefrontends + - engineimages + - engines + - instancemanagers + - nodes + - orphans + - recurringjobs + - replicas + - settings + - sharemanagers + - snapshots + - supportbundles + - systembackups + - systemrestores + - volumeattachments + - volumes + verbs: + - "*" + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - "*" + - apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-uninstall-bind +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-uninstall-role +subjects: + - kind: ServiceAccount + name: longhorn-uninstall-service-account + namespace: longhorn-system +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: longhorn-uninstall + namespace: longhorn-system +spec: + activeDeadlineSeconds: 900 + backoffLimit: 1 + template: + metadata: + name: longhorn-uninstall + spec: + containers: + - command: + - longhorn-manager + - uninstall + - --force + env: + - name: LONGHORN_NAMESPACE + value: longhorn-system + image: docker.io/longhornio/longhorn-manager:v1.12.0 + imagePullPolicy: IfNotPresent + name: longhorn-uninstall + restartPolicy: Never + serviceAccountName: longhorn-uninstall-service-account +--- diff --git a/deployments/k3s/single-node/README.md b/deployments/k3s/single-node/README.md new file mode 100644 index 0000000..853dfbf --- /dev/null +++ b/deployments/k3s/single-node/README.md @@ -0,0 +1,201 @@ +# ClawManager K3s Single-Node Manifest + +This directory contains the all-in-one K3s manifest for a single-node +ClawManager deployment: + +- `clawmanager.yaml`: ClawManager workloads plus static HostPath PVs. +- No Longhorn dependency. +- No dynamic StorageClass dependency. + +Use this profile only when ClawManager workloads and user runtimes can run on +one storage node. For multi-node clusters, use `deployments/k3s/cluster` instead. + +## Storage Model + +The manifest uses static PV/PVC binding: + +- StorageClass name: `manual` +- System HostPath root: `/data/clawmanager/system` +- Runtime HostPath root: `/data/clawreef` +- Storage node label: `clawmanager.io/storage-node=true` + +The `manual` value is only a matching name between bundled PVs and PVCs. It does +not require a Kubernetes `StorageClass` object or dynamic provisioner. + +The bundled PVs use `persistentVolumeReclaimPolicy: Retain`, so deleting the +manifest does not erase HostPath data. + +## Install + +### 1. Enter This Directory + +```sh +cd deployments/k3s/single-node +``` + +Check that `kubectl` points to the target cluster: + +```sh +kubectl config current-context +kubectl get nodes -o wide +``` + +Expected result: the target node is `Ready`. + +### 2. Choose And Label One Storage Node + +Label exactly one node before installing: + +```sh +kubectl label node clawmanager.io/storage-node=true --overwrite +kubectl get nodes -l clawmanager.io/storage-node=true +``` + +Expected result: only one node is listed. + +If more than one node has this label, remove the label from the extra nodes: + +```sh +kubectl label node clawmanager.io/storage-node- +``` + +### 3. Check For Old Installation State + +```sh +kubectl get ns clawmanager-system clawmanager-user-1 --ignore-not-found +kubectl get pv | grep clawmanager || true +kubectl get pvc -A | grep clawmanager || true +``` + +Expected result for a fresh install: no old ClawManager namespace, PV, or PVC +remains. + +If old resources exist, run the uninstall flow in this README before installing +again. + +### 4. Check HostPath Directories + +The kubelet can create the HostPath directories automatically, but checking the +target path first makes permission and disk issues easier to diagnose: + +```sh +df -h /data || true +ls -ld /data || true +``` + +If `/data` is not the right disk for this node, edit these paths in +`clawmanager.yaml` before applying it: + +- `/data/clawmanager/system/mysql` +- `/data/clawmanager/system/minio` +- `/data/clawmanager/system/redis` +- `/data/clawmanager/system/workspaces` +- `K8S_PV_HOST_PATH_PREFIX=/data/clawreef` + +### 5. Check Image Availability + +The manifest uses the images listed in `clawmanager.yaml`. Make sure the node +can pull from the referenced registry before installing: + +```sh +grep -n 'image:' clawmanager.yaml | head -n 40 +``` + +If this is an offline or private-registry deployment, load or push the images to +the registry used by the manifest before continuing. + +### 6. Apply The Manifest + +```sh +kubectl apply -f clawmanager.yaml +``` + +### 7. Check ClawManager + +```sh +kubectl -n clawmanager-system get pvc +kubectl -n clawmanager-system rollout status deployment/clawmanager-app --timeout=15m +kubectl -n clawmanager-system rollout status deployment/openclaw-runtime --timeout=15m +kubectl -n clawmanager-system rollout status deployment/hermes-runtime --timeout=15m +kubectl -n clawmanager-system get deploy,pod,pvc -o wide +``` + +Expected result: + +- `mysql-data`, `redis-data`, `minio-data`, and `clawmanager-workspaces` are `Bound`. +- `clawmanager-app`, `openclaw-runtime`, and `hermes-runtime` are available. +- MySQL, Redis, MinIO, and skill scanner pods are running. + +If PVCs stay `Pending`, check the static PV binding: + +```sh +kubectl get pv | grep clawmanager +kubectl -n clawmanager-system describe pvc +``` + +If pods stay `Pending` with node affinity errors, check the storage node label: + +```sh +kubectl get nodes -l clawmanager.io/storage-node=true +kubectl describe pod -n clawmanager-system +``` + +If pods stay `ContainerCreating`, check recent events: + +```sh +kubectl -n clawmanager-system get events --sort-by=.lastTimestamp | tail -n 80 +``` + +## Uninstall + +Use this order for a disposable single-node test deployment. + +### 1. Delete ClawManager Resources + +```sh +kubectl delete -f clawmanager.yaml --ignore-not-found +kubectl delete namespace clawmanager-user-1 --ignore-not-found +kubectl delete namespace clawmanager-system --ignore-not-found +``` + +Check: + +```sh +kubectl get ns clawmanager-system clawmanager-user-1 --ignore-not-found +kubectl get pv | grep clawmanager || true +kubectl get pvc -A | grep clawmanager || true +``` + +The PVs may remain because they use `Retain`. This is expected. + +### 2. Remove Retained PVs When Data Is No Longer Needed + +Only run this after confirming the data can be discarded: + +```sh +kubectl delete pv \ + clawmanager-mysql-pv \ + clawmanager-minio-pv \ + clawmanager-redis-pv \ + clawmanager-workspaces-pv \ + --ignore-not-found +``` + +### 3. Remove HostPath Data When Data Is No Longer Needed + +Run this on the labeled storage node only after confirming the data can be +discarded: + +```sh +rm -rf /data/clawmanager/system /data/clawreef +``` + +## Common Issues + +| Symptom | Likely Cause | Check Or Fix | +| :--- | :--- | :--- | +| PVC stays `Pending` | Static PV and PVC did not bind | `kubectl get pv`, then `kubectl describe pvc -n clawmanager-system ` | +| Pod stays `Pending` | Missing or wrong `clawmanager.io/storage-node=true` label | `kubectl get nodes -l clawmanager.io/storage-node=true` | +| Pod is `ImagePullBackOff` | Node cannot pull an image | `kubectl describe pod -n ` | +| Pod is `ContainerCreating` | HostPath mount or permission issue | `kubectl -n clawmanager-system get events --sort-by=.lastTimestamp` | +| Runtime creation fails | Runtime HostPath root is unavailable | Check `K8S_PV_HOST_PATH_PREFIX` and disk space on the labeled node | diff --git a/deployments/k3s/single-node/clawmanager.yaml b/deployments/k3s/single-node/clawmanager.yaml new file mode 100644 index 0000000..33bf43c --- /dev/null +++ b/deployments/k3s/single-node/clawmanager.yaml @@ -0,0 +1,1656 @@ +# ClawManager single-node install profile. +# Official validation target: HostPath on one labeled storage node. +# Before installing, label exactly one node: +# kubectl label node clawmanager.io/storage-node=true --overwrite +# Replace paths or StorageClass names only if you understand the single-node constraints. +apiVersion: v1 +kind: Namespace +metadata: + name: clawmanager-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: clawmanager-secrets + namespace: clawmanager-system +type: Opaque +stringData: + mysql-root-password: root123 + mysql-password: clawreef123 + jwt-secret: change-me-in-production + runtime-agent-control-token: change-me-runtime-control-token + runtime-agent-report-token: change-me-runtime-report-token + openclaw-gateway-token: change-me-openclaw-gateway-token + minio-root-user: minioadmin + minio-root-password: minioadmin123 + minio-access-key: minioadmin + minio-secret-key: minioadmin123 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: clawmanager-mysql-init + namespace: clawmanager-system +data: + 001_init_schema.sql: | + CREATE DATABASE IF NOT EXISTS clawmanager CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + USE clawmanager; + + CREATE TABLE IF NOT EXISTS users ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(255) UNIQUE NOT NULL, + email VARCHAR(320) UNIQUE NOT NULL, + password_hash VARCHAR(255) NOT NULL, + role ENUM('admin', 'user') DEFAULT 'user', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + last_login TIMESTAMP, + INDEX idx_username (username), + INDEX idx_role (role) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instances ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT, + type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu', + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + status ENUM('creating', 'running', 'stopped', 'error', 'deleting') DEFAULT 'creating', + cpu_cores DECIMAL(10,2) NOT NULL, + memory_gb INT NOT NULL, + disk_gb INT NOT NULL, + gpu_enabled BOOLEAN DEFAULT FALSE, + gpu_type VARCHAR(100), + gpu_count INT DEFAULT 0, + os_type VARCHAR(50) NOT NULL, + os_version VARCHAR(50) NOT NULL, + image_registry VARCHAR(255), + image_tag VARCHAR(100), + storage_class VARCHAR(50) DEFAULT 'standard', + mount_path VARCHAR(255) DEFAULT '/data', + pod_name VARCHAR(255), + pod_namespace VARCHAR(255), + pod_ip VARCHAR(45), + access_url VARCHAR(500), + access_token VARCHAR(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + started_at TIMESTAMP, + stopped_at TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id), + INDEX idx_status (status), + INDEX idx_type (type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS persistent_volumes ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + pvc_name VARCHAR(255) UNIQUE NOT NULL, + pvc_namespace VARCHAR(255) NOT NULL, + storage_size_gb INT NOT NULL, + storage_class VARCHAR(50), + mount_path VARCHAR(255), + status ENUM('pending', 'bound', 'released', 'failed') DEFAULT 'pending', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + UNIQUE KEY uk_pvc_name_namespace (pvc_name, pvc_namespace) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backups ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + backup_name VARCHAR(255) NOT NULL, + backup_size_gb INT, + backup_path VARCHAR(500), + status ENUM('creating', 'completed', 'failed', 'deleted') DEFAULT 'creating', + backup_type ENUM('manual', 'scheduled') DEFAULT 'manual', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP, + expires_at TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backup_schedules ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + schedule_name VARCHAR(255), + cron_expression VARCHAR(100) NOT NULL, + retention_days INT DEFAULT 30, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS user_quotas ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL UNIQUE, + max_instances INT DEFAULT 10, + max_cpu_cores DECIMAL(10,2) DEFAULT 40, + max_memory_gb INT DEFAULT 100, + max_storage_gb INT DEFAULT 500, + max_gpu_count INT DEFAULT 2, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_usage ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + cpu_usage_percent DECIMAL(5,2), + memory_usage_gb DECIMAL(10,2), + disk_usage_gb DECIMAL(10,2), + gpu_usage_percent DECIMAL(5,2), + uptime_seconds INT, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_recorded (instance_id, recorded_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS audit_logs ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT, + action VARCHAR(100) NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_id INT, + details JSON, + ip_address VARCHAR(45), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL, + INDEX idx_user_id (user_id), + INDEX idx_action (action), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + INSERT INTO users (username, email, password_hash, role, is_active) + SELECT 'admin', 'admin@clawmanager.local', '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi', 'admin', TRUE + WHERE NOT EXISTS (SELECT 1 FROM users WHERE username = 'admin'); + + INSERT INTO user_quotas (user_id, max_instances, max_cpu_cores, max_memory_gb, max_storage_gb, max_gpu_count) + SELECT id, 100, 200, 1000, 5000, 10 FROM users + WHERE username = 'admin' + AND NOT EXISTS (SELECT 1 FROM user_quotas WHERE user_id = users.id); + 002_add_webtop_instance_type.sql: | + USE clawmanager; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 003_add_system_image_settings.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS system_image_settings ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_type VARCHAR(50) NOT NULL, + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + display_name VARCHAR(255) NOT NULL, + image VARCHAR(500) NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_instance_type (instance_type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 004_fix_seeded_admin_password.sql: | + USE clawmanager; + UPDATE users + SET password_hash = '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi' + WHERE username = 'admin' + AND password_hash = '$2a$10$N9qo8uLOickgx2ZMRZoMy.MqrzL9wGC3qD3Q.ZHqQH6t3q7l1L5uG'; + 005_update_openclaw_default_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + 006_add_openclaw_config_center.sql: | + USE clawmanager; + SET @openclaw_snapshot_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'openclaw_config_snapshot_id' + ); + SET @openclaw_snapshot_column_sql = IF( + @openclaw_snapshot_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN openclaw_config_snapshot_id INT NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE openclaw_snapshot_column_stmt FROM @openclaw_snapshot_column_sql; + EXECUTE openclaw_snapshot_column_stmt; + DEALLOCATE PREPARE openclaw_snapshot_column_stmt; + + CREATE TABLE IF NOT EXISTS openclaw_config_resources ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_key VARCHAR(100) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + tags_json LONGTEXT NOT NULL, + content_json LONGTEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_resource_key (user_id, resource_type, resource_key), + INDEX idx_openclaw_resource_user_type (user_id, resource_type), + INDEX idx_openclaw_resource_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundles ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_openclaw_bundle_user (user_id), + INDEX idx_openclaw_bundle_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_items ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + resource_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (resource_id) REFERENCES openclaw_config_resources(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_resource (bundle_id, resource_id), + INDEX idx_openclaw_bundle_item_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_injection_snapshots ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NULL, + user_id INT NOT NULL, + mode VARCHAR(20) NOT NULL, + bundle_id INT NULL, + selected_resource_ids_json LONGTEXT NOT NULL, + resolved_resources_json LONGTEXT NOT NULL, + rendered_manifest_json LONGTEXT NOT NULL, + rendered_env_json LONGTEXT NOT NULL, + secret_name VARCHAR(255) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + activated_at TIMESTAMP NULL, + INDEX idx_openclaw_snapshot_user_created (user_id, created_at), + INDEX idx_openclaw_snapshot_instance (instance_id), + INDEX idx_openclaw_snapshot_bundle (bundle_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 007_add_instance_agent_control_plane.sql: | + USE clawmanager; + SET @instance_agent_bootstrap_token_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'agent_bootstrap_token' + ); + SET @instance_agent_bootstrap_token_column_sql = IF( + @instance_agent_bootstrap_token_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN agent_bootstrap_token VARCHAR(255) NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE instance_agent_bootstrap_token_column_stmt FROM @instance_agent_bootstrap_token_column_sql; + EXECUTE instance_agent_bootstrap_token_column_stmt; + DEALLOCATE PREPARE instance_agent_bootstrap_token_column_stmt; + + CREATE TABLE IF NOT EXISTS instance_agents ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NOT NULL, + agent_version VARCHAR(50) NOT NULL, + protocol_version VARCHAR(50) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'online', + capabilities_json LONGTEXT NOT NULL, + host_info_json LONGTEXT NULL, + session_token VARCHAR(255) NULL, + session_expires_at TIMESTAMP NULL, + last_heartbeat_at TIMESTAMP NULL, + last_reported_at TIMESTAMP NULL, + last_seen_ip VARCHAR(45) NULL, + registered_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_agents_instance (instance_id), + UNIQUE KEY uk_instance_agents_session_token (session_token), + INDEX idx_instance_agents_agent_id (agent_id), + INDEX idx_instance_agents_status (status), + INDEX idx_instance_agents_last_heartbeat (last_heartbeat_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_runtime_status ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + infra_status VARCHAR(30) NOT NULL DEFAULT 'creating', + agent_status VARCHAR(30) NOT NULL DEFAULT 'offline', + openclaw_status VARCHAR(30) NOT NULL DEFAULT 'unknown', + openclaw_pid INT NULL, + openclaw_version VARCHAR(100) NULL, + current_config_revision_id INT NULL, + desired_config_revision_id INT NULL, + summary_json LONGTEXT NULL, + system_info_json LONGTEXT NULL, + health_json LONGTEXT NULL, + last_reported_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_runtime_status_instance (instance_id), + INDEX idx_instance_runtime_status_agent_status (agent_status), + INDEX idx_instance_runtime_status_openclaw_status (openclaw_status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_desired_state ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + desired_power_state VARCHAR(30) NOT NULL DEFAULT 'running', + desired_config_revision_id INT NULL, + desired_runtime_action VARCHAR(50) NULL, + updated_by INT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_desired_state_instance (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_commands ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NULL, + command_type VARCHAR(50) NOT NULL, + payload_json LONGTEXT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + idempotency_key VARCHAR(255) NOT NULL, + issued_by INT NULL, + issued_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + dispatched_at TIMESTAMP NULL, + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + timeout_seconds INT NOT NULL DEFAULT 300, + result_json LONGTEXT NULL, + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (issued_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_commands_idempotency (instance_id, idempotency_key), + INDEX idx_instance_commands_instance_status (instance_id, status), + INDEX idx_instance_commands_agent_status (agent_id, status), + INDEX idx_instance_commands_issued_at (issued_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_config_revisions ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + source_snapshot_id INT NULL, + source_bundle_id INT NULL, + revision_no INT NOT NULL, + content_json LONGTEXT NOT NULL, + checksum VARCHAR(255) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'published', + published_by INT NULL, + published_at TIMESTAMP NULL, + activated_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (published_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_config_revision_unique (instance_id, revision_no), + INDEX idx_instance_config_revision_instance (instance_id, revision_no), + INDEX idx_instance_config_revision_status (status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 008_add_skill_management.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS skill_blobs ( + id INT AUTO_INCREMENT PRIMARY KEY, + content_hash VARCHAR(128) NOT NULL, + archive_hash VARCHAR(128) NOT NULL, + object_key VARCHAR(512) NOT NULL, + file_name VARCHAR(255) NOT NULL, + media_type VARCHAR(100) NOT NULL DEFAULT 'application/gzip', + size_bytes BIGINT NOT NULL DEFAULT 0, + scan_status VARCHAR(30) NOT NULL DEFAULT 'pending', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_skill_blobs_content_hash (content_hash), + INDEX idx_skill_blobs_scan_status (scan_status), + INDEX idx_skill_blobs_risk_level (risk_level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + skill_key VARCHAR(120) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + current_version_id INT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + status VARCHAR(30) NOT NULL DEFAULT 'active', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_skills_user_key (user_id, skill_key), + INDEX idx_skills_user_status (user_id, status), + INDEX idx_skills_risk_level (risk_level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skill_versions ( + id INT AUTO_INCREMENT PRIMARY KEY, + skill_id INT NOT NULL, + blob_id INT NOT NULL, + version_no INT NOT NULL, + manifest_json LONGTEXT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE RESTRICT, + UNIQUE KEY uk_skill_versions_skill_version (skill_id, version_no), + UNIQUE KEY uk_skill_versions_skill_blob (skill_id, blob_id), + INDEX idx_skill_versions_skill_id (skill_id, version_no) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + skill_id INT NOT NULL, + skill_version_id INT NULL, + source_type VARCHAR(40) NOT NULL DEFAULT 'discovered_in_instance', + install_path VARCHAR(1024) NULL, + observed_hash VARCHAR(128) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'active', + last_seen_at TIMESTAMP NULL, + removed_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (skill_version_id) REFERENCES skill_versions(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_skills_instance_skill (instance_id, skill_id), + INDEX idx_instance_skills_instance (instance_id, status), + INDEX idx_instance_skills_skill (skill_id, status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skill_scan_results ( + id INT AUTO_INCREMENT PRIMARY KEY, + blob_id INT NOT NULL, + engine VARCHAR(60) NOT NULL, + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + status VARCHAR(30) NOT NULL DEFAULT 'completed', + summary TEXT NULL, + findings_json LONGTEXT NULL, + scanned_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE CASCADE, + INDEX idx_skill_scan_results_blob (blob_id, scanned_at), + INDEX idx_skill_scan_results_risk (risk_level, scanned_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 009_cpu_cores_decimal.sql: | + USE clawmanager; + ALTER TABLE instances MODIFY COLUMN cpu_cores DECIMAL(10,2) NOT NULL; + ALTER TABLE user_quotas MODIFY COLUMN max_cpu_cores DECIMAL(10,2) DEFAULT 40; + 011_add_hermes_instance_type.sql: | + USE clawmanager; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 012_update_agents_runtime_default_images.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' + WHERE instance_type = 'hermes' + AND image IN ( + 'registry.example.com/hermes-webtop:latest', + 'lscr.io/linuxserver/webtop:ubuntu-xfce' + ); + 013_normalize_agents_runtime_image_names.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/openclaw:latest'; + + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/hermes:latest'; + 019_add_instance_runtime_type.sql: | + USE clawmanager; + SET @instance_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'runtime_type' + ); + SET @instance_runtime_type_column_sql = IF( + @instance_runtime_type_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER type', + 'SELECT 1' + ); + PREPARE instance_runtime_type_column_stmt FROM @instance_runtime_type_column_sql; + EXECUTE instance_runtime_type_column_stmt; + DEALLOCATE PREPARE instance_runtime_type_column_stmt; + 020_add_system_image_runtime_type.sql: | + USE clawmanager; + SET @system_image_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'runtime_type' + ); + SET @system_image_runtime_type_column_sql = IF( + @system_image_runtime_type_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER instance_type', + 'SELECT 1' + ); + PREPARE system_image_runtime_type_column_stmt FROM @system_image_runtime_type_column_sql; + EXECUTE system_image_runtime_type_column_stmt; + DEALLOCATE PREPARE system_image_runtime_type_column_stmt; + 021_add_openclaw_shell_runtime_image.sql: | + USE clawmanager; + SET @system_image_is_enabled_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'is_enabled' + ); + SET @system_image_is_enabled_column_sql = IF( + @system_image_is_enabled_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN is_enabled BOOLEAN NOT NULL DEFAULT TRUE', + 'SELECT 1' + ); + PREPARE system_image_is_enabled_column_stmt FROM @system_image_is_enabled_column_sql; + EXECUTE system_image_is_enabled_column_stmt; + DEALLOCATE PREPARE system_image_is_enabled_column_stmt; + + SET @system_image_unique_instance_type_index_name = ( + SELECT INDEX_NAME + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'instance_type' + AND NON_UNIQUE = 0 + AND INDEX_NAME <> 'PRIMARY' + LIMIT 1 + ); + SET @system_image_unique_instance_type_index_sql = IF( + @system_image_unique_instance_type_index_name IS NOT NULL, + CONCAT( + 'ALTER TABLE system_image_settings DROP INDEX `', + REPLACE(@system_image_unique_instance_type_index_name, '`', '``'), + '`' + ), + 'SELECT 1' + ); + PREPARE system_image_unique_instance_type_index_stmt FROM @system_image_unique_instance_type_index_sql; + EXECUTE system_image_unique_instance_type_index_stmt; + DEALLOCATE PREPARE system_image_unique_instance_type_index_stmt; + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'desktop', + 'OpenClaw Desktop', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'desktop' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'shell', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + 022_add_openclaw_config_bundle_skills.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + skill_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id), + INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 023_normalize_runtime_config_mounts.sql: | + USE clawmanager; + UPDATE instances + SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP + WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); + 031_add_hermes_lite_runtime_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + 'registry.example.com/your-custom-shell-image:latest' + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'hermes', + 'shell', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND is_enabled = FALSE + ); +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager-mysql-pv + labels: + app: clawmanager + managed-by: clawmanager + clawmanager.io/storage-profile: single-node + clawmanager.io/storage-node: "true" +spec: + capacity: + storage: 5Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: clawmanager.io/storage-node + operator: In + values: + - "true" + hostPath: + path: /data/clawmanager/system/mysql + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: manual + volumeName: clawmanager-mysql-pv +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager-minio-pv + labels: + app: clawmanager + managed-by: clawmanager + clawmanager.io/storage-profile: single-node + clawmanager.io/storage-node: "true" +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: clawmanager.io/storage-node + operator: In + values: + - "true" + hostPath: + path: /data/clawmanager/system/minio + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: manual + volumeName: clawmanager-minio-pv +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager-redis-pv + labels: + app: clawmanager + managed-by: clawmanager + clawmanager.io/storage-profile: single-node + clawmanager.io/storage-node: "true" +spec: + capacity: + storage: 1Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: clawmanager.io/storage-node + operator: In + values: + - "true" + hostPath: + path: /data/clawmanager/system/redis + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: manual + volumeName: clawmanager-redis-pv +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager-workspaces-pv + labels: + app: clawmanager + managed-by: clawmanager + clawmanager.io/storage-profile: single-node + clawmanager.io/storage-node: "true" +spec: + capacity: + storage: 50Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: clawmanager.io/storage-node + operator: In + values: + - "true" + hostPath: + path: /data/clawmanager/system/workspaces + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clawmanager-workspaces + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Gi + storageClassName: manual + volumeName: clawmanager-workspaces-pv +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: mysql + template: + metadata: + labels: + app: mysql + spec: + containers: + - name: mysql + image: mysql:8.4.8 + imagePullPolicy: IfNotPresent + args: + - "--innodb-use-native-aio=0" + ports: + - containerPort: 3306 + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-root-password + - name: MYSQL_DATABASE + value: clawmanager + - name: MYSQL_USER + value: clawmanager + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + - name: mysql-init + mountPath: /docker-entrypoint-initdb.d + readinessProbe: + exec: + command: ["sh", "-c", "mysqladmin ping -h 127.0.0.1 -uroot -p$MYSQL_ROOT_PASSWORD"] + initialDelaySeconds: 20 + periodSeconds: 10 + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: mysql-data + - name: mysql-init + configMap: + name: clawmanager-mysql-init +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: clawmanager-system +spec: + selector: + app: mysql + ports: + - name: mysql + port: 3306 + targetPort: 3306 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio:latest + imagePullPolicy: IfNotPresent + args: + - server + - /data + - --console-address + - ":9001" + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-root-password + ports: + - name: api + containerPort: 9000 + - name: console + containerPort: 9001 + volumeMounts: + - name: minio-data + mountPath: /data + readinessProbe: + tcpSocket: + port: api + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: minio-data + persistentVolumeClaim: + claimName: minio-data +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: clawmanager-system +spec: + selector: + app: minio + ports: + - name: api + port: 9000 + targetPort: api + - name: console + port: 9001 + targetPort: console +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-team-redis + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: clawmanager-team-redis + template: + metadata: + labels: + app: clawmanager-team-redis + spec: + containers: + - name: redis + image: redis:7-alpine + imagePullPolicy: IfNotPresent + args: + - redis-server + - --appendonly + - "yes" + ports: + - name: redis + containerPort: 6379 + readinessProbe: + tcpSocket: + port: redis + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: redis-data + mountPath: /data + volumes: + - name: redis-data + persistentVolumeClaim: + claimName: redis-data +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-redis + namespace: clawmanager-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-team-redis + namespace: clawmanager-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: clawmanager-runtime-manager +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - apiGroups: + - apps + resources: + - deployments + - deployments/scale + verbs: + - get + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager-runtime-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clawmanager-runtime-manager +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager-app-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +# Explicit lease permissions for control-plane leader election. The +# cluster-admin binding above already covers this; this Role documents the +# requirement and keeps leader election working if RBAC is tightened later. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: clawmanager-app-leaderelection + namespace: clawmanager-system +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: clawmanager-app-leaderelection + namespace: clawmanager-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: clawmanager-app-leaderelection +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: skill-scanner + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: skill-scanner + template: + metadata: + labels: + app: skill-scanner + spec: + containers: + - name: skill-scanner + image: ghcr.io/yuan-lab-llm/skill-scanner:latest + imagePullPolicy: IfNotPresent + command: + - /opt/skill-scanner-venv/bin/skill-scanner-api + - --host + - 0.0.0.0 + - --port + - "8000" + env: + - name: SKILL_SCANNER_LLM_API_KEY + value: "" + - name: SKILL_SCANNER_LLM_MODEL + value: "" + - name: SKILL_SCANNER_LLM_BASE_URL + value: "" + - name: SKILL_SCANNER_META_LLM_API_KEY + value: "" + - name: SKILL_SCANNER_META_LLM_MODEL + value: "" + - name: SKILL_SCANNER_META_LLM_BASE_URL + value: "" + ports: + - name: http + containerPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: skill-scanner + namespace: clawmanager-system +spec: + selector: + app: skill-scanner + ports: + - name: http + port: 8000 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-app + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: clawmanager-app + template: + metadata: + labels: + app: clawmanager-app + spec: + serviceAccountName: clawmanager-app + containers: + - name: clawmanager-app + image: ghcr.io/yuan-lab-llm/clawmanager:latest + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 8443 + - name: proxy + containerPort: 9001 + env: + - name: SERVER_ADDRESS + value: ":9001" + - name: SERVER_MODE + value: "release" + - name: DB_HOST + value: "mysql" + - name: DB_PORT + value: "3306" + - name: DB_USER + value: "clawmanager" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + - name: DB_NAME + value: "clawmanager" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: jwt-secret + - name: OBJECT_STORAGE_ENDPOINT + value: "minio.clawmanager-system.svc.cluster.local:9000" + - name: OBJECT_STORAGE_REGION + value: "" + - name: OBJECT_STORAGE_ACCESS_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-access-key + - name: OBJECT_STORAGE_SECRET_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-secret-key + - name: OBJECT_STORAGE_BUCKET + value: "clawmanager-skills" + - name: OBJECT_STORAGE_USE_SSL + value: "false" + - name: OBJECT_STORAGE_BASE_PATH + value: "skills" + - name: OBJECT_STORAGE_FORCE_PATH_STYLE + value: "true" + - name: K8S_MODE + value: "incluster" + - name: K8S_NAMESPACE + value: "clawmanager" + - name: K8S_STORAGE_CLASS + value: "manual" + - name: CLAWMANAGER_STORAGE_PROFILE + value: "single-node" + - name: K8S_HOSTPATH_FALLBACK_ENABLED + value: "true" + - name: K8S_PVC_BIND_TIMEOUT + value: "2m" + - name: K8S_CONTROL_PLANE_STORAGE_CLASS + value: "manual" + - name: K8S_INSTANCE_STORAGE_CLASS + value: "manual" + - name: K8S_WORKSPACE_STORAGE_CLASS + value: "manual" + - name: K8S_WORKSPACE_ACCESS_MODE + value: "ReadWriteMany" + - name: K8S_PV_HOST_PATH_PREFIX + value: "/data/clawreef" + - name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB + value: "500" + - name: PLATFORM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: TEAM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_DESKTOP_DIRECT_PROXY + value: "true" + # Leader election for control-plane singleton background loops. + # Required before scaling replicas > 1 so SyncService / Team event + # consumers / stale-task monitor run on exactly one replica. + - name: CLAWMANAGER_LEADER_ELECTION + value: "true" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLAWMANAGER_TEAM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_TEAM_MANAGER_BASE_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: RUNTIME_NAMESPACE + value: "clawmanager-system" + - name: RUNTIME_WORKSPACE_ROOT + value: "/workspaces" + - name: RUNTIME_WORKSPACE_PVC_CLAIM + value: "clawmanager-workspaces" + - name: RUNTIME_MAX_GATEWAYS_PER_POD + value: "100" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_SCHEDULER_ENABLED + value: "true" + - name: RUNTIME_HEARTBEAT_TIMEOUT + value: "10s" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: SKILL_SCANNER_ENABLED + value: "true" + - name: SKILL_SCANNER_BASE_URL + value: "http://skill-scanner.clawmanager-system.svc.cluster.local:8000" + - name: SKILL_SCANNER_TIMEOUT_SECONDS + value: "120" + - name: SKILL_SCANNER_NAMESPACE + value: "clawmanager-system" + - name: SKILL_SCANNER_DEPLOYMENT + value: "skill-scanner" + volumeMounts: + - name: tls-cert + mountPath: /var/run/clawreef-tls + readOnly: true + - name: workspaces + mountPath: /workspaces + readinessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 30 + periodSeconds: 20 + volumes: + - name: tls-cert + secret: + secretName: clawmanager-tls + optional: true + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openclaw-runtime + namespace: clawmanager-system + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw +spec: + replicas: 1 + selector: + matchLabels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + template: + metadata: + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + imagePullPolicy: Always + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: openclaw + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: openclaw-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_COMMAND + value: "/usr/local/bin/openclaw gateway run --allow-unconfigured --auth token --bind lan --force" + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-runtime + namespace: clawmanager-system + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes +spec: + replicas: 1 + selector: + matchLabels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + template: + metadata: + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + imagePullPolicy: IfNotPresent + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: hermes + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: hermes-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + - name: CLAWMANAGER_AGENT_PORT + value: "19090" + - name: CLAWMANAGER_GATEWAY_PORT_START + value: "20000" + - name: CLAWMANAGER_GATEWAY_PORT_END + value: "20099" + - name: HERMES_TUI_DIR + value: /usr/local/lib/hermes-agent/ui-tui + - name: CLAWMANAGER_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: CLAWMANAGER_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-frontend + namespace: clawmanager-system +spec: + type: NodePort + selector: + app: clawmanager-app + ports: + - name: https + port: 443 + targetPort: https + nodePort: 30443 +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-gateway + namespace: clawmanager-system +spec: + type: ClusterIP + selector: + app: clawmanager-app + ports: + - name: api + port: 9001 + targetPort: 9001 +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-egress-proxy + namespace: clawmanager-system +spec: + type: ClusterIP + selector: + app: clawmanager-app + ports: + - name: proxy + port: 3128 + targetPort: 9001 diff --git a/deployments/k8s/clawmanager-apply.sh b/deployments/k8s/clawmanager-apply.sh new file mode 100644 index 0000000..87b567c --- /dev/null +++ b/deployments/k8s/clawmanager-apply.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Edit these values before deploying, or override them with environment vars: +# TENANT_SUFFIX=-hxc NODE_PORT=32443 APP_IMAGE=10.130.14.23:5000/clawmanager-hxc-app:team-update-20260701 ./clawmanager-apply.sh +# OPENCLAW_RUNTIME_IMAGE=10.130.14.23:5000/openclaw-lite:latest HERMES_RUNTIME_IMAGE=10.130.14.23:5000/hermes-lite:latest ./clawmanager-apply.sh +# +# TENANT_SUFFIX examples: +# empty = clawmanager-system +# -abc = clawmanager-abc-system +TENANT_SUFFIX="${TENANT_SUFFIX--hxc}" +NODE_PORT="${NODE_PORT:-32443}" +APP_IMAGE="${APP_IMAGE:-10.130.14.23:5000/clawmanager-hxc-app:team-update-20260701}" +OPENCLAW_RUNTIME_IMAGE="${OPENCLAW_RUNTIME_IMAGE:-ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest}" +HERMES_RUNTIME_IMAGE="${HERMES_RUNTIME_IMAGE:-ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest}" + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MANIFEST="${1:-${ROOT}/clawmanager-tenant.yaml}" + +case "${TENANT_SUFFIX}" in + ""|-*) ;; + *) TENANT_SUFFIX="-${TENANT_SUFFIX}" ;; +esac + +if [[ ! -f "${MANIFEST}" ]]; then + echo "ERROR: manifest not found: ${MANIFEST}" >&2 + echo >&2 + echo "Put clawmanager-tenant.yaml in the same directory as this script, or pass it explicitly:" >&2 + echo " ./clawmanager-apply.sh /path/to/clawmanager-tenant.yaml" >&2 + exit 1 +fi + +SYSTEM_NAMESPACE="clawmanager${TENANT_SUFFIX}-system" +RENDERED_MANIFEST="$(mktemp)" +trap 'rm -f "${RENDERED_MANIFEST}"' EXIT + +sed "s|{TENANT_SUFFIX}|${TENANT_SUFFIX}|g;s|{NODE_PORT}|${NODE_PORT}|g;s|{APP_IMAGE}|${APP_IMAGE}|g;s|{OPENCLAW_RUNTIME_IMAGE}|${OPENCLAW_RUNTIME_IMAGE}|g;s|{HERMES_RUNTIME_IMAGE}|${HERMES_RUNTIME_IMAGE}|g" \ + "${MANIFEST}" > "${RENDERED_MANIFEST}" + +kubectl apply -f "${RENDERED_MANIFEST}" + +# NFS volumes are mounted by kubelet on the node, not inside a Pod. Some +# clusters do not resolve *.svc.cluster.local from the node mount namespace, so +# patch NFS volumes to the workspace-store ClusterIP after the Service exists. +if kubectl -n "${SYSTEM_NAMESPACE}" get svc workspace-store >/dev/null 2>&1; then + WORKSPACE_STORE_IP="$(kubectl -n "${SYSTEM_NAMESPACE}" get svc workspace-store -o jsonpath='{.spec.clusterIP}')" + if [[ -n "${WORKSPACE_STORE_IP}" && "${WORKSPACE_STORE_IP}" != "None" ]]; then + echo "Patching workspace NFS server to workspace-store ClusterIP: ${WORKSPACE_STORE_IP}" + + kubectl -n "${SYSTEM_NAMESPACE}" set env deployment/clawmanager-app \ + "RUNTIME_WORKSPACE_NFS_SERVER=${WORKSPACE_STORE_IP}" \ + "RUNTIME_WORKSPACE_NFS_PATH=/" + + patch_workspace_volume() { + local deployment="$1" + local volume_index="" + local current_index + local volume_name + + current_index=0 + while IFS= read -r volume_name; do + if [[ "${volume_name}" == "workspaces" ]]; then + volume_index="${current_index}" + break + fi + current_index=$((current_index + 1)) + done < <(kubectl -n "${SYSTEM_NAMESPACE}" get deployment "${deployment}" -o jsonpath='{range .spec.template.spec.volumes[*]}{.name}{"\n"}{end}') + + if [[ -z "${volume_index}" ]]; then + echo "WARNING: deployment/${deployment} has no workspaces volume; skipping NFS patch" >&2 + return 0 + fi + + kubectl -n "${SYSTEM_NAMESPACE}" patch deployment "${deployment}" --type=json -p \ + "[{\"op\":\"replace\",\"path\":\"/spec/template/spec/volumes/${volume_index}/nfs/server\",\"value\":\"${WORKSPACE_STORE_IP}\"},{\"op\":\"replace\",\"path\":\"/spec/template/spec/volumes/${volume_index}/nfs/path\",\"value\":\"/\"}]" + } + + patch_workspace_volume clawmanager-app + + for deployment in openclaw-runtime hermes-runtime; do + if kubectl -n "${SYSTEM_NAMESPACE}" get deployment "${deployment}" >/dev/null 2>&1; then + patch_workspace_volume "${deployment}" + fi + done + fi +fi diff --git a/deployments/k8s/clawmanager-team-profiles-test.yaml b/deployments/k8s/clawmanager-team-profiles-test.yaml new file mode 100644 index 0000000..60222cc --- /dev/null +++ b/deployments/k8s/clawmanager-team-profiles-test.yaml @@ -0,0 +1,175 @@ +# Test-only override for validating local ClawManager Team profile UI/injection. +# Apply the normal manifest first, then apply this file: +# +# kubectl apply -f deployments/k8s/clawmanager.yaml +# docker build -t clawmanager:team-profiles-test . +# kubectl apply -f deployments/k8s/clawmanager-team-profiles-test.yaml +# +# This changes only the ClawManager app image. OpenClaw/Hermes member images +# remain configured by system settings and the base manifest. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-app + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: clawmanager-app + template: + metadata: + labels: + app: clawmanager-app + spec: + serviceAccountName: clawmanager-app + containers: + - name: clawmanager-app + image: clawmanager:team-profiles-test + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 8443 + - name: proxy + containerPort: 9001 + env: + - name: SERVER_ADDRESS + value: ":9001" + - name: SERVER_MODE + value: "release" + - name: DB_HOST + value: "mysql" + - name: DB_PORT + value: "3306" + - name: DB_USER + value: "clawmanager" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + - name: DB_NAME + value: "clawmanager" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: jwt-secret + - name: OBJECT_STORAGE_ENDPOINT + value: "minio.clawmanager-system.svc.cluster.local:9000" + - name: OBJECT_STORAGE_REGION + value: "" + - name: OBJECT_STORAGE_ACCESS_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-access-key + - name: OBJECT_STORAGE_SECRET_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-secret-key + - name: OBJECT_STORAGE_BUCKET + value: "clawmanager-skills" + - name: OBJECT_STORAGE_USE_SSL + value: "false" + - name: OBJECT_STORAGE_BASE_PATH + value: "skills" + - name: OBJECT_STORAGE_FORCE_PATH_STYLE + value: "true" + - name: K8S_MODE + value: "incluster" + - name: K8S_NAMESPACE + value: "clawmanager" + - name: K8S_STORAGE_CLASS + value: "manual" + - name: K8S_PV_HOST_PATH_PREFIX + value: "/data/clawreef" + - name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB + value: "500" + - name: CLAWMANAGER_DESKTOP_DIRECT_PROXY + value: "false" + - name: CLAWMANAGER_LEADER_ELECTION + value: "true" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PLATFORM_REDIS_URL + value: "redis://clawmanager-team-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: TEAM_REDIS_URL + value: "redis://clawmanager-team-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_TEAM_REDIS_URL + value: "redis://clawmanager-team-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_TEAM_MANAGER_BASE_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: RUNTIME_NAMESPACE + value: "clawmanager-system" + - name: RUNTIME_WORKSPACE_ROOT + value: "/workspaces" + - name: RUNTIME_WORKSPACE_NFS_SERVER + value: "workspace-store.clawmanager-system.svc.cluster.local" + - name: RUNTIME_WORKSPACE_NFS_PATH + value: "/" + - name: RUNTIME_MAX_GATEWAYS_PER_POD + value: "100" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_SCHEDULER_ENABLED + value: "true" + - name: RUNTIME_HEARTBEAT_TIMEOUT + value: "10s" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: SKILL_SCANNER_ENABLED + value: "true" + - name: SKILL_SCANNER_BASE_URL + value: "http://skill-scanner.clawmanager-system.svc.cluster.local:8000" + - name: SKILL_SCANNER_TIMEOUT_SECONDS + value: "120" + - name: SKILL_SCANNER_NAMESPACE + value: "clawmanager-system" + - name: SKILL_SCANNER_DEPLOYMENT + value: "skill-scanner" + volumeMounts: + - name: tls-cert + mountPath: /var/run/clawreef-tls + readOnly: true + readinessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 30 + periodSeconds: 20 + volumes: + - name: tls-cert + secret: + secretName: clawmanager-tls + optional: true diff --git a/deployments/k8s/clawmanager-tenant.yaml b/deployments/k8s/clawmanager-tenant.yaml new file mode 100644 index 0000000..97b3000 --- /dev/null +++ b/deployments/k8s/clawmanager-tenant.yaml @@ -0,0 +1,1530 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: clawmanager{TENANT_SUFFIX}-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: clawmanager-secrets + namespace: clawmanager{TENANT_SUFFIX}-system +type: Opaque +stringData: + mysql-root-password: root123 + mysql-password: clawreef123 + jwt-secret: change-me-in-production + runtime-agent-control-token: change-me-runtime-control-token + runtime-agent-report-token: change-me-runtime-report-token + openclaw-gateway-token: change-me-openclaw-gateway-token + minio-root-user: minioadmin + minio-root-password: minioadmin123 + minio-access-key: minioadmin + minio-secret-key: minioadmin123 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: clawmanager-mysql-init + namespace: clawmanager{TENANT_SUFFIX}-system +data: + 001_init_schema.sql: | + CREATE DATABASE IF NOT EXISTS clawmanager CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + USE clawmanager; + + CREATE TABLE IF NOT EXISTS users ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(255) UNIQUE NOT NULL, + email VARCHAR(320) UNIQUE NOT NULL, + password_hash VARCHAR(255) NOT NULL, + role ENUM('admin', 'user') DEFAULT 'user', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + last_login TIMESTAMP, + INDEX idx_username (username), + INDEX idx_role (role) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instances ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT, + type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu', + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + status ENUM('creating', 'running', 'stopped', 'error', 'deleting') DEFAULT 'creating', + cpu_cores DECIMAL(10,2) NOT NULL, + memory_gb INT NOT NULL, + disk_gb INT NOT NULL, + gpu_enabled BOOLEAN DEFAULT FALSE, + gpu_type VARCHAR(100), + gpu_count INT DEFAULT 0, + os_type VARCHAR(50) NOT NULL, + os_version VARCHAR(50) NOT NULL, + image_registry VARCHAR(255), + image_tag VARCHAR(100), + storage_class VARCHAR(50) DEFAULT 'standard', + mount_path VARCHAR(255) DEFAULT '/data', + pod_name VARCHAR(255), + pod_namespace VARCHAR(255), + pod_ip VARCHAR(45), + access_url VARCHAR(500), + access_token VARCHAR(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + started_at TIMESTAMP, + stopped_at TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id), + INDEX idx_status (status), + INDEX idx_type (type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS persistent_volumes ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + pvc_name VARCHAR(255) UNIQUE NOT NULL, + pvc_namespace VARCHAR(255) NOT NULL, + storage_size_gb INT NOT NULL, + storage_class VARCHAR(50), + mount_path VARCHAR(255), + status ENUM('pending', 'bound', 'released', 'failed') DEFAULT 'pending', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + UNIQUE KEY uk_pvc_name_namespace (pvc_name, pvc_namespace) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backups ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + backup_name VARCHAR(255) NOT NULL, + backup_size_gb INT, + backup_path VARCHAR(500), + status ENUM('creating', 'completed', 'failed', 'deleted') DEFAULT 'creating', + backup_type ENUM('manual', 'scheduled') DEFAULT 'manual', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP, + expires_at TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backup_schedules ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + schedule_name VARCHAR(255), + cron_expression VARCHAR(100) NOT NULL, + retention_days INT DEFAULT 30, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS user_quotas ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL UNIQUE, + max_instances INT DEFAULT 10, + max_cpu_cores DECIMAL(10,2) DEFAULT 40, + max_memory_gb INT DEFAULT 100, + max_storage_gb INT DEFAULT 500, + max_gpu_count INT DEFAULT 2, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_usage ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + cpu_usage_percent DECIMAL(5,2), + memory_usage_gb DECIMAL(10,2), + disk_usage_gb DECIMAL(10,2), + gpu_usage_percent DECIMAL(5,2), + uptime_seconds INT, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_recorded (instance_id, recorded_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS audit_logs ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT, + action VARCHAR(100) NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_id INT, + details JSON, + ip_address VARCHAR(45), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL, + INDEX idx_user_id (user_id), + INDEX idx_action (action), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + INSERT INTO users (username, email, password_hash, role, is_active) + SELECT 'admin', 'admin@clawmanager.local', '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi', 'admin', TRUE + WHERE NOT EXISTS (SELECT 1 FROM users WHERE username = 'admin'); + + INSERT INTO user_quotas (user_id, max_instances, max_cpu_cores, max_memory_gb, max_storage_gb, max_gpu_count) + SELECT id, 100, 200, 1000, 5000, 10 FROM users + WHERE username = 'admin' + AND NOT EXISTS (SELECT 1 FROM user_quotas WHERE user_id = users.id); + 002_add_webtop_instance_type.sql: | + USE clawmanager; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 003_add_system_image_settings.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS system_image_settings ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_type VARCHAR(50) NOT NULL, + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + display_name VARCHAR(255) NOT NULL, + image VARCHAR(500) NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_instance_type (instance_type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 004_fix_seeded_admin_password.sql: | + USE clawmanager; + UPDATE users + SET password_hash = '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi' + WHERE username = 'admin' + AND password_hash = '$2a$10$N9qo8uLOickgx2ZMRZoMy.MqrzL9wGC3qD3Q.ZHqQH6t3q7l1L5uG'; + 005_update_openclaw_default_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = '10.130.14.23:5000/openclaw:2026.6.5' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + 006_add_openclaw_config_center.sql: | + USE clawmanager; + SET @openclaw_snapshot_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'openclaw_config_snapshot_id' + ); + SET @openclaw_snapshot_column_sql = IF( + @openclaw_snapshot_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN openclaw_config_snapshot_id INT NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE openclaw_snapshot_column_stmt FROM @openclaw_snapshot_column_sql; + EXECUTE openclaw_snapshot_column_stmt; + DEALLOCATE PREPARE openclaw_snapshot_column_stmt; + + CREATE TABLE IF NOT EXISTS openclaw_config_resources ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_key VARCHAR(100) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + tags_json LONGTEXT NOT NULL, + content_json LONGTEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_resource_key (user_id, resource_type, resource_key), + INDEX idx_openclaw_resource_user_type (user_id, resource_type), + INDEX idx_openclaw_resource_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundles ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_openclaw_bundle_user (user_id), + INDEX idx_openclaw_bundle_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_items ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + resource_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (resource_id) REFERENCES openclaw_config_resources(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_resource (bundle_id, resource_id), + INDEX idx_openclaw_bundle_item_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_injection_snapshots ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NULL, + user_id INT NOT NULL, + mode VARCHAR(20) NOT NULL, + bundle_id INT NULL, + selected_resource_ids_json LONGTEXT NOT NULL, + resolved_resources_json LONGTEXT NOT NULL, + rendered_manifest_json LONGTEXT NOT NULL, + rendered_env_json LONGTEXT NOT NULL, + secret_name VARCHAR(255) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + activated_at TIMESTAMP NULL, + INDEX idx_openclaw_snapshot_user_created (user_id, created_at), + INDEX idx_openclaw_snapshot_instance (instance_id), + INDEX idx_openclaw_snapshot_bundle (bundle_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 007_add_instance_agent_control_plane.sql: | + USE clawmanager; + SET @instance_agent_bootstrap_token_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'agent_bootstrap_token' + ); + SET @instance_agent_bootstrap_token_column_sql = IF( + @instance_agent_bootstrap_token_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN agent_bootstrap_token VARCHAR(255) NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE instance_agent_bootstrap_token_column_stmt FROM @instance_agent_bootstrap_token_column_sql; + EXECUTE instance_agent_bootstrap_token_column_stmt; + DEALLOCATE PREPARE instance_agent_bootstrap_token_column_stmt; + + CREATE TABLE IF NOT EXISTS instance_agents ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NOT NULL, + agent_version VARCHAR(50) NOT NULL, + protocol_version VARCHAR(50) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'online', + capabilities_json LONGTEXT NOT NULL, + host_info_json LONGTEXT NULL, + session_token VARCHAR(255) NULL, + session_expires_at TIMESTAMP NULL, + last_heartbeat_at TIMESTAMP NULL, + last_reported_at TIMESTAMP NULL, + last_seen_ip VARCHAR(45) NULL, + registered_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_agents_instance (instance_id), + UNIQUE KEY uk_instance_agents_session_token (session_token), + INDEX idx_instance_agents_agent_id (agent_id), + INDEX idx_instance_agents_status (status), + INDEX idx_instance_agents_last_heartbeat (last_heartbeat_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_runtime_status ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + infra_status VARCHAR(30) NOT NULL DEFAULT 'creating', + agent_status VARCHAR(30) NOT NULL DEFAULT 'offline', + openclaw_status VARCHAR(30) NOT NULL DEFAULT 'unknown', + openclaw_pid INT NULL, + openclaw_version VARCHAR(100) NULL, + current_config_revision_id INT NULL, + desired_config_revision_id INT NULL, + summary_json LONGTEXT NULL, + system_info_json LONGTEXT NULL, + health_json LONGTEXT NULL, + last_reported_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_runtime_status_instance (instance_id), + INDEX idx_instance_runtime_status_agent_status (agent_status), + INDEX idx_instance_runtime_status_openclaw_status (openclaw_status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_desired_state ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + desired_power_state VARCHAR(30) NOT NULL DEFAULT 'running', + desired_config_revision_id INT NULL, + desired_runtime_action VARCHAR(50) NULL, + updated_by INT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_desired_state_instance (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_commands ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NULL, + command_type VARCHAR(50) NOT NULL, + payload_json LONGTEXT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + idempotency_key VARCHAR(255) NOT NULL, + issued_by INT NULL, + issued_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + dispatched_at TIMESTAMP NULL, + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + timeout_seconds INT NOT NULL DEFAULT 300, + result_json LONGTEXT NULL, + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (issued_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_commands_idempotency (instance_id, idempotency_key), + INDEX idx_instance_commands_instance_status (instance_id, status), + INDEX idx_instance_commands_agent_status (agent_id, status), + INDEX idx_instance_commands_issued_at (issued_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_config_revisions ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + source_snapshot_id INT NULL, + source_bundle_id INT NULL, + revision_no INT NOT NULL, + content_json LONGTEXT NOT NULL, + checksum VARCHAR(255) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'published', + published_by INT NULL, + published_at TIMESTAMP NULL, + activated_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (published_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_config_revision_unique (instance_id, revision_no), + INDEX idx_instance_config_revision_instance (instance_id, revision_no), + INDEX idx_instance_config_revision_status (status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 008_add_skill_management.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS skill_blobs ( + id INT AUTO_INCREMENT PRIMARY KEY, + content_hash VARCHAR(128) NOT NULL, + archive_hash VARCHAR(128) NOT NULL, + object_key VARCHAR(512) NOT NULL, + file_name VARCHAR(255) NOT NULL, + media_type VARCHAR(100) NOT NULL DEFAULT 'application/gzip', + size_bytes BIGINT NOT NULL DEFAULT 0, + scan_status VARCHAR(30) NOT NULL DEFAULT 'pending', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_skill_blobs_content_hash (content_hash), + INDEX idx_skill_blobs_scan_status (scan_status), + INDEX idx_skill_blobs_risk_level (risk_level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + skill_key VARCHAR(120) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + current_version_id INT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + status VARCHAR(30) NOT NULL DEFAULT 'active', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_skills_user_key (user_id, skill_key), + INDEX idx_skills_user_status (user_id, status), + INDEX idx_skills_risk_level (risk_level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skill_versions ( + id INT AUTO_INCREMENT PRIMARY KEY, + skill_id INT NOT NULL, + blob_id INT NOT NULL, + version_no INT NOT NULL, + manifest_json LONGTEXT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE RESTRICT, + UNIQUE KEY uk_skill_versions_skill_version (skill_id, version_no), + UNIQUE KEY uk_skill_versions_skill_blob (skill_id, blob_id), + INDEX idx_skill_versions_skill_id (skill_id, version_no) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + skill_id INT NOT NULL, + skill_version_id INT NULL, + source_type VARCHAR(40) NOT NULL DEFAULT 'discovered_in_instance', + install_path VARCHAR(1024) NULL, + observed_hash VARCHAR(128) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'active', + last_seen_at TIMESTAMP NULL, + removed_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (skill_version_id) REFERENCES skill_versions(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_skills_instance_skill (instance_id, skill_id), + INDEX idx_instance_skills_instance (instance_id, status), + INDEX idx_instance_skills_skill (skill_id, status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skill_scan_results ( + id INT AUTO_INCREMENT PRIMARY KEY, + blob_id INT NOT NULL, + engine VARCHAR(60) NOT NULL, + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + status VARCHAR(30) NOT NULL DEFAULT 'completed', + summary TEXT NULL, + findings_json LONGTEXT NULL, + scanned_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE CASCADE, + INDEX idx_skill_scan_results_blob (blob_id, scanned_at), + INDEX idx_skill_scan_results_risk (risk_level, scanned_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 009_cpu_cores_decimal.sql: | + USE clawmanager; + ALTER TABLE instances MODIFY COLUMN cpu_cores DECIMAL(10,2) NOT NULL; + ALTER TABLE user_quotas MODIFY COLUMN max_cpu_cores DECIMAL(10,2) DEFAULT 40; + 011_add_hermes_instance_type.sql: | + USE clawmanager; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 012_update_agents_runtime_default_images.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = '10.130.14.23:5000/openclaw:2026.6.5' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + + UPDATE system_image_settings + SET image = '10.130.14.23:5000/hermes:2026.6.5' + WHERE instance_type = 'hermes' + AND image IN ( + 'registry.example.com/hermes-webtop:latest', + 'lscr.io/linuxserver/webtop:ubuntu-xfce' + ); + 013_normalize_agents_runtime_image_names.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = '10.130.14.23:5000/openclaw:2026.6.5' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/openclaw:latest'; + + UPDATE system_image_settings + SET image = '10.130.14.23:5000/hermes:2026.6.5' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/hermes:latest'; + 019_add_instance_runtime_type.sql: | + USE clawmanager; + SET @instance_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'runtime_type' + ); + SET @instance_runtime_type_column_sql = IF( + @instance_runtime_type_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER type', + 'SELECT 1' + ); + PREPARE instance_runtime_type_column_stmt FROM @instance_runtime_type_column_sql; + EXECUTE instance_runtime_type_column_stmt; + DEALLOCATE PREPARE instance_runtime_type_column_stmt; + 020_add_system_image_runtime_type.sql: | + USE clawmanager; + SET @system_image_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'runtime_type' + ); + SET @system_image_runtime_type_column_sql = IF( + @system_image_runtime_type_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER instance_type', + 'SELECT 1' + ); + PREPARE system_image_runtime_type_column_stmt FROM @system_image_runtime_type_column_sql; + EXECUTE system_image_runtime_type_column_stmt; + DEALLOCATE PREPARE system_image_runtime_type_column_stmt; + 021_add_openclaw_shell_runtime_image.sql: | + USE clawmanager; + SET @system_image_is_enabled_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'is_enabled' + ); + SET @system_image_is_enabled_column_sql = IF( + @system_image_is_enabled_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN is_enabled BOOLEAN NOT NULL DEFAULT TRUE', + 'SELECT 1' + ); + PREPARE system_image_is_enabled_column_stmt FROM @system_image_is_enabled_column_sql; + EXECUTE system_image_is_enabled_column_stmt; + DEALLOCATE PREPARE system_image_is_enabled_column_stmt; + + SET @system_image_unique_instance_type_index_name = ( + SELECT INDEX_NAME + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'instance_type' + AND NON_UNIQUE = 0 + AND INDEX_NAME <> 'PRIMARY' + LIMIT 1 + ); + SET @system_image_unique_instance_type_index_sql = IF( + @system_image_unique_instance_type_index_name IS NOT NULL, + CONCAT( + 'ALTER TABLE system_image_settings DROP INDEX `', + REPLACE(@system_image_unique_instance_type_index_name, '`', '``'), + '`' + ), + 'SELECT 1' + ); + PREPARE system_image_unique_instance_type_index_stmt FROM @system_image_unique_instance_type_index_sql; + EXECUTE system_image_unique_instance_type_index_stmt; + DEALLOCATE PREPARE system_image_unique_instance_type_index_stmt; + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'desktop', + 'OpenClaw Desktop', + '10.130.14.23:5000/openclaw:2026.6.5', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'desktop' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'shell', + 'OpenClaw Shell', + '10.130.14.23:5000/openclaw-shell:2026.6.5', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + 022_add_openclaw_config_bundle_skills.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + skill_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id), + INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 023_normalize_runtime_config_mounts.sql: | + USE clawmanager; + UPDATE instances + SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP + WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager{TENANT_SUFFIX}-mysql-pv +spec: + capacity: + storage: 5Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + hostPath: + path: /data/clawmanager{TENANT_SUFFIX}/system/mysql + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-data + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: manual + volumeName: clawmanager{TENANT_SUFFIX}-mysql-pv +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager{TENANT_SUFFIX}-minio-pv +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + hostPath: + path: /data/clawmanager{TENANT_SUFFIX}/system/minio + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-data + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: manual + volumeName: clawmanager{TENANT_SUFFIX}-minio-pv +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + replicas: 1 + selector: + matchLabels: + app: mysql + template: + metadata: + labels: + app: mysql + spec: + containers: + - name: mysql + image: 10.130.14.23:5000/mysql:2026.6.5 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3306 + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-root-password + - name: MYSQL_DATABASE + value: clawmanager + - name: MYSQL_USER + value: clawmanager + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + - name: mysql-init + mountPath: /docker-entrypoint-initdb.d + readinessProbe: + exec: + command: ["sh", "-c", "mysqladmin ping -h 127.0.0.1 -uroot -p$MYSQL_ROOT_PASSWORD"] + initialDelaySeconds: 20 + periodSeconds: 10 + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: mysql-data + - name: mysql-init + configMap: + name: clawmanager-mysql-init +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + selector: + app: mysql + ports: + - name: mysql + port: 3306 + targetPort: 3306 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: 10.130.14.23:5000/minio:2026.6.5 + imagePullPolicy: IfNotPresent + args: + - server + - /data + - --console-address + - ":9001" + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-root-password + ports: + - name: api + containerPort: 9000 + - name: console + containerPort: 9001 + volumeMounts: + - name: minio-data + mountPath: /data + readinessProbe: + tcpSocket: + port: api + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: minio-data + persistentVolumeClaim: + claimName: minio-data +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + selector: + app: minio + ports: + - name: api + port: 9000 + targetPort: api + - name: console + port: 9001 + targetPort: console +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-team-redis + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + replicas: 1 + selector: + matchLabels: + app: clawmanager-team-redis + template: + metadata: + labels: + app: clawmanager-team-redis + spec: + containers: + - name: redis + image: redis:7-alpine + imagePullPolicy: IfNotPresent + args: + - redis-server + - --appendonly + - "yes" + ports: + - name: redis + containerPort: 6379 + readinessProbe: + tcpSocket: + port: redis + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: redis-data + mountPath: /data + volumes: + - name: redis-data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-team-redis + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager{TENANT_SUFFIX}-workspaces-pv +spec: + capacity: + storage: 50Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + hostPath: + path: /data/clawmanager{TENANT_SUFFIX}/workspaces + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clawmanager-workspaces + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Gi + storageClassName: manual + volumeName: clawmanager{TENANT_SUFFIX}-workspaces-pv +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: workspace-store + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + replicas: 1 + selector: + matchLabels: + app: workspace-store + template: + metadata: + labels: + app: workspace-store + spec: + containers: + - name: nfs-server + image: itsthenetwork/nfs-server-alpine:12 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: SHARED_DIRECTORY + value: /exports/workspaces + ports: + - name: nfs + containerPort: 2049 + volumeMounts: + - name: workspaces + mountPath: /exports/workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: v1 +kind: Service +metadata: + name: workspace-store + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + selector: + app: workspace-store + ports: + - name: nfs + port: 2049 + targetPort: nfs +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: clawmanager-app + namespace: clawmanager{TENANT_SUFFIX}-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager{TENANT_SUFFIX}-app-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager{TENANT_SUFFIX}-system +--- +# Explicit lease permissions for control-plane leader election. The +# cluster-admin binding above already covers this; this Role documents the +# requirement and keeps leader election working if RBAC is tightened later. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: clawmanager-app-leaderelection + namespace: clawmanager{TENANT_SUFFIX}-system +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: clawmanager-app-leaderelection + namespace: clawmanager{TENANT_SUFFIX}-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: clawmanager-app-leaderelection +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager{TENANT_SUFFIX}-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: skill-scanner + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + replicas: 1 + selector: + matchLabels: + app: skill-scanner + template: + metadata: + labels: + app: skill-scanner + spec: + containers: + - name: skill-scanner + image: 10.130.14.23:5000/skill-scanner:2026.6.5 + imagePullPolicy: IfNotPresent + command: + - /opt/skill-scanner-venv/bin/skill-scanner-api + - --host + - 0.0.0.0 + - --port + - "8000" + env: + - name: SKILL_SCANNER_LLM_API_KEY + value: "" + - name: SKILL_SCANNER_LLM_MODEL + value: "" + - name: SKILL_SCANNER_LLM_BASE_URL + value: "" + - name: SKILL_SCANNER_META_LLM_API_KEY + value: "" + - name: SKILL_SCANNER_META_LLM_MODEL + value: "" + - name: SKILL_SCANNER_META_LLM_BASE_URL + value: "" + ports: + - name: http + containerPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: skill-scanner + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + selector: + app: skill-scanner + ports: + - name: http + port: 8000 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-app + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + replicas: 1 + selector: + matchLabels: + app: clawmanager-app + template: + metadata: + labels: + app: clawmanager-app + spec: + serviceAccountName: clawmanager-app + containers: + - name: clawmanager-app + image: {APP_IMAGE} + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 8443 + - name: proxy + containerPort: 9001 + env: + - name: SERVER_ADDRESS + value: ":9001" + - name: SERVER_MODE + value: "release" + - name: DB_HOST + value: "mysql" + - name: DB_PORT + value: "3306" + - name: DB_USER + value: "clawmanager" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + - name: DB_NAME + value: "clawmanager" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: jwt-secret + - name: OBJECT_STORAGE_ENDPOINT + value: "minio:9000" + - name: OBJECT_STORAGE_REGION + value: "" + - name: OBJECT_STORAGE_ACCESS_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-access-key + - name: OBJECT_STORAGE_SECRET_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-secret-key + - name: OBJECT_STORAGE_BUCKET + value: "clawmanager-skills" + - name: OBJECT_STORAGE_USE_SSL + value: "false" + - name: OBJECT_STORAGE_BASE_PATH + value: "skills" + - name: OBJECT_STORAGE_FORCE_PATH_STYLE + value: "true" + - name: K8S_MODE + value: "incluster" + - name: K8S_NAMESPACE + value: "clawmanager{TENANT_SUFFIX}" + - name: K8S_STORAGE_CLASS + value: "longhorn" + - name: CLAWMANAGER_STORAGE_PROFILE + value: "cluster" + - name: K8S_HOSTPATH_FALLBACK_ENABLED + value: "false" + - name: K8S_PVC_BIND_TIMEOUT + value: "2m" + - name: K8S_CONTROL_PLANE_STORAGE_CLASS + value: "manual" + - name: K8S_INSTANCE_STORAGE_CLASS + value: "longhorn" + - name: K8S_WORKSPACE_STORAGE_CLASS + value: "manual" + - name: K8S_WORKSPACE_ACCESS_MODE + value: "ReadWriteMany" + - name: K8S_PV_HOST_PATH_PREFIX + value: "/data/clawmanager{TENANT_SUFFIX}/instances" + - name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB + value: "500" + - name: PLATFORM_REDIS_URL + value: "redis://clawmanager-team-redis.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local:6379/0" + - name: TEAM_REDIS_URL + value: "redis://clawmanager-team-redis.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_DESKTOP_DIRECT_PROXY + value: "false" + # Leader election for control-plane singleton background loops. + # Required before scaling replicas > 1 so SyncService / Team event + # consumers / stale-task monitor run on exactly one replica. + - name: CLAWMANAGER_LEADER_ELECTION + value: "true" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLAWMANAGER_TEAM_REDIS_URL + value: "redis://clawmanager-team-redis.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_TEAM_MANAGER_BASE_URL + value: "http://clawmanager-gateway.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local:9001" + - name: RUNTIME_NAMESPACE + value: "clawmanager{TENANT_SUFFIX}-system" + - name: RUNTIME_WORKSPACE_ROOT + value: "/workspaces" + - name: RUNTIME_WORKSPACE_NFS_SERVER + value: "workspace-store.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local" + - name: RUNTIME_WORKSPACE_NFS_PATH + value: "/" + - name: RUNTIME_MAX_GATEWAYS_PER_POD + value: "100" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_SCHEDULER_ENABLED + value: "true" + - name: RUNTIME_HEARTBEAT_TIMEOUT + value: "10s" + - name: OPENCLAW_RUNTIME_IMAGE + value: "{OPENCLAW_RUNTIME_IMAGE}" + - name: HERMES_RUNTIME_IMAGE + value: "{HERMES_RUNTIME_IMAGE}" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: SKILL_SCANNER_ENABLED + value: "true" + - name: SKILL_SCANNER_BASE_URL + value: "http://skill-scanner.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local:8000" + - name: SKILL_SCANNER_TIMEOUT_SECONDS + value: "120" + - name: SKILL_SCANNER_NAMESPACE + value: "clawmanager{TENANT_SUFFIX}-system" + - name: SKILL_SCANNER_DEPLOYMENT + value: "skill-scanner" + volumeMounts: + - name: tls-cert + mountPath: /var/run/clawreef-tls + readOnly: true + - name: workspaces + mountPath: /workspaces + readinessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 30 + periodSeconds: 20 + volumes: + - name: tls-cert + secret: + secretName: clawmanager-tls + optional: true + - name: workspaces + nfs: + server: workspace-store.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local + path: / +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openclaw-runtime + namespace: clawmanager{TENANT_SUFFIX}-system + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw +spec: + replicas: 1 + selector: + matchLabels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + template: + metadata: + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + spec: + containers: + - name: runtime + image: {OPENCLAW_RUNTIME_IMAGE} + imagePullPolicy: IfNotPresent + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: openclaw + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: openclaw-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: {OPENCLAW_RUNTIME_IMAGE} + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_COMMAND + value: "/usr/local/bin/openclaw gateway run --allow-unconfigured --auth token --bind lan --force" + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + nfs: + server: workspace-store.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local + path: / +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-runtime + namespace: clawmanager{TENANT_SUFFIX}-system + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes +spec: + replicas: 1 + selector: + matchLabels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + template: + metadata: + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + spec: + containers: + - name: runtime + image: {HERMES_RUNTIME_IMAGE} + imagePullPolicy: IfNotPresent + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: hermes + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: hermes-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: {HERMES_RUNTIME_IMAGE} + - name: CLAWMANAGER_AGENT_PORT + value: "19090" + - name: CLAWMANAGER_GATEWAY_PORT_START + value: "20000" + - name: CLAWMANAGER_GATEWAY_PORT_END + value: "20099" + - name: HERMES_TUI_DIR + value: /usr/local/lib/hermes-agent/ui-tui + - name: CLAWMANAGER_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: CLAWMANAGER_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + nfs: + server: workspace-store.clawmanager{TENANT_SUFFIX}-system.svc.cluster.local + path: / +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-frontend + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + type: NodePort + selector: + app: clawmanager-app + ports: + - name: https + port: 443 + targetPort: https + nodePort: {NODE_PORT} +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-gateway + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + type: ClusterIP + selector: + app: clawmanager-app + ports: + - name: api + port: 9001 + targetPort: 9001 +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-egress-proxy + namespace: clawmanager{TENANT_SUFFIX}-system +spec: + type: ClusterIP + selector: + app: clawmanager-app + ports: + - name: proxy + port: 3128 + targetPort: 9001 diff --git a/deployments/k8s/cluster/README.md b/deployments/k8s/cluster/README.md new file mode 100644 index 0000000..58409db --- /dev/null +++ b/deployments/k8s/cluster/README.md @@ -0,0 +1,408 @@ +# ClawManager Kubernetes Cluster Manifest + +This directory contains the all-in-one Kubernetes manifest for a test or small +cluster deployment: + +- `clawmanager.yaml`: Longhorn v1.12.0 plus ClawManager workloads. +- `uninstall.yaml`: Longhorn v1.12.0 uninstall job. +- StorageClasses `longhorn` and `longhorn-rwx`. + +The manifest includes Longhorn. Follow the install and uninstall order below; +do not delete the whole manifest as the first uninstall step. + +## Install + +### 1. Enter This Directory + +```sh +cd deployments/k8s/cluster +``` + +Check that `kubectl` points to the target cluster: + +```sh +kubectl config current-context +kubectl get nodes -o wide +``` + +Expected result: every node that may run Longhorn or ClawManager workloads is +`Ready`. + +If a node is not `Ready`, fix the Kubernetes node first. Longhorn will not be +stable on an unhealthy cluster. + +### 2. Check For Old Installation State + +```sh +kubectl get ns longhorn-system clawmanager-system clawmanager-user-1 --ignore-not-found +kubectl get sc longhorn longhorn-rwx --ignore-not-found +kubectl get crd | grep longhorn.io || true +``` + +Expected result for a fresh install: no `longhorn-system`, +`clawmanager-system`, `clawmanager-user-1`, Longhorn StorageClass, or Longhorn +CRD remains. + +If old Longhorn or ClawManager resources exist, run the uninstall flow in this +README before installing again. + +### 3. Check Image Availability + +The manifest uses the images listed in `clawmanager.yaml`. Make sure every node +can pull from the referenced registry before installing: + +```sh +grep -n 'image:' clawmanager.yaml | head -n 40 +``` + +If this is an offline or private-registry deployment, load or push the images to +the registry used by the manifest before continuing. + +If pods later show `ImagePullBackOff`, locate the failing image with: + +```sh +kubectl get pods -A | grep ImagePullBackOff || true +kubectl describe pod -n +``` + +Then verify registry access from the node that runs the pod. + +### 4. Prepare Every Node + +Longhorn uses iSCSI for block volumes and NFS for RWX volumes. Install the +required packages on every node that may run workloads. + +Ubuntu or Debian: + +```sh +apt-get update +apt-get install -y open-iscsi nfs-common cryptsetup dmsetup +systemctl enable --now iscsid || systemctl enable --now open-iscsi +``` + +RHEL, CentOS, Rocky, or AlmaLinux: + +```sh +yum install -y iscsi-initiator-utils nfs-utils cryptsetup device-mapper +systemctl enable --now iscsid +``` + +Check on every node: + +```sh +command -v iscsiadm +command -v mount.nfs +command -v mount.nfs4 +systemctl is-active iscsid || systemctl is-active open-iscsi +``` + +If `mount.nfs` or `mount.nfs4` is missing, pods that mount +`clawmanager-workspaces` can stay in `ContainerCreating` with an event like: + +```text +bad option; you might need a /sbin/mount. helper program +``` + +### 5. Check `multipathd` + +Longhorn volumes can be blocked by `multipathd` if Longhorn devices are not +blacklisted. + +Check on every node: + +```sh +systemctl is-active multipathd || true +multipath -t 2>/dev/null | grep -F 'devnode "^sd[a-z0-9]+"' || true +``` + +Expected result: either `multipathd` is not active, or the blacklist contains: + +```text +devnode "^sd[a-z0-9]+" +``` + +If `multipathd` is active and the blacklist is missing, either disable it if +the node does not need multipath: + +```sh +systemctl disable --now multipathd +``` + +Or add the Longhorn blacklist to `/etc/multipath.conf`, then restart +`multipathd`: + +```conf +blacklist { + devnode "^sd[a-z0-9]+" +} +``` + +```sh +systemctl restart multipathd +multipath -t | grep -F 'devnode "^sd[a-z0-9]+"' +``` + +Common symptom when this check is skipped: + +```text +Waiting for volume share to be available +is apparently in use by the system +``` + +### 6. Check Pod Network Health + +Longhorn manager, CSI, and share-manager pods require cross-node Pod networking. +First check for duplicate Pod IPs: + +```sh +kubectl get pods -A \ + -o custom-columns=IP:.status.podIP,NS:.metadata.namespace,NAME:.metadata.name,NODE:.spec.nodeName \ + --no-headers \ +| awk '$1 != "" { pods[$1] = pods[$1] "\n" $0; count[$1]++ } END { for (ip in count) if (count[ip] > 1) print pods[ip] }' +``` + +Expected result: no output. + +If this command prints any IP, fix the CNI before installing. For Calico, common +checks are: + +```sh +kubectl -n calico-system get pods -o wide +kubectl get blockaffinities.crd.projectcalico.org 2>/dev/null || true +kubectl get ippools.crd.projectcalico.org 2>/dev/null || true +``` + +If a single DaemonSet or Deployment pod has a stale or wrong IP, deleting that +pod and letting its controller recreate it is often enough: + +```sh +kubectl -n delete pod +``` + +Do not continue until the duplicate IP check has no output. + +### 7. Match Longhorn Replica Count To Storage Nodes + +The manifest defaults Longhorn StorageClasses to `numberOfReplicas: "3"`. +Use this default only when at least three Longhorn storage nodes are available. + +Check the current value: + +```sh +grep -n 'numberOfReplicas:' clawmanager.yaml +``` + +For a two-node test cluster, change both StorageClass values to `2` before +installing: + +```sh +sed -i.bak 's/numberOfReplicas: "3"/numberOfReplicas: "2"/g' clawmanager.yaml +grep -n 'numberOfReplicas:' clawmanager.yaml +``` + +If this is not adjusted, volumes may still work but will stay `degraded` with +replica scheduling errors. + +### 8. Apply The Manifest + +```sh +kubectl apply -f clawmanager.yaml +``` + +### 9. Check Longhorn + +Wait for Longhorn manager and CSI components: + +```sh +kubectl -n longhorn-system rollout status daemonset/longhorn-manager --timeout=10m +kubectl -n longhorn-system rollout status deployment/longhorn-driver-deployer --timeout=10m +kubectl -n longhorn-system rollout status daemonset/longhorn-csi-plugin --timeout=10m +kubectl -n longhorn-system get deploy,ds,pod -o wide +kubectl get csidriver driver.longhorn.io +kubectl get sc longhorn longhorn-rwx +``` + +Expected result: + +- `longhorn-manager` is ready on every node. +- `longhorn-driver-deployer` is `1/1`. +- `longhorn-csi-plugin` is ready on every node. +- `driver.longhorn.io`, `longhorn`, and `longhorn-rwx` exist. + +If `longhorn-driver-deployer` is `CrashLoopBackOff`, inspect it: + +```sh +kubectl -n longhorn-system logs -l app=longhorn-driver-deployer --all-containers --tail=200 +kubectl -n longhorn-system describe pod -l app=longhorn-driver-deployer +``` + +If the log contains `connect: connection refused` for +`http://longhorn-backend:9500/v1`, check duplicate Pod IPs and cross-node Pod +networking again. If the log mentions `MountPropagation`, also check that the +Longhorn manager pod has `mountPropagation: Bidirectional` on +`/var/lib/longhorn/`. + +### 10. Check ClawManager + +```sh +kubectl -n clawmanager-system get pvc +kubectl -n clawmanager-system rollout status deployment/clawmanager-app --timeout=15m +kubectl -n clawmanager-system rollout status deployment/openclaw-runtime --timeout=15m +kubectl -n clawmanager-system rollout status deployment/hermes-runtime --timeout=15m +kubectl -n clawmanager-system get deploy,pod,pvc -o wide +``` + +Expected result: + +- All PVCs are `Bound`. +- `clawmanager-app`, `openclaw-runtime`, and `hermes-runtime` are available. +- MySQL, Redis, MinIO, and skill scanner pods are running. + +If PVCs stay `Pending`, check CSI and StorageClass: + +```sh +kubectl get sc longhorn longhorn-rwx +kubectl -n longhorn-system get pods | grep csi +kubectl -n clawmanager-system describe pvc +``` + +If pods stay `ContainerCreating`, check recent events: + +```sh +kubectl -n clawmanager-system get events --sort-by=.lastTimestamp | tail -n 80 +``` + +### 11. Check Longhorn Volumes + +```sh +kubectl -n longhorn-system get volumes.longhorn.io \ + -o custom-columns=NAME:.metadata.name,STATE:.status.state,ROBUSTNESS:.status.robustness,REPLICAS:.spec.numberOfReplicas,SHARESTATE:.status.shareState,ENDPOINT:.status.shareEndpoint +``` + +Expected result: volumes are `healthy`. The workspace RWX volume should also +show `SHARESTATE` as `running` and an NFS endpoint. + +If volumes are `degraded`, check whether `numberOfReplicas` is greater than the +number of available storage nodes. + +If the RWX volume is stuck at `shareState: starting`, inspect the share-manager: + +```sh +kubectl -n longhorn-system get pods | grep share-manager +kubectl -n longhorn-system logs --tail=200 +kubectl -n longhorn-system describe pod +``` + +Common causes are missing NFS client packages, `multipathd` interference, or +cross-node Pod network problems. + +## Uninstall + +Use this order for a disposable test deployment. The Longhorn uninstall job must +run before deleting the all-in-one manifest. + +### 1. Delete ClawManager Workloads First + +```sh +kubectl delete namespace clawmanager-user-1 --ignore-not-found +kubectl delete namespace clawmanager-system --ignore-not-found +``` + +Check: + +```sh +kubectl get ns clawmanager-user-1 clawmanager-system --ignore-not-found +kubectl get pvc -A | grep clawmanager || true +``` + +If a namespace stays `Terminating`, inspect the remaining resources: + +```sh +kubectl get all,pvc -n clawmanager-system 2>/dev/null || true +kubectl get events -n clawmanager-system --sort-by=.lastTimestamp 2>/dev/null | tail -n 80 || true +``` + +### 2. Enable Longhorn Deletion + +```sh +kubectl -n longhorn-system patch settings.longhorn.io deleting-confirmation-flag \ + --type=merge \ + -p '{"value":"true"}' +``` + +Check: + +```sh +kubectl -n longhorn-system get settings.longhorn.io deleting-confirmation-flag -o yaml +``` + +Expected result: `value: "true"`. + +### 3. Run The Longhorn Uninstall Job + +```sh +kubectl create -f uninstall.yaml +kubectl wait --for=condition=complete job/longhorn-uninstall -n longhorn-system --timeout=10m +``` + +Check: + +```sh +kubectl -n longhorn-system logs job/longhorn-uninstall --tail=200 +``` + +If the job already exists from a previous attempt: + +```sh +kubectl delete -f uninstall.yaml --ignore-not-found +kubectl create -f uninstall.yaml +kubectl wait --for=condition=complete job/longhorn-uninstall -n longhorn-system --timeout=10m +``` + +### 4. Delete The All-In-One Manifest + +```sh +kubectl delete -f clawmanager.yaml --ignore-not-found +kubectl delete -f uninstall.yaml --ignore-not-found +``` + +Final check: + +```sh +kubectl get ns longhorn-system clawmanager-system clawmanager-user-1 --ignore-not-found +kubectl get sc longhorn longhorn-rwx --ignore-not-found +kubectl get crd | grep longhorn.io || true +``` + +Expected result: no Longhorn or ClawManager resources remain. + +## Recovery For A Stuck Uninstall + +If `kubectl delete -f clawmanager.yaml` was run first and the uninstall is now +stuck, check whether the Longhorn webhook service is gone: + +```sh +kubectl -n longhorn-system get svc longhorn-admission-webhook --ignore-not-found +kubectl get validatingwebhookconfiguration longhorn-webhook-validator --ignore-not-found +kubectl get mutatingwebhookconfiguration longhorn-webhook-mutator --ignore-not-found +``` + +If the webhook service is gone but webhook configurations remain, remove the +stale webhook registrations: + +```sh +kubectl delete validatingwebhookconfiguration longhorn-webhook-validator --ignore-not-found +kubectl delete mutatingwebhookconfiguration longhorn-webhook-mutator --ignore-not-found +``` + +If Longhorn CRDs still cannot be deleted after that, use this only when you +intend to wipe Longhorn state from the cluster: + +```sh +NAMESPACE=longhorn-system +for crd in $(kubectl get crd -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep longhorn.io); do + kubectl -n "${NAMESPACE}" get "${crd}" -o yaml | sed 's/- longhorn.io//g' | kubectl apply -f - + kubectl -n "${NAMESPACE}" delete "${crd}" --all --ignore-not-found + kubectl delete "crd/${crd}" --ignore-not-found +done +``` diff --git a/deployments/k8s/cluster/clawmanager.yaml b/deployments/k8s/cluster/clawmanager.yaml new file mode 100644 index 0000000..d556a5d --- /dev/null +++ b/deployments/k8s/cluster/clawmanager.yaml @@ -0,0 +1,6496 @@ +# ClawManager cluster deployment bundle. +# Includes Longhorn v1.12.0 from: +# https://raw.githubusercontent.com/longhorn/longhorn/v1.12.0/deploy/longhorn.yaml +# +# Apply this file on a clean Kubernetes cluster: +# kubectl apply -f +# +# Notes: +# - Longhorn RWX volumes require NFSv4 client support on all workload nodes. +# Install nfs-common on Ubuntu/Debian or nfs-utils on RHEL-compatible systems. +# - ClawManager uses StorageClass longhorn for RWO data and longhorn-rwx for workspace RWX data. +--- +# Builtin: "helm template" does not respect --create-namespace +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system +--- +# Source: longhorn/templates/priorityclass.yaml +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: "longhorn-critical" + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" +globalDefault: false +preemptionPolicy: PreemptLowerPriority +value: 1000000000 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-ui-service-account + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +--- +# Source: longhorn/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-support-bundle + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +--- +# Source: longhorn/templates/default-resource.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-resource + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +data: + default-resource.yaml: |- +--- +# Source: longhorn/templates/default-setting.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-setting + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +data: + default-setting.yaml: |- + priority-class: "longhorn-critical" + disable-revision-counter: "{\"v1\":\"true\"}" +--- +# Source: longhorn/templates/storageclass.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-storageclass + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +data: + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: longhorn + annotations: + storageclass.kubernetes.io/is-default-class: "true" + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: "Delete" + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "3" + staleReplicaTimeout: "30" + fromBackup: "" + fsType: "ext4" + dataLocality: "disabled" + unmapMarkSnapChainRemoved: "ignored" + disableRevisionCounter: "true" + dataEngine: "v1" + backupTargetName: "default" +--- +# Source: longhorn/templates/crds.yaml +# Generated crds.yaml from github.com/longhorn/longhorn-manager/k8s/pkg/apis and the crds.yaml will be copied to longhorn/longhorn chart/templates and cannot be directly used by kubectl apply. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backingimagedatasources.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageDataSource + listKind: BackingImageDataSourceList + plural: backingimagedatasources + shortNames: + - lhbids + singular: backingimagedatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system generated UUID of the provisioned backing image file + jsonPath: .spec.uuid + name: UUID + type: string + - description: The current state of the pod used to provision the backing image + file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size + jsonPath: .status.size + name: Size + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image + data source object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackingImageDataSourceSpec defines the desired state of the + Longhorn backing image data source + properties: + checksum: + type: string + diskPath: + type: string + diskUUID: + type: string + fileTransferred: + type: boolean + nodeID: + type: string + parameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + - clone + type: string + uuid: + type: string + type: object + status: + description: BackingImageDataSourceStatus defines the observed state of + the Longhorn backing image data source + properties: + checksum: + type: string + currentState: + type: string + ip: + type: string + message: + type: string + ownerID: + type: string + progress: + type: integer + runningParameters: + additionalProperties: + type: string + nullable: true + type: object + size: + format: int64 + type: integer + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backingimagemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageManager + listKind: BackingImageManagerList + plural: backingimagemanagers + shortNames: + - lhbim + singular: backingimagemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager + object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackingImageManagerSpec defines the desired state of the + Longhorn backing image manager + properties: + backingImages: + additionalProperties: + type: string + type: object + diskPath: + type: string + diskUUID: + type: string + image: + type: string + nodeID: + type: string + type: object + status: + description: BackingImageManagerStatus defines the observed state of the + Longhorn backing image manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImageFileMap: + additionalProperties: + properties: + currentChecksum: + type: string + message: + type: string + name: + type: string + progress: + type: integer + realSize: + format: int64 + type: integer + senderManagerAddress: + type: string + sendingReference: + type: integer + size: + format: int64 + type: integer + state: + type: string + uuid: + type: string + virtualSize: + format: int64 + type: integer + type: object + nullable: true + type: object + currentState: + type: string + ip: + type: string + ownerID: + type: string + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backingimages.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImage + listKind: BackingImageList + plural: backingimages + shortNames: + - lhbi + singular: backingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system generated UUID + jsonPath: .status.uuid + name: UUID + type: string + - description: The source of the backing image file data + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size in each disk + jsonPath: .status.size + name: Size + type: string + - description: The virtual size of the image (may be larger than file size) + jsonPath: .status.virtualSize + name: VirtualSize + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackingImageSpec defines the desired state of the Longhorn + backing image + properties: + checksum: + type: string + dataEngine: + default: v1 + enum: + - v1 + - v2 + type: string + diskFileSpecMap: + additionalProperties: + properties: + dataEngine: + enum: + - v1 + - v2 + type: string + evictionRequested: + type: boolean + type: object + type: object + diskSelector: + items: + type: string + type: array + disks: + additionalProperties: + type: string + description: Deprecated. We are now using DiskFileSpecMap to assign + different spec to the file on different disks. + type: object + minNumberOfCopies: + type: integer + nodeSelector: + items: + type: string + type: array + secret: + type: string + secretNamespace: + type: string + sourceParameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + - restore + - clone + type: string + type: object + status: + description: BackingImageStatus defines the observed state of the Longhorn + backing image status + properties: + checksum: + type: string + diskFileStatusMap: + additionalProperties: + properties: + dataEngine: + enum: + - v1 + - v2 + type: string + lastStateTransitionTime: + type: string + message: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + diskLastRefAtMap: + additionalProperties: + type: string + nullable: true + type: object + ownerID: + type: string + realSize: + description: Real size of image in bytes, which may be smaller than + the size when the file is a sparse file. Will be zero until known + (e.g. while a backing image is uploading) + format: int64 + type: integer + size: + format: int64 + type: integer + uuid: + type: string + v2FirstCopyDisk: + type: string + v2FirstCopyStatus: + description: It is pending -> in-progress -> ready/failed + type: string + virtualSize: + description: Virtual size of image in bytes, which may be larger than + physical size. Will be zero until known (e.g. while a backing image + is uploading) + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backupbackingimages.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupBackingImage + listKind: BackupBackingImageList + plural: backupbackingimages + shortNames: + - lhbbi + singular: backupbackingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .status.backingImage + name: BackingImage + type: string + - description: The backing image size + jsonPath: .status.size + name: Size + type: string + - description: The backing image backup upload finished time + jsonPath: .status.backupCreatedAt + name: BackupCreatedAt + type: string + - description: The backing image backup state + jsonPath: .status.state + name: State + type: string + - description: The last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupBackingImage is where Longhorn stores backing image backup + object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackupBackingImageSpec defines the desired state of the Longhorn + backing image backup + properties: + backingImage: + description: The backing image name. + type: string + backupTargetName: + description: The backup target name. + nullable: true + type: string + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + type: object + syncRequestedAt: + description: The time to request run sync the remote backing image + backup. + format: date-time + nullable: true + type: string + userCreated: + description: Is this CR created by user through API or UI. + type: boolean + required: + - backingImage + - userCreated + type: object + status: + description: BackupBackingImageStatus defines the observed state of the + Longhorn backing image backup + properties: + backingImage: + description: The backing image name. + type: string + backupCreatedAt: + description: The backing image backup upload finished time. + type: string + checksum: + description: The checksum of the backing image. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the backing image backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of backing image backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backing image backup was synced + with the remote backup target. + format: date-time + nullable: true + type: string + managerAddress: + description: The address of the backing image manager that runs backing + image backup. + type: string + messages: + additionalProperties: + type: string + description: The error messages when listing or inspecting backing + image backup. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to + reconcile this CR. + type: string + progress: + description: The backing image backup progress. + type: integer + secret: + description: Record the secret if this backup backing image is encrypted + type: string + secretNamespace: + description: Record the secret namespace if this backup backing image + is encrypted + type: string + size: + description: The backing image size. + format: int64 + type: integer + state: + description: |- + The backing image backup creation state. + Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The backing image backup URL. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backups.longhorn.io +spec: + group: longhorn.io + names: + kind: Backup + listKind: BackupList + plural: backups + shortNames: + - lhb + singular: backup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup target name + jsonPath: .status.backupTargetName + name: BackupTarget + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackupSpec defines the desired state of the Longhorn backup + properties: + backupBlockSize: + description: The backup block size. 0 means the legacy default size + 2MiB, and -1 indicate the block size is invalid. + enum: + - "-1" + - "2097152" + - "16777216" + format: int64 + type: string + backupMode: + description: |- + The backup mode of this backup. + Can be "full" or "incremental" + enum: + - full + - incremental + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + type: object + snapshotName: + description: The snapshot name. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupStatus defines the observed state of the Longhorn backup + properties: + backupCreatedAt: + description: The snapshot backup upload finished time. + type: string + backupTargetName: + description: The backup target name. + type: string + compressionMethod: + description: Compression method + type: string + error: + description: The error message when taking the snapshot backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backup was synced with the remote + backup target. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when calling longhorn engine on listing + or inspecting backups. + nullable: true + type: object + newlyUploadDataSize: + description: Size in bytes of newly uploaded data + type: string + ownerID: + description: The node ID on which the controller is responsible to + reconcile this backup CR. + type: string + progress: + description: The snapshot backup progress. + type: integer + reUploadedDataSize: + description: Size in bytes of reuploaded data + type: string + replicaAddress: + description: The address of the replica that runs snapshot backup. + type: string + size: + description: The snapshot size. + type: string + snapshotCreatedAt: + description: The snapshot creation time. + type: string + snapshotName: + description: The snapshot name. + type: string + state: + description: |- + The backup creation state. + Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The snapshot backup URL. + type: string + volumeBackingImageName: + description: The volume's backing image name. + type: string + volumeCreated: + description: The volume creation time. + type: string + volumeName: + description: The volume name. + type: string + volumeSize: + description: The volume size. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backuptargets.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupTarget + listKind: BackupTargetList + plural: backuptargets + shortNames: + - lhbt + singular: backuptarget + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackupTargetSpec defines the desired state of the Longhorn + backup target + properties: + backupTargetURL: + description: The backup target URL. + type: string + credentialSecret: + description: The backup target credential secret. + type: string + pollInterval: + description: The interval that the cluster needs to run sync with + the backup target. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup target. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupTargetStatus defines the observed state of the Longhorn + backup target + properties: + available: + description: Available indicates if the remote backup target is available + or not. + type: boolean + conditions: + description: Records the reason on why the backup target is unavailable. + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + lastSyncedAt: + description: The last time that the controller synced with the remote + backup target. + format: date-time + nullable: true + type: string + ownerID: + description: The node ID on which the controller is responsible to + reconcile this backup target CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: backupvolumes.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupVolume + listKind: BackupVolumeList + plural: backupvolumes + shortNames: + - lhbv + singular: backupvolume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target name + jsonPath: .spec.backupTargetName + name: BackupTarget + type: string + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BackupVolumeSpec defines the desired state of the Longhorn + backup volume + properties: + backupTargetName: + description: The backup target name that the backup volume was synced. + nullable: true + type: string + syncRequestedAt: + description: The time to request run sync the remote backup volume. + format: date-time + nullable: true + type: string + volumeName: + description: The volume name that the backup volume was used to backup. + type: string + type: object + status: + description: BackupVolumeStatus defines the observed state of the Longhorn + backup volume + properties: + backingImageChecksum: + description: the backing image checksum. + type: string + backingImageName: + description: The backing image name. + type: string + createdAt: + description: The backup volume creation time. + type: string + dataStored: + description: The backup volume block count. + type: string + labels: + additionalProperties: + type: string + description: The backup volume labels. + nullable: true + type: object + lastBackupAt: + description: The latest volume backup time. + type: string + lastBackupName: + description: The latest volume backup name. + type: string + lastModificationTime: + description: The backup volume config last modification time. + format: date-time + nullable: true + type: string + lastSyncedAt: + description: The last time that the backup volume was synced into + the cluster. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when call longhorn engine on list + or inspect backup volumes. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to + reconcile this backup volume CR. + type: string + size: + description: The backup volume size. + type: string + storageClassName: + description: the storage class name of pv/pvc binding with the volume. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: enginefrontends.longhorn.io +spec: + group: longhorn.io + names: + kind: EngineFrontend + listKind: EngineFrontendList + plural: enginefrontends + shortNames: + - lhef + singular: enginefrontend + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The data engine of the engine frontend + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the engine frontend + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine frontend is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine frontend + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineFrontend is where Longhorn stores engine frontend object + for v2 data engine initiator. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EngineFrontendSpec defines the desired state of the Longhorn + engine frontend (v2 initiator) + properties: + active: + type: boolean + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + disableFrontend: + type: boolean + engineName: + description: EngineName is the name of the v2 engine target (required + for EngineFrontend instance creation) + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - ublk + - "" + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + salvageRequested: + type: boolean + size: + description: |- + Size is the desired size of the frontend device in bytes, as requested + by the volume owner. The EngineFrontend controller drives the frontend + device toward this size independently of the engine's target size. + format: int64 + type: string + targetIP: + description: TargetIP is the IP address of the v2 engine target + type: string + targetPort: + description: TargetPort is the port of the v2 engine target + type: integer + ublkNumberOfQueue: + description: ublkNumberOfQueue controls the number of queues for ublk + frontend. + type: integer + ublkQueueDepth: + description: ublkQueueDepth controls the depth of each queue for ublk + frontend. + type: integer + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineFrontendStatus defines the observed state of the Longhorn + engine frontend + properties: + activePath: + description: ActivePath is the currently active frontend path address. + type: string + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentSize: + description: |- + CurrentSize is the current size of the frontend device in bytes, as + observed from the data plane. It is 0 while the engine frontend is not + running. + format: int64 + type: string + currentState: + type: string + endpoint: + description: Endpoint is the initiator endpoint (e.g., /dev/longhorn/vol-name) + type: string + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + paths: + description: Paths describes the currently known frontend multipath + state. + items: + properties: + anaState: + type: string + engineName: + type: string + nguid: + type: string + nqn: + type: string + targetIP: + type: string + targetPort: + type: integer + type: object + type: array + port: + type: integer + preferredPath: + description: PreferredPath is the preferred frontend path address. + type: string + salvageExecuted: + type: boolean + started: + type: boolean + starting: + type: boolean + storageIP: + type: string + switchoverPhase: + description: SwitchoverPhase is the last completed switchover phase + reported by the data plane. + type: string + targetIP: + description: TargetIP is the currently connected IP address of the + v2 engine target + type: string + targetPort: + description: TargetPort is the currently connected port of the v2 + engine target + type: integer + ublkID: + format: int32 + type: integer + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: engineimages.longhorn.io +spec: + group: longhorn.io + names: + kind: EngineImage + listKind: EngineImageList + plural: engineimages + shortNames: + - lhei + singular: engineimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Compatibility of the engine image + jsonPath: .status.incompatible + name: Incompatible + type: boolean + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EngineImageSpec defines the desired state of the Longhorn + engine image + properties: + image: + minLength: 1 + type: string + required: + - image + type: object + status: + description: EngineImageStatus defines the observed state of the Longhorn + engine image + properties: + buildDate: + type: string + cliAPIMinVersion: + type: integer + cliAPIVersion: + type: integer + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + controllerAPIMinVersion: + type: integer + controllerAPIVersion: + type: integer + dataFormatMinVersion: + type: integer + dataFormatVersion: + type: integer + gitCommit: + type: string + incompatible: + type: boolean + noRefSince: + type: string + nodeDeploymentMap: + additionalProperties: + type: boolean + nullable: true + type: object + ownerID: + type: string + refCount: + type: integer + state: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: engines.longhorn.io +spec: + group: longhorn.io + names: + kind: Engine + listKind: EngineList + plural: engines + shortNames: + - lhe + singular: engine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The data engine of the engine + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EngineSpec defines the desired state of the Longhorn engine + properties: + active: + type: boolean + backupVolume: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + disableFrontend: + type: boolean + frontend: + enum: + - blockdev + - iscsi + - nvmf + - ublk + - "" + type: string + image: + type: string + logRequested: + type: boolean + nodeID: + type: string + rebuildConcurrentSyncLimit: + description: |- + RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run + concurrently during a single replica rebuild. + It is determined by the global setting or the volume spec field with the same name. + maximum: 5 + minimum: 0 + type: integer + replicaAddressMap: + additionalProperties: + type: string + type: object + requestedBackupRestore: + type: string + requestedDataSource: + type: string + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + ublkNumberOfQueue: + description: ublkNumberOfQueue controls the number of queues for ublk + frontend. + type: integer + ublkQueueDepth: + description: ublkQueueDepth controls the depth of each queue for ublk + frontend. + type: integer + unmapMarkSnapChainRemovedEnabled: + type: boolean + upgradedReplicaAddressMap: + additionalProperties: + type: string + type: object + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineStatus defines the observed state of the Longhorn engine + properties: + backupStatus: + additionalProperties: + properties: + backupURL: + type: string + error: + type: string + progress: + type: integer + replicaAddress: + type: string + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + cloneStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isCloning: + type: boolean + progress: + type: integer + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentReplicaAddressMap: + additionalProperties: + type: string + nullable: true + type: object + currentSize: + format: int64 + type: string + currentState: + type: string + endpoint: + type: string + instanceManagerName: + type: string + ip: + type: string + isExpanding: + type: boolean + lastExpansionError: + type: string + lastExpansionFailedAt: + type: string + lastRestoredBackup: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + purgeStatus: + additionalProperties: + properties: + error: + type: string + isPurging: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + rebuildConcurrentSyncLimit: + description: |- + RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run + concurrently during a single replica rebuild. + It is determined by the global setting or the volume spec field with the same name. + minimum: 0 + type: integer + rebuildStatus: + additionalProperties: + properties: + appliedRebuildingMBps: + format: int64 + type: integer + error: + type: string + fromReplicaAddress: + description: Deprecated. We are now using FromReplicaAddressList + to list all source replicas. + type: string + fromReplicaAddressList: + items: + type: string + type: array + isRebuilding: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + replicaModeMap: + additionalProperties: + type: string + nullable: true + type: object + replicaTransitionTimeMap: + additionalProperties: + type: string + description: |- + ReplicaTransitionTimeMap records the time a replica in ReplicaModeMap transitions from one mode to another (or + from not being in the ReplicaModeMap to being in it). This information is sometimes required by other controllers + (e.g. the volume controller uses it to determine the correct value for replica.Spec.lastHealthyAt). + type: object + restoreStatus: + additionalProperties: + properties: + backupURL: + type: string + currentRestoringBackup: + type: string + error: + type: string + filename: + type: string + isRestoring: + type: boolean + lastRestored: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + salvageExecuted: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + snapshots: + additionalProperties: + properties: + children: + additionalProperties: + type: boolean + nullable: true + type: object + created: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + name: + type: string + parent: + type: string + removed: + type: boolean + size: + type: string + usercreated: + type: boolean + type: object + nullable: true + type: object + snapshotsError: + type: string + started: + type: boolean + starting: + type: boolean + storageIP: + type: string + ublkID: + format: int32 + type: integer + unmapMarkSnapChainRemovedEnabled: + type: boolean + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: instancemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: InstanceManager + listKind: InstanceManagerList + plural: instancemanagers + shortNames: + - lhim + singular: instancemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The data engine of the instance manager + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: InstanceManagerSpec defines the desired state of the Longhorn + instance manager + properties: + dataEngine: + type: string + dataEngineSpec: + properties: + v2: + properties: + cpuMask: + type: string + type: object + type: object + image: + type: string + nodeID: + type: string + type: + enum: + - aio + - engine + - replica + type: string + type: object + status: + description: InstanceManagerStatus defines the observed state of the Longhorn + instance manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImages: + additionalProperties: + properties: + currentChecksum: + type: string + diskUUID: + type: string + message: + type: string + name: + type: string + progress: + type: integer + size: + format: int64 + type: integer + state: + type: string + uuid: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentState: + type: string + dataEngineStatus: + properties: + v2: + properties: + cpuMask: + type: string + interruptModeEnabled: + description: |- + InterruptModeEnabled indicates whether the V2 data engine is running in + interrupt mode (true) or polling mode (false). Set by Longhorn manager; + read-only to users. + enum: + - "" + - "true" + - "false" + type: string + type: object + type: object + instanceEngineFrontends: + additionalProperties: + properties: + spec: + properties: + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + activePath: + type: string + conditions: + additionalProperties: + type: boolean + nullable: true + type: object + endpoint: + type: string + errorMsg: + type: string + frontend: + type: string + listen: + type: string + paths: + items: + properties: + anaState: + type: string + engineName: + type: string + nguid: + type: string + nqn: + type: string + targetIP: + type: string + targetPort: + type: integer + type: object + type: array + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + preferredPath: + type: string + resourceVersion: + format: int64 + type: integer + state: + type: string + targetPortEnd: + format: int32 + type: integer + targetPortStart: + format: int32 + type: integer + type: + type: string + ublkID: + format: int32 + type: integer + uuid: + type: string + type: object + type: object + nullable: true + type: object + instanceEngines: + additionalProperties: + properties: + spec: + properties: + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + activePath: + type: string + conditions: + additionalProperties: + type: boolean + nullable: true + type: object + endpoint: + type: string + errorMsg: + type: string + frontend: + type: string + listen: + type: string + paths: + items: + properties: + anaState: + type: string + engineName: + type: string + nguid: + type: string + nqn: + type: string + targetIP: + type: string + targetPort: + type: integer + type: object + type: array + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + preferredPath: + type: string + resourceVersion: + format: int64 + type: integer + state: + type: string + targetPortEnd: + format: int32 + type: integer + targetPortStart: + format: int32 + type: integer + type: + type: string + ublkID: + format: int32 + type: integer + uuid: + type: string + type: object + type: object + nullable: true + type: object + instanceReplicas: + additionalProperties: + properties: + spec: + properties: + dataEngine: + type: string + name: + type: string + type: object + status: + properties: + activePath: + type: string + conditions: + additionalProperties: + type: boolean + nullable: true + type: object + endpoint: + type: string + errorMsg: + type: string + frontend: + type: string + listen: + type: string + paths: + items: + properties: + anaState: + type: string + engineName: + type: string + nguid: + type: string + nqn: + type: string + targetIP: + type: string + targetPort: + type: integer + type: object + type: array + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + preferredPath: + type: string + resourceVersion: + format: int64 + type: integer + state: + type: string + targetPortEnd: + format: int32 + type: integer + targetPortStart: + format: int32 + type: integer + type: + type: string + ublkID: + format: int32 + type: integer + uuid: + type: string + type: object + type: object + nullable: true + type: object + ip: + type: string + ownerID: + type: string + proxyApiMinVersion: + type: integer + proxyApiVersion: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: nodes.longhorn.io +spec: + group: longhorn.io + names: + kind: Node + listKind: NodeList + plural: nodes + shortNames: + - lhn + singular: node + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for + the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeSpec defines the desired state of the Longhorn node + properties: + allowScheduling: + type: boolean + disks: + additionalProperties: + properties: + allowScheduling: + type: boolean + diskDriver: + enum: + - "" + - auto + - aio + - nvme + type: string + diskType: + enum: + - filesystem + - block + type: string + evictionRequested: + type: boolean + path: + type: string + storageReserved: + format: int64 + type: integer + tags: + items: + type: string + type: array + type: object + type: object + evictionRequested: + type: boolean + instanceManagerCPURequest: + type: integer + name: + type: string + tags: + items: + type: string + type: array + type: object + status: + description: NodeStatus defines the observed state of the Longhorn node + properties: + autoEvicting: + type: boolean + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskDriver: + type: string + diskName: + type: string + diskPath: + type: string + diskType: + type: string + diskUUID: + type: string + filesystemType: + type: string + healthData: + additionalProperties: + properties: + attributes: + items: + properties: + id: + type: integer + name: + type: string + rawString: + type: string + rawValue: + format: int64 + type: integer + threshold: + type: integer + value: + type: integer + whenFailed: + type: string + worst: + type: integer + type: object + type: array + capacity: + format: int64 + type: integer + diskName: + type: string + diskType: + type: string + firmwareVersion: + type: string + healthStatus: + enum: + - FAILED + - PASSED + - UNKNOWN + - WARNING + type: string + modelName: + type: string + serialNumber: + type: string + source: + enum: + - SMART + - SPDK + type: string + temperature: + type: integer + type: object + type: object + healthDataLastCollectedAt: + format: date-time + type: string + instanceManagerName: + type: string + scheduledBackingImage: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + scheduledReplica: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + storageAvailable: + format: int64 + type: integer + storageMaximum: + format: int64 + type: integer + storageScheduled: + format: int64 + type: integer + type: object + nullable: true + type: object + region: + type: string + snapshotCheckStatus: + properties: + lastPeriodicCheckedAt: + format: date-time + type: string + type: object + zone: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: orphans.longhorn.io +spec: + group: longhorn.io + names: + kind: Orphan + listKind: OrphanList + plural: orphans + shortNames: + - lho + singular: orphan + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The type of the orphan + jsonPath: .spec.orphanType + name: Type + type: string + - description: The node that the orphan is on + jsonPath: .spec.nodeID + name: Node + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Orphan is where Longhorn stores orphan object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: OrphanSpec defines the desired state of the Longhorn orphaned + data + properties: + dataEngine: + description: |- + The type of data engine for instance orphan. + Can be "v1", "v2". + enum: + - v1 + - v2 + type: string + nodeID: + description: The node ID on which the controller is responsible to + reconcile this orphan CR. + type: string + orphanType: + description: |- + The type of the orphaned data. + Can be "replica". + type: string + parameters: + additionalProperties: + type: string + description: The parameters of the orphaned data + type: object + type: object + status: + description: OrphanStatus defines the observed state of the Longhorn orphaned + data + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: recurringjobs.longhorn.io +spec: + group: longhorn.io + names: + kind: RecurringJob + listKind: RecurringJobList + plural: recurringjobs + shortNames: + - lhrj + singular: recurringjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be + added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", + "snapshot-delete", "backup", "backup-force-create", "filesystem-trim" or "system-backup" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: RecurringJobSpec defines the desired state of the Longhorn + recurring job + properties: + concurrency: + description: The concurrency of taking the snapshot/backup. + type: integer + cron: + description: The cron setting. + type: string + groups: + description: The recurring job group. + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: The label of the snapshot/backup. + type: object + name: + description: The recurring job name. + type: string + parameters: + additionalProperties: + type: string + description: |- + The parameters of the snapshot/backup. + Support parameters: "full-backup-interval", "volume-backup-policy". + type: object + retain: + description: The retain count of the snapshot/backup. + type: integer + task: + description: |- + The recurring job task. + Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create", "filesystem-trim" or "system-backup". + enum: + - snapshot + - snapshot-force-create + - snapshot-cleanup + - snapshot-delete + - backup + - backup-force-create + - filesystem-trim + - system-backup + type: string + type: object + status: + description: RecurringJobStatus defines the observed state of the Longhorn + recurring job + properties: + executionCount: + description: The number of jobs that have been triggered. + type: integer + ownerID: + description: The owner ID which is responsible to reconcile this recurring + job CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: replicas.longhorn.io +spec: + group: longhorn.io + names: + kind: Replica + listKind: ReplicaList + plural: replicas + shortNames: + - lhr + singular: replica + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The data engine of the replica + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ReplicaSpec defines the desired state of the Longhorn replica + properties: + active: + type: boolean + backingImage: + type: string + dataDirectoryName: + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + desireState: + type: string + diskID: + type: string + diskPath: + type: string + engineName: + type: string + evictionRequested: + type: boolean + failedAt: + description: |- + FailedAt is set when a running replica fails or when a running engine is unable to use a replica for any reason. + FailedAt indicates the time the failure occurred. When FailedAt is set, a replica is likely to have useful + (though possibly stale) data. A replica with FailedAt set must be rebuilt from a non-failed replica (or it can + be used in a salvage if all replicas are failed). FailedAt is cleared before a rebuild or salvage. FailedAt may + be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume + controller acknowledges the change. + type: string + hardNodeAffinity: + type: string + healthyAt: + description: |- + HealthyAt is set the first time a replica becomes read/write in an engine after creation or rebuild. HealthyAt + indicates the time the last successful rebuild occurred. When HealthyAt is set, a replica is likely to have + useful (though possibly stale) data. HealthyAt is cleared before a rebuild. HealthyAt may be later than the + corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller + acknowledges the change. + type: string + image: + type: string + lastFailedAt: + description: |- + LastFailedAt is always set at the same time as FailedAt. Unlike FailedAt, LastFailedAt is never cleared. + LastFailedAt is not a reliable indicator of the state of a replica's data. For example, a replica with + LastFailedAt may already be healthy and in use again. However, because it is never cleared, it can be compared to + LastHealthyAt to help prevent dangerous replica deletion in some corner cases. LastFailedAt may be later than the + corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller + acknowledges the change. + type: string + lastHealthyAt: + description: |- + LastHealthyAt is set every time a replica becomes read/write in an engine. Unlike HealthyAt, LastHealthyAt is + never cleared. LastHealthyAt is not a reliable indicator of the state of a replica's data. For example, a + replica with LastHealthyAt set may be in the middle of a rebuild. However, because it is never cleared, it can be + compared to LastFailedAt to help prevent dangerous replica deletion in some corner cases. LastHealthyAt may be + later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume + controller acknowledges the change. + type: string + logRequested: + type: boolean + migrationEngineName: + description: |- + MigrationEngineName is indicating the migrating engine which current connected to this replica. This is only + used for live migration of v2 data engine + type: string + nodeID: + type: string + rebuildRetryCount: + type: integer + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + unmapMarkDiskChainRemovedEnabled: + type: boolean + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: ReplicaStatus defines the observed state of the Longhorn + replica + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentState: + type: string + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + salvageExecuted: + type: boolean + started: + type: boolean + starting: + type: boolean + storageIP: + type: string + ublkID: + format: int32 + type: integer + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: settings.longhorn.io +spec: + group: longhorn.io + names: + kind: Setting + listKind: SettingList + plural: settings + shortNames: + - lhs + singular: setting + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - description: The setting is applied + jsonPath: .status.applied + name: Applied + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + description: The status of the setting. + properties: + applied: + description: The setting is applied. + type: boolean + required: + - applied + type: object + value: + description: |- + The value of the setting. + - It can be a non-JSON formatted string that is applied to all the applicable data engines listed in the setting definition. + - It can be a JSON formatted string that contains values for applicable data engines listed in the setting definition's Default. + type: string + required: + - value + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: sharemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: ShareManager + listKind: ShareManagerList + plural: sharemanagers + shortNames: + - lhsm + singular: sharemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - description: The current image of the share manager + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ShareManagerSpec defines the desired state of the Longhorn + share manager + properties: + image: + description: Share manager image used for creating a share manager + pod + type: string + type: object + status: + description: ShareManagerStatus defines the observed state of the Longhorn + share manager + properties: + currentImage: + description: The image currently used by the share manager pod + type: string + endpoint: + description: NFS endpoint that can access the mounted filesystem of + the volume + type: string + ownerID: + description: The node ID on which the controller is responsible to + reconcile this share manager resource + type: string + state: + description: The state of the share manager resource + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: snapshots.longhorn.io +spec: + group: longhorn.io + names: + kind: Snapshot + listKind: SnapshotList + plural: snapshots + shortNames: + - lhsnap + singular: snapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The volume that this snapshot belongs to + jsonPath: .spec.volume + name: Volume + type: string + - description: Timestamp when the point-in-time snapshot was taken + jsonPath: .status.creationTime + name: CreationTime + type: string + - description: Indicates if the snapshot is ready to be used to restore/backup + a volume + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the minimum size of volume required to rehydrate from + this snapshot + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The actual size of the snapshot + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Snapshot is the Schema for the snapshots API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SnapshotSpec defines the desired state of Longhorn Snapshot + properties: + createSnapshot: + description: require creating a new snapshot + type: boolean + labels: + additionalProperties: + type: string + description: The labels of snapshot + nullable: true + type: object + volume: + description: |- + the volume that this snapshot belongs to. + This field is immutable after creation. + type: string + required: + - volume + type: object + status: + description: SnapshotStatus defines the observed state of Longhorn Snapshot + properties: + checksum: + type: string + checksumCalculatedAt: + description: |- + ChecksumCalculatedAt is the RFC3339 timestamp indicating when the checksum + for this snapshot was last calculated or updated. + type: string + children: + additionalProperties: + type: boolean + nullable: true + type: object + creationTime: + type: string + error: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + markRemoved: + type: boolean + ownerID: + type: string + parent: + type: string + readyToUse: + type: boolean + requestedTime: + type: string + restoreSize: + format: int64 + type: integer + size: + format: int64 + type: integer + userCreated: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: supportbundles.longhorn.io +spec: + group: longhorn.io + names: + kind: SupportBundle + listKind: SupportBundleList + plural: supportbundles + shortNames: + - lhbundle + singular: supportbundle + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the support bundle + jsonPath: .status.state + name: State + type: string + - description: The issue URL + jsonPath: .spec.issueURL + name: Issue + type: string + - description: A brief description of the issue + jsonPath: .spec.description + name: Description + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SupportBundle is where Longhorn stores support bundle object + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SupportBundleSpec defines the desired state of the Longhorn + SupportBundle + properties: + description: + description: A brief description of the issue + type: string + issueURL: + description: The issue URL + nullable: true + type: string + nodeID: + description: The preferred responsible controller node ID. + type: string + required: + - description + type: object + status: + description: SupportBundleStatus defines the observed state of the Longhorn + SupportBundle + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + type: array + filename: + type: string + filesize: + format: int64 + type: integer + image: + description: The support bundle manager image + type: string + managerIP: + description: The support bundle manager IP + type: string + ownerID: + description: The current responsible controller node ID + type: string + progress: + type: integer + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: systembackups.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemBackup + listKind: SystemBackupList + plural: systembackups + shortNames: + - lhsb + singular: systembackup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system backup Longhorn version + jsonPath: .status.version + name: Version + type: string + - description: The system backup state + jsonPath: .status.state + name: State + type: string + - description: The system backup creation time + jsonPath: .status.createdAt + name: Created + type: string + - description: The last time that the system backup was synced into the cluster + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemBackup is where Longhorn stores system backup object + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SystemBackupSpec defines the desired state of the Longhorn + SystemBackup + properties: + volumeBackupPolicy: + description: |- + The create volume backup policy + Can be "if-not-present", "always" or "disabled" + nullable: true + type: string + type: object + status: + description: SystemBackupStatus defines the observed state of the Longhorn + SystemBackup + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + createdAt: + description: The system backup creation time. + format: date-time + type: string + gitCommit: + description: The saved Longhorn manager git commit. + nullable: true + type: string + lastSyncedAt: + description: The last time that the system backup was synced into + the cluster. + format: date-time + nullable: true + type: string + managerImage: + description: The saved manager image. + type: string + ownerID: + description: The node ID of the responsible controller to reconcile + this SystemBackup. + type: string + state: + description: The system backup state. + type: string + version: + description: The saved Longhorn version. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: systemrestores.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemRestore + listKind: SystemRestoreList + plural: systemrestores + shortNames: + - lhsr + singular: systemrestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system restore state + jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemRestore is where Longhorn stores system restore object + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SystemRestoreSpec defines the desired state of the Longhorn + SystemRestore + properties: + systemBackup: + description: The system backup name in the object store. + type: string + required: + - systemBackup + type: object + status: + description: SystemRestoreStatus defines the observed state of the Longhorn + SystemRestore + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + description: The node ID of the responsible controller to reconcile + this SystemRestore. + type: string + sourceURL: + description: The source system backup URL. + type: string + state: + description: The system restore state. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: volumeattachments.longhorn.io +spec: + group: longhorn.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + shortNames: + - lhva + singular: volumeattachment + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: VolumeAttachment stores attachment information of a Longhorn + volume + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: VolumeAttachmentSpec defines the desired state of Longhorn + VolumeAttachment + properties: + attachmentTickets: + additionalProperties: + properties: + generation: + description: |- + A sequence number representing a specific generation of the desired state. + Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate + different attachments of the same volume. + type: string + nodeID: + description: The node that this attachment is requesting + type: string + parameters: + additionalProperties: + type: string + description: Optional additional parameter for this attachment + type: object + type: + type: string + type: object + type: object + volume: + description: The name of Longhorn volume of this VolumeAttachment + type: string + required: + - volume + type: object + status: + description: VolumeAttachmentStatus defines the observed state of Longhorn + VolumeAttachment + properties: + attachmentTicketStatuses: + additionalProperties: + properties: + conditions: + description: Record any error when trying to fulfill this attachment + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + generation: + description: |- + A sequence number representing a specific generation of the desired state. + Populated by the system. Read-only. + format: int64 + type: integer + id: + description: The unique ID of this attachment. Used to differentiate + different attachments of the same volume. + type: string + satisfied: + description: Indicate whether this attachment ticket has been + satisfied + type: boolean + required: + - conditions + - satisfied + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + longhorn-manager: "" + name: volumes.longhorn.io +spec: + group: longhorn.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + shortNames: + - lhv + singular: volume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The data engine of the volume + jsonPath: .spec.dataEngine + name: Data Engine + type: string + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - description: The engine switchover state + jsonPath: .status.switchoverState + name: Switchover + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: VolumeSpec defines the desired state of the Longhorn volume + properties: + Standby: + type: boolean + accessMode: + enum: + - rwo + - rwop + - rwx + type: string + backingImage: + type: string + x-kubernetes-validations: + - message: BackingImage is immutable + rule: self == oldSelf + backupBlockSize: + description: BackupBlockSize indicate the block size to create backups. + The block size is immutable. + enum: + - "2097152" + - "16777216" + format: int64 + type: string + backupCompressionMethod: + enum: + - none + - lz4 + - gzip + type: string + backupTargetName: + description: The backup target name that the volume will be backed + up to or is synced. + type: string + cloneMode: + enum: + - "" + - full-copy + - linked-clone + type: string + dataEngine: + enum: + - v1 + - v2 + type: string + dataLocality: + enum: + - disabled + - best-effort + - strict-local + type: string + dataSource: + type: string + disableFrontend: + type: boolean + diskSelector: + items: + type: string + type: array + encrypted: + type: boolean + x-kubernetes-validations: + - message: Encrypted is immutable + rule: self == oldSelf + engineNodeID: + description: |- + engineNodeID defines the node where the backend engine (target) runs. + If empty, falls back to NodeID. + type: string + freezeFilesystemForSnapshot: + description: Setting that freezes the filesystem on the root partition + before a snapshot is created. + enum: + - ignored + - enabled + - disabled + type: string + fromBackup: + type: string + frontend: + enum: + - blockdev + - iscsi + - nvmf + - ublk + - "" + type: string + image: + type: string + lastAttachedBy: + type: string + migratable: + type: boolean + migrationNodeID: + type: string + nodeID: + description: nodeID defines the node where the volume is attached + (where the frontend initiator runs). + type: string + nodeSelector: + items: + type: string + type: array + numberOfReplicas: + type: integer + offlineRebuilding: + description: |- + Specifies whether Longhorn should rebuild replicas while the detached volume is degraded. + - ignored: Use the global setting for offline replica rebuilding. + - enabled: Enable offline rebuilding for this volume, regardless of the global setting. + - disabled: Disable offline rebuilding for this volume, regardless of the global setting + enum: + - ignored + - disabled + - enabled + type: string + rebuildConcurrentSyncLimit: + description: |- + RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run + concurrently during a single replica rebuild. + When set to 0, it means following the global setting. + maximum: 5 + minimum: 0 + type: integer + replicaAutoBalance: + enum: + - ignored + - disabled + - least-effort + - best-effort + type: string + replicaDiskSoftAntiAffinity: + description: Replica disk soft anti affinity of the volume. Set enabled + to allow replicas to be scheduled in the same disk. + enum: + - ignored + - enabled + - disabled + type: string + replicaRebuildingBandwidthLimit: + description: ReplicaRebuildingBandwidthLimit controls the maximum + write bandwidth (in megabytes per second) allowed on the destination + replica during the rebuilding process. Set this value to 0 to disable + bandwidth limiting. + format: int64 + minimum: 0 + type: integer + replicaSoftAntiAffinity: + description: Replica soft anti affinity of the volume. Set enabled + to allow replicas to be scheduled on the same node. + enum: + - ignored + - enabled + - disabled + type: string + replicaZoneSoftAntiAffinity: + description: Replica zone soft anti affinity of the volume. Set enabled + to allow replicas to be scheduled in the same zone. + enum: + - ignored + - enabled + - disabled + type: string + restoreVolumeRecurringJob: + enum: + - ignored + - enabled + - disabled + type: string + revisionCounterDisabled: + type: boolean + size: + format: int64 + type: string + snapshotDataIntegrity: + enum: + - ignored + - disabled + - enabled + - fast-check + type: string + snapshotHashingRequestedAt: + description: |- + SnapshotHashingRequestedAt is the RFC3339 timestamp (e.g., "2026-03-16T10:30:00Z") when an on-demand snapshot checksum calculation is requested. + When this value is set and is later than LastOnDemandSnapshotHashingCompleteAt, the system will calculate checksums + for all user snapshots. + + If SnapshotHashingRequestedAt differs from LastOnDemandSnapshotHashingCompleteAt, it indicates that a hashing request + is still in progress, and a new request will be rejected. + type: string + snapshotMaxCount: + type: integer + snapshotMaxSize: + format: int64 + type: string + staleReplicaTimeout: + type: integer + ublkNumberOfQueue: + description: ublkNumberOfQueue controls the number of queues for ublk + frontend. + type: integer + ublkQueueDepth: + description: ublkQueueDepth controls the depth of each queue for ublk + frontend. + type: integer + unmapMarkSnapChainRemoved: + enum: + - ignored + - disabled + - enabled + type: string + type: object + status: + description: VolumeStatus defines the observed state of the Longhorn volume + properties: + actualSize: + format: int64 + type: integer + cloneStatus: + properties: + attemptCount: + type: integer + nextAllowedAttemptAt: + type: string + snapshot: + type: string + sourceVolume: + type: string + state: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: |- + Status is the status of the condition. + Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentEngineNodeID: + description: the node that the engine (target) is currently running + on. + type: string + currentImage: + type: string + currentMigrationNodeID: + description: the node that this volume is currently migrating to + type: string + currentNodeID: + type: string + expansionRequired: + type: boolean + frontendDisabled: + type: boolean + isStandby: + type: boolean + kubernetesStatus: + properties: + lastPVCRefAt: + type: string + lastPodRefAt: + type: string + namespace: + description: determine if PVC/Namespace is history or not + type: string + pvName: + type: string + pvStatus: + type: string + pvcName: + type: string + workloadsStatus: + description: determine if Pod/Workload is history or not + items: + properties: + podName: + type: string + podStatus: + type: string + workloadName: + type: string + workloadType: + type: string + type: object + nullable: true + type: array + type: object + lastAutoSalvagedAt: + type: string + lastBackup: + type: string + lastBackupAt: + type: string + lastDegradedAt: + type: string + lastOnDemandSnapshotHashingCompleteAt: + description: |- + LastOnDemandSnapshotHashingCompleteAt is the RFC3339 timestamp (e.g., "2026-03-16T10:30:00Z") when the + most recent on-demand snapshot checksum calculation completed. + When this value matches SnapshotHashingRequestedAt, the requested on-demand checksum calculation is considered complete. + type: string + ownerID: + type: string + remountRequestedAt: + type: string + restoreInitiated: + type: boolean + restoreRequired: + type: boolean + robustness: + type: string + shareEndpoint: + type: string + shareState: + type: string + state: + type: string + switchoverState: + description: |- + SwitchoverState describes the current progress of a v2 engine live switchover. + Empty when no switchover is in progress. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: longhorn/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-role + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "delete", "deletecollection"] +- apiGroups: [""] + resources: ["secrets", "services", "endpoints", "configmaps", "serviceaccounts", "pods/log"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events", "persistentvolumes", "persistentvolumeclaims", "persistentvolumeclaims/status", "nodes"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["apps"] + resources: ["daemonsets", "statefulsets", "deployments", "replicasets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list", "watch"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "volumeattributesclasses", "csinodes", "csidrivers", "csistoragecapacities"] + verbs: ["*"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["*"] +- apiGroups: ["longhorn.io"] + resources: ["volumes", "volumes/status", "enginefrontends", "enginefrontends/status", "engines", "engines/status", "replicas", "replicas/status", "settings", "settings/status", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", + "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch"] +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] +- apiGroups: ["apiregistration.k8s.io"] + resources: ["apiservices"] + verbs: ["list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "create", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "list", "watch"] +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterrolebindings", "clusterroles"] + verbs: ["*"] +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-bind + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: longhorn-system +--- +# Source: longhorn/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-support-bundle + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: longhorn-support-bundle + namespace: longhorn-system +--- +# Source: longhorn/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhorn + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +rules: +- apiGroups: [""] + resources: ["pods", "pods/log", "events", "secrets", "services", "endpoints", "configmaps", "serviceaccounts", "persistentvolumeclaims", "persistentvolumeclaims/status"] + verbs: ["*"] +- apiGroups: ["apps"] + resources: ["daemonsets", "deployments", "statefulsets", "replicasets"] + verbs: ["*"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["*"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["*"] +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["*"] +--- +# Source: longhorn/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhorn + namespace: longhorn-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhorn +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: longhorn-system +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-manager + name: longhorn-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-manager + ports: + - name: manager + port: 9500 + targetPort: manager +--- +# Source: longhorn/templates/deployment-ui.yaml +kind: Service +apiVersion: v1 +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-ui + name: longhorn-frontend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + app: longhorn-ui + ports: + - name: http + port: 80 + targetPort: http + nodePort: null +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-admission-webhook + name: longhorn-admission-webhook + namespace: longhorn-system +spec: + type: ClusterIP + selector: + longhorn.io/admission-webhook: longhorn-admission-webhook + ports: + - name: admission-webhook + port: 9502 + targetPort: admission-wh +--- +# Source: longhorn/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-recovery-backend + name: longhorn-recovery-backend + namespace: longhorn-system +spec: + type: ClusterIP + selector: + longhorn.io/recovery-backend: longhorn-recovery-backend + ports: + - name: recovery-backend + port: 9503 + targetPort: recov-backend +--- +# Source: longhorn/templates/daemonset-sa.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-manager + name: longhorn-manager + namespace: longhorn-system +spec: + selector: + matchLabels: + app: longhorn-manager + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-manager + spec: + containers: + - name: longhorn-manager + image: docker.io/longhornio/longhorn-manager:v1.12.0 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - longhorn-manager + - -d + - daemon + - --engine-image + - "docker.io/longhornio/longhorn-engine:v1.12.0" + - --instance-manager-image + - "docker.io/longhornio/longhorn-instance-manager:v1.12.0" + - --share-manager-image + - "docker.io/longhornio/longhorn-share-manager:v1.12.0" + - --backing-image-manager-image + - "docker.io/longhornio/backing-image-manager:v1.12.0" + - --support-bundle-manager-image + - "docker.io/longhornio/support-bundle-kit:v0.0.86" + - --manager-image + - "docker.io/longhornio/longhorn-manager:v1.12.0" + - --service-account + - longhorn-service-account + - --upgrade-version-check + ports: + - containerPort: 9500 + name: manager + - containerPort: 9502 + name: admission-wh + - containerPort: 9503 + name: recov-backend + readinessProbe: + httpGet: + path: /v1/healthz + port: 9502 + scheme: HTTPS + volumeMounts: + - name: boot + mountPath: /host/boot/ + readOnly: true + - name: dev + mountPath: /host/dev/ + - name: proc + mountPath: /host/proc/ + readOnly: true + - name: etc + mountPath: /host/etc/ + readOnly: true + - name: longhorn + mountPath: /var/lib/longhorn/ + mountPropagation: Bidirectional + - name: longhorn-grpc-tls + mountPath: /tls-files/ + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: LONGHORN_DISTRO + value: "longhorn" + + - name: pre-pull-share-manager-image + imagePullPolicy: IfNotPresent + image: docker.io/longhornio/longhorn-share-manager:v1.12.0 + command: ["sh", "-c", "echo share-manager image pulled && sleep infinity"] + volumes: + - name: boot + hostPath: + path: /boot/ + - name: dev + hostPath: + path: /dev/ + - name: proc + hostPath: + path: /proc/ + - name: etc + hostPath: + path: /etc/ + - name: longhorn + hostPath: + path: /var/lib/longhorn/ + - name: longhorn-grpc-tls + secret: + secretName: longhorn-grpc-tls + optional: true + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + updateStrategy: + rollingUpdate: + maxUnavailable: 100% +--- +# Source: longhorn/templates/deployment-driver.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhorn-driver-deployer + namespace: longhorn-system + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 +spec: + replicas: 1 + selector: + matchLabels: + app: longhorn-driver-deployer + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-driver-deployer + spec: + initContainers: + - name: wait-longhorn-manager + image: docker.io/longhornio/longhorn-manager:v1.12.0 + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] + containers: + - name: longhorn-driver-deployer + image: docker.io/longhornio/longhorn-manager:v1.12.0 + imagePullPolicy: IfNotPresent + command: + - longhorn-manager + - -d + - deploy-driver + - --manager-image + - "docker.io/longhornio/longhorn-manager:v1.12.0" + - --manager-url + - http://longhorn-backend:9500/v1 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: CSI_ATTACHER_IMAGE + value: "docker.io/longhornio/csi-attacher:v4.12.0" + - name: CSI_PROVISIONER_IMAGE + value: "docker.io/longhornio/csi-provisioner:v5.3.0-20260514" + - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE + value: "docker.io/longhornio/csi-node-driver-registrar:v2.17.0" + - name: CSI_RESIZER_IMAGE + value: "docker.io/longhornio/csi-resizer:v2.1.0-20260514" + - name: CSI_SNAPSHOTTER_IMAGE + value: "docker.io/longhornio/csi-snapshotter:v8.5.0-20260514" + - name: CSI_LIVENESS_PROBE_IMAGE + value: "docker.io/longhornio/livenessprobe:v2.19.0" + + priorityClassName: "longhorn-critical" + serviceAccountName: longhorn-service-account + securityContext: + runAsUser: 0 +--- +# Source: longhorn/templates/deployment-ui.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-ui + name: longhorn-ui + namespace: longhorn-system +spec: + replicas: 2 + selector: + matchLabels: + app: longhorn-ui + template: + metadata: + labels: + app.kubernetes.io/name: longhorn + app.kubernetes.io/instance: longhorn + app.kubernetes.io/version: v1.12.0 + app: longhorn-ui + spec: + serviceAccountName: longhorn-ui-service-account + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-ui + topologyKey: kubernetes.io/hostname + weight: 1 + containers: + - name: longhorn-ui + image: docker.io/longhornio/longhorn-ui:v1.12.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: nginx-cache + mountPath: /var/cache/nginx/ + - name: nginx-config + mountPath: /var/config/nginx/ + - name: var-run + mountPath: /var/run/ + ports: + - containerPort: 8000 + name: http + env: + - name: LONGHORN_MANAGER_IP + value: "http://longhorn-backend:9500" + - name: LONGHORN_UI_PORT + value: "8000" + + volumes: + - emptyDir: {} + name: nginx-cache + - emptyDir: {} + name: nginx-config + - emptyDir: {} + name: var-run + priorityClassName: "longhorn-critical" +--- +# Source: longhorn/templates/validate-psp-install.yaml +# +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: longhorn-rwx + annotations: + storageclass.kubernetes.io/is-default-class: "false" +provisioner: driver.longhorn.io +allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + numberOfReplicas: "3" + staleReplicaTimeout: "30" + fromBackup: "" + fsType: "ext4" +--- +# ClawManager cluster install profile. +# Official validation target: Longhorn CSI example. +# Defaults use longhorn for RWO data and longhorn-rwx for RWX workspaces. +# These StorageClass names are examples and may be replaced by any compatible CSI classes. +apiVersion: v1 +kind: Namespace +metadata: + name: clawmanager-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: clawmanager-secrets + namespace: clawmanager-system +type: Opaque +stringData: + mysql-root-password: root123 + mysql-password: clawreef123 + jwt-secret: change-me-in-production + runtime-agent-control-token: change-me-runtime-control-token + runtime-agent-report-token: change-me-runtime-report-token + openclaw-gateway-token: change-me-openclaw-gateway-token + minio-root-user: minioadmin + minio-root-password: minioadmin123 + minio-access-key: minioadmin + minio-secret-key: minioadmin123 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: clawmanager-mysql-init + namespace: clawmanager-system +data: + 001_init_schema.sql: | + CREATE DATABASE IF NOT EXISTS clawmanager CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + USE clawmanager; + + CREATE TABLE IF NOT EXISTS users ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(255) UNIQUE NOT NULL, + email VARCHAR(320) UNIQUE NOT NULL, + password_hash VARCHAR(255) NOT NULL, + role ENUM('admin', 'user') DEFAULT 'user', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + last_login TIMESTAMP, + INDEX idx_username (username), + INDEX idx_role (role) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instances ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT, + type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu', + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + status ENUM('creating', 'running', 'stopped', 'error', 'deleting') DEFAULT 'creating', + cpu_cores DECIMAL(10,2) NOT NULL, + memory_gb INT NOT NULL, + disk_gb INT NOT NULL, + gpu_enabled BOOLEAN DEFAULT FALSE, + gpu_type VARCHAR(100), + gpu_count INT DEFAULT 0, + os_type VARCHAR(50) NOT NULL, + os_version VARCHAR(50) NOT NULL, + image_registry VARCHAR(255), + image_tag VARCHAR(100), + storage_class VARCHAR(50) DEFAULT 'standard', + mount_path VARCHAR(255) DEFAULT '/data', + pod_name VARCHAR(255), + pod_namespace VARCHAR(255), + pod_ip VARCHAR(45), + access_url VARCHAR(500), + access_token VARCHAR(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + started_at TIMESTAMP, + stopped_at TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id), + INDEX idx_status (status), + INDEX idx_type (type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS persistent_volumes ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + pvc_name VARCHAR(255) UNIQUE NOT NULL, + pvc_namespace VARCHAR(255) NOT NULL, + storage_size_gb INT NOT NULL, + storage_class VARCHAR(50), + mount_path VARCHAR(255), + status ENUM('pending', 'bound', 'released', 'failed') DEFAULT 'pending', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + UNIQUE KEY uk_pvc_name_namespace (pvc_name, pvc_namespace) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backups ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + backup_name VARCHAR(255) NOT NULL, + backup_size_gb INT, + backup_path VARCHAR(500), + status ENUM('creating', 'completed', 'failed', 'deleted') DEFAULT 'creating', + backup_type ENUM('manual', 'scheduled') DEFAULT 'manual', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP, + expires_at TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backup_schedules ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + schedule_name VARCHAR(255), + cron_expression VARCHAR(100) NOT NULL, + retention_days INT DEFAULT 30, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS user_quotas ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL UNIQUE, + max_instances INT DEFAULT 10, + max_cpu_cores DECIMAL(10,2) DEFAULT 40, + max_memory_gb INT DEFAULT 100, + max_storage_gb INT DEFAULT 500, + max_gpu_count INT DEFAULT 2, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_usage ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + cpu_usage_percent DECIMAL(5,2), + memory_usage_gb DECIMAL(10,2), + disk_usage_gb DECIMAL(10,2), + gpu_usage_percent DECIMAL(5,2), + uptime_seconds INT, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_recorded (instance_id, recorded_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS audit_logs ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT, + action VARCHAR(100) NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_id INT, + details JSON, + ip_address VARCHAR(45), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL, + INDEX idx_user_id (user_id), + INDEX idx_action (action), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + INSERT INTO users (username, email, password_hash, role, is_active) + SELECT 'admin', 'admin@clawmanager.local', '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi', 'admin', TRUE + WHERE NOT EXISTS (SELECT 1 FROM users WHERE username = 'admin'); + + INSERT INTO user_quotas (user_id, max_instances, max_cpu_cores, max_memory_gb, max_storage_gb, max_gpu_count) + SELECT id, 100, 200, 1000, 5000, 10 FROM users + WHERE username = 'admin' + AND NOT EXISTS (SELECT 1 FROM user_quotas WHERE user_id = users.id); + 002_add_webtop_instance_type.sql: | + USE clawmanager; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 003_add_system_image_settings.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS system_image_settings ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_type VARCHAR(50) NOT NULL, + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + display_name VARCHAR(255) NOT NULL, + image VARCHAR(500) NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_instance_type (instance_type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 004_fix_seeded_admin_password.sql: | + USE clawmanager; + UPDATE users + SET password_hash = '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi' + WHERE username = 'admin' + AND password_hash = '$2a$10$N9qo8uLOickgx2ZMRZoMy.MqrzL9wGC3qD3Q.ZHqQH6t3q7l1L5uG'; + 005_update_openclaw_default_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + 006_add_openclaw_config_center.sql: | + USE clawmanager; + SET @openclaw_snapshot_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'openclaw_config_snapshot_id' + ); + SET @openclaw_snapshot_column_sql = IF( + @openclaw_snapshot_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN openclaw_config_snapshot_id INT NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE openclaw_snapshot_column_stmt FROM @openclaw_snapshot_column_sql; + EXECUTE openclaw_snapshot_column_stmt; + DEALLOCATE PREPARE openclaw_snapshot_column_stmt; + + CREATE TABLE IF NOT EXISTS openclaw_config_resources ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_key VARCHAR(100) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + tags_json LONGTEXT NOT NULL, + content_json LONGTEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_resource_key (user_id, resource_type, resource_key), + INDEX idx_openclaw_resource_user_type (user_id, resource_type), + INDEX idx_openclaw_resource_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundles ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_openclaw_bundle_user (user_id), + INDEX idx_openclaw_bundle_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_items ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + resource_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (resource_id) REFERENCES openclaw_config_resources(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_resource (bundle_id, resource_id), + INDEX idx_openclaw_bundle_item_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_injection_snapshots ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NULL, + user_id INT NOT NULL, + mode VARCHAR(20) NOT NULL, + bundle_id INT NULL, + selected_resource_ids_json LONGTEXT NOT NULL, + resolved_resources_json LONGTEXT NOT NULL, + rendered_manifest_json LONGTEXT NOT NULL, + rendered_env_json LONGTEXT NOT NULL, + secret_name VARCHAR(255) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + activated_at TIMESTAMP NULL, + INDEX idx_openclaw_snapshot_user_created (user_id, created_at), + INDEX idx_openclaw_snapshot_instance (instance_id), + INDEX idx_openclaw_snapshot_bundle (bundle_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 007_add_instance_agent_control_plane.sql: | + USE clawmanager; + SET @instance_agent_bootstrap_token_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'agent_bootstrap_token' + ); + SET @instance_agent_bootstrap_token_column_sql = IF( + @instance_agent_bootstrap_token_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN agent_bootstrap_token VARCHAR(255) NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE instance_agent_bootstrap_token_column_stmt FROM @instance_agent_bootstrap_token_column_sql; + EXECUTE instance_agent_bootstrap_token_column_stmt; + DEALLOCATE PREPARE instance_agent_bootstrap_token_column_stmt; + + CREATE TABLE IF NOT EXISTS instance_agents ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NOT NULL, + agent_version VARCHAR(50) NOT NULL, + protocol_version VARCHAR(50) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'online', + capabilities_json LONGTEXT NOT NULL, + host_info_json LONGTEXT NULL, + session_token VARCHAR(255) NULL, + session_expires_at TIMESTAMP NULL, + last_heartbeat_at TIMESTAMP NULL, + last_reported_at TIMESTAMP NULL, + last_seen_ip VARCHAR(45) NULL, + registered_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_agents_instance (instance_id), + UNIQUE KEY uk_instance_agents_session_token (session_token), + INDEX idx_instance_agents_agent_id (agent_id), + INDEX idx_instance_agents_status (status), + INDEX idx_instance_agents_last_heartbeat (last_heartbeat_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_runtime_status ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + infra_status VARCHAR(30) NOT NULL DEFAULT 'creating', + agent_status VARCHAR(30) NOT NULL DEFAULT 'offline', + openclaw_status VARCHAR(30) NOT NULL DEFAULT 'unknown', + openclaw_pid INT NULL, + openclaw_version VARCHAR(100) NULL, + current_config_revision_id INT NULL, + desired_config_revision_id INT NULL, + summary_json LONGTEXT NULL, + system_info_json LONGTEXT NULL, + health_json LONGTEXT NULL, + last_reported_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_runtime_status_instance (instance_id), + INDEX idx_instance_runtime_status_agent_status (agent_status), + INDEX idx_instance_runtime_status_openclaw_status (openclaw_status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_desired_state ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + desired_power_state VARCHAR(30) NOT NULL DEFAULT 'running', + desired_config_revision_id INT NULL, + desired_runtime_action VARCHAR(50) NULL, + updated_by INT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_desired_state_instance (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_commands ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NULL, + command_type VARCHAR(50) NOT NULL, + payload_json LONGTEXT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + idempotency_key VARCHAR(255) NOT NULL, + issued_by INT NULL, + issued_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + dispatched_at TIMESTAMP NULL, + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + timeout_seconds INT NOT NULL DEFAULT 300, + result_json LONGTEXT NULL, + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (issued_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_commands_idempotency (instance_id, idempotency_key), + INDEX idx_instance_commands_instance_status (instance_id, status), + INDEX idx_instance_commands_agent_status (agent_id, status), + INDEX idx_instance_commands_issued_at (issued_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_config_revisions ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + source_snapshot_id INT NULL, + source_bundle_id INT NULL, + revision_no INT NOT NULL, + content_json LONGTEXT NOT NULL, + checksum VARCHAR(255) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'published', + published_by INT NULL, + published_at TIMESTAMP NULL, + activated_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (published_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_config_revision_unique (instance_id, revision_no), + INDEX idx_instance_config_revision_instance (instance_id, revision_no), + INDEX idx_instance_config_revision_status (status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 008_add_skill_management.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS skill_blobs ( + id INT AUTO_INCREMENT PRIMARY KEY, + content_hash VARCHAR(128) NOT NULL, + archive_hash VARCHAR(128) NOT NULL, + object_key VARCHAR(512) NOT NULL, + file_name VARCHAR(255) NOT NULL, + media_type VARCHAR(100) NOT NULL DEFAULT 'application/gzip', + size_bytes BIGINT NOT NULL DEFAULT 0, + scan_status VARCHAR(30) NOT NULL DEFAULT 'pending', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_skill_blobs_content_hash (content_hash), + INDEX idx_skill_blobs_scan_status (scan_status), + INDEX idx_skill_blobs_risk_level (risk_level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + skill_key VARCHAR(120) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + current_version_id INT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + status VARCHAR(30) NOT NULL DEFAULT 'active', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_skills_user_key (user_id, skill_key), + INDEX idx_skills_user_status (user_id, status), + INDEX idx_skills_risk_level (risk_level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skill_versions ( + id INT AUTO_INCREMENT PRIMARY KEY, + skill_id INT NOT NULL, + blob_id INT NOT NULL, + version_no INT NOT NULL, + manifest_json LONGTEXT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE RESTRICT, + UNIQUE KEY uk_skill_versions_skill_version (skill_id, version_no), + UNIQUE KEY uk_skill_versions_skill_blob (skill_id, blob_id), + INDEX idx_skill_versions_skill_id (skill_id, version_no) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + skill_id INT NOT NULL, + skill_version_id INT NULL, + source_type VARCHAR(40) NOT NULL DEFAULT 'discovered_in_instance', + install_path VARCHAR(1024) NULL, + observed_hash VARCHAR(128) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'active', + last_seen_at TIMESTAMP NULL, + removed_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (skill_version_id) REFERENCES skill_versions(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_skills_instance_skill (instance_id, skill_id), + INDEX idx_instance_skills_instance (instance_id, status), + INDEX idx_instance_skills_skill (skill_id, status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skill_scan_results ( + id INT AUTO_INCREMENT PRIMARY KEY, + blob_id INT NOT NULL, + engine VARCHAR(60) NOT NULL, + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + status VARCHAR(30) NOT NULL DEFAULT 'completed', + summary TEXT NULL, + findings_json LONGTEXT NULL, + scanned_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE CASCADE, + INDEX idx_skill_scan_results_blob (blob_id, scanned_at), + INDEX idx_skill_scan_results_risk (risk_level, scanned_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 009_cpu_cores_decimal.sql: | + USE clawmanager; + ALTER TABLE instances MODIFY COLUMN cpu_cores DECIMAL(10,2) NOT NULL; + ALTER TABLE user_quotas MODIFY COLUMN max_cpu_cores DECIMAL(10,2) DEFAULT 40; + 011_add_hermes_instance_type.sql: | + USE clawmanager; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 012_update_agents_runtime_default_images.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' + WHERE instance_type = 'hermes' + AND image IN ( + 'registry.example.com/hermes-webtop:latest', + 'lscr.io/linuxserver/webtop:ubuntu-xfce' + ); + 013_normalize_agents_runtime_image_names.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/openclaw:latest'; + + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/hermes:latest'; + 019_add_instance_runtime_type.sql: | + USE clawmanager; + SET @instance_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'runtime_type' + ); + SET @instance_runtime_type_column_sql = IF( + @instance_runtime_type_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER type', + 'SELECT 1' + ); + PREPARE instance_runtime_type_column_stmt FROM @instance_runtime_type_column_sql; + EXECUTE instance_runtime_type_column_stmt; + DEALLOCATE PREPARE instance_runtime_type_column_stmt; + 020_add_system_image_runtime_type.sql: | + USE clawmanager; + SET @system_image_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'runtime_type' + ); + SET @system_image_runtime_type_column_sql = IF( + @system_image_runtime_type_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER instance_type', + 'SELECT 1' + ); + PREPARE system_image_runtime_type_column_stmt FROM @system_image_runtime_type_column_sql; + EXECUTE system_image_runtime_type_column_stmt; + DEALLOCATE PREPARE system_image_runtime_type_column_stmt; + 021_add_openclaw_shell_runtime_image.sql: | + USE clawmanager; + SET @system_image_is_enabled_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'is_enabled' + ); + SET @system_image_is_enabled_column_sql = IF( + @system_image_is_enabled_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN is_enabled BOOLEAN NOT NULL DEFAULT TRUE', + 'SELECT 1' + ); + PREPARE system_image_is_enabled_column_stmt FROM @system_image_is_enabled_column_sql; + EXECUTE system_image_is_enabled_column_stmt; + DEALLOCATE PREPARE system_image_is_enabled_column_stmt; + + SET @system_image_unique_instance_type_index_name = ( + SELECT INDEX_NAME + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'instance_type' + AND NON_UNIQUE = 0 + AND INDEX_NAME <> 'PRIMARY' + LIMIT 1 + ); + SET @system_image_unique_instance_type_index_sql = IF( + @system_image_unique_instance_type_index_name IS NOT NULL, + CONCAT( + 'ALTER TABLE system_image_settings DROP INDEX `', + REPLACE(@system_image_unique_instance_type_index_name, '`', '``'), + '`' + ), + 'SELECT 1' + ); + PREPARE system_image_unique_instance_type_index_stmt FROM @system_image_unique_instance_type_index_sql; + EXECUTE system_image_unique_instance_type_index_stmt; + DEALLOCATE PREPARE system_image_unique_instance_type_index_stmt; + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'desktop', + 'OpenClaw Desktop', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'desktop' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'shell', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + 022_add_openclaw_config_bundle_skills.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + skill_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id), + INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 023_normalize_runtime_config_mounts.sql: | + USE clawmanager; + UPDATE instances + SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP + WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); + 031_add_hermes_lite_runtime_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + 'registry.example.com/your-custom-shell-image:latest' + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'hermes', + 'shell', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND is_enabled = FALSE + ); +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: longhorn +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: longhorn +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clawmanager-workspaces + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Gi + storageClassName: longhorn-rwx +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: mysql + template: + metadata: + labels: + app: mysql + spec: + containers: + - name: mysql + image: mysql:8.4.8 + imagePullPolicy: IfNotPresent + args: + - "--innodb-use-native-aio=0" + ports: + - containerPort: 3306 + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-root-password + - name: MYSQL_DATABASE + value: clawmanager + - name: MYSQL_USER + value: clawmanager + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + - name: mysql-init + mountPath: /docker-entrypoint-initdb.d + readinessProbe: + exec: + command: ["sh", "-c", "mysqladmin ping -h 127.0.0.1 -uroot -p$MYSQL_ROOT_PASSWORD"] + initialDelaySeconds: 20 + periodSeconds: 10 + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: mysql-data + - name: mysql-init + configMap: + name: clawmanager-mysql-init +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: clawmanager-system +spec: + selector: + app: mysql + ports: + - name: mysql + port: 3306 + targetPort: 3306 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio:latest + imagePullPolicy: IfNotPresent + args: + - server + - /data + - --console-address + - ":9001" + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-root-password + ports: + - name: api + containerPort: 9000 + - name: console + containerPort: 9001 + volumeMounts: + - name: minio-data + mountPath: /data + readinessProbe: + tcpSocket: + port: api + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: minio-data + persistentVolumeClaim: + claimName: minio-data +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: clawmanager-system +spec: + selector: + app: minio + ports: + - name: api + port: 9000 + targetPort: api + - name: console + port: 9001 + targetPort: console +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-team-redis + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: clawmanager-team-redis + template: + metadata: + labels: + app: clawmanager-team-redis + spec: + containers: + - name: redis + image: redis:7-alpine + imagePullPolicy: IfNotPresent + args: + - redis-server + - --appendonly + - "yes" + ports: + - name: redis + containerPort: 6379 + readinessProbe: + tcpSocket: + port: redis + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: redis-data + mountPath: /data + volumes: + - name: redis-data + persistentVolumeClaim: + claimName: redis-data +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-redis + namespace: clawmanager-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-team-redis + namespace: clawmanager-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: clawmanager-runtime-manager +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - apiGroups: + - apps + resources: + - deployments + - deployments/scale + verbs: + - get + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager-runtime-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clawmanager-runtime-manager +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager-app-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: skill-scanner + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: skill-scanner + template: + metadata: + labels: + app: skill-scanner + spec: + containers: + - name: skill-scanner + image: ghcr.io/yuan-lab-llm/skill-scanner:latest + imagePullPolicy: IfNotPresent + command: + - /opt/skill-scanner-venv/bin/skill-scanner-api + - --host + - 0.0.0.0 + - --port + - "8000" + env: + - name: SKILL_SCANNER_LLM_API_KEY + value: "" + - name: SKILL_SCANNER_LLM_MODEL + value: "" + - name: SKILL_SCANNER_LLM_BASE_URL + value: "" + - name: SKILL_SCANNER_META_LLM_API_KEY + value: "" + - name: SKILL_SCANNER_META_LLM_MODEL + value: "" + - name: SKILL_SCANNER_META_LLM_BASE_URL + value: "" + ports: + - name: http + containerPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: skill-scanner + namespace: clawmanager-system +spec: + selector: + app: skill-scanner + ports: + - name: http + port: 8000 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-app + namespace: clawmanager-system +spec: + replicas: 3 + selector: + matchLabels: + app: clawmanager-app + template: + metadata: + labels: + app: clawmanager-app + spec: + serviceAccountName: clawmanager-app + containers: + - name: clawmanager-app + image: ghcr.io/yuan-lab-llm/clawmanager:latest + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 8443 + - name: proxy + containerPort: 9001 + env: + - name: SERVER_ADDRESS + value: ":9001" + - name: SERVER_MODE + value: "release" + - name: DB_HOST + value: "mysql" + - name: DB_PORT + value: "3306" + - name: DB_USER + value: "clawmanager" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + - name: DB_NAME + value: "clawmanager" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: jwt-secret + - name: OBJECT_STORAGE_ENDPOINT + value: "minio.clawmanager-system.svc.cluster.local:9000" + - name: OBJECT_STORAGE_REGION + value: "" + - name: OBJECT_STORAGE_ACCESS_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-access-key + - name: OBJECT_STORAGE_SECRET_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-secret-key + - name: OBJECT_STORAGE_BUCKET + value: "clawmanager-skills" + - name: OBJECT_STORAGE_USE_SSL + value: "false" + - name: OBJECT_STORAGE_BASE_PATH + value: "skills" + - name: OBJECT_STORAGE_FORCE_PATH_STYLE + value: "true" + - name: K8S_MODE + value: "incluster" + - name: K8S_NAMESPACE + value: "clawmanager" + - name: K8S_STORAGE_CLASS + value: "longhorn" + - name: CLAWMANAGER_STORAGE_PROFILE + value: "cluster" + - name: K8S_HOSTPATH_FALLBACK_ENABLED + value: "false" + - name: K8S_PVC_BIND_TIMEOUT + value: "2m" + - name: K8S_CONTROL_PLANE_STORAGE_CLASS + value: "longhorn" + - name: K8S_INSTANCE_STORAGE_CLASS + value: "longhorn" + - name: K8S_WORKSPACE_STORAGE_CLASS + value: "longhorn-rwx" + - name: K8S_WORKSPACE_ACCESS_MODE + value: "ReadWriteMany" + - name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB + value: "500" + - name: PLATFORM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: TEAM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_TEAM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_TEAM_MANAGER_BASE_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: RUNTIME_NAMESPACE + value: "clawmanager-system" + - name: RUNTIME_WORKSPACE_ROOT + value: "/workspaces" + - name: RUNTIME_WORKSPACE_PVC_CLAIM + value: "clawmanager-workspaces" + - name: RUNTIME_MAX_GATEWAYS_PER_POD + value: "100" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_SCHEDULER_ENABLED + value: "true" + - name: RUNTIME_HEARTBEAT_TIMEOUT + value: "10s" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: SKILL_SCANNER_ENABLED + value: "true" + - name: SKILL_SCANNER_BASE_URL + value: "http://skill-scanner.clawmanager-system.svc.cluster.local:8000" + - name: SKILL_SCANNER_TIMEOUT_SECONDS + value: "120" + - name: SKILL_SCANNER_NAMESPACE + value: "clawmanager-system" + - name: SKILL_SCANNER_DEPLOYMENT + value: "skill-scanner" + volumeMounts: + - name: tls-cert + mountPath: /var/run/clawreef-tls + readOnly: true + - name: workspaces + mountPath: /workspaces + readinessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 30 + periodSeconds: 20 + volumes: + - name: tls-cert + secret: + secretName: clawmanager-tls + optional: true + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openclaw-runtime + namespace: clawmanager-system + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw +spec: + replicas: 1 + selector: + matchLabels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + template: + metadata: + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + imagePullPolicy: Always + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: openclaw + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: openclaw-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_COMMAND + value: "/usr/local/bin/openclaw gateway run --allow-unconfigured --auth token --bind lan --force" + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-runtime + namespace: clawmanager-system + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes +spec: + replicas: 1 + selector: + matchLabels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + template: + metadata: + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + imagePullPolicy: IfNotPresent + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: hermes + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: hermes-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + - name: CLAWMANAGER_AGENT_PORT + value: "19090" + - name: CLAWMANAGER_GATEWAY_PORT_START + value: "20000" + - name: CLAWMANAGER_GATEWAY_PORT_END + value: "20099" + - name: HERMES_TUI_DIR + value: /usr/local/lib/hermes-agent/ui-tui + - name: CLAWMANAGER_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: CLAWMANAGER_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-frontend + namespace: clawmanager-system +spec: + type: NodePort + selector: + app: clawmanager-app + ports: + - name: https + port: 443 + targetPort: https + nodePort: 30443 +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-gateway + namespace: clawmanager-system +spec: + type: ClusterIP + selector: + app: clawmanager-app + ports: + - name: api + port: 9001 + targetPort: 9001 +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-egress-proxy + namespace: clawmanager-system +spec: + type: ClusterIP + selector: + app: clawmanager-app + ports: + - name: proxy + port: 3128 + targetPort: 9001 diff --git a/deployments/k8s/cluster/uninstall.yaml b/deployments/k8s/cluster/uninstall.yaml new file mode 100644 index 0000000..e87daa5 --- /dev/null +++ b/deployments/k8s/cluster/uninstall.yaml @@ -0,0 +1,148 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-uninstall-service-account + namespace: longhorn-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-uninstall-role +rules: + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" + - apiGroups: + - "" + resources: + - pods + - persistentvolumes + - persistentvolumeclaims + - nodes + - configmaps + - secrets + - services + - endpoints + verbs: + - "*" + - apiGroups: + - apps + resources: + - daemonsets + - statefulsets + - deployments + verbs: + - "*" + - apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - "*" + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - "*" + - apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - watch + - list + - apiGroups: + - storage.k8s.io + resources: + - csidrivers + - storageclasses + - volumeattachments + verbs: + - "*" + - apiGroups: + - longhorn.io + resources: + - backingimagedatasources + - backingimagemanagers + - backingimages + - backupbackingimages + - backups + - backuptargets + - backupvolumes + - enginefrontends + - engineimages + - engines + - instancemanagers + - nodes + - orphans + - recurringjobs + - replicas + - settings + - sharemanagers + - snapshots + - supportbundles + - systembackups + - systemrestores + - volumeattachments + - volumes + verbs: + - "*" + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - "*" + - apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-uninstall-bind +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-uninstall-role +subjects: + - kind: ServiceAccount + name: longhorn-uninstall-service-account + namespace: longhorn-system +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: longhorn-uninstall + namespace: longhorn-system +spec: + activeDeadlineSeconds: 900 + backoffLimit: 1 + template: + metadata: + name: longhorn-uninstall + spec: + containers: + - command: + - longhorn-manager + - uninstall + - --force + env: + - name: LONGHORN_NAMESPACE + value: longhorn-system + image: docker.io/longhornio/longhorn-manager:v1.12.0 + imagePullPolicy: IfNotPresent + name: longhorn-uninstall + restartPolicy: Never + serviceAccountName: longhorn-uninstall-service-account +--- diff --git a/deployments/k8s/single-node/README.md b/deployments/k8s/single-node/README.md new file mode 100644 index 0000000..99d3598 --- /dev/null +++ b/deployments/k8s/single-node/README.md @@ -0,0 +1,201 @@ +# ClawManager Kubernetes Single-Node Manifest + +This directory contains the all-in-one Kubernetes manifest for a single-node +ClawManager deployment: + +- `clawmanager.yaml`: ClawManager workloads plus static HostPath PVs. +- No Longhorn dependency. +- No dynamic StorageClass dependency. + +Use this profile only when ClawManager workloads and user runtimes can run on +one storage node. For multi-node clusters, use `deployments/k8s/cluster` instead. + +## Storage Model + +The manifest uses static PV/PVC binding: + +- StorageClass name: `manual` +- System HostPath root: `/data/clawmanager/system` +- Runtime HostPath root: `/data/clawreef` +- Storage node label: `clawmanager.io/storage-node=true` + +The `manual` value is only a matching name between bundled PVs and PVCs. It does +not require a Kubernetes `StorageClass` object or dynamic provisioner. + +The bundled PVs use `persistentVolumeReclaimPolicy: Retain`, so deleting the +manifest does not erase HostPath data. + +## Install + +### 1. Enter This Directory + +```sh +cd deployments/k8s/single-node +``` + +Check that `kubectl` points to the target cluster: + +```sh +kubectl config current-context +kubectl get nodes -o wide +``` + +Expected result: the target node is `Ready`. + +### 2. Choose And Label One Storage Node + +Label exactly one node before installing: + +```sh +kubectl label node clawmanager.io/storage-node=true --overwrite +kubectl get nodes -l clawmanager.io/storage-node=true +``` + +Expected result: only one node is listed. + +If more than one node has this label, remove the label from the extra nodes: + +```sh +kubectl label node clawmanager.io/storage-node- +``` + +### 3. Check For Old Installation State + +```sh +kubectl get ns clawmanager-system clawmanager-user-1 --ignore-not-found +kubectl get pv | grep clawmanager || true +kubectl get pvc -A | grep clawmanager || true +``` + +Expected result for a fresh install: no old ClawManager namespace, PV, or PVC +remains. + +If old resources exist, run the uninstall flow in this README before installing +again. + +### 4. Check HostPath Directories + +The kubelet can create the HostPath directories automatically, but checking the +target path first makes permission and disk issues easier to diagnose: + +```sh +df -h /data || true +ls -ld /data || true +``` + +If `/data` is not the right disk for this node, edit these paths in +`clawmanager.yaml` before applying it: + +- `/data/clawmanager/system/mysql` +- `/data/clawmanager/system/minio` +- `/data/clawmanager/system/redis` +- `/data/clawmanager/system/workspaces` +- `K8S_PV_HOST_PATH_PREFIX=/data/clawreef` + +### 5. Check Image Availability + +The manifest uses the images listed in `clawmanager.yaml`. Make sure the node +can pull from the referenced registry before installing: + +```sh +grep -n 'image:' clawmanager.yaml | head -n 40 +``` + +If this is an offline or private-registry deployment, load or push the images to +the registry used by the manifest before continuing. + +### 6. Apply The Manifest + +```sh +kubectl apply -f clawmanager.yaml +``` + +### 7. Check ClawManager + +```sh +kubectl -n clawmanager-system get pvc +kubectl -n clawmanager-system rollout status deployment/clawmanager-app --timeout=15m +kubectl -n clawmanager-system rollout status deployment/openclaw-runtime --timeout=15m +kubectl -n clawmanager-system rollout status deployment/hermes-runtime --timeout=15m +kubectl -n clawmanager-system get deploy,pod,pvc -o wide +``` + +Expected result: + +- `mysql-data`, `redis-data`, `minio-data`, and `clawmanager-workspaces` are `Bound`. +- `clawmanager-app`, `openclaw-runtime`, and `hermes-runtime` are available. +- MySQL, Redis, MinIO, and skill scanner pods are running. + +If PVCs stay `Pending`, check the static PV binding: + +```sh +kubectl get pv | grep clawmanager +kubectl -n clawmanager-system describe pvc +``` + +If pods stay `Pending` with node affinity errors, check the storage node label: + +```sh +kubectl get nodes -l clawmanager.io/storage-node=true +kubectl describe pod -n clawmanager-system +``` + +If pods stay `ContainerCreating`, check recent events: + +```sh +kubectl -n clawmanager-system get events --sort-by=.lastTimestamp | tail -n 80 +``` + +## Uninstall + +Use this order for a disposable single-node test deployment. + +### 1. Delete ClawManager Resources + +```sh +kubectl delete -f clawmanager.yaml --ignore-not-found +kubectl delete namespace clawmanager-user-1 --ignore-not-found +kubectl delete namespace clawmanager-system --ignore-not-found +``` + +Check: + +```sh +kubectl get ns clawmanager-system clawmanager-user-1 --ignore-not-found +kubectl get pv | grep clawmanager || true +kubectl get pvc -A | grep clawmanager || true +``` + +The PVs may remain because they use `Retain`. This is expected. + +### 2. Remove Retained PVs When Data Is No Longer Needed + +Only run this after confirming the data can be discarded: + +```sh +kubectl delete pv \ + clawmanager-mysql-pv \ + clawmanager-minio-pv \ + clawmanager-redis-pv \ + clawmanager-workspaces-pv \ + --ignore-not-found +``` + +### 3. Remove HostPath Data When Data Is No Longer Needed + +Run this on the labeled storage node only after confirming the data can be +discarded: + +```sh +rm -rf /data/clawmanager/system /data/clawreef +``` + +## Common Issues + +| Symptom | Likely Cause | Check Or Fix | +| :--- | :--- | :--- | +| PVC stays `Pending` | Static PV and PVC did not bind | `kubectl get pv`, then `kubectl describe pvc -n clawmanager-system ` | +| Pod stays `Pending` | Missing or wrong `clawmanager.io/storage-node=true` label | `kubectl get nodes -l clawmanager.io/storage-node=true` | +| Pod is `ImagePullBackOff` | Node cannot pull an image | `kubectl describe pod -n ` | +| Pod is `ContainerCreating` | HostPath mount or permission issue | `kubectl -n clawmanager-system get events --sort-by=.lastTimestamp` | +| Runtime creation fails | Runtime HostPath root is unavailable | Check `K8S_PV_HOST_PATH_PREFIX` and disk space on the labeled node | diff --git a/deployments/k8s/single-node/clawmanager.yaml b/deployments/k8s/single-node/clawmanager.yaml new file mode 100644 index 0000000..33bf43c --- /dev/null +++ b/deployments/k8s/single-node/clawmanager.yaml @@ -0,0 +1,1656 @@ +# ClawManager single-node install profile. +# Official validation target: HostPath on one labeled storage node. +# Before installing, label exactly one node: +# kubectl label node clawmanager.io/storage-node=true --overwrite +# Replace paths or StorageClass names only if you understand the single-node constraints. +apiVersion: v1 +kind: Namespace +metadata: + name: clawmanager-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: clawmanager-secrets + namespace: clawmanager-system +type: Opaque +stringData: + mysql-root-password: root123 + mysql-password: clawreef123 + jwt-secret: change-me-in-production + runtime-agent-control-token: change-me-runtime-control-token + runtime-agent-report-token: change-me-runtime-report-token + openclaw-gateway-token: change-me-openclaw-gateway-token + minio-root-user: minioadmin + minio-root-password: minioadmin123 + minio-access-key: minioadmin + minio-secret-key: minioadmin123 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: clawmanager-mysql-init + namespace: clawmanager-system +data: + 001_init_schema.sql: | + CREATE DATABASE IF NOT EXISTS clawmanager CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + USE clawmanager; + + CREATE TABLE IF NOT EXISTS users ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(255) UNIQUE NOT NULL, + email VARCHAR(320) UNIQUE NOT NULL, + password_hash VARCHAR(255) NOT NULL, + role ENUM('admin', 'user') DEFAULT 'user', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + last_login TIMESTAMP, + INDEX idx_username (username), + INDEX idx_role (role) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instances ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT, + type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu', + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + status ENUM('creating', 'running', 'stopped', 'error', 'deleting') DEFAULT 'creating', + cpu_cores DECIMAL(10,2) NOT NULL, + memory_gb INT NOT NULL, + disk_gb INT NOT NULL, + gpu_enabled BOOLEAN DEFAULT FALSE, + gpu_type VARCHAR(100), + gpu_count INT DEFAULT 0, + os_type VARCHAR(50) NOT NULL, + os_version VARCHAR(50) NOT NULL, + image_registry VARCHAR(255), + image_tag VARCHAR(100), + storage_class VARCHAR(50) DEFAULT 'standard', + mount_path VARCHAR(255) DEFAULT '/data', + pod_name VARCHAR(255), + pod_namespace VARCHAR(255), + pod_ip VARCHAR(45), + access_url VARCHAR(500), + access_token VARCHAR(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + started_at TIMESTAMP, + stopped_at TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id), + INDEX idx_status (status), + INDEX idx_type (type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS persistent_volumes ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + pvc_name VARCHAR(255) UNIQUE NOT NULL, + pvc_namespace VARCHAR(255) NOT NULL, + storage_size_gb INT NOT NULL, + storage_class VARCHAR(50), + mount_path VARCHAR(255), + status ENUM('pending', 'bound', 'released', 'failed') DEFAULT 'pending', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + UNIQUE KEY uk_pvc_name_namespace (pvc_name, pvc_namespace) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backups ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + backup_name VARCHAR(255) NOT NULL, + backup_size_gb INT, + backup_path VARCHAR(500), + status ENUM('creating', 'completed', 'failed', 'deleted') DEFAULT 'creating', + backup_type ENUM('manual', 'scheduled') DEFAULT 'manual', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP, + expires_at TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS backup_schedules ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + schedule_name VARCHAR(255), + cron_expression VARCHAR(100) NOT NULL, + retention_days INT DEFAULT 30, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_id (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS user_quotas ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL UNIQUE, + max_instances INT DEFAULT 10, + max_cpu_cores DECIMAL(10,2) DEFAULT 40, + max_memory_gb INT DEFAULT 100, + max_storage_gb INT DEFAULT 500, + max_gpu_count INT DEFAULT 2, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_user_id (user_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_usage ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + cpu_usage_percent DECIMAL(5,2), + memory_usage_gb DECIMAL(10,2), + disk_usage_gb DECIMAL(10,2), + gpu_usage_percent DECIMAL(5,2), + uptime_seconds INT, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + INDEX idx_instance_recorded (instance_id, recorded_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS audit_logs ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT, + action VARCHAR(100) NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_id INT, + details JSON, + ip_address VARCHAR(45), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL, + INDEX idx_user_id (user_id), + INDEX idx_action (action), + INDEX idx_created_at (created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + INSERT INTO users (username, email, password_hash, role, is_active) + SELECT 'admin', 'admin@clawmanager.local', '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi', 'admin', TRUE + WHERE NOT EXISTS (SELECT 1 FROM users WHERE username = 'admin'); + + INSERT INTO user_quotas (user_id, max_instances, max_cpu_cores, max_memory_gb, max_storage_gb, max_gpu_count) + SELECT id, 100, 200, 1000, 5000, 10 FROM users + WHERE username = 'admin' + AND NOT EXISTS (SELECT 1 FROM user_quotas WHERE user_id = users.id); + 002_add_webtop_instance_type.sql: | + USE clawmanager; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 003_add_system_image_settings.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS system_image_settings ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_type VARCHAR(50) NOT NULL, + runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + display_name VARCHAR(255) NOT NULL, + image VARCHAR(500) NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_instance_type (instance_type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 004_fix_seeded_admin_password.sql: | + USE clawmanager; + UPDATE users + SET password_hash = '$2a$10$pbenze514mwv3pvQySQBVOsF5J4DBXL2kVo1hLa8JFhQu5x3AKvBi' + WHERE username = 'admin' + AND password_hash = '$2a$10$N9qo8uLOickgx2ZMRZoMy.MqrzL9wGC3qD3Q.ZHqQH6t3q7l1L5uG'; + 005_update_openclaw_default_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + 006_add_openclaw_config_center.sql: | + USE clawmanager; + SET @openclaw_snapshot_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'openclaw_config_snapshot_id' + ); + SET @openclaw_snapshot_column_sql = IF( + @openclaw_snapshot_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN openclaw_config_snapshot_id INT NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE openclaw_snapshot_column_stmt FROM @openclaw_snapshot_column_sql; + EXECUTE openclaw_snapshot_column_stmt; + DEALLOCATE PREPARE openclaw_snapshot_column_stmt; + + CREATE TABLE IF NOT EXISTS openclaw_config_resources ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + resource_type VARCHAR(50) NOT NULL, + resource_key VARCHAR(100) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + tags_json LONGTEXT NOT NULL, + content_json LONGTEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_resource_key (user_id, resource_type, resource_key), + INDEX idx_openclaw_resource_user_type (user_id, resource_type), + INDEX idx_openclaw_resource_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundles ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + version INT NOT NULL DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_openclaw_bundle_user (user_id), + INDEX idx_openclaw_bundle_user_enabled (user_id, enabled) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_items ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + resource_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (resource_id) REFERENCES openclaw_config_resources(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_resource (bundle_id, resource_id), + INDEX idx_openclaw_bundle_item_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS openclaw_injection_snapshots ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NULL, + user_id INT NOT NULL, + mode VARCHAR(20) NOT NULL, + bundle_id INT NULL, + selected_resource_ids_json LONGTEXT NOT NULL, + resolved_resources_json LONGTEXT NOT NULL, + rendered_manifest_json LONGTEXT NOT NULL, + rendered_env_json LONGTEXT NOT NULL, + secret_name VARCHAR(255) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + activated_at TIMESTAMP NULL, + INDEX idx_openclaw_snapshot_user_created (user_id, created_at), + INDEX idx_openclaw_snapshot_instance (instance_id), + INDEX idx_openclaw_snapshot_bundle (bundle_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 007_add_instance_agent_control_plane.sql: | + USE clawmanager; + SET @instance_agent_bootstrap_token_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'agent_bootstrap_token' + ); + SET @instance_agent_bootstrap_token_column_sql = IF( + @instance_agent_bootstrap_token_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN agent_bootstrap_token VARCHAR(255) NULL AFTER access_token', + 'SELECT 1' + ); + PREPARE instance_agent_bootstrap_token_column_stmt FROM @instance_agent_bootstrap_token_column_sql; + EXECUTE instance_agent_bootstrap_token_column_stmt; + DEALLOCATE PREPARE instance_agent_bootstrap_token_column_stmt; + + CREATE TABLE IF NOT EXISTS instance_agents ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NOT NULL, + agent_version VARCHAR(50) NOT NULL, + protocol_version VARCHAR(50) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'online', + capabilities_json LONGTEXT NOT NULL, + host_info_json LONGTEXT NULL, + session_token VARCHAR(255) NULL, + session_expires_at TIMESTAMP NULL, + last_heartbeat_at TIMESTAMP NULL, + last_reported_at TIMESTAMP NULL, + last_seen_ip VARCHAR(45) NULL, + registered_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_agents_instance (instance_id), + UNIQUE KEY uk_instance_agents_session_token (session_token), + INDEX idx_instance_agents_agent_id (agent_id), + INDEX idx_instance_agents_status (status), + INDEX idx_instance_agents_last_heartbeat (last_heartbeat_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_runtime_status ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + infra_status VARCHAR(30) NOT NULL DEFAULT 'creating', + agent_status VARCHAR(30) NOT NULL DEFAULT 'offline', + openclaw_status VARCHAR(30) NOT NULL DEFAULT 'unknown', + openclaw_pid INT NULL, + openclaw_version VARCHAR(100) NULL, + current_config_revision_id INT NULL, + desired_config_revision_id INT NULL, + summary_json LONGTEXT NULL, + system_info_json LONGTEXT NULL, + health_json LONGTEXT NULL, + last_reported_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + UNIQUE KEY uk_instance_runtime_status_instance (instance_id), + INDEX idx_instance_runtime_status_agent_status (agent_status), + INDEX idx_instance_runtime_status_openclaw_status (openclaw_status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_desired_state ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + desired_power_state VARCHAR(30) NOT NULL DEFAULT 'running', + desired_config_revision_id INT NULL, + desired_runtime_action VARCHAR(50) NULL, + updated_by INT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_desired_state_instance (instance_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_commands ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + agent_id VARCHAR(255) NULL, + command_type VARCHAR(50) NOT NULL, + payload_json LONGTEXT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + idempotency_key VARCHAR(255) NOT NULL, + issued_by INT NULL, + issued_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + dispatched_at TIMESTAMP NULL, + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + timeout_seconds INT NOT NULL DEFAULT 300, + result_json LONGTEXT NULL, + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (issued_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_commands_idempotency (instance_id, idempotency_key), + INDEX idx_instance_commands_instance_status (instance_id, status), + INDEX idx_instance_commands_agent_status (agent_id, status), + INDEX idx_instance_commands_issued_at (issued_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_config_revisions ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + source_snapshot_id INT NULL, + source_bundle_id INT NULL, + revision_no INT NOT NULL, + content_json LONGTEXT NOT NULL, + checksum VARCHAR(255) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'published', + published_by INT NULL, + published_at TIMESTAMP NULL, + activated_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (published_by) REFERENCES users(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_config_revision_unique (instance_id, revision_no), + INDEX idx_instance_config_revision_instance (instance_id, revision_no), + INDEX idx_instance_config_revision_status (status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 008_add_skill_management.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS skill_blobs ( + id INT AUTO_INCREMENT PRIMARY KEY, + content_hash VARCHAR(128) NOT NULL, + archive_hash VARCHAR(128) NOT NULL, + object_key VARCHAR(512) NOT NULL, + file_name VARCHAR(255) NOT NULL, + media_type VARCHAR(100) NOT NULL DEFAULT 'application/gzip', + size_bytes BIGINT NOT NULL DEFAULT 0, + scan_status VARCHAR(30) NOT NULL DEFAULT 'pending', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_skill_blobs_content_hash (content_hash), + INDEX idx_skill_blobs_scan_status (scan_status), + INDEX idx_skill_blobs_risk_level (risk_level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + skill_key VARCHAR(120) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT NULL, + current_version_id INT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + status VARCHAR(30) NOT NULL DEFAULT 'active', + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + last_scanned_at TIMESTAMP NULL, + last_scan_result_id INT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE KEY uk_skills_user_key (user_id, skill_key), + INDEX idx_skills_user_status (user_id, status), + INDEX idx_skills_risk_level (risk_level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skill_versions ( + id INT AUTO_INCREMENT PRIMARY KEY, + skill_id INT NOT NULL, + blob_id INT NOT NULL, + version_no INT NOT NULL, + manifest_json LONGTEXT NULL, + source_type VARCHAR(30) NOT NULL DEFAULT 'uploaded', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE RESTRICT, + UNIQUE KEY uk_skill_versions_skill_version (skill_id, version_no), + UNIQUE KEY uk_skill_versions_skill_blob (skill_id, blob_id), + INDEX idx_skill_versions_skill_id (skill_id, version_no) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS instance_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + skill_id INT NOT NULL, + skill_version_id INT NULL, + source_type VARCHAR(40) NOT NULL DEFAULT 'discovered_in_instance', + install_path VARCHAR(1024) NULL, + observed_hash VARCHAR(128) NULL, + status VARCHAR(30) NOT NULL DEFAULT 'active', + last_seen_at TIMESTAMP NULL, + removed_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (skill_version_id) REFERENCES skill_versions(id) ON DELETE SET NULL, + UNIQUE KEY uk_instance_skills_instance_skill (instance_id, skill_id), + INDEX idx_instance_skills_instance (instance_id, status), + INDEX idx_instance_skills_skill (skill_id, status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + CREATE TABLE IF NOT EXISTS skill_scan_results ( + id INT AUTO_INCREMENT PRIMARY KEY, + blob_id INT NOT NULL, + engine VARCHAR(60) NOT NULL, + risk_level VARCHAR(30) NOT NULL DEFAULT 'unknown', + status VARCHAR(30) NOT NULL DEFAULT 'completed', + summary TEXT NULL, + findings_json LONGTEXT NULL, + scanned_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE CASCADE, + INDEX idx_skill_scan_results_blob (blob_id, scanned_at), + INDEX idx_skill_scan_results_risk (risk_level, scanned_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 009_cpu_cores_decimal.sql: | + USE clawmanager; + ALTER TABLE instances MODIFY COLUMN cpu_cores DECIMAL(10,2) NOT NULL; + ALTER TABLE user_quotas MODIFY COLUMN max_cpu_cores DECIMAL(10,2) DEFAULT 40; + 011_add_hermes_instance_type.sql: | + USE clawmanager; + ALTER TABLE instances + MODIFY COLUMN type ENUM('openclaw', 'ubuntu', 'debian', 'centos', 'custom', 'webtop', 'hermes') DEFAULT 'ubuntu'; + 012_update_agents_runtime_default_images.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE instance_type = 'openclaw' + AND image IN ( + 'ericpearlee/openclaw:v2026.3.24', + 'ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest' + ); + + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' + WHERE instance_type = 'hermes' + AND image IN ( + 'registry.example.com/hermes-webtop:latest', + 'lscr.io/linuxserver/webtop:ubuntu-xfce' + ); + 013_normalize_agents_runtime_image_names.sql: | + USE clawmanager; + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/openclaw:latest'; + + UPDATE system_image_settings + SET image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes:latest' + WHERE image = 'ghcr.io/Yuan-lab-LLM/AgentsRuntime/hermes:latest'; + 019_add_instance_runtime_type.sql: | + USE clawmanager; + SET @instance_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'runtime_type' + ); + SET @instance_runtime_type_column_sql = IF( + @instance_runtime_type_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER type', + 'SELECT 1' + ); + PREPARE instance_runtime_type_column_stmt FROM @instance_runtime_type_column_sql; + EXECUTE instance_runtime_type_column_stmt; + DEALLOCATE PREPARE instance_runtime_type_column_stmt; + 020_add_system_image_runtime_type.sql: | + USE clawmanager; + SET @system_image_runtime_type_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'runtime_type' + ); + SET @system_image_runtime_type_column_sql = IF( + @system_image_runtime_type_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM(''desktop'', ''shell'') NOT NULL DEFAULT ''desktop'' AFTER instance_type', + 'SELECT 1' + ); + PREPARE system_image_runtime_type_column_stmt FROM @system_image_runtime_type_column_sql; + EXECUTE system_image_runtime_type_column_stmt; + DEALLOCATE PREPARE system_image_runtime_type_column_stmt; + 021_add_openclaw_shell_runtime_image.sql: | + USE clawmanager; + SET @system_image_is_enabled_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'is_enabled' + ); + SET @system_image_is_enabled_column_sql = IF( + @system_image_is_enabled_column_exists = 0, + 'ALTER TABLE system_image_settings ADD COLUMN is_enabled BOOLEAN NOT NULL DEFAULT TRUE', + 'SELECT 1' + ); + PREPARE system_image_is_enabled_column_stmt FROM @system_image_is_enabled_column_sql; + EXECUTE system_image_is_enabled_column_stmt; + DEALLOCATE PREPARE system_image_is_enabled_column_stmt; + + SET @system_image_unique_instance_type_index_name = ( + SELECT INDEX_NAME + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'system_image_settings' + AND COLUMN_NAME = 'instance_type' + AND NON_UNIQUE = 0 + AND INDEX_NAME <> 'PRIMARY' + LIMIT 1 + ); + SET @system_image_unique_instance_type_index_sql = IF( + @system_image_unique_instance_type_index_name IS NOT NULL, + CONCAT( + 'ALTER TABLE system_image_settings DROP INDEX `', + REPLACE(@system_image_unique_instance_type_index_name, '`', '``'), + '`' + ), + 'SELECT 1' + ); + PREPARE system_image_unique_instance_type_index_stmt FROM @system_image_unique_instance_type_index_sql; + EXECUTE system_image_unique_instance_type_index_stmt; + DEALLOCATE PREPARE system_image_unique_instance_type_index_stmt; + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'desktop', + 'OpenClaw Desktop', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'desktop' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'openclaw', + 'shell', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND is_enabled = FALSE + ); + 022_add_openclaw_config_bundle_skills.sql: | + USE clawmanager; + CREATE TABLE IF NOT EXISTS openclaw_config_bundle_skills ( + id INT AUTO_INCREMENT PRIMARY KEY, + bundle_id INT NOT NULL, + skill_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + required BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bundle_id) REFERENCES openclaw_config_bundles(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id), + INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 023_normalize_runtime_config_mounts.sql: | + USE clawmanager; + UPDATE instances + SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP + WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); + 031_add_hermes_lite_runtime_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + 'registry.example.com/your-custom-shell-image:latest' + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'hermes', + 'shell', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND is_enabled = FALSE + ); +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager-mysql-pv + labels: + app: clawmanager + managed-by: clawmanager + clawmanager.io/storage-profile: single-node + clawmanager.io/storage-node: "true" +spec: + capacity: + storage: 5Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: clawmanager.io/storage-node + operator: In + values: + - "true" + hostPath: + path: /data/clawmanager/system/mysql + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: manual + volumeName: clawmanager-mysql-pv +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager-minio-pv + labels: + app: clawmanager + managed-by: clawmanager + clawmanager.io/storage-profile: single-node + clawmanager.io/storage-node: "true" +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: clawmanager.io/storage-node + operator: In + values: + - "true" + hostPath: + path: /data/clawmanager/system/minio + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: manual + volumeName: clawmanager-minio-pv +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager-redis-pv + labels: + app: clawmanager + managed-by: clawmanager + clawmanager.io/storage-profile: single-node + clawmanager.io/storage-node: "true" +spec: + capacity: + storage: 1Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: clawmanager.io/storage-node + operator: In + values: + - "true" + hostPath: + path: /data/clawmanager/system/redis + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-data + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: manual + volumeName: clawmanager-redis-pv +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager-workspaces-pv + labels: + app: clawmanager + managed-by: clawmanager + clawmanager.io/storage-profile: single-node + clawmanager.io/storage-node: "true" +spec: + capacity: + storage: 50Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: clawmanager.io/storage-node + operator: In + values: + - "true" + hostPath: + path: /data/clawmanager/system/workspaces + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clawmanager-workspaces + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Gi + storageClassName: manual + volumeName: clawmanager-workspaces-pv +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: mysql + template: + metadata: + labels: + app: mysql + spec: + containers: + - name: mysql + image: mysql:8.4.8 + imagePullPolicy: IfNotPresent + args: + - "--innodb-use-native-aio=0" + ports: + - containerPort: 3306 + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-root-password + - name: MYSQL_DATABASE + value: clawmanager + - name: MYSQL_USER + value: clawmanager + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + - name: mysql-init + mountPath: /docker-entrypoint-initdb.d + readinessProbe: + exec: + command: ["sh", "-c", "mysqladmin ping -h 127.0.0.1 -uroot -p$MYSQL_ROOT_PASSWORD"] + initialDelaySeconds: 20 + periodSeconds: 10 + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: mysql-data + - name: mysql-init + configMap: + name: clawmanager-mysql-init +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: clawmanager-system +spec: + selector: + app: mysql + ports: + - name: mysql + port: 3306 + targetPort: 3306 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio:latest + imagePullPolicy: IfNotPresent + args: + - server + - /data + - --console-address + - ":9001" + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-root-password + ports: + - name: api + containerPort: 9000 + - name: console + containerPort: 9001 + volumeMounts: + - name: minio-data + mountPath: /data + readinessProbe: + tcpSocket: + port: api + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: minio-data + persistentVolumeClaim: + claimName: minio-data +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: clawmanager-system +spec: + selector: + app: minio + ports: + - name: api + port: 9000 + targetPort: api + - name: console + port: 9001 + targetPort: console +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-team-redis + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: clawmanager-team-redis + template: + metadata: + labels: + app: clawmanager-team-redis + spec: + containers: + - name: redis + image: redis:7-alpine + imagePullPolicy: IfNotPresent + args: + - redis-server + - --appendonly + - "yes" + ports: + - name: redis + containerPort: 6379 + readinessProbe: + tcpSocket: + port: redis + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: redis-data + mountPath: /data + volumes: + - name: redis-data + persistentVolumeClaim: + claimName: redis-data +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-redis + namespace: clawmanager-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-team-redis + namespace: clawmanager-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: clawmanager-runtime-manager +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - apiGroups: + - apps + resources: + - deployments + - deployments/scale + verbs: + - get + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager-runtime-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clawmanager-runtime-manager +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager-app-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +# Explicit lease permissions for control-plane leader election. The +# cluster-admin binding above already covers this; this Role documents the +# requirement and keeps leader election working if RBAC is tightened later. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: clawmanager-app-leaderelection + namespace: clawmanager-system +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: clawmanager-app-leaderelection + namespace: clawmanager-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: clawmanager-app-leaderelection +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: skill-scanner + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: skill-scanner + template: + metadata: + labels: + app: skill-scanner + spec: + containers: + - name: skill-scanner + image: ghcr.io/yuan-lab-llm/skill-scanner:latest + imagePullPolicy: IfNotPresent + command: + - /opt/skill-scanner-venv/bin/skill-scanner-api + - --host + - 0.0.0.0 + - --port + - "8000" + env: + - name: SKILL_SCANNER_LLM_API_KEY + value: "" + - name: SKILL_SCANNER_LLM_MODEL + value: "" + - name: SKILL_SCANNER_LLM_BASE_URL + value: "" + - name: SKILL_SCANNER_META_LLM_API_KEY + value: "" + - name: SKILL_SCANNER_META_LLM_MODEL + value: "" + - name: SKILL_SCANNER_META_LLM_BASE_URL + value: "" + ports: + - name: http + containerPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: skill-scanner + namespace: clawmanager-system +spec: + selector: + app: skill-scanner + ports: + - name: http + port: 8000 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawmanager-app + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: clawmanager-app + template: + metadata: + labels: + app: clawmanager-app + spec: + serviceAccountName: clawmanager-app + containers: + - name: clawmanager-app + image: ghcr.io/yuan-lab-llm/clawmanager:latest + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 8443 + - name: proxy + containerPort: 9001 + env: + - name: SERVER_ADDRESS + value: ":9001" + - name: SERVER_MODE + value: "release" + - name: DB_HOST + value: "mysql" + - name: DB_PORT + value: "3306" + - name: DB_USER + value: "clawmanager" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: mysql-password + - name: DB_NAME + value: "clawmanager" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: jwt-secret + - name: OBJECT_STORAGE_ENDPOINT + value: "minio.clawmanager-system.svc.cluster.local:9000" + - name: OBJECT_STORAGE_REGION + value: "" + - name: OBJECT_STORAGE_ACCESS_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-access-key + - name: OBJECT_STORAGE_SECRET_KEY + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: minio-secret-key + - name: OBJECT_STORAGE_BUCKET + value: "clawmanager-skills" + - name: OBJECT_STORAGE_USE_SSL + value: "false" + - name: OBJECT_STORAGE_BASE_PATH + value: "skills" + - name: OBJECT_STORAGE_FORCE_PATH_STYLE + value: "true" + - name: K8S_MODE + value: "incluster" + - name: K8S_NAMESPACE + value: "clawmanager" + - name: K8S_STORAGE_CLASS + value: "manual" + - name: CLAWMANAGER_STORAGE_PROFILE + value: "single-node" + - name: K8S_HOSTPATH_FALLBACK_ENABLED + value: "true" + - name: K8S_PVC_BIND_TIMEOUT + value: "2m" + - name: K8S_CONTROL_PLANE_STORAGE_CLASS + value: "manual" + - name: K8S_INSTANCE_STORAGE_CLASS + value: "manual" + - name: K8S_WORKSPACE_STORAGE_CLASS + value: "manual" + - name: K8S_WORKSPACE_ACCESS_MODE + value: "ReadWriteMany" + - name: K8S_PV_HOST_PATH_PREFIX + value: "/data/clawreef" + - name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB + value: "500" + - name: PLATFORM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: TEAM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_DESKTOP_DIRECT_PROXY + value: "true" + # Leader election for control-plane singleton background loops. + # Required before scaling replicas > 1 so SyncService / Team event + # consumers / stale-task monitor run on exactly one replica. + - name: CLAWMANAGER_LEADER_ELECTION + value: "true" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLAWMANAGER_TEAM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: CLAWMANAGER_TEAM_MANAGER_BASE_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: RUNTIME_NAMESPACE + value: "clawmanager-system" + - name: RUNTIME_WORKSPACE_ROOT + value: "/workspaces" + - name: RUNTIME_WORKSPACE_PVC_CLAIM + value: "clawmanager-workspaces" + - name: RUNTIME_MAX_GATEWAYS_PER_POD + value: "100" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_SCHEDULER_ENABLED + value: "true" + - name: RUNTIME_HEARTBEAT_TIMEOUT + value: "10s" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: SKILL_SCANNER_ENABLED + value: "true" + - name: SKILL_SCANNER_BASE_URL + value: "http://skill-scanner.clawmanager-system.svc.cluster.local:8000" + - name: SKILL_SCANNER_TIMEOUT_SECONDS + value: "120" + - name: SKILL_SCANNER_NAMESPACE + value: "clawmanager-system" + - name: SKILL_SCANNER_DEPLOYMENT + value: "skill-scanner" + volumeMounts: + - name: tls-cert + mountPath: /var/run/clawreef-tls + readOnly: true + - name: workspaces + mountPath: /workspaces + readinessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: https + initialDelaySeconds: 30 + periodSeconds: 20 + volumes: + - name: tls-cert + secret: + secretName: clawmanager-tls + optional: true + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openclaw-runtime + namespace: clawmanager-system + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw +spec: + replicas: 1 + selector: + matchLabels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + template: + metadata: + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + imagePullPolicy: Always + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: openclaw + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: openclaw-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_COMMAND + value: "/usr/local/bin/openclaw gateway run --allow-unconfigured --auth token --bind lan --force" + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-runtime + namespace: clawmanager-system + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes +spec: + replicas: 1 + selector: + matchLabels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + template: + metadata: + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + imagePullPolicy: IfNotPresent + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: hermes + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: hermes-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + - name: CLAWMANAGER_AGENT_PORT + value: "19090" + - name: CLAWMANAGER_GATEWAY_PORT_START + value: "20000" + - name: CLAWMANAGER_GATEWAY_PORT_END + value: "20099" + - name: HERMES_TUI_DIR + value: /usr/local/lib/hermes-agent/ui-tui + - name: CLAWMANAGER_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: CLAWMANAGER_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-frontend + namespace: clawmanager-system +spec: + type: NodePort + selector: + app: clawmanager-app + ports: + - name: https + port: 443 + targetPort: https + nodePort: 30443 +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-gateway + namespace: clawmanager-system +spec: + type: ClusterIP + selector: + app: clawmanager-app + ports: + - name: api + port: 9001 + targetPort: 9001 +--- +apiVersion: v1 +kind: Service +metadata: + name: clawmanager-egress-proxy + namespace: clawmanager-system +spec: + type: ClusterIP + selector: + app: clawmanager-app + ports: + - name: proxy + port: 3128 + targetPort: 9001 diff --git a/deployments/nginx/nginx.conf b/deployments/nginx/nginx.conf new file mode 100644 index 0000000..ba31cf1 --- /dev/null +++ b/deployments/nginx/nginx.conf @@ -0,0 +1,194 @@ +load_module modules/ngx_http_js_module.so; + +worker_processes auto; + +# Expose the instance-access JWT secret to njs (process.env) so the edge +# gateway can verify desktop access tokens locally. Values come from the +# container environment; they are not logged. +env INSTANCE_ACCESS_TOKEN_SECRET; +env JWT_SECRET; + +events { + worker_connections 8192; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + js_import desktop from /etc/nginx/njs/desktop_auth.js; + + # Cluster DNS resolver, templated by start.sh from /etc/resolv.conf so the + # desktop location can resolve per-instance Service FQDNs at request time. + resolver __DNS_RESOLVER__ valid=10s ipv6=off; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + gzip on; + gzip_comp_level 6; + gzip_min_length 1024; + gzip_proxied any; + gzip_vary on; + gzip_types + application/javascript + application/json + application/xml + image/svg+xml + text/css + text/javascript + text/plain + text/xml; + keepalive_timeout 75; + keepalive_requests 10000; + server_tokens off; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + map $desktop_target $desktop_proxy_mode { + "deny" "deny"; + "http://127.0.0.1:9001" "fallback"; + default "direct"; + } + + log_format desktop_proxy '$remote_addr - $status "$request_method $desktop_clean_uri $server_protocol" ' + 'mode=$desktop_proxy_mode upstream="$upstream_addr" ' + 'rt=$request_time urt=$upstream_response_time bytes=$body_bytes_sent'; + + upstream clawreef_backend { + server 127.0.0.1:9001; + keepalive 128; + } + + server { + listen 8443 ssl; + http2 on; + server_name _; + + ssl_certificate /etc/nginx/tls/tls.crt; + ssl_certificate_key /etc/nginx/tls/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers off; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + + root /usr/share/nginx/html; + index index.html; + + # Allow runtime workspace archives through the API. + # Default nginx client_max_body_size (1 MiB) would reject normal + # workspace uploads at the edge with 413 Content Too Large before + # the request reaches the backend. start.sh rewrites this value from + # CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB when set. + client_max_body_size 500m; + + location = /healthz { + access_log off; + add_header Content-Type text/plain; + return 200 'ok'; + } + + location ^~ /assets/ { + try_files $uri =404; + access_log off; + expires 1y; + add_header Cache-Control "public, max-age=31536000, immutable"; + } + + location ~* ^/(?:lobster_logo|lobster_transparent|openclaw|hermes)\.png$ { + try_files $uri =404; + access_log off; + expires 30d; + add_header Cache-Control "public, max-age=2592000"; + } + + location ~* ^/vendor-icons/.+\.(?:png|jpg|jpeg|gif|webp|ico|svg)$ { + try_files $uri =404; + access_log off; + expires 30d; + add_header Cache-Control "public, max-age=2592000"; + } + + location ~ ^/s/ { + proxy_pass http://clawreef_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_connect_timeout 15s; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_buffering off; + proxy_request_buffering off; + } + + location ~ ^/api/v1/instances/(?[0-9]+)/proxy(?/.*)?$ { + # njs verifies the instance-access JWT locally and chooses the + # proxy target: the instance Service (direct), the in-process + # control-plane proxy (gray fallback), or "deny". + js_set $desktop_target desktop.resolveTarget; + # Forward the original URI to the upstream but with the JWT "token" + # query param stripped, so the access token never reaches webtop. + js_set $desktop_clean_uri desktop.cleanUri; + + access_log /var/log/nginx/access.log desktop_proxy; + add_header X-ClawManager-Desktop-Proxy $desktop_proxy_mode always; + + error_page 463 = @desktop_denied; + if ($desktop_target = "deny") { + return 463; + } + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Prefix /api/v1/instances/$inst_id/proxy; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # Browsers may include the original iframe URL (with ?token=...) + # as Referer on subresource requests. Do not forward it to webtop. + proxy_set_header Referer ""; + + proxy_ssl_verify off; + proxy_ssl_server_name on; + + proxy_connect_timeout 15s; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_buffering off; + proxy_request_buffering off; + + proxy_pass $desktop_target$desktop_clean_uri; + } + + location /api/ { + proxy_pass http://clawreef_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_connect_timeout 15s; + proxy_read_timeout 3600s; + } + + location / { + try_files $uri $uri/ /index.html; + } + + location @desktop_denied { + add_header Content-Type text/plain always; + return 401 "Access token expired or invalid"; + } + } +} diff --git a/deployments/nginx/njs/desktop_auth.js b/deployments/nginx/njs/desktop_auth.js new file mode 100644 index 0000000..d53ad0b --- /dev/null +++ b/deployments/nginx/njs/desktop_auth.js @@ -0,0 +1,176 @@ +// desktop_auth.js +// +// njs helper used by the in-pod nginx to decide where a desktop proxy request +// should be sent. It locally verifies the ClawManager instance-access JWT +// (HS256, same secret as the Go control plane) and returns a proxy_pass target: +// +// "deny" -> token missing/invalid/expired/mismatched +// "https://" -> direct connection to the instance Service +// (taken from the token's "upstream" claim) +// "http://127.0.0.1:9001" -> fall back to the in-process control-plane +// proxy (gray rollout / legacy tokens without +// an "upstream" claim) +// +// The function runs in the main request context (js_set), so query args and +// cookies of the original request are available. + +var crypto = require('crypto'); + +var CONTROL_PLANE_FALLBACK = 'http://127.0.0.1:9001'; +var DENY = 'deny'; + +function secret() { + return process.env.INSTANCE_ACCESS_TOKEN_SECRET || process.env.JWT_SECRET || ''; +} + +// Normalize base64url / base64 to pad-less standard base64 for comparison. +function toStdB64NoPad(s) { + return String(s).replace(/-/g, '+').replace(/_/g, '/').replace(/=+$/, ''); +} + +function b64urlToString(s) { + var std = String(s).replace(/-/g, '+').replace(/_/g, '/'); + while (std.length % 4 !== 0) { + std += '='; + } + return Buffer.from(std, 'base64').toString(); +} + +function readCookieToken(r) { + var cookie = r.headersIn['Cookie']; + if (!cookie) { + return ''; + } + + var name = 'instance_access_' + r.variables.inst_id; + var parts = cookie.split(';'); + for (var i = 0; i < parts.length; i++) { + var kv = parts[i].trim(); + var eq = kv.indexOf('='); + if (eq > 0 && kv.substring(0, eq) === name) { + return kv.substring(eq + 1); + } + } + return ''; +} + +function readQueryToken(r) { + if (r.args && r.args.token) { + return r.args.token; + } + return ''; +} + +function readToken(r) { + var cookieToken = readCookieToken(r); + if (cookieToken) { + return cookieToken; + } + return readQueryToken(r); +} + +function resolveTarget(r) { + var key = secret(); + if (!key) { + r.error('desktop_auth: missing JWT secret in environment'); + return DENY; + } + + var token = readToken(r); + if (!token) { + return DENY; + } + + var segments = token.split('.'); + if (segments.length !== 3) { + return DENY; + } + + var signingInput = segments[0] + '.' + segments[1]; + var expected = crypto.createHmac('sha256', key).update(signingInput).digest('base64'); + if (toStdB64NoPad(expected) !== toStdB64NoPad(segments[2])) { + return DENY; + } + + var payload; + try { + payload = JSON.parse(b64urlToString(segments[1])); + } catch (e) { + return DENY; + } + + if (payload.token_type !== 'instance_access') { + return DENY; + } + + if (payload.exp && (Date.now() / 1000) >= Number(payload.exp)) { + return DENY; + } + + if (String(payload.instance_id) !== String(r.variables.inst_id)) { + return DENY; + } + + if (payload.upstream) { + return 'https://' + payload.upstream; + } + + return CONTROL_PLANE_FALLBACK; +} + +// cleanUri returns the original request URI (path + query) with only the +// ClawManager instance-access JWT stripped. Runtime apps such as Hermes may use +// their own "token" query parameter for websocket/session auth; when the +// ClawManager token came from the HttpOnly cookie, that runtime token must be +// preserved and forwarded to the upstream gateway. +function cleanUri(r) { + var uri = r.variables.request_uri || r.uri || '/'; + var q = uri.indexOf('?'); + if (q < 0) { + return uri; + } + + var queryToken = readQueryToken(r); + if (!queryToken) { + return uri; + } + var cookieToken = readCookieToken(r); + if (cookieToken && cookieToken !== queryToken) { + return uri; + } + + var path = uri.substring(0, q); + var query = uri.substring(q + 1); + var parts = query.split('&'); + var kept = []; + for (var i = 0; i < parts.length; i++) { + if (parts[i] === '') { + continue; + } + var eq = parts[i].indexOf('='); + var name = eq < 0 ? parts[i] : parts[i].substring(0, eq); + var value = eq < 0 ? '' : parts[i].substring(eq + 1); + if (name === 'token' && queryValueEquals(value, queryToken)) { + continue; + } + kept.push(parts[i]); + } + + if (kept.length === 0) { + return path; + } + return path + '?' + kept.join('&'); +} + +function queryValueEquals(rawValue, expected) { + if (rawValue === expected) { + return true; + } + try { + return decodeURIComponent(rawValue.replace(/\+/g, ' ')) === expected; + } catch (e) { + return false; + } +} + +export default { resolveTarget, cleanUri }; diff --git a/docs/AIGateway/aigateway-audit.png b/docs/AIGateway/aigateway-audit.png new file mode 100644 index 0000000..3ccb30d Binary files /dev/null and b/docs/AIGateway/aigateway-audit.png differ diff --git a/docs/AIGateway/aigateway-home.png b/docs/AIGateway/aigateway-home.png new file mode 100644 index 0000000..1fb7b9d Binary files /dev/null and b/docs/AIGateway/aigateway-home.png differ diff --git a/docs/AIGateway/aigateway-model.png b/docs/AIGateway/aigateway-model.png new file mode 100644 index 0000000..0f02ea9 Binary files /dev/null and b/docs/AIGateway/aigateway-model.png differ diff --git a/docs/AIGateway/costs.png b/docs/AIGateway/costs.png new file mode 100644 index 0000000..92eaaf0 Binary files /dev/null and b/docs/AIGateway/costs.png differ diff --git a/docs/AIGateway/risks.png b/docs/AIGateway/risks.png new file mode 100644 index 0000000..197b258 Binary files /dev/null and b/docs/AIGateway/risks.png differ diff --git a/docs/admin-user-guide.md b/docs/admin-user-guide.md new file mode 100644 index 0000000..478e94c --- /dev/null +++ b/docs/admin-user-guide.md @@ -0,0 +1,39 @@ +# Admin and User Guide + +This guide maps the main product surfaces for administrators and end users. It is the best starting point when you want to understand how ClawManager is experienced in day-to-day use rather than how it is deployed. + +## Admin Experience + +Administrators use ClawManager to: + +- manage users, quotas, and platform-wide policies +- review instances and cluster-level operations +- govern AI Gateway models, audit trails, cost analysis, and risk rules +- manage Security Center and `skill-scanner` operations +- prepare reusable resources that users can apply to workspaces + +## User Experience + +End users use ClawManager to: + +- create or access OpenClaw workspaces +- create a template-based Team and describe a shared goal in the Team chat +- open workspaces through the portal experience +- inspect runtime status, agent signals, and recent command activity +- attach or remove skills from an instance when permitted +- consume platform-governed AI access through AI Gateway + +## Product Areas + +- [AI Gateway Guide](./aigateway.md) +- [Agent Control Plane Guide](./agent-control-plane.md) +- [Resource Management Guide](./resource-management.md) +- [Security / Skill Scanner Guide](./security-skill-scanner.md) +- [Team Workspace Quick Guide](./team-workspaces-guide_en.md) + +## Suggested Walkthrough + +1. Start with the AI Gateway overview if your team cares most about model governance. +2. Review Agent Control Plane if your focus is runtime visibility and operations. +3. Review Resource Management and Security Center if you want reusable channels, skills, and scan-backed workflows. +4. Create a Team from a role template when a task benefits from coordinated planning, delivery, and review. diff --git a/docs/agent-control-plane.md b/docs/agent-control-plane.md new file mode 100644 index 0000000..5c04bbe --- /dev/null +++ b/docs/agent-control-plane.md @@ -0,0 +1,47 @@ +# Agent Control Plane Guide + +Agent Control Plane is the runtime orchestration layer for managed runtime instances in ClawManager. It allows the platform to understand live runtime state, distribute commands, and keep each managed workspace aligned with the desired state defined by the control plane. + +## Core Responsibilities + +- agent bootstrap and registration for managed runtime instances +- authenticated session lifecycle between the runtime agent and the platform +- heartbeat-driven runtime and health reporting +- desired power state and desired config revision tracking +- command dispatch and completion tracking for runtime operations + +## Runtime Signals + +The control plane keeps a runtime view that includes: + +- agent identity, version, and last heartbeat +- runtime status and v1 compatibility status fields +- current and desired config revision +- reported summary data such as agent, runtime, and skill counts +- recent command history and execution outcomes + +## Typical Commands + +Examples of platform-driven runtime actions include: + +- start, stop, and restart operations +- config revision apply and reload +- health checks and system info collection +- skill install, update, removal, quarantine, and inventory refresh + +## Where It Shows Up in the Product + +- instance detail views for agent status and runtime summaries +- runtime command history and execution feedback +- workflows that apply config revisions or skill-related changes to a workspace + +## Related Guides + +- [ClawManager Agent V2 Development Guide](./clawmanager-agent-v2-development-guide.md) +- [Runtime Agent Integration Guide](./runtime-agent-integration-guide.md) +- [Hermes Lite / Pro Agent Development Guide](./hermes-lite-pro-agent-development.md) +- [Hermes Runtime Image and Agent Development Guide](./hermes-runtime-agent-development.md) +- [Admin and User Guide](./admin-user-guide.md) +- [Resource Management Guide](./resource-management.md) +- [Security / Skill Scanner Guide](./security-skill-scanner.md) +- [Developer Guide](./developer-guide.md) diff --git a/docs/agent-runtime-development-spec.md b/docs/agent-runtime-development-spec.md new file mode 100644 index 0000000..dcfaf86 --- /dev/null +++ b/docs/agent-runtime-development-spec.md @@ -0,0 +1,512 @@ +# Agent Runtime 开发规范 + +本文是 ClawManager 托管 runtime 的 agent 侧开发规范,适用于 OpenClaw、Hermes,以及后续新增的任意 runtime。开发新 runtime 时,应先满足本文的通用要求,再补充 runtime 自己的启动命令、配置文件和健康检查细节。 + +相关协议和背景请同时参考: + +- `docs/clawmanager-agent-v2-development-guide.md` +- `docs/clawmanager-agent-v2-contract.md` +- `docs/runtime-agent-integration-guide.md` +- `docs/hermes-lite-pro-agent-development.md` +- `docs/hermes-runtime-agent-development.md` + +## 1. 范围和目标 + +一个托管 runtime 镜像不能只启动业务进程。它必须内置一个常驻 runtime agent,由 agent 在 Pod 内负责: + +- Runtime Pod 注册、heartbeat、metrics、gateway 状态上报。 +- 接收 ClawManager 下发的 create、delete、drain、restart 等控制指令。 +- 管理本 Pod 内多个 gateway 子进程。 +- 管理端口池、进程、workspace、UID/GID、资源限制和健康检查。 +- 把 ClawManager 注入的 AI Gateway、实例 token、代理信任和 runtime 配置写入每个实例自己的 workspace。 + +一句话原则:agent 是 Pod 内 gateway 管理器,不是同步启动脚本。 + +## 2. 架构约束 + +每个 Runtime Pod 内只能有一个 runtime agent 控制进程。一个用户实例对应一个 gateway 子进程和一个 workspace。 + +```mermaid +flowchart LR + CM["ClawManager Backend"] + POD["Runtime Pod"] + AG["runtime-agent"] + GW1["gateway: instance 101"] + GW2["gateway: instance 102"] + WS["/workspaces shared volume"] + + CM -- "POST /v1/gateways" --> AG + CM -- "DELETE /v1/gateways/{id}" --> AG + CM -- "POST /v1/drain" --> AG + AG -- "register / heartbeat / metrics / gateways report" --> CM + AG --> GW1 + AG --> GW2 + GW1 --> WS + GW2 --> WS +``` + +浏览器只访问 ClawManager 的 HTTPS 入口。ClawManager 再反代到 Pod 内部 gateway。runtime agent 不应把每个 gateway 的自签 HTTPS 地址直接暴露给前端。 + +## 3. 必须遵守的原则 + +| 原则 | 要求 | +| --- | --- | +| 异步创建 | `POST /v1/gateways` 必须快速返回,通常 1 到 3 秒内返回 `status=starting`,不能阻塞等待业务 gateway 完全启动。 | +| 幂等创建 | 以 `instance_id + generation` 为主要幂等键,重复请求必须返回同一个 gateway 元数据,不能重复启动进程。 | +| 状态以真实进程为准 | `running` 只能在进程存在且健康检查通过时上报,不能只读旧缓存。 | +| 端口统一管理 | 端口由 agent 本地分配器加锁分配,启动前检查监听占用,停止后释放。 | +| 配置按 workspace 隔离 | 每个实例的 runtime 配置写入自己的 workspace,不能写全局配置。 | +| 不覆盖用户配置 | 修改 runtime 配置必须做 JSON merge,保留已有字段,数组 append 后去重。 | +| 不写死环境地址 | 不写死外部 IP、域名、CIDR,必须从 env 或 ClawManager 请求中读取。 | +| 内部服务优先 | runtime 内部访问 ClawManager 使用 Kubernetes Service DNS,不使用浏览器访问的外部地址。 | +| 时间必须可信 | token、设备签名、JWT 和 WebSocket 校验依赖时间,agent 必须检测时钟偏移,不能在明显偏移时报告健康。 | +| 安全默认开启 | 控制接口校验 token,路径校验不能越权,命令执行不能拼 shell 字符串。 | + +## 4. Runtime Pod 环境变量 + +agent 至少支持以下环境变量: + +| 变量 | 说明 | +| --- | --- | +| `CLAWMANAGER_RUNTIME_TYPE` | runtime 类型,例如 `openclaw`、`hermes`。 | +| `RUNTIME_WORKSPACE_ROOT` | workspace 根目录,默认 `/workspaces`。 | +| `RUNTIME_GATEWAY_PORT_START` | gateway 端口起始值,默认 `20000`。 | +| `RUNTIME_GATEWAY_PORT_END` | gateway 端口结束值,默认 `20099`。 | +| `RUNTIME_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用。 | +| `RUNTIME_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用。 | +| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址,推荐 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。 | +| `RUNTIME_AGENT_LISTEN_ADDR` | agent 控制接口监听地址,默认 `0.0.0.0:19090`。 | +| `RUNTIME_AGENT_PUBLIC_PORT` | 注册给 ClawManager 的 agent 端口,默认 `19090`。 | +| `CLAWMANAGER_RUNTIME_IMAGE_REF` | 当前 runtime 镜像标识。 | +| `POD_NAME` / `POD_NAMESPACE` / `POD_IP` / `NODE_NAME` | 通过 Downward API 注入的 Pod 身份。 | +| `CLAWMANAGER_TRUSTED_PROXY_CIDRS` | ClawManager app 或 gateway Pod 的来源网段,供 runtime 的 trusted proxy 配置使用。 | +| `CLAWMANAGER_CONTROL_UI_ORIGIN` | ClawManager 内部 Control UI origin,默认可由 `CLAWMANAGER_BACKEND_URL` 推导。 | + +要求: + +- `CLAWMANAGER_BACKEND_URL` 必须是内部服务地址,不能使用 `https://172.16.1.12:39443` 这类浏览器外部入口。 +- 只有对浏览器生成页面 URL 时才使用外部入口。runtime gateway 的信任来源和 LLM base URL 都应使用内部服务 DNS。 +- agent 启动时要校验必填 env,缺失时上报 `unhealthy`,并在 `/v1/health` 返回 503。 + +## 5. Gateway 创建协议 + +ClawManager 调用: + +```http +POST /v1/gateways +X-ClawManager-Control-Token: ${RUNTIME_AGENT_CONTROL_TOKEN} +Content-Type: application/json +``` + +请求字段以 `docs/clawmanager-agent-v2-contract.md` 为准,当前应支持: + +```json +{ + "instance_id": 123, + "user_id": 45, + "agent_type": "hermes", + "workspace_path": "/workspaces/hermes/user-45/instance-123", + "port_range": { + "start": 20000, + "end": 20099 + }, + "uid": 200123, + "gid": 200123, + "cpu_cores": 2, + "memory_mb": 4096, + "disk_quota_mb": 20480, + "generation": 7, + "environment": { + "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm", + "CLAWMANAGER_LLM_API_KEY": "instance-token", + "CLAWMANAGER_LLM_MODEL": "[\"auto\"]", + "CLAWMANAGER_LLM_PROVIDER": "openai-compatible", + "CLAWMANAGER_INSTANCE_TOKEN": "instance-token", + "OPENAI_API_KEY": "instance-token" + } +} +``` + +成功响应必须快速返回: + +```json +{ + "gateway_id": "gw-123-7", + "instance_id": 123, + "port": 20017, + "pid": null, + "status": "starting", + "workspace_path": "/workspaces/hermes/user-45/instance-123" +} +``` + +创建流程: + +1. 校验控制 token、`agent_type`、`instance_id`、`user_id`、`generation`、`workspace_path`、端口范围。 +2. 检查当前 Pod 是否 draining。draining 时返回 409 或 503。 +3. 按 `instance_id + generation` 查询本地状态。已存在 `starting` 或 `running` 时直接返回已有元数据。 +4. 若收到更高 generation,停止并释放同实例旧 generation gateway。 +5. 加锁分配端口,先标记 reserved。 +6. 创建 workspace 和 home 目录,设置 owner 为请求中的 `uid/gid`。 +7. 写入或合并 runtime 配置。 +8. 记录 gateway 元数据为 `starting`。 +9. 后台启动进程,立即返回。 +10. 后台健康检查通过后,上报 `/api/v1/runtime-agent/gateways/report`,状态为 `running`。 +11. 启动失败或健康检查失败时,上报 `error`,释放端口,并保留错误信息。 + +禁止事项: + +- 禁止在 HTTP handler 里直接运行前台命令并等待它退出。 +- 禁止同一个 create 请求触发多个 gateway 进程。 +- 禁止把请求里的任意字符串拼接成 shell 命令。 +- 禁止创建不在端口范围内的监听端口。 + +## 6. 端口和资源管理 + +agent 必须实现并发安全的端口分配器: + +```text +reserve(instance_id, generation, count) -> port_set +commit(instance_id, generation, port_set) +release(instance_id, generation) +list_used() +``` + +要求: + +- 分配时持有互斥锁。 +- 在锁内检查 agent 内存状态和系统监听端口。 +- runtime 如果需要主端口、control 端口、browser 端口或 sidecar 端口,必须一次分配一组端口。 +- 启动失败、健康检查失败、DELETE 成功后必须释放端口。 +- `used_slots` 来自真实 `starting` 和健康 `running` gateway 数量,不能来自旧缓存。 + +资源隔离要求: + +- 每个 gateway 使用独立进程组。 +- 业务进程最终以请求里的 `uid/gid` 运行。 +- CPU、Memory 使用 cgroup 限制。 +- Disk 使用 filesystem quota 或周期扫描加错误上报。 +- 删除 gateway 不删除 workspace。 + +## 7. Workspace 和配置写入 + +workspace 固定格式: + +```text +/workspaces/{runtime}/user-{user_id}/instance-{instance_id} +``` + +agent 必须: + +- 对 workspace root 和请求路径执行 realpath。 +- 拒绝 `..`、符号链接逃逸、跨用户或跨实例路径。 +- 创建 `${workspace}/home`,并设置 `HOME=${workspace}/home`。 +- 密钥、token、provider 配置文件权限建议 `0600`。 +- owner 设置为该 gateway 的 `uid/gid`。 + +runtime 配置必须写在 workspace 内。例如: + +| Runtime | 推荐配置路径 | +| --- | --- | +| OpenClaw | `{workspace}/home/.openclaw/openclaw.json` | +| Hermes | `{workspace}/home/.hermes/hermes.json` 或 Hermes 原生配置路径 | + +JSON merge 规则: + +- 对象字段递归 merge。 +- 已存在字段默认保留。 +- 平台必须控制的字段可以覆盖,但要在代码中显式列出。 +- 数组字段 append 后去重,例如 `allowedOrigins`。 +- 不删除用户已有的未知字段。 +- 写入前先生成临时文件,再原子 rename。 + +## 8. AI Gateway 和模型凭证注入 + +ClawManager 会在 `POST /v1/gateways` 的 `environment` 字段中下发实例级 LLM 环境变量。agent 必须把这些变量传给 gateway 子进程,并按 runtime 需要写入配置文件。 + +允许转发给子进程的变量应使用白名单: + +```text +CLAWMANAGER_LLM_BASE_URL +CLAWMANAGER_LLM_API_KEY +CLAWMANAGER_LLM_MODEL +CLAWMANAGER_LLM_PROVIDER +CLAWMANAGER_INSTANCE_TOKEN +OPENAI_BASE_URL +OPENAI_API_BASE +OPENAI_API_KEY +OPENAI_MODEL +``` + +要求: + +- `CLAWMANAGER_LLM_BASE_URL` 使用 ClawManager 内部服务 DNS。 +- `CLAWMANAGER_LLM_API_KEY` 和 `OPENAI_API_KEY` 使用当前实例 token。 +- 不能要求用户在 runtime 页面手工填写 OpenAI key。 +- 不能把 API key、实例 token、bootstrap token 打到日志。 +- 如果 `environment` 里缺少 API key,agent 不应报告 gateway `running`,应上报 `error`,错误消息明确说明缺少实例 LLM token。 + +Hermes 落地时应把上述 env 转换成 Hermes 的 provider 配置,推荐语义: + +```json +{ + "models": { + "providers": { + "auto": { + "type": "openai-compatible", + "baseUrl": "${CLAWMANAGER_LLM_BASE_URL}", + "apiKey": "${CLAWMANAGER_LLM_API_KEY}" + } + }, + "primary": "auto/auto" + } +} +``` + +如果 Hermes 使用不同配置 schema,也必须满足同样的安全和可用性要求:通过 ClawManager AI Gateway 访问模型,不在镜像内写死厂商 key。 + +## 9. 代理信任和浏览器访问 + +runtime gateway 在 Pod 内部推荐使用 HTTP。浏览器访问链路应为: + +```text +Browser HTTPS -> ClawManager HTTPS -> ClawManager backend proxy -> Runtime Pod HTTP gateway +``` + +agent 写入 runtime 配置时必须配置: + +- Control UI 的 `basePath` 为 `/api/v1/instances/{instance_id}/proxy`。 +- 允许的 origin 使用 ClawManager 内部 origin,例如 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。 +- trusted proxies 使用 `CLAWMANAGER_TRUSTED_PROXY_CIDRS` 或等价 env,不写死 `100.68.x.x`。 +- 不使用外部地址 `https://172.16.1.12:39443` 作为 runtime 内部信任来源。 + +OpenClaw 参考配置: + +```json +{ + "gateway": { + "auth": { + "mode": "trusted-proxy" + }, + "controlUi": { + "allowedOrigins": [ + "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + ], + "basePath": "/api/v1/instances/123/proxy", + "dangerouslyDisableDeviceAuth": true + }, + "trustedProxies": [ + "100.68.0.0/16" + ] + } +} +``` + +说明: + +- `dangerouslyDisableDeviceAuth` 只能在 ClawManager trusted proxy 模式下使用。 +- 如果 runtime 不支持 trusted proxy 模式,必须明确实现一种等价机制,保证浏览器通过 ClawManager 代理即可自动进入,不需要用户手工粘贴 gateway token。 +- 如果出现 `origin not allowed`,优先检查 `allowedOrigins` 是否使用内部 origin,以及 WebSocket 代理是否正确传递 Origin。 +- 如果出现 `Proxy headers detected from untrusted address`,优先检查 trusted proxies CIDR。 + +## 10. 时间同步规范 + +runtime 侧不能忽略时间问题。以下问题通常与时钟偏移有关: + +- `device signature expired` +- token 未过期但被拒绝。 +- JWT、签名 URL、WebSocket 鉴权异常。 +- ClawManager 显示状态和 gateway 实际状态不一致。 + +agent 必须实现: + +- 注册和 heartbeat 响应如果带 `server_time`,记录本地时间与服务端时间偏移。 +- 若偏移超过 30 秒,在日志和 health 中上报 `clock_skew_warning`。 +- 若偏移超过 120 秒,不应报告 `ready` 或 gateway `running`,应上报 `unhealthy` 或 `error`。 +- 生成或校验任何带时间的签名时使用 UTC。 +- 不在容器里自行修改系统时间。时间同步由 Kubernetes 节点的 NTP、chrony 或宿主机配置负责。 + +运维要求: + +- 所有 Kubernetes 节点必须启用 NTP 或 chrony。 +- 发现签名过期类错误时,先比较浏览器机器、ClawManager Pod、runtime Pod、Kubernetes Node 的 UTC 时间。 +- agent 健康检查输出应包含当前本地时间、最近服务端时间和估算偏移,便于排查。 + +## 11. 状态上报和恢复语义 + +Runtime Pod 注册 payload 必须包含容量: + +```json +{ + "runtime_type": "hermes", + "capacity": 100, + "used_slots": 12, + "available_slots": 88, + "state": "ready", + "draining": false +} +``` + +Gateway report 每个 gateway 建议包含: + +```json +{ + "instance_id": 66, + "gateway_id": "gw-66-7", + "gateway_port": 20000, + "gateway_pid": 1234, + "state": "running", + "generation": 7, + "workspace_path": "/workspaces/hermes/user-1/instance-66", + "started_at": "2026-06-03T06:00:00Z", + "updated_at": "2026-06-03T06:00:10Z", + "error_message": "" +} +``` + +状态规则: + +- `starting`:进程准备中或健康检查尚未通过。 +- `running`:进程存在、端口监听、HTTP 健康检查通过、必要的代理和 LLM 检查通过。 +- `error`:启动失败、配置失败、健康检查失败、凭证缺失或端口冲突。 +- `stopped`:进程已停止,端口已释放。 +- `unhealthy`:进程存在但健康检查失败。 + +Pod 重启或升级时: + +- agent 启动后先初始化 workspace root、端口池、控制服务和报告客户端。 +- 控制服务可用后注册 Runtime Pod。 +- 只有初始化完成后才能上报 `ready`。 +- 不要把旧 Pod 的 gateway 当成运行中,除非进程确实存在且健康。 +- 收到 SIGTERM 时进入 draining,拒绝新 create,尽量上报最后状态。 +- 支持 ClawManager 下发 drain、stop、restart gateway 指令。 + +## 12. 健康检查 + +`/v1/health` 只有在 agent 能接受控制指令时才返回 2xx。以下情况必须返回 503: + +- 必填环境变量缺失。 +- workspace root 不可访问。 +- 端口池不可用。 +- report token 或 control token 缺失。 +- 时钟偏移超过健康阈值。 +- agent 正在关闭且不再接受请求。 + +gateway 从 `starting` 转为 `running` 前至少校验: + +- 端口已监听。 +- HTTP `/` 或 runtime 约定健康端点可访问。 +- 代理 base path 配置已写入。 +- 使用 ClawManager 内部 Origin 的 Control UI 或 WebSocket 握手不再返回 `origin not allowed`。 +- runtime 如果需要 LLM,使用实例 token 调用 ClawManager AI Gateway 可以返回模型列表或完成一次轻量探测。 + +健康检查失败时不要上报 `running`。应上报 `error`,并带短错误文本。 + +## 13. 安全规范 + +agent 控制接口必须校验: + +- `X-ClawManager-Control-Token`。 +- `instance_id`、`user_id`、`generation` 类型和值范围。 +- `agent_type` 必须等于当前 Pod runtime type。 +- `workspace_path` 不越权。 +- 端口只能来自允许范围。 +- `environment` 只允许白名单 key。 + +进程启动必须使用 argv 数组,不使用 shell 拼接: + +```text +execve(binary, ["hermes", "gateway", "run", "--port", "20017"], env) +``` + +禁止: + +- 把请求参数拼成 `sh -c`。 +- 日志打印 token、API key、channel secret。 +- 允许用户覆盖 agent 自己的控制 token 和上报 token。 +- 删除 gateway 时顺手删除 workspace。 +- 为了调通临时写死 IP、CIDR 或 `allowedOrigins=["*"]`,除非明确是本地开发环境。 + +## 14. Hermes Runtime 开发落地清单 + +开发 Hermes runtime 时,至少完成以下事项: + +| 项目 | 要求 | +| --- | --- | +| Runtime 类型 | `CLAWMANAGER_RUNTIME_TYPE=hermes`,注册和上报 payload 中都使用 `hermes`。 | +| Agent 控制服务 | 监听 `0.0.0.0:19090`,实现 `/v1/health`、`/v1/gateways`、`DELETE /v1/gateways/{id}`、`/v1/drain`。 | +| Gateway 启动 | 后台启动 Hermes gateway,使用实例 workspace、端口、UID/GID、HOME。 | +| 配置文件 | 写入 `{workspace}/home/.hermes/hermes.json` 或 Hermes 原生配置,必须 JSON merge。 | +| AI Gateway | 将 `CLAWMANAGER_LLM_*` 和 `OPENAI_*` 注入 Hermes provider 配置。 | +| 代理路径 | Hermes UI 或 WebSocket 必须支持 `/api/v1/instances/{instance_id}/proxy` base path。 | +| Trusted proxy | 信任 ClawManager 内部 proxy 来源,不信任任意外部来源。 | +| 自动登录 | 浏览器通过 ClawManager proxy 进入时不要求手工粘贴 token 或重复确认自签证书。 | +| 端口组 | 如果 Hermes 除主端口外还有 control/browser/sidecar 端口,按端口组分配。 | +| Skills | skill inventory 和状态上报按 `instance_id`、`workspace_path` 分组。 | +| 时间 | 实现 clock skew 检测;偏移过大时不报告 ready/running。 | +| 验收 | 100 个实例并发创建无端口冲突,Pod drain 和重启后状态正确。 | + +Hermes 启动命令由 Hermes 项目确定,但 agent 必须保证: + +- 命令参数固定且可审计。 +- 端口和 workspace 只能来自 agent 校验后的值。 +- LLM、proxy、auth 配置在进程启动前已经写入。 +- `POST /v1/gateways` 返回时进程可以仍在启动,但 metadata 必须已经持久化。 + +## 15. 测试和验收 + +单元测试至少覆盖: + +- create gateway 幂等。 +- 高 generation 替换低 generation。 +- 端口分配并发无冲突。 +- 端口被系统占用时跳过。 +- workspace path 越权被拒绝。 +- JSON merge 保留用户字段,数组去重。 +- LLM env 白名单转发。 +- 缺少 LLM token 时 gateway 不报告 running。 +- trusted proxy 和 allowed origin 配置生成。 +- clock skew 阈值判断。 + +集成测试至少覆盖: + +- Runtime Pod 注册为 `ready`,capacity、used_slots、available_slots 正确。 +- 创建 100 个 gateway,端口唯一,状态最终为 running。 +- 重复创建同一个 `instance_id + generation` 不重复启动进程。 +- ClawManager 反代 WebSocket 不出现 `origin not allowed`。 +- 通过 ClawManager proxy 进入 runtime 不要求手工粘贴 gateway token。 +- runtime 内模型调用经过 ClawManager AI Gateway,不出现 `No API key found`。 +- Pod drain 后拒绝新 create,已有 gateway 持续上报,DELETE 后释放端口。 +- Pod 重启后不误报旧 gateway running。 + +上线前 Kubernetes 验证建议: + +```bash +kubectl -n clawmanager-system get pods -l app.kubernetes.io/component=runtime-agent -o wide +kubectl -n clawmanager-system logs deploy/hermes-runtime --tail=200 +kubectl -n clawmanager-system exec deploy/hermes-runtime -- curl -fsS http://127.0.0.1:19090/v1/health +kubectl -n clawmanager-system exec deploy/hermes-runtime -- date -u +kubectl -n clawmanager-system exec deploy/clawmanager-app -- date -u +``` + +如果出现认证或签名问题,排查顺序: + +1. 时间偏移。 +2. 实例 token 是否下发到 `environment`。 +3. runtime 配置文件是否包含 provider 和 API key。 +4. ClawManager 内部 LLM base URL 是否可达。 +5. allowed origins 和 trusted proxies 是否使用内部服务地址和 env CIDR。 +6. gateway 是否真正健康,而不是缓存状态误报。 + +## 16. Do / Don't + +| Do | Don't | +| --- | --- | +| 用内部 Kubernetes Service DNS 配置 runtime 到 ClawManager 的访问。 | 写死 `172.16.1.12`、NodePort、外部 HTTPS 入口。 | +| `POST /v1/gateways` 快速返回 `starting`,后台拉起进程。 | 在 HTTP 请求里阻塞等待 `gateway run` 前台进程。 | +| 按 `instance_id + generation` 幂等。 | 重试时重复启动多个 gateway。 | +| 上报状态前检查真实进程和健康。 | 只读旧状态文件就上报 `running`。 | +| JSON merge runtime 配置。 | 覆盖用户的整个配置文件。 | +| 白名单转发 env。 | 把用户请求里的所有 env 原样传给进程。 | +| 使用 trusted proxy 让浏览器通过 ClawManager 自动进入。 | 要求用户手工粘贴 gateway token。 | +| 检测并上报 clock skew。 | 忽略时间,导致 `device signature expired` 反复出现。 | diff --git a/docs/aigateway.md b/docs/aigateway.md new file mode 100644 index 0000000..3710d1e --- /dev/null +++ b/docs/aigateway.md @@ -0,0 +1,77 @@ +## AI Gateway + +AI Gateway is the core governance plane in ClawManager for OpenClaw instances. It provides a controlled, secure, and auditable model access layer that hides provider-specific differences behind a single interface, while adding compliance controls and cost visibility on top. +![](./AIGateway/aigateway-home.png) + +### 1. Model Management +![](./AIGateway/aigateway-model.png) + +AI Gateway applies fine-grained model governance so users can access only authorized and active models. + +- Unified access and routing through a single OpenAI-compatible interface +- Provider onboarding with discovery and centralized endpoint configuration +- Tiered model governance with regular models and secure models for sensitive workloads +- Per-model activation, endpoint, credential reference, and pricing controls + +### 2. Audit and Trace +![](./AIGateway/aigateway-audit.png) +The platform keeps end-to-end records for compliance, incident response, and operational debugging. + +- End-to-end traceability with `trace_id`, session ID, and request ID correlation +- Persistent request and response payload logging, including streamed SSE responses +- Recorded risk hits, routing decisions, and final invocation status +- Search and review by user, model, instance, time window, or trace ID + +### 3. Cost Accounting +![](./AIGateway/costs.png) +Cost accounting is built in as a core capability rather than an add-on. + +- Prompt, completion, and total token tracking per invocation +- Support for reasoning and cached token classification where available +- Per-model pricing with configurable input and output rates and currency support +- Estimated cost calculation for external models and internal cost allocation for secure models +- User-, instance-, model-, and session-level cost analysis in the admin experience + +### 4. Risk Control +![](./AIGateway/risks.png) + +Requests can be evaluated before they reach the upstream model, helping teams apply data protection and compliance policies consistently. + +- Built-in rules for privacy, enterprise-sensitive data, finance, and security/compliance scenarios +- Regex-based and custom rule extensibility +- Automatic actions such as `block` and `route_secure_model` +- Transparent governance decisions recorded in the audit trail +- Rule testing and hit preview from the management console + +### 5. AI Gateway Model Selection and Routing Logic + +The AI Gateway routing decision is primarily driven by the `is_secure` flag rather than `provider_type`. + +1. If the request uses `model: "auto"`, the gateway selects the first active model with `is_secure=false`. +2. If the request specifies a model name, the gateway matches the active model by `display_name`. +3. The gateway scans all messages with the configured risk rules. +4. If risk evaluation triggers `route_secure_model`, the request is switched to the first active secure model. +5. The final route is determined by `is_secure`: + - `is_secure=true` routes to the internal secure path. + - `is_secure=false` routes to the regular external path. + +```mermaid +graph TD + A(["User Request"]) --> B{"Model Specified?"} + + B -- "auto" --> C["Match first is_secure=false model"] + B -- "specified" --> D["Match active model by display_name"] + + C --> E["Risk Control Scan"] + D --> E + + E -- "route_secure_model" --> F["Switch to first active secure model"] + E -- "no trigger" --> H["Execute final selected model"] + F --> H + + subgraph "Core Routing Logic" + H --> I{"is_secure flag"} + I -- "true" --> J["Internal secure route"] + I -- "false" --> K["External regular route"] + end +``` \ No newline at end of file diff --git a/docs/arm64-deployment.md b/docs/arm64-deployment.md new file mode 100644 index 0000000..d033010 --- /dev/null +++ b/docs/arm64-deployment.md @@ -0,0 +1,118 @@ +# ARM64 部署说明 + +## ARM64 平台支持 + +本项目支持在 ARM64 (aarch64) 架构上部署,如树莓派、ARM开发板等设备。 + +## 预构建 ARM64 镜像 + +以下镜像可用于 ARM64 部署: + +| 镜像 | 地址 | 说明 | +|------|------|------| +| ClawManager 主应用 | `ghcr.io/yuan-lab-llm/clawmanager:latest` | 官方多架构镜像,包含 ARM64 | +| Skill Scanner | `ghcr.io/yuan-lab-llm/skill-scanner:latest` | 官方镜像已支持 ARM64 | + +### 使用预构建镜像 + +```bash +# 拉取ARM64镜像 +docker pull ghcr.io/yuan-lab-llm/clawmanager:latest --platform linux/arm64 +docker pull ghcr.io/yuan-lab-llm/skill-scanner:latest --platform linux/arm64 +``` + +## 从源码构建 ARM64 镜像 + +### 后端静态编译 + +```bash +cd backend +CGO_ENABLED=0 go build -ldflags="-s -w -extldflags=-static" -o bin/clawreef-server ./cmd/server +``` + +### Docker 多平台构建 + +```bash +# 构建并推送多平台镜像 +docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/your-name/clawmanager:latest --push . +``` + +### skill-scanner ARM64 构建(可选) + +`ghcr.io/yuan-lab-llm/skill-scanner:latest` 已支持 ARM64 平台。 + +如需自定义镜像,可从源码构建: + +```bash +# 克隆仓库 +git clone https://github.com/Yuan-lab-LLM/skill-scanner.git +cd skill-scanner + +# 使用代理构建(国内网络需要) +export http_proxy="http://your-proxy:port" +export https_proxy="http://your-proxy:port" + +# 构建并推送ARM64镜像 +docker buildx build --platform linux/arm64 \ + -t ghcr.io/your-name/skill-scanner:latest \ + --push . +``` + +## Kubernetes ARM64 部署 + +### 修改镜像地址 + +在 `clawmanager.yaml` 中确认镜像地址支持 ARM64: + +```yaml +# 主应用 +image: ghcr.io/yuan-lab-llm/clawmanager:latest + +# skill-scanner(如果需要) +image: ghcr.io/yuan-lab-llm/skill-scanner:latest +``` + +### 数据库初始化 + +首次部署需要手动创建数据库用户: + +```bash +kubectl exec -it -n clawmanager-system mysql-xxx -- mysql -u root + +# 在MySQL中执行 +CREATE USER IF NOT EXISTS 'clawmanager'@'%' IDENTIFIED BY 'your-password'; +GRANT ALL PRIVILEGES ON *.* TO 'clawmanager'@'%' WITH GRANT OPTION; +CREATE DATABASE IF NOT EXISTS clawmanager CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +FLUSH PRIVILEGES; +``` + +## 已知问题 + +### ClawManager 官方镜像支持 ARM64 + +- **状态**: `ghcr.io/yuan-lab-llm/clawmanager:latest` 发布为多架构镜像 +- **支持平台**: `linux/amd64`、`linux/arm64` +- **部署方式**: ARM64 节点会自动拉取对应架构镜像,无需单独维护主应用镜像地址 + +### skill-scanner 官方镜像支持 ARM64 + +- **状态**: `ghcr.io/yuan-lab-llm/skill-scanner:latest` 已提供 ARM64 平台镜像 +- **部署方式**: ARM64 节点会自动拉取对应架构镜像 +- **建议**: 如需可复现部署,建议使用固定 tag 而非 `latest` + +### ARM64 设备性能注意 + +- 建议设备内存 >= 4GB +- 建议使用 SSD 存储(项目数据目录 `/mnt/Storage1`) +- 建议配置 swap 以避免内存不足 + +## 测试环境 + +以下环境已验证可正常运行: + +- **开发板**: S922X-Oes-Plus (Amlogic S922X) +- **系统**: Armbian OS 26.05.0 trixie +- **内核**: 6.12.80-ophub +- **内存**: 3.6GB RAM +- **存储**: 110GB SSD +- **集群**: k3s v1.34.6 diff --git a/docs/arm64-deployment_en.md b/docs/arm64-deployment_en.md new file mode 100644 index 0000000..3bdfe89 --- /dev/null +++ b/docs/arm64-deployment_en.md @@ -0,0 +1,118 @@ +# ARM64 Deployment Guide + +## ARM64 Platform Support + +This project supports deployment on ARM64 (aarch64) architecture devices such as Raspberry Pi and ARM development boards. + +## Pre-built ARM64 Images + +The following images can be used for ARM64 deployments: + +| Image | Address | Description | +|-------|---------|-------------| +| ClawManager Main App | `ghcr.io/yuan-lab-llm/clawmanager:latest` | Official multi-platform image with ARM64 support | +| Skill Scanner | `ghcr.io/yuan-lab-llm/skill-scanner:latest` | Official image now supports ARM64 | + +### Using Pre-built Images + +```bash +# Pull ARM64 images +docker pull ghcr.io/yuan-lab-llm/clawmanager:latest --platform linux/arm64 +docker pull ghcr.io/yuan-lab-llm/skill-scanner:latest --platform linux/arm64 +``` + +## Building ARM64 Images from Source + +### Backend Static Compilation + +```bash +cd backend +CGO_ENABLED=0 go build -ldflags="-s -w -extldflags=-static" -o bin/clawreef-server ./cmd/server +``` + +### Docker Multi-platform Build + +```bash +# Build and push multi-platform images +docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/your-name/clawmanager:latest --push . +``` + +### skill-scanner ARM64 Build (Optional) + +`ghcr.io/yuan-lab-llm/skill-scanner:latest` now supports ARM64. + +Build from source only if you need a customized image: + +```bash +# Clone the repo +git clone https://github.com/Yuan-lab-LLM/skill-scanner.git +cd skill-scanner + +# Use proxy if needed (for China networks) +export http_proxy="http://your-proxy:port" +export https_proxy="http://your-proxy:port" + +# Build and push ARM64 image +docker buildx build --platform linux/arm64 \ + -t ghcr.io/your-name/skill-scanner:latest \ + --push . +``` + +## Kubernetes ARM64 Deployment + +### Modify Image Addresses + +In `clawmanager.yaml`, make sure the image addresses support ARM64: + +```yaml +# Main app +image: ghcr.io/yuan-lab-llm/clawmanager:latest + +# skill-scanner (if needed) +image: ghcr.io/yuan-lab-llm/skill-scanner:latest +``` + +### Database Initialization + +Manual database user creation is required for first deployment: + +```bash +kubectl exec -it -n clawmanager-system mysql-xxx -- mysql -u root + +# Execute in MySQL +CREATE USER IF NOT EXISTS 'clawmanager'@'%' IDENTIFIED BY 'your-password'; +GRANT ALL PRIVILEGES ON *.* TO 'clawmanager'@'%' WITH GRANT OPTION; +CREATE DATABASE IF NOT EXISTS clawmanager CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +FLUSH PRIVILEGES; +``` + +## Known Issues + +### ClawManager Official Image Supports ARM64 + +- **Status**: `ghcr.io/yuan-lab-llm/clawmanager:latest` is published as a multi-platform image +- **Platforms**: `linux/amd64`, `linux/arm64` +- **Deployment impact**: ARM64 nodes automatically pull the matching main app image without a separate ARM-only tag + +### skill-scanner Official Image Supports ARM64 + +- **Status**: `ghcr.io/yuan-lab-llm/skill-scanner:latest` now provides ARM64 image +- **Deployment impact**: ARM64 nodes automatically pull the matching architecture image +- **Recommendation**: pin a specific tag instead of `latest` for reproducible deployments + +### ARM64 Device Performance Notes + +- Recommended device memory >= 4GB +- Recommended SSD storage (project data directory `/mnt/Storage1`) +- Recommended swap configuration to avoid OOM + +## Tested Environment + +The following environment has been verified to work: + +- **Board**: S922X-Oes-Plus (Amlogic S922X) +- **OS**: Armbian OS 26.05.0 trixie +- **Kernel**: 6.12.80-ophub +- **Memory**: 3.6GB RAM +- **Storage**: 110GB SSD +- **Cluster**: k3s v1.34.6 diff --git a/docs/clawmanager-agent-v2-contract.md b/docs/clawmanager-agent-v2-contract.md new file mode 100644 index 0000000..8879cd2 --- /dev/null +++ b/docs/clawmanager-agent-v2-contract.md @@ -0,0 +1,178 @@ +# clawmanager-agent V2 Contract + +## Purpose + +The agent runs inside each shared OpenClaw or Hermes runtime Pod. It manages gateway subprocesses, ports, cgroups, Linux users, workspace quota, health checks, skill/status reporting, and metrics. + +## Agent-To-Backend Reports + +All report requests use header `X-ClawManager-Agent-Token`. + +### POST /api/v1/runtime-agent/register + +Request body: + +```json +{ + "runtime_type": "openclaw", + "namespace": "clawmanager-system", + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "pod_uid": "pod-uid", + "pod_ip": "10.42.0.31", + "node_name": "node-a", + "deployment_name": "openclaw-runtime", + "image_ref": "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest", + "agent_endpoint": "http://10.42.0.31:19090", + "capacity": 100 +} +``` + +### POST /api/v1/runtime-agent/heartbeat + +Request body: + +```json +{ + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "runtime_type": "openclaw", + "state": "ready", + "used_slots": 37, + "draining": false +} +``` + +### POST /api/v1/runtime-agent/metrics/report + +Request body: + +```json +{ + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "cpu_millis_used": 13600, + "memory_bytes_used": 42949672960, + "disk_bytes_used": 214748364800, + "network_rx_bytes": 9223372, + "network_tx_bytes": 19223372, + "gateways": [ + { + "instance_id": 123, + "gateway_id": "gw-123-7", + "port": 20017, + "state": "running", + "last_health_at": "2026-06-01T10:00:00Z" + } + ] +} +``` + +## Backend-To-Agent Commands + +All command requests use header `X-ClawManager-Control-Token`. + +ClawManager treats redirects from agent command endpoints as errors. Agents should return direct 2xx, 4xx, or 5xx responses instead of 3xx redirects. + +### GET /v1/health + +The agent reports whether it can accept control-plane commands. + +Success status: HTTP 200. + +Success response body: + +```json +{ + "status": "ready" +} +``` + +ClawManager currently treats any HTTP 2xx response as success and does not require a response body for this endpoint. Any non-2xx status is treated as an error. + +### POST /v1/gateways + +The agent creates a gateway subprocess and returns the bound port. If no port is free in the requested range, return HTTP 409. + +Request body: + +```json +{ + "instance_id": 123, + "user_id": 45, + "agent_type": "openclaw", + "workspace_path": "/workspaces/openclaw/user-45/instance-123", + "port_range": { + "start": 20000, + "end": 20099 + }, + "uid": 200123, + "gid": 200123, + "cpu_cores": 2, + "memory_mb": 4096, + "disk_quota_mb": 20480, + "generation": 7 +} +``` + +Success status: HTTP 200 or HTTP 201. + +Success response body: + +```json +{ + "gateway_id": "gw-123-7", + "port": 20017, + "pid": 8842, + "status": "running" +} +``` + +### DELETE /v1/gateways/{gateway_id} + +The agent stops the gateway and releases cgroup, process, and port resources. + +`gateway_id` is URL path-escaped by ClawManager. Agents must decode the path segment before lookup. + +Success status: HTTP 200, HTTP 202, or HTTP 204. ClawManager does not require a response body. + +### POST /v1/drain + +The agent marks the Pod as draining and rejects new gateway creation. Existing gateways continue until ClawManager migrates them. + +Request body: + +```json +{ + "draining": true +} +``` + +Success status: HTTP 200 or HTTP 202. ClawManager does not require a response body. + +## Backend-To-Agent Error Responses + +For now, agents should return a plain text response body for command errors. + +When no port is free for `POST /v1/gateways`, return: + +```http +HTTP/1.1 409 Conflict +Content-Type: text/plain + +no free port +``` + +ClawManager maps this to `runtime agent conflict: no free port`. + +For general failures, return an appropriate non-2xx status with a short plain text body: + +```http +HTTP/1.1 500 Internal Server Error +Content-Type: text/plain + +failed to start gateway process +``` + +ClawManager includes both the status code and response body in the returned error. + +## Isolation Requirements + +Every gateway runs as UID/GID `200000 + instance_id`. The agent rejects workspace paths outside `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` after resolving symlinks. CPU and memory limits are enforced with cgroups. Workspace quota is enforced before writes. diff --git a/docs/clawmanager-agent-v2-development-guide.md b/docs/clawmanager-agent-v2-development-guide.md new file mode 100644 index 0000000..ec62088 --- /dev/null +++ b/docs/clawmanager-agent-v2-development-guide.md @@ -0,0 +1,888 @@ +# ClawManager Agent V2 开发规范 + +本文整理当前 ClawManager 代码中的最新 agent 约定,面向 OpenClaw、Hermes 以及后续新增的托管 runtime。开发或改造 runtime 镜像时,应优先遵守本文;字段级契约以 `docs/clawmanager-agent-v2-contract.md` 和后端代码为准。 + +## 1. 架构定位 + +最新架构中有两类 agent 通信面: + +1. **Runtime Pod Agent,也就是本文主线的 V2 agent** + 运行在共享 runtime Pod 内,负责本 Pod 内多个 gateway 子进程的创建、删除、端口、workspace、资源隔离、健康检查和状态上报。ClawManager 通过它把一个实例绑定到 `pod_ip + gateway_port`。 + +2. **Instance Agent Control Plane,旧的 `/api/v1/agent/*` 协议** + 面向单个实例的状态、命令、skill inventory、skill package 上传和配置 revision 下载。当前后端仍保留这套协议,runtime 可以继续复用它完成实例级状态与 skill 管理;但 gateway 生命周期管理必须走 Runtime Pod Agent。 + +一句话原则:**Runtime Pod Agent 是 Pod 内 gateway 管理器,不是一个阻塞式启动脚本。** + +## 2. 支持范围 + +当前 V2 托管 runtime 类型: + +| 类型 | 说明 | +| --- | --- | +| `openclaw` | OpenClaw runtime | +| `hermes` | Hermes runtime | + +当前平台默认值: + +| 项 | 默认值 | +| --- | --- | +| workspace root | `/workspaces` | +| workspace path | `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` | +| gateway 端口范围 | `20000-20099` | +| 单 Runtime Pod 容量 | `100` | +| 实例 UID/GID | `200000 + instance_id` | +| agent 控制端口 | `19090` | + +## 3. Runtime Pod Agent 职责 + +Runtime Pod Agent 必须实现: + +- 启动本地控制服务:`GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{gateway_id}`、`POST /v1/drain`。 +- 向 ClawManager 上报 runtime pod 注册、heartbeat、metrics、gateway 状态和 skill 状态。 +- 管理本 Pod 内 gateway 子进程生命周期。 +- 管理端口池,避免同一 Pod 内端口冲突。 +- 创建并校验 workspace,设置正确 UID/GID。 +- 将 ClawManager 下发的 LLM、代理、实例 token、runtime 配置写入实例自己的 workspace。 +- 实现 drain,拒绝新 gateway,保持已有 gateway 运行,等待 ClawManager 迁移和删除。 +- 避免任何敏感 token、API key、Authorization header 出现在日志中。 + +不应该由 Runtime Pod Agent 实现: + +- 直接对浏览器暴露 gateway 自签地址。 +- 管理 ClawManager 用户权限。 +- 删除用户 workspace。 +- 把外部 NodePort、外部 HTTPS 入口或固定内网 IP 写死进 runtime 配置。 + +## 4. 运行环境变量 + +当前 Runtime Deployment 构建器注入以下变量,agent 实现应优先支持: + +| 变量 | 说明 | +| --- | --- | +| `CLAWMANAGER_RUNTIME_TYPE` | `openclaw` 或 `hermes` | +| `CLAWMANAGER_AGENT_PORT` | agent 控制端口,当前为 `19090` | +| `CLAWMANAGER_GATEWAY_PORT_START` | gateway 起始端口 | +| `CLAWMANAGER_GATEWAY_PORT_END` | gateway 结束端口 | +| `CLAWMANAGER_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用 | +| `CLAWMANAGER_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用 | + +为了兼容早期文档和旧镜像,agent 建议同时识别以下别名: + +| 新变量 | 兼容别名 | +| --- | --- | +| `CLAWMANAGER_AGENT_CONTROL_TOKEN` | `RUNTIME_AGENT_CONTROL_TOKEN` | +| `CLAWMANAGER_AGENT_REPORT_TOKEN` | `RUNTIME_AGENT_REPORT_TOKEN` | +| `CLAWMANAGER_GATEWAY_PORT_START` | `RUNTIME_GATEWAY_PORT_START` | +| `CLAWMANAGER_GATEWAY_PORT_END` | `RUNTIME_GATEWAY_PORT_END` | +| `CLAWMANAGER_AGENT_PORT` | `RUNTIME_AGENT_PUBLIC_PORT` | + +建议支持的附加变量: + +| 变量 | 默认值/要求 | +| --- | --- | +| `RUNTIME_WORKSPACE_ROOT` | 默认 `/workspaces` | +| `RUNTIME_AGENT_LISTEN_ADDR` | 默认 `0.0.0.0:19090` | +| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址,推荐 Kubernetes Service DNS | +| `CLAWMANAGER_RUNTIME_IMAGE_REF` | 当前镜像标识,用于注册上报 | +| `POD_NAME` / `POD_NAMESPACE` / `POD_IP` / `NODE_NAME` | 建议通过 Downward API 注入 | +| `CLAWMANAGER_TRUSTED_PROXY_CIDRS` | ClawManager app/gateway 的可信代理网段 | + +`CLAWMANAGER_BACKEND_URL` 必须使用集群内部地址,例如: + +```text +http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001 +``` + +不要在 runtime 内部配置中写入浏览器外部入口,例如 `https://172.16.1.12:39443`。 + +## 5. Agent 到 ClawManager 的上报接口 + +所有 Runtime Pod Agent 上报请求必须带: + +```http +X-ClawManager-Agent-Token: ${CLAWMANAGER_AGENT_REPORT_TOKEN} +Content-Type: application/json +``` + +后端统一返回: + +```json +{ + "success": true, + "message": "...", + "data": {} +} +``` + +### 5.1 注册 Runtime Pod + +```http +POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/register +``` + +必填字段: + +- `runtime_type` +- `namespace` +- `pod_name` +- `deployment_name` +- `image_ref` + +推荐完整 payload: + +```json +{ + "runtime_type": "openclaw", + "namespace": "clawmanager-system", + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "pod_uid": "pod-uid", + "pod_ip": "10.42.0.31", + "node_name": "node-a", + "deployment_name": "openclaw-runtime", + "image_ref": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest", + "agent_endpoint": "http://10.42.0.31:19090", + "state": "ready", + "capacity": 100, + "used_slots": 0, + "draining": false, + "metrics": { + "agent_version": "0.1.0" + }, + "reported_at": "2026-06-08T09:00:00Z" +} +``` + +要求: + +- `agent_endpoint` 必须是 ClawManager Pod 可以访问的地址,通常是 `http://{POD_IP}:19090`,不能是 `127.0.0.1`。 +- `capacity <= 0` 时后端会按 `100` 处理,但 agent 应主动上报真实容量。 +- `state` 为空时后端按 `ready` 处理;agent 不应在初始化未完成时上报 ready。 +- `metrics` 必须是合法 JSON。 + +### 5.2 Heartbeat + +```http +POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/heartbeat +``` + +payload: + +```json +{ + "pod_id": 17, + "namespace": "clawmanager-system", + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "state": "ready", + "used_slots": 37, + "draining": false, + "reported_at": "2026-06-08T09:00:02Z" +} +``` + +要求: + +- 优先使用注册返回的 `data.pod.id` 作为 `pod_id`。 +- 没有 `pod_id` 时必须带 `namespace + pod_name`。 +- `state` 必填。 +- `used_slots` 必须来自当前真实 `starting/running` gateway 数量,不要使用过期缓存。 +- 建议 heartbeat 间隔 2 秒;如果环境压力较大,可放宽,但必须小于 ClawManager 的 heartbeat 超时窗口。 + +### 5.3 Metrics + +```http +POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/metrics/report +``` + +payload: + +```json +{ + "pod_id": 17, + "cpu_millis_used": 13600, + "memory_bytes_used": 42949672960, + "disk_bytes_used": 214748364800, + "network_rx_bytes": 9223372, + "network_tx_bytes": 19223372, + "metrics": { + "gateway_count": 37, + "load_1m": 2.4 + }, + "reported_at": "2026-06-08T09:00:05Z" +} +``` + +要求: + +- CPU 单位是 millicore。 +- memory/disk/network 单位是 bytes。 +- network 字段必须是单调递增 counter,不要填瞬时速率。 +- `metrics` 是扩展 JSON,必须合法。 + +### 5.4 Gateway Report + +```http +POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/gateways/report +``` + +payload: + +```json +{ + "pod_id": 17, + "gateways": [ + { + "instance_id": 123, + "gateway_id": "gw-123-7", + "gateway_port": 20017, + "gateway_pid": 8842, + "state": "running", + "generation": 7, + "health_at": "2026-06-08T09:00:05Z" + } + ] +} +``` + +要求: + +- `gateways` 必填。 +- 每个 gateway 的 `instance_id`、`state`、`generation` 必填。 +- 后端只接受“当前绑定 Pod + 当前 generation”的上报;旧 Pod 或旧 generation 上报会被忽略。 +- `state=running` 或 `state=healthy` 会把绑定更新为 running。 +- `starting`、`stopped`、`error`、`unhealthy` 等状态会更新绑定状态。 +- 错误状态必须带 `error_message`,不要只上报空状态。 + +### 5.5 Skills Report + +```http +POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/skills/report +``` + +当前后端接收并发布任意 JSON payload。推荐按实例分组: + +```json +{ + "pod_id": 17, + "runtime_type": "openclaw", + "mode": "full", + "reported_at": "2026-06-08T09:00:05Z", + "instances": [ + { + "instance_id": 123, + "workspace_path": "/workspaces/openclaw/user-45/instance-123", + "skills": [ + { + "skill_id": "weather", + "skill_version": "1.0.0", + "identifier": "weather", + "install_path": "/workspaces/openclaw/user-45/instance-123/.openclaw/skills/weather", + "content_md5": "0123456789abcdef0123456789abcdef", + "source": "runtime", + "type": "agent-skill" + } + ] + } + ] +} +``` + +skill inventory 必须按 `instance_id` 和 `workspace_path` 隔离,不要把一个实例的 skill 混到另一个实例。 + +## 6. ClawManager 到 Agent 的控制接口 + +所有控制请求必须校验: + +```http +X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN} +``` + +token 错误返回 `401`。不要返回 `3xx` redirect;后端会把 redirect 当成失败。 + +### 6.1 Health + +```http +GET /v1/health +``` + +任意 HTTP 2xx 都会被后端视为成功。建议响应: + +```json +{ + "status": "ready" +} +``` + +以下情况必须返回 `503`: + +- 必填环境变量缺失。 +- workspace root 不可访问。 +- 端口池不可用。 +- control/report token 缺失。 +- clock skew 超过健康阈值。 +- agent 正在关闭,不再接受新请求。 + +### 6.2 Create Gateway + +```http +POST /v1/gateways +``` + +请求: + +```json +{ + "instance_id": 123, + "user_id": 45, + "agent_type": "openclaw", + "workspace_path": "/workspaces/openclaw/user-45/instance-123", + "port_range": { + "start": 20000, + "end": 20099 + }, + "uid": 200123, + "gid": 200123, + "cpu_cores": 2, + "memory_mb": 4096, + "disk_quota_mb": 20480, + "generation": 7, + "environment": { + "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm", + "CLAWMANAGER_LLM_API_KEY": "instance-token", + "CLAWMANAGER_LLM_MODEL": "[\"auto\"]", + "CLAWMANAGER_LLM_PROVIDER": "openai-compatible", + "CLAWMANAGER_INSTANCE_TOKEN": "instance-token", + "OPENAI_API_KEY": "instance-token" + } +} +``` + +响应: + +```json +{ + "gateway_id": "gw-123-7", + "port": 20017, + "pid": 8842, + "status": "starting" +} +``` + +实现要求: + +- handler 必须快速返回,通常 1 到 3 秒内返回 `starting` 或已有 gateway 元数据。 +- 不能在 HTTP handler 内阻塞等待前台 gateway 进程完整运行。 +- `agent_type` 必须等于当前 Pod runtime type。 +- Pod draining 时返回 `409` 或 `503`,拒绝新建。 +- 以 `instance_id + generation` 为幂等键;重复请求不能重复启动进程。 +- 收到更高 generation 时,停止同实例旧 generation gateway 并释放端口。 +- 端口必须来自请求的 `port_range`。 +- 没有可用端口时返回: + +```http +HTTP/1.1 409 Conflict +Content-Type: text/plain + +no free port +``` + +- 启动失败、健康检查失败、配置写入失败时,上报 gateway `error` 并释放端口。 + +### 6.3 Delete Gateway + +```http +DELETE /v1/gateways/{gateway_id} +``` + +要求: + +- `gateway_id` 是 URL path escaped,agent 必须 decode 后再查找。 +- 优雅停止进程组,超时后强制 kill。 +- 释放端口、cgroup、进程表和本地状态。 +- 不删除 workspace。 +- gateway 不存在时建议返回 `204`,保持删除幂等。 +- 成功状态可以是 `200`、`202` 或 `204`。 + +### 6.4 Drain + +```http +POST /v1/drain +``` + +请求: + +```json +{ + "draining": true +} +``` + +要求: + +- 设置本地 draining 状态。 +- 立即拒绝新的 `POST /v1/gateways`。 +- 已有 gateway 不主动停止,继续健康检查和上报。 +- 立即补发 heartbeat,`state=draining, draining=true`。 +- 等 ClawManager 迁移实例后,逐个收到 DELETE 再清理。 + +## 7. Gateway 生命周期 + +推荐状态机: + +```text +requested -> reserved_port -> starting -> running + -> error +running -> stopping -> stopped +running -> unhealthy -> running/error +``` + +创建流程: + +1. 校验 token、runtime type、instance/user/generation、workspace、端口范围。 +2. 如果 draining,拒绝创建。 +3. 查本地状态,处理幂等。 +4. 如果 generation 更高,清理旧 generation。 +5. 加锁分配端口,标记为 reserved。 +6. 创建 workspace 和 home,设置 owner 为请求中的 `uid/gid`。 +7. 写入 runtime 配置和 LLM/proxy 配置。 +8. 持久化 gateway 元数据为 `starting`。 +9. 后台启动 gateway 子进程,立即返回。 +10. 健康检查通过后上报 `running`。 +11. 失败时上报 `error`,记录短错误信息并释放端口。 + +`running` 必须以真实进程和真实健康检查为准,不能只读旧状态文件。 + +## 8. 端口管理 + +agent 必须实现并发安全的端口分配器: + +```text +reserve(instance_id, generation, count) -> port_set +commit(instance_id, generation, port_set) +release(instance_id, generation) +list_used() +``` + +要求: + +- 分配时持有互斥锁。 +- 在锁内检查 agent 内存状态和系统监听端口。 +- 启动失败、健康检查失败、DELETE 成功后必须释放端口。 +- 如果 runtime 需要多个端口,必须一次性分配端口组,避免部分成功。 +- 同一 Pod 内端口不能冲突;不同 Pod 可以复用相同端口,因为路由使用 `pod_ip + port`。 + +## 9. Workspace 与资源隔离 + +workspace 固定格式: + +```text +/workspaces/{runtime}/user-{user_id}/instance-{instance_id} +``` + +agent 必须: + +- 对 workspace root 和请求路径执行 realpath。 +- 拒绝 `..`、绝对路径逃逸、符号链接逃逸、跨用户、跨实例路径。 +- 创建 `${workspace}/home`,并设置 `HOME=${workspace}/home`。 +- owner 设置为请求中的 `uid/gid`,当前默认 `200000 + instance_id`。 +- token、provider、配置文件权限建议 `0600`。 +- 删除 gateway 时不删除 workspace。 + +资源限制要求: + +- 每个 gateway 使用独立进程组。 +- 业务进程最终以实例 UID/GID 运行。 +- CPU/Memory 使用 cgroup 限制。 +- Disk 优先使用 filesystem quota;不支持时至少周期扫描并在超限时上报错误。 +- 资源限制无法启用时必须在 report 或错误消息中体现,不要静默忽略。 + +## 10. Runtime 配置和 LLM 注入 + +ClawManager 会通过 `POST /v1/gateways` 的 `environment` 字段下发实例级 LLM 环境变量。agent 只能转发白名单变量: + +```text +CLAWMANAGER_LLM_BASE_URL +CLAWMANAGER_LLM_API_KEY +CLAWMANAGER_LLM_MODEL +CLAWMANAGER_LLM_PROVIDER +CLAWMANAGER_INSTANCE_TOKEN +OPENAI_BASE_URL +OPENAI_API_BASE +OPENAI_API_KEY +OPENAI_MODEL +``` + +要求: + +- LLM base URL 使用 ClawManager 内部 Service DNS。 +- API key 使用当前实例 token。 +- 不要求用户在 runtime 页面手工填写 OpenAI key。 +- 不打印 token/API key。 +- 如果缺少 LLM key,不能上报 gateway `running`;应上报 `error` 并说明缺少实例 LLM token。 + +runtime 配置必须写在实例 workspace 内,例如: + +| Runtime | 推荐路径 | +| --- | --- | +| OpenClaw | `{workspace}/home/.openclaw/openclaw.json` | +| Hermes | `{workspace}/home/.hermes/hermes.json` 或 Hermes 原生配置路径 | + +JSON merge 规则: + +- 对象字段递归 merge。 +- 保留用户已有未知字段。 +- 平台必须控制的字段可以覆盖,但要在代码中显式列出。 +- 数组 append 后去重,例如 `allowedOrigins`。 +- 写入前先生成临时文件,再原子 rename。 + +## 11. 代理和浏览器访问 + +浏览器访问链路必须是: + +```text +Browser HTTPS -> ClawManager HTTPS -> ClawManager backend proxy -> Runtime Pod HTTP gateway +``` + +agent 写 runtime 配置时必须保证: + +- gateway 支持 `/api/v1/instances/{instance_id}/proxy` base path。 +- allowed origin 使用 ClawManager 内部 origin,例如 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。 +- trusted proxies 来自 `CLAWMANAGER_TRUSTED_PROXY_CIDRS` 或等价 env。 +- 不使用外部地址作为 runtime 内部 trusted origin。 +- 通过 ClawManager proxy 访问时,不应要求用户手工粘贴 gateway token。 + +## 12. 时间同步 + +agent 必须处理 clock skew: + +- 注册和 heartbeat 响应如果带 `server_time`,记录本地时间和服务端时间偏移。 +- 偏移超过 30 秒时记录 warning,并在 health 中暴露 `clock_skew_warning`。 +- 偏移超过 120 秒时不应上报 `ready` 或 gateway `running`。 +- 所有签名、token 时间和上报时间使用 UTC。 +- 不在容器里修改系统时间,时间同步由 Kubernetes 节点 NTP/chrony 负责。 + +常见症状包括 `device signature expired`、JWT/WebSocket 鉴权异常、签名 URL 过期和 gateway 实际状态与 ClawManager 展示不一致。 + +## 13. Instance Agent v1 兼容协议 + +当前后端仍提供 `/api/v1/agent/*`,用于实例级状态、命令和 skill 管理。托管 runtime 如果需要复用这套能力,应由具体实例的 gateway/agent 逻辑使用实例级环境变量。 + +ClawManager 会为支持托管集成的实例注入: + +| 变量 | 说明 | +| --- | --- | +| `CLAWMANAGER_AGENT_ENABLED` | `true` 时启用实例 agent | +| `CLAWMANAGER_AGENT_BASE_URL` | ClawManager API base URL,不带 `/api/v1/agent` 后缀 | +| `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | 首次注册使用 | +| `CLAWMANAGER_AGENT_INSTANCE_ID` | 当前实例 ID | +| `CLAWMANAGER_AGENT_PROTOCOL_VERSION` | 当前为 `v1` | +| `CLAWMANAGER_AGENT_PERSISTENT_DIR` | 当前实例持久化目录 | +| `CLAWMANAGER_AGENT_DISK_LIMIT_BYTES` | 实例磁盘配额 | + +### 13.1 Register + +```http +POST {base}/api/v1/agent/register +Authorization: Bearer ${CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN} +``` + +payload: + +```json +{ + "instance_id": 123, + "agent_id": "openclaw-123-main", + "agent_version": "0.1.0", + "protocol_version": "v1", + "capabilities": [ + "runtime.status", + "runtime.health", + "metrics.report", + "skills.inventory", + "skills.upload", + "commands.poll", + "llm.gateway" + ], + "host_info": { + "runtime_type": "openclaw", + "runtime_name": "OpenClaw", + "image": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest", + "persistent_dir": "/config" + } +} +``` + +响应 `data` 包含: + +- `session_token` +- `session_expires_at` +- `heartbeat_interval_seconds`,当前默认 `15` +- `command_poll_interval_seconds`,当前默认 `5` +- `server_time` + +session token 每次成功 heartbeat 会续期 24 小时。收到 `401` 时删除本地 session,并用 bootstrap token 重新注册。 + +### 13.2 Heartbeat + +```http +POST {base}/api/v1/agent/heartbeat +Authorization: Bearer ${session_token} +``` + +payload: + +```json +{ + "agent_id": "openclaw-123-main", + "timestamp": "2026-06-08T09:00:15Z", + "openclaw_status": "running", + "current_config_revision_id": null, + "summary": { + "runtime_type": "openclaw", + "runtime_status": "running", + "runtime_pid": 245, + "openclaw_pid": 245, + "skill_count": 8, + "disk_used_bytes": 2147483648, + "disk_limit_bytes": 10737418240 + } +} +``` + +兼容要求: + +- `openclaw_status`、`openclaw_pid`、`openclaw_version` 是历史字段名;非 OpenClaw runtime 也需要填自己的主进程状态、PID 和版本,直到协议升级。 +- 后端 45 秒内收到 heartbeat 展示 online,45 到 120 秒展示 stale,超过 120 秒展示 offline。 +- heartbeat 响应里的 `has_pending_command=true` 时,应立即拉取命令。 + +### 13.3 State Report + +```http +POST {base}/api/v1/agent/state/report +Authorization: Bearer ${session_token} +``` + +payload: + +```json +{ + "agent_id": "openclaw-123-main", + "reported_at": "2026-06-08T09:00:20Z", + "runtime": { + "openclaw_status": "running", + "openclaw_pid": 245, + "openclaw_version": "openclaw-2026.5.4", + "current_config_revision_id": null + }, + "system_info": { + "runtime_type": "openclaw", + "cpu": { + "cores": 2, + "load": { + "1m": 0.64, + "5m": 0.52, + "15m": 0.4 + } + }, + "memory": { + "mem_total_bytes": 4294967296, + "mem_available_bytes": 2147483648 + }, + "disk": { + "root_total_bytes": 10737418240, + "root_free_bytes": 8589934592 + }, + "network": { + "interfaces": [ + { + "name": "eth0", + "status": "up", + "addresses": ["10.42.0.12"], + "rx_bytes": 123456789, + "tx_bytes": 98765432 + } + ] + } + }, + "health": { + "runtime_process": "ok", + "agent": "ok", + "metrics_collector": "ok" + } +} +``` + +network 的 `rx_bytes` / `tx_bytes` 必须是单调递增 counter。 + +### 13.4 Commands + +拉取: + +```http +GET {base}/api/v1/agent/commands/next +Authorization: Bearer ${session_token} +``` + +开始: + +```http +POST {base}/api/v1/agent/commands/{id}/start +Authorization: Bearer ${session_token} +``` + +完成: + +```http +POST {base}/api/v1/agent/commands/{id}/finish +Authorization: Bearer ${session_token} +``` + +finish 只允许: + +- `succeeded` +- `failed` + +当前支持的命令类型: + +```text +start_openclaw +stop_openclaw +restart_openclaw +collect_system_info +apply_config_revision +reload_config +health_check +install_skill +update_skill +uninstall_skill +remove_skill +disable_skill +quarantine_skill +handle_skill_risk +sync_skill_inventory +refresh_skill_inventory +collect_skill_package +``` + +未知命令必须 finish 为 `failed`,`error_message` 写明 `unsupported command type: `。 + +### 13.5 Skills + +inventory: + +```http +POST {base}/api/v1/agent/skills/inventory +Authorization: Bearer ${session_token} +``` + +upload: + +```http +POST {base}/api/v1/agent/skills/upload +Authorization: Bearer ${session_token} +Content-Type: multipart/form-data +``` + +上传表单字段: + +- `file` +- `agent_id` +- `skill_id` +- `skill_version` +- `identifier` +- `content_md5` +- `source` + +要求: + +- `identifier` 和 `content_md5` 必须稳定。 +- `mode=full` 的 inventory 代表全量结果,平台会用它对齐实例 skill 状态。 +- `content_md5` 按目录内容指纹计算,不是 zip 文件 MD5;算法见 `docs/skill-content-md5-spec.md`。 +- 打包和解压必须防止路径逃逸。 + +## 14. 安全规范 + +- 控制接口只接受正确 `X-ClawManager-Control-Token`。 +- 上报接口只使用 `X-ClawManager-Agent-Token`。 +- 实例级接口使用 Bearer bootstrap/session token。 +- 日志不得打印 token、API key、完整 Authorization header、channel secret。 +- 启动进程使用 argv 数组,不使用 shell 拼接: + +```text +execve(binary, ["openclaw", "gateway", "run", "--port", "20017"], env) +``` + +- 不允许用户环境变量覆盖 agent 自己的 control/report token。 +- 不允许 `allowedOrigins=["*"]` 作为生产配置。 +- 所有用户输入路径都要边界校验。 +- 命令执行必须有超时,超过 `timeout_seconds` 主动终止并 finish failed。 + +## 15. 测试要求 + +单元测试至少覆盖: + +- Runtime type 只接受 `openclaw` / `hermes`。 +- create gateway 幂等。 +- 更高 generation 替换旧 generation。 +- 端口并发分配无冲突。 +- 端口被系统占用时跳过。 +- 端口耗尽返回 `409 no free port`。 +- workspace path 越权被拒绝。 +- JSON merge 保留用户字段,数组去重。 +- LLM env 白名单转发。 +- 缺少 LLM token 时 gateway 不报 `running`。 +- trusted proxy 和 allowed origin 配置生成。 +- clock skew 阈值判断。 +- redirect 响应不被当成成功。 +- DELETE gateway id path escape/decode。 + +集成测试至少覆盖: + +- Runtime Pod 注册为 `ready`,capacity/used_slots/draining 正确。 +- heartbeat 更新 Pod 状态。 +- metrics 上报后管理端 Runtime Pods 页面能看到指标。 +- 创建 100 个 gateway 端口唯一,第 101 个触发扩容或 no capacity。 +- 重复 create 同一 `instance_id + generation` 不重复启动。 +- gateway report 旧 generation 不覆盖新 generation。 +- ClawManager 通过 proxy 能访问 gateway 页面和 WebSocket。 +- 通过 ClawManager proxy 访问时不要求用户手工粘贴 gateway token。 +- Pod drain 后拒绝新 create,已有 gateway 持续上报,DELETE 后释放端口。 +- Pod 重启后不误报旧 gateway running。 +- instance agent register/heartbeat/state/commands/skills 全链路可用。 +- 日志中没有 token 和 API key。 + +## 16. Do / Don't + +| Do | Don't | +| --- | --- | +| 使用 Kubernetes Service DNS 访问 ClawManager backend | 写死外部 NodePort、`172.16.1.12` 或浏览器 HTTPS 入口 | +| `POST /v1/gateways` 快速返回 `starting` | 在 HTTP handler 中阻塞等待前台进程退出 | +| 以 `instance_id + generation` 幂等 | 重试时重复启动多个 gateway | +| 以真实进程和健康检查决定 `running` | 只读旧状态文件就上报 `running` | +| JSON merge runtime 配置 | 覆盖用户整个配置文件 | +| 只转发白名单 env | 把用户请求里的全部 env 原样传给进程 | +| trusted proxy 模式下通过 ClawManager 自动进入 | 要求用户手工粘贴 gateway token | +| 检测并上报 clock skew | 忽略时间导致签名和 WebSocket 鉴权问题反复出现 | +| DELETE gateway 不删除 workspace | 清理进程时顺手删除用户数据 | + +## 17. 代码位置 + +后端关键实现: + +- Runtime Pod 上报接口:`backend/internal/handlers/runtime_agent_handler.go` +- ClawManager 调 agent 客户端:`backend/internal/services/runtime_agent_client.go` +- Runtime 调度器:`backend/internal/services/runtime_scheduler.go` +- Runtime Deployment 构建:`backend/internal/services/k8s/runtime_deployment_service.go` +- Runtime 常量与路径:`backend/internal/services/runtime_capacity.go` +- 实例 agent handler:`backend/internal/handlers/agent_handler.go` +- 实例 agent service:`backend/internal/services/instance_agent_service.go` +- 实例命令 service:`backend/internal/services/instance_command_service.go` +- 实例 runtime 状态 service:`backend/internal/services/instance_runtime_status_service.go` + +相关文档: + +- `docs/clawmanager-agent-v2-contract.md` +- `docs/runtime-agent-integration-guide.md` +- `docs/hermes-lite-pro-agent-development.md` +- `docs/hermes-runtime-agent-development.md` +- `docs/skill-content-md5-spec.md` diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..f4a95d5 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,109 @@ +# Deployment Guide + +ClawManager is packaged as a Kubernetes-first platform. This guide is the operational entry point for deploying the control plane, locating the relevant manifests in the repository, and understanding which services are expected to come up in a working environment. + +## Deployment Paths + +Choose the deployment path that matches your environment: + +- K3s single-node HostPath: [`deployments/k3s/single-node/clawmanager.yaml`](../deployments/k3s/single-node/clawmanager.yaml) +- K3s multi-node CSI/RWX: [`deployments/k3s/cluster/clawmanager.yaml`](../deployments/k3s/cluster/clawmanager.yaml) +- Kubernetes single-node HostPath: [`deployments/k8s/single-node/clawmanager.yaml`](../deployments/k8s/single-node/clawmanager.yaml) +- Kubernetes multi-node CSI/RWX: [`deployments/k8s/cluster/clawmanager.yaml`](../deployments/k8s/cluster/clawmanager.yaml) +- End-to-end first-use walkthrough: [User Guide](./use_guide_en.md) + +The cluster profile is validated with Longhorn as the example CSI implementation. It uses `longhorn` for RWO control-plane and instance volumes, and `longhorn-rwx` for the RWX workspace volume. These names are not a project requirement; replace them with compatible StorageClasses for your storage provider. + +## What Gets Deployed + +- ClawManager frontend and backend +- MySQL for application state +- MinIO for object storage-backed features +- `skill-scanner` for skill analysis workflows +- Kubernetes Services used for portal, gateway, and supporting traffic paths + +## Repository Entry Points + +- Kubernetes single-node manifest: [`deployments/k8s/single-node/clawmanager.yaml`](../deployments/k8s/single-node/clawmanager.yaml) +- Kubernetes cluster manifest: [`deployments/k8s/cluster/clawmanager.yaml`](../deployments/k8s/cluster/clawmanager.yaml) +- K3s single-node manifest: [`deployments/k3s/single-node/clawmanager.yaml`](../deployments/k3s/single-node/clawmanager.yaml) +- K3s cluster manifest: [`deployments/k3s/cluster/clawmanager.yaml`](../deployments/k3s/cluster/clawmanager.yaml) +- Container startup script: [`deployments/container/start.sh`](../deployments/container/start.sh) +- Nginx config: [`deployments/nginx/nginx.conf`](../deployments/nginx/nginx.conf) + +## Deployment Workflow + +1. Choose the Kubernetes distribution: `k3s` or `k8s`. +2. Choose the storage profile: `single-node` or `cluster`. +3. Check the storage prerequisites for that profile. +4. Review the bundled manifest and adjust secrets, images, StorageClass names, and ingress exposure for your environment. +5. Deploy the platform components into the cluster. +6. Wait for the core services to become ready. +7. Validate frontend access, AI Gateway management pages, Security Center connectivity, and runtime creation flows. + +Single-node example: + +```bash +kubectl get nodes +kubectl label node clawmanager.io/storage-node=true --overwrite +kubectl apply -f deployments/k8s/single-node/clawmanager.yaml +kubectl get pvc -n clawmanager-system +kubectl get pods -n clawmanager-system +``` + +Cluster example: + +```bash +kubectl get storageclass longhorn longhorn-rwx +kubectl apply -f deployments/k8s/cluster/clawmanager.yaml +kubectl get pvc -n clawmanager-system +kubectl get pods -n clawmanager-system +``` + +## Storage Profiles + +### Single-Node + +The `single-node` profile is the official HostPath validation path. Label exactly one node with `clawmanager.io/storage-node=true` before installation. The manifest pins HostPath PVs through node affinity and runs `clawmanager-app` as a single replica. MySQL, Redis, MinIO, workspace, and runtime data are all backed by persistent volumes; durable data must not use `emptyDir`. + +### Cluster + +The `cluster` profile is the official multi-node CSI/RWX validation path. The bundled manifest uses Longhorn as an example only: + +- `longhorn`: RWO MySQL, Redis, MinIO, and instance volumes +- `longhorn-rwx`: RWX workspace volume shared by ClawManager and runtime Pods + +Set these environment variables in the ClawManager app deployment when replacing the storage provider: + +- `CLAWMANAGER_STORAGE_PROFILE=cluster` +- `K8S_HOSTPATH_FALLBACK_ENABLED=false` +- `K8S_PVC_BIND_TIMEOUT=2m` +- `K8S_CONTROL_PLANE_STORAGE_CLASS=` +- `K8S_INSTANCE_STORAGE_CLASS=` +- `K8S_WORKSPACE_STORAGE_CLASS=` +- `K8S_WORKSPACE_ACCESS_MODE=ReadWriteMany` + +Unsupported combinations: + +- multi-node HostPath as a production or shared workspace strategy +- `local-path` or other node-local storage pretending to provide RWX across nodes +- cluster-internal Service DNS such as `workspace-store.clawmanager-system.svc.cluster.local` as an NFS server for kubelet-mounted PVs +- durable MySQL, Redis, MinIO, workspace, or object data on `emptyDir` +- cluster profile with implicit HostPath fallback + +## Operational Notes + +- ClawManager is designed around in-cluster services and platform-mediated access rather than direct pod exposure. +- Resource Management features depend on object storage and `skill-scanner` being available. +- Runtime workspace `.openclaw` and `.hermes` archive import/export size is controlled by `CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB`. The default is `500` MiB; set the env var on the ClawManager app deployment when a larger or smaller limit is needed. +- For install issues, collect `kubectl get storageclass`, `kubectl get pvc -n clawmanager-system`, `kubectl get pods -n clawmanager-system`, `kubectl get events -n clawmanager-system --sort-by=.lastTimestamp`, and `kubectl describe pvc -n clawmanager-system ` output before filing an issue. +- Production environments should review images, credentials, TLS, persistence, and networking policies before rollout. + +## Related Guides + +- [Admin and User Guide](./admin-user-guide.md) +- [Agent Control Plane Guide](./agent-control-plane.md) +- [AI Gateway Guide](./aigateway.md) +- [Security / Skill Scanner Guide](./security-skill-scanner.md) +- [Resource Management Guide](./resource-management.md) +- [Developer Guide](./developer-guide.md) diff --git a/docs/developer-guide.md b/docs/developer-guide.md new file mode 100644 index 0000000..f31b569 --- /dev/null +++ b/docs/developer-guide.md @@ -0,0 +1,30 @@ +# Developer Guide + +This guide is the codebase orientation page for contributors. ClawManager spans frontend, backend, deployment assets, and supporting product documentation, so the fastest way to get productive is to start from the subsystem you want to change. + +## Repository Map + +- `frontend/`: React application, admin surfaces, portal views, and product UI +- `backend/`: Go services, handlers, repositories, migrations, and platform logic +- `deployments/`: Kubernetes manifests, container bootstrap, and nginx config +- `docs/`: product-facing guides and screenshots + +## Suggested Entry Points + +- AI governance work: [`docs/aigateway.md`](./aigateway.md) +- runtime orchestration work: [Agent Control Plane Guide](./agent-control-plane.md) +- reusable resource workflows: [Resource Management Guide](./resource-management.md) +- security scanning work: [Security / Skill Scanner Guide](./security-skill-scanner.md) + +## Common Areas of Change + +- frontend pages and navigation for product surfaces such as AI Gateway, Security Center, and Config Center +- backend services for agents, commands, resources, and scanning +- migrations and repository logic when new control-plane state is introduced +- deployment manifests when platform components or images change + +## Related Guides + +- [Deployment Guide](./deployment.md) +- [Admin and User Guide](./admin-user-guide.md) +- [AI Gateway Guide](./aigateway.md) diff --git a/docs/hermes-lite-pro-agent-development.md b/docs/hermes-lite-pro-agent-development.md new file mode 100644 index 0000000..82ef047 --- /dev/null +++ b/docs/hermes-lite-pro-agent-development.md @@ -0,0 +1,227 @@ +# Hermes Lite / Pro Agent 开发说明 + +本文写给 Hermes agent 侧开发者,用来区分 ClawManager 中 Hermes Lite 和 Hermes Pro 两种形态。先读本文,再分别参考: + +- Lite: `docs/agent-runtime-development-spec.md` 和 `docs/clawmanager-agent-v2-contract.md` +- Pro: `docs/hermes-runtime-agent-development.md` 和 `docs/agent-control-plane.md` + +一句话原则: + +- **Hermes Lite 是 Runtime Pod Agent V2**:一个共享 `hermes-runtime` Pod 内只有一个 pod-level agent,它负责创建、删除和上报多个 Hermes gateway 子进程。 +- **Hermes Pro 是 Instance Agent Control Plane**:每个 Hermes Pro 实例有自己的桌面容器和 instance-level agent,它只负责这个实例自己的状态、命令、skill 和 metrics。 + +不要只根据镜像名或 `hermes` 这个 runtime 类型判断开发目标。Hermes Lite 和 Hermes Pro 都叫 Hermes,但 agent 协议、进程模型、目录和端口完全不同。 + +## 1. 模式对照 + +| 项目 | Hermes Lite | Hermes Pro | +| --- | --- | --- | +| ClawManager mode | `lite` | `pro` | +| Runtime backend | `gateway` | `desktop` | +| Kubernetes 资源 | 共享 Hermes runtime Deployment/Pod | 每个实例一个专属 Deployment + Service | +| agent 定位 | Pod 级 gateway 管理器 | 实例级状态和命令 agent | +| agent 入站端口 | 必须监听 `0.0.0.0:19090` | 不需要实现 runtime-pod 控制端口 | +| ClawManager 调 agent | `GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{id}`、`POST /v1/drain` | 不适用 | +| agent 上报 ClawManager | `/api/v1/runtime-agent/*` | `/api/v1/agent/*` | +| 用户实例进程 | 每个 Lite 实例是一个 gateway 子进程 | 整个桌面容器就是实例 | +| 用户访问端口 | agent 分配 `20000-20099` 中的 gateway 端口 | Webtop/KasmVNC `3001` | +| workspace | `/workspaces/hermes/user-{user_id}/instance-{instance_id}` | `/config/.hermes` | +| 典型实现参考 | 按 OpenClaw Lite 的 Runtime Pod Agent V2 做 | 按当前 Hermes Webtop guide 做 | + +## 2. 如何判断当前该跑哪种 agent + +建议 Hermes 镜像里可以放同一个 `hermes-agent` 二进制,但必须拆成两个清晰入口: + +```bash +hermes-agent runtime-pod # Lite +hermes-agent instance # Pro +``` + +也可以自动检测环境变量: + +| 检测条件 | 应进入的模式 | +| --- | --- | +| `CLAWMANAGER_RUNTIME_TYPE=hermes` 且存在 `CLAWMANAGER_AGENT_PORT` 或 `RUNTIME_AGENT_PUBLIC_PORT` | Lite runtime-pod agent | +| `CLAWMANAGER_AGENT_ENABLED=true` 且存在 `CLAWMANAGER_AGENT_INSTANCE_ID` 和 `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | Pro instance agent | + +如果两组变量同时存在,优先按明确的启动参数决定;没有启动参数时建议拒绝启动并打印清晰错误,避免一个进程同时承担两套协议。 + +## 3. Hermes Lite 应该怎么开发 + +Hermes Lite 要仿照当前 OpenClaw Lite 做。差异只有 runtime 名称、Hermes gateway 启动命令、Hermes 配置文件位置和健康检查细节。 + +Lite agent 是共享 Pod 内的控制进程,不是某个用户实例内部的 agent。它启动后必须: + +- 监听 `0.0.0.0:19090`。 +- 校验 `X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN}`。 +- 实现 `GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{gateway_id}`、`POST /v1/drain`。 +- 向 `${CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/*` 注册、heartbeat、metrics、gateway 状态和 skill inventory。 +- 管理本 Pod 内多个 Hermes gateway 子进程,端口默认从 `20000-20099` 分配。 +- 按 `instance_id + generation` 做幂等创建,重复请求不能启动多个进程。 +- `POST /v1/gateways` 必须快速返回 `starting`,后台继续启动和健康检查。 +- gateway 健康后再上报 `running`;缺少 LLM token、workspace 越权、端口冲突或 Hermes 配置失败时上报 `error`。 +- 每个实例使用独立 workspace,不能共用一个全局 `/config/.hermes`。 + +Lite runtime Pod 会收到这些环境变量: + +| 变量 | 用途 | +| --- | --- | +| `CLAWMANAGER_RUNTIME_TYPE=hermes` | runtime 类型,注册和上报都用 `hermes` | +| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址 | +| `CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME` | 当前 runtime Deployment 名 | +| `CLAWMANAGER_RUNTIME_IMAGE_REF` | 当前 runtime 镜像 | +| `CLAWMANAGER_AGENT_PORT=19090` | agent 控制端口 | +| `CLAWMANAGER_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用 | +| `CLAWMANAGER_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用 | +| `RUNTIME_WORKSPACE_ROOT=/workspaces` | workspace 根目录 | +| `RUNTIME_AGENT_LISTEN_ADDR=0.0.0.0:19090` | agent 监听地址 | +| `RUNTIME_AGENT_PUBLIC_PORT=19090` | agent 对 ClawManager 暴露的端口 | +| `RUNTIME_GATEWAY_PORT_START` / `RUNTIME_GATEWAY_PORT_END` | gateway 端口范围 | +| `POD_NAME` / `POD_NAMESPACE` / `POD_IP` / `NODE_NAME` | Pod 身份和调度信息 | +| `CLAWMANAGER_TRUSTED_PROXY_CIDRS` | ClawManager 反代来源网段,用于 Hermes trusted proxy | + +ClawManager 创建 Lite 实例时会调用 runtime-pod agent: + +```http +POST /v1/gateways +X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN} +Content-Type: application/json +``` + +```json +{ + "instance_id": 123, + "user_id": 45, + "generation": 7, + "agent_type": "hermes", + "workspace_path": "/workspaces/hermes/user-45/instance-123", + "gateway_port_start": 20000, + "gateway_port_end": 20099, + "uid": 200123, + "gid": 200123, + "environment": { + "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm", + "CLAWMANAGER_LLM_API_KEY": "instance-token", + "CLAWMANAGER_LLM_MODEL": "[{\"id\":\"gpt-4.1\"}]", + "CLAWMANAGER_INSTANCE_TOKEN": "instance-token", + "OPENAI_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm", + "OPENAI_API_KEY": "instance-token" + } +} +``` + +Lite agent 的建议处理流程: + +1. 校验控制 token、`agent_type=hermes`、`workspace_path` 必须在 `/workspaces/hermes/` 下。 +2. 用 `instance_id + generation` 查本地状态;如果已经创建过,返回同一个 gateway 元数据。 +3. 创建 workspace,并把 owner 设置为请求里的 `uid/gid`。 +4. 分配未占用 gateway 端口,记录端口、pid、workspace、generation。 +5. 把请求里的 `environment` 白名单变量传给 Hermes gateway 子进程。 +6. 写入 Hermes gateway 需要的 LLM、workspace、trusted proxy 和 instance token 配置。 +7. 后台启动 Hermes gateway,例如 `hermes gateway run --port ${port}`,实际命令由 Hermes 项目确定。 +8. 立即返回 `status=starting`。 +9. 后台轮询健康检查,成功后上报 `/api/v1/runtime-agent/gateways/report` 为 `running`。 +10. 删除时停止对应进程组并释放端口,但不要删除 workspace。 + +Lite 下浏览器访问链路是: + +```text +Browser -> ClawManager -> Runtime Pod HTTP gateway +``` + +ClawManager 会向 Lite runtime gateway 代理请求注入实例 token。Hermes gateway 不应该要求用户手工粘贴 gateway token;它需要信任来自 ClawManager 的代理来源,并正确处理 `Authorization`、`X-Forwarded-*` 和内部 base URL。 + +## 4. Hermes Pro 应该怎么开发 + +Hermes Pro 继续按现有 Hermes Webtop 文档实现。Pro 是专属桌面实例,不需要实现 Lite 的 `/v1/gateways` 协议,也不需要在一个 Pod 里管理多个用户实例。 + +Pro image 的固定约定: + +- 基于 Webtop/KasmVNC。 +- 对外服务端口是 `3001`。 +- 持久化目录是 `/config/.hermes`。 +- ClawManager 会把 `SUBFOLDER` 设置为 `/api/v1/instances/{instance_id}/proxy/`。 +- agent 作为 s6 longrun 或等价守护进程运行在这个实例容器里。 + +Pro instance agent 会收到这些环境变量: + +| 变量 | 用途 | +| --- | --- | +| `CLAWMANAGER_AGENT_ENABLED=true` | 启用实例 agent | +| `CLAWMANAGER_AGENT_BASE_URL` | ClawManager API base URL,不带 `/api/v1/agent` 后缀 | +| `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | 首次注册使用的一次性 token | +| `CLAWMANAGER_AGENT_INSTANCE_ID` | 当前实例 ID | +| `CLAWMANAGER_AGENT_PROTOCOL_VERSION=v1` | instance agent 协议版本 | +| `CLAWMANAGER_AGENT_PERSISTENT_DIR=/config/.hermes` | Hermes 持久化目录 | +| `CLAWMANAGER_AGENT_DISK_LIMIT_BYTES` | 实例磁盘配额 | + +Pro agent 的职责: + +- 读取 `CLAWMANAGER_AGENT_*` 变量。 +- 向 `${CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/register` 注册。 +- 使用 bootstrap 后返回的 session token 持续 heartbeat。 +- 上报 runtime state、health、system metrics、config revision 和 skill inventory。 +- 轮询并执行 ClawManager 下发的 instance command。 +- 管理 `/config/.hermes/skills` 里的 skill 下载、安装、上传和 inventory。 +- 所有状态都只代表这个 Pro 实例自身。 + +Pro agent 不应该实现: + +- `POST /v1/gateways` +- gateway 端口池 +- 多实例 workspace 隔离 +- `/api/v1/runtime-agent/*` runtime pod 上报 +- Lite 的 drain 和 gateway 迁移逻辑 + +## 5. OpenClaw 对照开发法 + +如果你已经看过 OpenClaw 的实现,可以这样映射: + +| OpenClaw 概念 | Hermes Lite 应替换为 | +| --- | --- | +| `CLAWMANAGER_RUNTIME_TYPE=openclaw` | `CLAWMANAGER_RUNTIME_TYPE=hermes` | +| `agent_type=openclaw` | `agent_type=hermes` | +| `/workspaces/openclaw/user-{uid}/instance-{id}` | `/workspaces/hermes/user-{uid}/instance-{id}` | +| OpenClaw gateway 启动命令 | Hermes gateway 启动命令 | +| OpenClaw workspace 配置文件 | Hermes workspace 内的实例级配置文件 | +| OpenClaw 健康检查 | Hermes gateway 健康检查 | + +不要把 OpenClaw Pro/Desktop 的 agent 逻辑直接套到 Hermes Lite。Lite 的关键不是“启动一个桌面”,而是“在共享 Pod 中稳定管理多个 gateway 子进程”。 + +## 6. 验收清单 + +Hermes Lite 验收: + +- runtime pod agent 可以注册为 `runtime_type=hermes`。 +- `GET /v1/health` 返回 ready 状态和真实容量。 +- 创建 100 个 Hermes gateway 时端口唯一,状态最终变为 `running`。 +- 重复 `POST /v1/gateways` 不会启动重复进程。 +- 删除 gateway 会停止进程并释放端口,不会删除 workspace。 +- `POST /v1/drain` 后拒绝新 create,已有 gateway 持续上报。 +- Pod 重启后不会把不存在的旧 gateway 误报为 `running`。 +- 缺少 LLM token 时 gateway 不报告 `running`,错误信息明确。 +- 通过 ClawManager proxy 访问 Hermes Lite 不需要用户手工粘贴 token。 +- 每个 Lite 实例的 skill inventory、配置和 workspace 互相隔离。 + +Hermes Pro 验收: + +- Pro 实例的 Webtop 端口 `3001` 可通过 ClawManager proxy 访问。 +- `/config/.hermes` 持久化,实例重启后数据仍在。 +- instance agent 能注册、heartbeat、上报 state 和 metrics。 +- instance command 能 poll、start、finish 并返回失败原因。 +- `/config/.hermes/skills` 的 skill 下载、安装、上传和 inventory 可用。 +- Pro agent 不监听或实现 `/v1/gateways`。 + +## 7. 常见错误 + +| 错误 | 正确做法 | +| --- | --- | +| 只实现 Pro instance agent,然后期望 Lite 可用 | Lite 必须实现 Runtime Pod Agent V2 | +| Lite agent 监听 `3001` | Lite agent 监听 `19090`,gateway 子进程监听 `20000-20099` | +| Lite 所有实例共用 `/config/.hermes` | Lite 每个实例使用自己的 `/workspaces/hermes/user-{uid}/instance-{id}` | +| `POST /v1/gateways` 阻塞到 Hermes gateway 完全启动 | 快速返回 `starting`,后台健康检查后上报 `running` | +| 重试 create 时启动多个 gateway | 按 `instance_id + generation` 幂等 | +| 把外部 HTTPS/NodePort 写进 runtime 内部配置 | runtime 内部用 ClawManager backend service DNS | +| 要求用户粘贴 gateway token | 使用 ClawManager 注入的 instance token 和 trusted proxy | +| Lite 上报 `/api/v1/agent/*` 当作生命周期依据 | Lite 生命周期必须上报 `/api/v1/runtime-agent/*` | +| Pro 实现 `/v1/gateways` 和端口池 | Pro 只实现 instance-level `/api/v1/agent/*` | diff --git a/docs/hermes-runtime-agent-development.md b/docs/hermes-runtime-agent-development.md new file mode 100644 index 0000000..25ce32c --- /dev/null +++ b/docs/hermes-runtime-agent-development.md @@ -0,0 +1,828 @@ +# Hermes Runtime Image and Agent Development Guide + +This guide is for Hermes image and agent developers. It explains how to build a ClawManager-managed Hermes runtime image on top of Webtop, and how the embedded Hermes agent should integrate with ClawManager Agent Control Plane so Hermes can behave like OpenClaw: report live runtime status, report health and system metrics, sync skill inventory, upload skill packages, and poll runtime commands. + +> Mode note: this document describes the Hermes Pro Webtop/instance-agent path. For Hermes Lite, implement Runtime Pod Agent V2 instead. See [Hermes Lite / Pro Agent 开发说明](./hermes-lite-pro-agent-development.md). + +## Goals + +A Hermes image must satisfy two layers of requirements: + +- Desktop access layer: keep the Webtop/KasmVNC runtime model. ClawManager accesses the desktop through the instance Service on port `3001`. +- Runtime agent layer: run a long-lived Hermes agent inside the image. The agent registers with ClawManager, sends heartbeats, reports state, syncs skills, and executes platform-issued commands. + +Current Hermes runtime defaults in ClawManager: + +- Port: `3001` +- PVC mount path: `/config` +- Persistent directory: `/config/.hermes` +- Default title: `Hermes Runtime` +- Proxy path: ClawManager rewrites `SUBFOLDER` to `/api/v1/instances/{instance_id}/proxy/` when the instance is created. + +Do not change the port, PVC mount path, or persistent directory in the image. ClawManager mounts the instance PVC at `/config` so Webtop desktop files such as `/config/Desktop` and Hermes runtime files under `/config/.hermes` persist together. + +## Image Build Requirements + +Use a LinuxServer Webtop image as the base image, for example: + +```dockerfile +FROM lscr.io/linuxserver/webtop:ubuntu-xfce + +USER root + +# 1. Install Hermes runtime dependencies. +# Keep this as an example. The Hermes project should own the real install steps. +# RUN apt-get update && apt-get install -y ... && rm -rf /var/lib/apt/lists/* + +# 2. Install Hermes itself. +# COPY hermes /opt/hermes + +# 3. Install the ClawManager Hermes agent. +COPY hermes-agent /usr/local/bin/hermes-agent +RUN chmod +x /usr/local/bin/hermes-agent + +# 4. Register an s6 longrun service so the agent starts with the Webtop container. +COPY root/ / + +ENV TITLE="Hermes Runtime" +ENV SUBFOLDER="/" + +EXPOSE 3001 +``` + +Webtop uses s6 overlay for process supervision. The Hermes agent can run as a longrun service: + +```text +root/ + etc/ + s6-overlay/ + s6-rc.d/ + hermes-agent/ + type + run + user/ + contents.d/ + hermes-agent +``` + +`type`: + +```text +longrun +``` + +`run`: + +```bash +#!/usr/bin/with-contenv bash +set -euo pipefail + +if [ "${CLAWMANAGER_AGENT_ENABLED:-false}" != "true" ]; then + echo "ClawManager Hermes agent disabled" + sleep infinity +fi + +exec /usr/local/bin/hermes-agent +``` + +The Hermes agent must not bind to `3001`. Port `3001` belongs to the Webtop desktop entrypoint. The agent only needs outbound HTTP access to ClawManager. + +## Environment Variables Injected by ClawManager + +The Hermes image must read configuration from environment variables. Do not hardcode ClawManager URLs, instance IDs, tokens, or persistent paths into the image. + +Base Webtop variables: + +| Variable | Description | +| --- | --- | +| `TITLE` | Desktop title. Hermes defaults to `Hermes Runtime`. | +| `SUBFOLDER` | Reverse proxy subpath. ClawManager rewrites it at runtime. | +| `HTTP_PROXY` / `HTTPS_PROXY` | Injected when platform egress proxy is enabled. | +| `NO_PROXY` | Platform-internal services and localhost are added automatically. | + +Agent Control Plane variables: + +| Variable | Description | +| --- | --- | +| `CLAWMANAGER_AGENT_ENABLED` | Start the Hermes agent when set to `true`. | +| `CLAWMANAGER_AGENT_BASE_URL` | ClawManager API base URL, without the `/api/v1/agent` suffix. | +| `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | One-time bootstrap token used for initial registration. | +| `CLAWMANAGER_AGENT_INSTANCE_ID` | Current ClawManager instance ID. | +| `CLAWMANAGER_AGENT_PROTOCOL_VERSION` | Current protocol version, `v1`. | +| `CLAWMANAGER_AGENT_PERSISTENT_DIR` | Persistent directory. Hermes uses `/config/.hermes`. | +| `CLAWMANAGER_AGENT_DISK_LIMIT_BYTES` | Instance disk quota in bytes. | + +Runtime resource bootstrap variables: + +| Variable | Description | +| --- | --- | +| `CLAWMANAGER_HERMES_CHANNELS_JSON` | Channel configuration injected at instance creation time. | +| `CLAWMANAGER_HERMES_SKILLS_JSON` | Skill configuration injected at instance creation time. | +| `CLAWMANAGER_HERMES_BOOTSTRAP_MANIFEST_JSON` | Manifest for the current bootstrap payload. | +| `CLAWMANAGER_RUNTIME_CHANNELS_JSON` | Generic runtime alias for channel configuration. | +| `CLAWMANAGER_RUNTIME_SKILLS_JSON` | Generic runtime alias for skill configuration. | +| `CLAWMANAGER_RUNTIME_BOOTSTRAP_MANIFEST_JSON` | Generic runtime alias for the bootstrap manifest. | + +For compatibility with the existing OpenClaw resource center, ClawManager also keeps the original `CLAWMANAGER_OPENCLAW_*` variables. Hermes agents should prefer `CLAWMANAGER_HERMES_*`, then fall back to `CLAWMANAGER_RUNTIME_*`, and finally to `CLAWMANAGER_OPENCLAW_*`. + +Platform-side note: some Agent Control Plane payload fields still use historical OpenClaw names. Until those fields are renamed into generic runtime fields, Hermes agents should reuse compatible fields such as `openclaw_status`, `openclaw_pid`, and `openclaw_version` to describe Hermes runtime state. + +## Channel and Skill Bootstrap Consumption + +Hermes agents must handle two kinds of injection: + +- Runtime bootstrap injection: channels, configuration skills, session templates, agents, scheduled tasks, and related resources selected at instance creation time are injected through environment variables. +- Platform skill installation: reusable platform skills selected at instance creation time are first attached to the instance. ClawManager then sends an `install_skill` command. The agent must download and install the skill package. + +### Read Order + +At startup, read bootstrap payloads in this priority order. Use the first non-empty value: + +| Resource | Preferred variable | Fallback variables | +| --- | --- | --- | +| Manifest | `CLAWMANAGER_HERMES_BOOTSTRAP_MANIFEST_JSON` | `CLAWMANAGER_RUNTIME_BOOTSTRAP_MANIFEST_JSON`, `CLAWMANAGER_OPENCLAW_BOOTSTRAP_MANIFEST_JSON` | +| Channels | `CLAWMANAGER_HERMES_CHANNELS_JSON` | `CLAWMANAGER_RUNTIME_CHANNELS_JSON`, `CLAWMANAGER_OPENCLAW_CHANNELS_JSON` | +| Config Skills | `CLAWMANAGER_HERMES_SKILLS_JSON` | `CLAWMANAGER_RUNTIME_SKILLS_JSON`, `CLAWMANAGER_OPENCLAW_SKILLS_JSON` | +| Session Templates | `CLAWMANAGER_HERMES_SESSION_TEMPLATES_JSON` | `CLAWMANAGER_RUNTIME_SESSION_TEMPLATES_JSON`, `CLAWMANAGER_OPENCLAW_SESSION_TEMPLATES_JSON` | +| Agents | `CLAWMANAGER_HERMES_AGENTS_JSON` | `CLAWMANAGER_RUNTIME_AGENTS_JSON`, `CLAWMANAGER_OPENCLAW_AGENTS_JSON` | +| Scheduled Tasks | `CLAWMANAGER_HERMES_SCHEDULED_TASKS_JSON` | `CLAWMANAGER_RUNTIME_SCHEDULED_TASKS_JSON`, `CLAWMANAGER_OPENCLAW_SCHEDULED_TASKS_JSON` | + +If a variable is missing or empty, treat it as an empty config. Do not fail agent startup for missing optional bootstrap payloads. If a variable exists but contains invalid JSON, log a clear error and report `health.bootstrap_config` or `health.config_loader` as `error` in the next state report. + +Recommended local bootstrap state: + +```text +/config/.hermes/hermes-agent/bootstrap/ + manifest.json + channels.json + skills.json + applied-state.json +``` + +Do not print channel tokens, secrets, webhooks, bootstrap tokens, session tokens, or AI Gateway API keys in logs. + +### Channel Injection + +`CLAWMANAGER_HERMES_CHANNELS_JSON` is a JSON object keyed by resource key. Example: + +```json +{ + "feishu": { + "enabled": true, + "domain": "feishu", + "defaultAccount": "main", + "accounts": { + "main": { + "appId": "cli_xxx", + "appSecret": "secret", + "enabled": true + } + }, + "requireMention": true + }, + "telegram": { + "enabled": true, + "botToken": "123456:xxx", + "dmPolicy": "open", + "allowFrom": ["*"] + } +} +``` + +Hermes agent behavior: + +1. Use the top-level key as the runtime channel ID, for example `feishu`, `telegram`, `slack`, or `dingtalk-connector`. Resource-specific Feishu/Lark keys such as `feishu-ops` must be folded into the `feishu.accounts` map before injection. +2. Skip channels with `enabled=false`, but keep their config on disk so future updates can re-enable them. +3. Convert channel config into Hermes-native notification or messaging configuration. A reasonable default path is `/config/.hermes/channels.json`, unless Hermes has a native config location. +4. Preserve unknown fields so future ClawManager extensions are not lost. +5. If Hermes does not support a channel type yet, mark that channel as unsupported in `health.channels` and keep registration and heartbeats running. + +### Config Skill Injection + +`CLAWMANAGER_HERMES_SKILLS_JSON` is a list of configuration resources. It is not a zip package. It is used to inject resource-center configuration skills into the runtime at instance creation time. Example: + +```json +{ + "schemaVersion": 1, + "items": [ + { + "id": 5, + "type": "skill", + "key": "support-bot", + "name": "Support Bot", + "version": 1, + "tags": ["skill"], + "content": { + "schemaVersion": 1, + "kind": "skill", + "format": "skill/custom@v1", + "dependsOn": [], + "config": { + "prompt": "help" + } + } + } + ] +} +``` + +Hermes agent behavior: + +1. Iterate over `items` and use `key` as the stable skill identifier. +2. Read `content.config` and translate it into executable Hermes skill configuration. +3. If Hermes stores skills as directories, write the generated config to `/config/.hermes/skills/{key}/skill.json`. Keep the raw `content` as well for debugging. +4. Calculate `content_md5` for the written skill directory and include it in the next `skills/inventory` report. +5. For skills generated from bootstrap config, inventory `source` should be `injected_by_clawmanager` or `bootstrap_config`. If `source` is `injected_by_clawmanager` and `skill_id` uses a platform external ID, use the format `skill-{id}`, for example `skill-5`. + +Config skills and platform skill package installation are separate paths: + +- `CLAWMANAGER_HERMES_SKILLS_JSON`: read and apply during startup. Do not wait for a command. +- `install_skill` command: download and install a platform-uploaded zip skill package at runtime. + +### Bootstrap Manifest + +`CLAWMANAGER_HERMES_BOOTSTRAP_MANIFEST_JSON` describes the payloads injected for the current bootstrap. Example: + +```json +{ + "schemaVersion": 1, + "mode": "manual", + "resources": [ + { "id": 5, "type": "skill", "key": "support-bot", "name": "Support Bot", "version": 1 } + ], + "payloads": [ + { "env": "CLAWMANAGER_HERMES_CHANNELS_JSON", "count": 1 }, + { "env": "CLAWMANAGER_HERMES_SKILLS_JSON", "count": 1 } + ] +} +``` + +The agent can use the manifest as an idempotency key. If the manifest hash has not changed, skip reapplying the same bootstrap payload. If it changes, reapply channel and config skill payloads, then send one state report and one full skill inventory report. + +### Skills Selected During Instance Creation + +When a user selects existing platform skills while creating a Hermes instance, ClawManager attaches those skills to the instance and creates `install_skill` commands. The Hermes agent must implement this command. Otherwise the UI selection only exists in platform records and the skill will not be installed inside the instance. + +Example `install_skill` payload: + +```json +{ + "skill_id": "skill-12", + "skill_version": "skill-version-34", + "target_name": "weather-tool", + "content_md5": "d41d8cd98f00b204e9800998ecf8427e" +} +``` + +Processing steps: + +1. Download the version package: + +```http +GET {CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/skills/versions/{skill_version}/download +Authorization: Bearer {session_token} +``` + +2. Validate the zip. Reject absolute paths, `../`, multiple top-level directories, and any path that would extract outside `/config/.hermes/skills`. +3. Extract to `/config/.hermes/skills/{target_name}`. Prefer extracting into a temp directory and then atomically replacing the target directory. +4. Recalculate directory `content_md5` and compare it with the command payload. Finish the command as `failed` if it does not match. +5. Send one `skills/inventory` report. For the installed skill, set `source` to `injected_by_clawmanager` and `skill_id` to the command payload `skill_id`. +6. Finish the command with `install_path`, `skill_id`, `skill_version`, and `content_md5` in `result`. + +## Agent Lifecycle + +Hermes agent startup flow: + +1. Read `CLAWMANAGER_AGENT_*` environment variables. +2. Read and apply runtime bootstrap payloads if present. +3. If no local session token is available, register with the bootstrap token. +4. Store the returned session token in `/config/.hermes/hermes-agent/session.json`. +5. Send heartbeats using the interval returned by the server. +6. Poll commands using the command poll interval returned by the server. If heartbeat returns `has_pending_command=true`, poll once immediately. +7. Periodically send full state reports and skill inventory reports. +8. If the session token expires or an API returns HTTP 401, register again with the bootstrap token. + +Recommended local agent state directory: + +```text +/config/.hermes/hermes-agent/ + session.json + state.json + logs/ + cache/ + bootstrap/ +``` + +## Registration + +Request: + +```http +POST {CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/register +Authorization: Bearer {CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN} +Content-Type: application/json +``` + +Body example: + +```json +{ + "instance_id": 123, + "agent_id": "hermes-123-main", + "agent_version": "0.1.0", + "protocol_version": "v1", + "capabilities": [ + "runtime.status", + "runtime.health", + "metrics.report", + "skills.inventory", + "skills.upload", + "commands.poll" + ], + "host_info": { + "runtime": "hermes", + "desktop_base": "webtop", + "persistent_dir": "/config/.hermes", + "port": 3001, + "arch": "amd64" + } +} +``` + +The response field `data.session_token` is the token for subsequent Agent API calls. Cache it locally, but never write it to logs. + +## Heartbeat + +Request: + +```http +POST {CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/heartbeat +Authorization: Bearer {session_token} +Content-Type: application/json +``` + +Body example: + +```json +{ + "agent_id": "hermes-123-main", + "timestamp": "2026-04-27T14:30:00Z", + "openclaw_status": "running", + "summary": { + "runtime": "hermes", + "hermes_status": "running", + "hermes_pid": 245, + "skill_count": 8, + "active_skill_count": 8, + "disk_used_bytes": 2147483648, + "disk_limit_bytes": 10737418240 + } +} +``` + +Compatibility requirements: + +- `openclaw_status` is still the platform compatibility field. Fill it with the Hermes main process status. +- Recommended status values: `starting`, `running`, `stopped`, `error`, `unknown`. +- Default heartbeat interval is roughly 15 seconds, but use `heartbeat_interval_seconds` from the registration response. +- ClawManager considers the agent online when heartbeat is received within 45 seconds, stale between 45 and 120 seconds, and offline after 120 seconds. + +## Full State Report + +Heartbeat is the lightweight online signal. Complete runtime status, system metrics, and health information are reported through state reports. + +Request: + +```http +POST {CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/state/report +Authorization: Bearer {session_token} +Content-Type: application/json +``` + +Body example: + +```json +{ + "agent_id": "hermes-123-main", + "reported_at": "2026-04-27T14:30:00Z", + "runtime": { + "openclaw_status": "running", + "openclaw_pid": 245, + "openclaw_version": "hermes-0.4.0" + }, + "system_info": { + "runtime": "hermes", + "os": "ubuntu", + "desktop_base": "webtop", + "sampled_at": "2026-04-27T14:30:00Z", + "cpu": { + "cores": 2, + "load": { + "1m": 0.64, + "5m": 0.52, + "15m": 0.40 + } + }, + "memory": { + "mem_total_bytes": 4294967296, + "mem_available_bytes": 2147483648 + }, + "disk": { + "mount_path": "/config/.hermes", + "root_total_bytes": 10737418240, + "root_free_bytes": 8589934592 + }, + "network": { + "interfaces": [ + { + "name": "eth0", + "status": "up", + "addresses": ["10.42.0.12"], + "rx_bytes": 123456789, + "tx_bytes": 98765432 + } + ] + } + }, + "health": { + "hermes_process": "ok", + "desktop": "ok", + "agent": "ok", + "metrics_collector": "ok", + "bootstrap_config": "ok", + "channels": "ok", + "metrics_sample_interval_seconds": 5, + "last_skill_scan_at": "2026-04-27T14:29:30Z" + } +} +``` + +Reporting guidance: + +- Heartbeat: send at the server-provided interval. +- State report: send once immediately after startup, then every 5 to 10 seconds when possible. +- Send an extra state report after Hermes main process state changes, skill inventory changes, bootstrap config changes, or command completion. + +## Metrics Reporting Contract + +ClawManager does not provide a separate CPU, memory, disk, or network metrics endpoint. Hermes agent must include every sample in the `system_info` field of the state report. The backend stores this JSON as-is, and the instance detail page reads the fields below to render recent trends. + +### Required Fields + +| Path | Type | Unit | Description | +| --- | --- | --- | --- | +| `system_info.sampled_at` | string | ISO 8601 UTC | Agent sampling time. | +| `system_info.cpu.cores` | number | cores | CPU cores available to the container. | +| `system_info.cpu.load.1m` | number | load average | 1-minute load average. | +| `system_info.cpu.load.5m` | number | load average | 5-minute load average. | +| `system_info.cpu.load.15m` | number | load average | 15-minute load average. | +| `system_info.memory.mem_total_bytes` | number | bytes | Container memory limit or system total memory. | +| `system_info.memory.mem_available_bytes` | number | bytes | Currently available memory. | +| `system_info.disk.root_total_bytes` | number | bytes | Total capacity of the filesystem containing `/config/.hermes`. | +| `system_info.disk.root_free_bytes` | number | bytes | Free capacity of the filesystem containing `/config/.hermes`. | +| `system_info.network.interfaces[].name` | string | none | Network interface name, for example `eth0`. | +| `system_info.network.interfaces[].status` | string | none | Suggested values: `up` or `down`. | +| `system_info.network.interfaces[].rx_bytes` | number | bytes | Monotonic received byte counter. | +| `system_info.network.interfaces[].tx_bytes` | number | bytes | Monotonic transmitted byte counter. | + +The frontend calculates CPU percentage as `load.1m / cores * 100`, capped to 0..100. The agent does not need to report `cpu_percent`. + +The frontend calculates memory percentage as `(mem_total_bytes - mem_available_bytes) / mem_total_bytes * 100`, and disk percentage as `(root_total_bytes - root_free_bytes) / root_total_bytes * 100`. + +Network rates are calculated by the frontend from adjacent `rx_bytes` and `tx_bytes` samples. Report monotonic counters, not instantaneous rates. If counters reset after a container restart, the frontend will resume calculation from the next valid sample. + +### Sampling Sources + +- CPU load: read the first three values from `/proc/loadavg`. +- CPU cores: prefer cgroup quota. For cgroup v2, read `/sys/fs/cgroup/cpu.max`. If there is no quota, fall back to `/proc/cpuinfo` or the language runtime. +- Memory: prefer cgroup memory limit and current usage. For cgroup v2, read `/sys/fs/cgroup/memory.max` and `/sys/fs/cgroup/memory.current`. Compute `mem_available_bytes = memory.max - memory.current`, floored at 0. If no cgroup limit exists, use `/proc/meminfo` `MemTotal` and `MemAvailable`. +- Disk: call `statvfs` on `/config/.hermes`. Keep the field names `root_total_bytes` and `root_free_bytes`, but interpret them as the filesystem containing the Hermes persistent directory. +- Network: read `/proc/net/dev`. Exclude `lo` by default and keep business interfaces such as `eth0`. + +### Reporting Frequency + +- Send one state report with complete `system_info` immediately after successful startup. +- During normal operation, sample and report every 5 seconds to match the instance detail page polling cadence. +- If overhead is a concern, 10 seconds is acceptable. Do not sample faster than every 2 seconds. +- When receiving `collect_system_info` or `health_check`, sample immediately, send a state report, then finish the command. + +### Command Result Guidance + +`collect_system_info` finish `result` can reuse the same snapshot: + +```json +{ + "agent_id": "hermes-123-main", + "status": "succeeded", + "finished_at": "2026-04-27T14:31:05Z", + "result": { + "sampled_at": "2026-04-27T14:31:05Z", + "system_info": { + "cpu": { + "cores": 2, + "load": { + "1m": 0.70, + "5m": 0.55, + "15m": 0.42 + } + }, + "memory": { + "mem_total_bytes": 4294967296, + "mem_available_bytes": 2013265920 + }, + "disk": { + "mount_path": "/config/.hermes", + "root_total_bytes": 10737418240, + "root_free_bytes": 8589934592 + }, + "network": { + "interfaces": [ + { + "name": "eth0", + "status": "up", + "rx_bytes": 124000000, + "tx_bytes": 99000000 + } + ] + } + } + }, + "error_message": "" +} +``` + +`health_check` finish `result` should include `health` and `system_info.sampled_at`. If sampling fails, still send available fields in the state report and set `health.metrics_collector` to `error` with a short `health.metrics_error` message. + +### Metrics Acceptance + +1. Hermes agent sends two consecutive state reports roughly 5 seconds apart. +2. `GET /api/v1/instances/{instance_id}/runtime` returns `data.runtime.system_info.cpu`, `memory`, `disk`, and `network`. +3. The ClawManager instance detail page starts showing CPU, Memory, Disk, and Network metrics within 10 seconds. +4. Creating network traffic or disk writes changes the corresponding trend in later samples. + +## Skill Inventory Sync + +Hermes agent must discover skills installed inside the instance and report inventory to ClawManager. + +Recommended skill root: + +```text +/config/.hermes/skills +``` + +Optional environment override: + +```text +HERMES_SKILL_DIRS=/config/.hermes/skills +``` + +Each skill should be managed as a directory. For every skill, calculate `content_md5` and report it: + +```http +POST {CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/skills/inventory +Authorization: Bearer {session_token} +Content-Type: application/json +``` + +Body example: + +```json +{ + "agent_id": "hermes-123-main", + "reported_at": "2026-04-27T14:30:00Z", + "mode": "full", + "trigger": "startup", + "skills": [ + { + "skill_id": "hermes-weather", + "skill_version": "1.2.0", + "identifier": "hermes-weather", + "install_path": "/config/.hermes/skills/hermes-weather", + "content_md5": "d41d8cd98f00b204e9800998ecf8427e", + "source": "discovered_in_instance", + "type": "hermes-skill", + "size_bytes": 20480, + "file_count": 12, + "metadata": { + "runtime": "hermes", + "manifest": "skill.json" + } + } + ] +} +``` + +`mode` semantics: + +- `full`: complete inventory. ClawManager marks instance skills missing from this report as removed. +- `incremental`: partial update. Only skills in this report are updated. + +Recommended behavior: + +- Send one `full` inventory after startup. +- Use file watching or periodic scanning for later `incremental` updates. +- When the platform sends `sync_skill_inventory` or `refresh_skill_inventory`, run a `full` scan. + +### content_md5 Calculation + +ClawManager `content_md5` is a skill directory content fingerprint, not a zip file MD5. The full algorithm is defined in [Skill Content MD5 Calculation Spec](skill-content-md5-spec.md). + +The most common mistake is top-level directory handling: + +- During inventory, calculate against the contents of `/config/.hermes/skills/{skill_name}`. +- During upload, the zip must contain one top-level directory named `{skill_name}/`. +- ClawManager strips the zip top-level `{skill_name}/` once before validation. +- Do not strip internal directories such as `src/`, `lib/`, or `dist/`. + +For example, if the local file is `/config/.hermes/skills/weather/src/main.py`, the relative path used for MD5 must be `src/main.py`, not `weather/src/main.py` and not `main.py`. + +The agent must use the same directory content and the same algorithm for inventory and `collect_skill_package` upload. Otherwise ClawManager will return `skill package md5 mismatch`. + +## Skill Package Upload + +When ClawManager finds a skill blob without object content, it sends a `collect_skill_package` command. Hermes agent should zip the corresponding skill directory and upload it. + +Request: + +```http +POST {CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/skills/upload +Authorization: Bearer {session_token} +Content-Type: multipart/form-data +``` + +Form fields: + +| Field | Description | +| --- | --- | +| `file` | Zip package. It must contain exactly one top-level skill directory. | +| `agent_id` | Current agent ID. | +| `skill_id` | Skill ID from inventory. | +| `skill_version` | Skill version from inventory. | +| `identifier` | Skill name or key. | +| `content_md5` | Directory fingerprint reported in inventory. | +| `source` | Usually `discovered_in_instance` or `injected_by_clawmanager`. | + +Zip structure example: + +```text +hermes-weather/ + skill.json + main.py + README.md +``` + +Do not upload multiple top-level directories. Do not put loose files at the zip root. ClawManager rejects both formats. + +## Command Polling and Execution + +Hermes agent polls commands with the session token: + +```http +GET {CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/commands/next +Authorization: Bearer {session_token} +``` + +If `data.command` is `null`, no command is pending. If a command exists, the agent must: + +1. Call the start endpoint. +2. Execute the command. +3. Call the finish endpoint with the result. +4. Always finish failed commands with `status=failed`. + +Start: + +```http +POST {CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/commands/{id}/start +Authorization: Bearer {session_token} +Content-Type: application/json +``` + +```json +{ + "agent_id": "hermes-123-main", + "started_at": "2026-04-27T14:31:00Z" +} +``` + +Finish: + +```http +POST {CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/commands/{id}/finish +Authorization: Bearer {session_token} +Content-Type: application/json +``` + +```json +{ + "agent_id": "hermes-123-main", + "status": "succeeded", + "finished_at": "2026-04-27T14:31:05Z", + "result": { + "message": "skill inventory refreshed", + "skill_count": 8 + }, + "error_message": "" +} +``` + +Current platform command types: + +- `collect_system_info` +- `health_check` +- `sync_skill_inventory` +- `refresh_skill_inventory` +- `collect_skill_package` +- `install_skill` +- `update_skill` +- `uninstall_skill` +- `remove_skill` +- `disable_skill` +- `quarantine_skill` +- `handle_skill_risk` +- `start_openclaw` +- `stop_openclaw` +- `restart_openclaw` +- `apply_config_revision` +- `reload_config` + +Minimum Hermes implementation: + +- `collect_system_info` +- `health_check` +- `sync_skill_inventory` +- `refresh_skill_inventory` +- `collect_skill_package` +- `install_skill` + +Commands containing `openclaw` are currently compatibility names. Hermes may ignore `start_openclaw`, `stop_openclaw`, and `restart_openclaw` until Hermes-specific or generic runtime commands are added. + +## Skill Installation and Version Download + +If Hermes supports platform-managed skill installation, command payloads may include a skill version identifier. The agent can download the zip package through: + +```http +GET {CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/skills/versions/{external_version_id}/download +Authorization: Bearer {session_token} +``` + +After downloading: + +1. Validate the zip path boundaries. +2. Extract it under `/config/.hermes/skills`. +3. Recalculate `content_md5`. +4. Send `skills/inventory`. +5. Finish the command with install path, skill ID, version, and `content_md5`. + +## Local Development + +Use the following variables to run the agent locally: + +```bash +export CLAWMANAGER_AGENT_ENABLED=true +export CLAWMANAGER_AGENT_BASE_URL=http://127.0.0.1:8080 +export CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN=agt_boot_xxx +export CLAWMANAGER_AGENT_INSTANCE_ID=123 +export CLAWMANAGER_AGENT_PROTOCOL_VERSION=v1 +export CLAWMANAGER_AGENT_PERSISTENT_DIR=/config/.hermes +export CLAWMANAGER_AGENT_DISK_LIMIT_BYTES=10737418240 +``` + +For local bootstrap testing, also set: + +```bash +export CLAWMANAGER_HERMES_CHANNELS_JSON='{}' +export CLAWMANAGER_HERMES_SKILLS_JSON='{"schemaVersion":1,"items":[]}' +export CLAWMANAGER_HERMES_BOOTSTRAP_MANIFEST_JSON='{"schemaVersion":1,"mode":"manual","payloads":[]}' +``` + +Never commit real tokens, channel secrets, Gateway API keys, or downloaded session tokens into images, repositories, or logs. At most, log a short token prefix and suffix for debugging. + +## Acceptance Checklist + +Before delivering a Hermes image, verify: + +- The Webtop desktop is reachable through the ClawManager instance proxy. +- `/config` is mounted and both `/config/Desktop` and `/config/.hermes` persist across restarts. +- The agent registers and starts heartbeat within 30 seconds. +- The instance detail page shows agent online, runtime running, and an updated last report time. +- CPU, memory, disk, and network metrics are visible and refresh continuously. +- Channel bootstrap payloads are applied or clearly reported as unsupported. +- Config skill bootstrap payloads are applied and included in inventory. +- Skill inventory syncs after changes under the Hermes skill directory. +- For discovered skills without stored object content, `collect_skill_package` causes the agent to upload a valid zip package. +- `content_md5` in inventory and package upload match the ClawManager specification. +- Command execution calls start and finish, including clear `error_message` on failure. +- Network interruption, ClawManager restart, or session expiration causes retry and re-registration. + +## Platform-Side Companion Checklist + +ClawManager must keep the following capabilities for Hermes to work end to end: + +- Inject `CLAWMANAGER_AGENT_*` variables for `hermes` during instance creation and start. +- Allow `hermes` instances to register with the Agent Control Plane. +- Inject `CLAWMANAGER_LLM_*` and OpenAI-compatible variables so Hermes can access models through ClawManager AI Gateway. +- Inject Hermes and generic runtime bootstrap variables for channels, skills, and related resources. +- Mount persistent storage at `/config`, while keeping Hermes runtime state under `/config/.hermes`. +- Support `.hermes` import and export. +- Keep compatibility fields such as `openclaw_status`, `openclaw_pid`, and `openclaw_version` until generic runtime fields are introduced. +- Add Hermes-specific runtime control commands, or generic `start_runtime`, `stop_runtime`, and `restart_runtime`, if runtime process control becomes required. diff --git a/docs/images/hermes.png b/docs/images/hermes.png new file mode 100644 index 0000000..dd48aa9 Binary files /dev/null and b/docs/images/hermes.png differ diff --git a/docs/images/openclaw.png b/docs/images/openclaw.png new file mode 100644 index 0000000..a8484df Binary files /dev/null and b/docs/images/openclaw.png differ diff --git a/docs/lite-mode-development-progress-2026-06-11.md b/docs/lite-mode-development-progress-2026-06-11.md new file mode 100644 index 0000000..859e8eb --- /dev/null +++ b/docs/lite-mode-development-progress-2026-06-11.md @@ -0,0 +1,98 @@ +# Lite 模式与 Share Link 开发进度汇报 + +> 汇报时间:2026-06-11 下午 + +## 技术实现概览 + +本次开发的核心是把实例形态从原来的运行时类型,抽象成用户可理解的 `Lite / Pro` 两种模式。 + +- `Lite`:使用共享 Runtime Pod,在 Pod 内为每个实例创建独立 gateway 进程,适合轻量、低资源成本的使用场景。 +- `Pro`:使用独立的 Kubernetes Deployment / Service / PVC,适合需要完整桌面运行环境和独立资源的场景。 + +后端以 `instance_mode` 作为产品层模式,内部再映射到不同 runtime backend: + +- `lite` -> `gateway` +- `pro` -> `desktop` + +这样前端、普通实例创建、Team 成员创建、管理端展示和外部访问都统一使用 Lite / Pro 语义;底层调度、代理、资源控制仍然按 gateway / desktop 执行。 + +Share Link 能力基于实例外部访问表实现,生成 `/s//` 形式的短链接。短链 code 只在创建时返回,数据库只保存 hash;访问时由 ClawManager 解析短链,再复用现有实例代理链路转发到对应 Lite gateway 或 Pro service。当前支持普通分享链接和密码访问,并支持固定时间、自定义时间和永久有效几类过期策略。 + +## Lite / Pro 架构图 + +```mermaid +flowchart LR + user["用户 / 外部访问者"] --> web["ClawManager Web"] + user --> shortLink["Share Link
/s/<code>/"] + + web --> api["ClawManager Backend API"] + shortLink --> publicAccess["短链解析与鉴权
Share Link / Password"] + publicAccess --> proxy["统一实例代理"] + api --> instanceService["Instance Service
instance_mode: lite / pro"] + instanceService --> mode{"实例模式"} + + mode -->|Lite| liteScheduler["Lite Scheduler
runtime_type = gateway"] + mode -->|Pro| proLifecycle["Pro Lifecycle
runtime_type = desktop"] + + liteScheduler --> runtimePod["共享 Runtime Pod"] + runtimePod --> runtimeAgent["Runtime Pod Agent V2"] + runtimeAgent --> gatewayA["Gateway 进程
Lite 实例 A"] + runtimeAgent --> gatewayB["Gateway 进程
Lite 实例 B"] + runtimeAgent --> gatewayN["Gateway 进程
Lite 实例 N"] + + proLifecycle --> deployment["专属 Deployment"] + deployment --> service["专属 Service"] + deployment --> pvc["专属 PVC"] + service --> desktop["Desktop Runtime
Pro 实例"] + + proxy -->|Lite| gatewayA + proxy -->|Lite| gatewayB + proxy -->|Pro| service + + team["Team 创建"] --> teamService["Team Service"] + teamService --> teamMemberMode{"成员实例模式"} + teamMemberMode -->|Lite 成员| liteScheduler + teamMemberMode -->|Pro 成员| proLifecycle + + aiGateway["AI Gateway / 审计 / 费用 / 风险"] --> attribution["实例归因
instance_id / instance_mode / runtime_type"] + gatewayA --> attribution + gatewayB --> attribution + desktop --> attribution +``` + +## 目标 + +- 新增 Lite 实例模式,让用户在创建普通实例时可以选择 Lite / Pro。 +- 保留 Pro 模式的完整桌面实例能力,同时让 Lite 模式复用共享运行时,降低资源占用和启动成本。 +- 将 Lite / Pro 模式贯通到 Team 场景,让 Team 成员实例也可以按角色选择 Lite 或 Pro。 +- 增加 Share Link 能力,支持将实例访问入口分享给外部用户。 +- 建立自动化测试覆盖,保证普通实例、Team 实例和分享访问能力可以持续回归。 + +## 进展 + +- 普通实例创建已经支持 Lite / Pro 模式选择。 +- Lite 实例创建时会进入共享 gateway runtime 链路,Pro 实例创建时会进入独立 desktop runtime 链路。 +- 前端创建页、实例详情页、实例列表和管理端已完成 Lite / Pro 的展示和交互区分。 +- Lite 模式下已简化资源配置流程,不再要求用户配置专属 CPU、内存、存储和 GPU;Pro 模式继续保留专属资源配置能力。 +- Team 创建流程已经支持为成员选择 Lite / Pro 实例模式。 +- Team 成员实例已能按选择的模式创建,并保留后续接入 Team 协作链路所需的基础配置。 +- Share Link 已完成基础能力,包括短链接分享、密码访问、过期时间选择和复制入口。 +- Share Link 访问已统一走 ClawManager 代理,能够兼容 Lite gateway 和 Pro service 两种后端形态。 +- 相关改动已部署到 172.16.1.12 环境,并完成一轮自动化回归。 +- 当前自动化测试已覆盖普通实例 Lite / Pro 模式、Share Link 基础行为、管理端模式展示以及 Team Lite 创建链路。 +- 联调中发现,部分 Lite runtime / agent 能力仍需补齐,主要集中在 Team 任务消费和 Hermes Lite 会话稳定性上。 + +## 计划 + +- 继续推进 agent / runtime 侧补齐 Lite 模式能力,重点包括 Team 任务接收、执行状态回写和会话稳定性。 +- 在 agent / runtime 修复后,补充完整的 Team Lite 任务执行 e2e,覆盖从建队、成员实例创建、任务下发到状态回写的闭环。 +- 继续完善 Hermes Lite 的联调验证,确保 Lite 模式下聊天、访问和实例生命周期稳定。 +- 对 Share Link 增加更完整的回归覆盖,包括短链访问、密码访问、过期策略、禁用/重新生成等场景。 +- 在 172.16.1.12 环境继续做端到端回归,确认普通实例、Team 实例和 Share Link 在 Lite / Pro 两种模式下都能稳定运行。 +- 最后整理验收说明,明确 Lite / Pro 的支持范围、已验证场景和仍需 agent 侧配合的事项。 + +## 简短汇报口径 + +这次开发不是单独做 Team Lite,而是把实例模式整体升级为 Lite / Pro 两种形态。Lite 走共享 runtime gateway,Pro 走独立 desktop runtime,对用户层统一暴露 Lite / Pro,对后端仍按不同 runtime backend 调度。 + +目前普通实例创建、实例展示、管理端识别、Team 成员创建以及 Share Link 分享能力都已经完成基础支持,并已在 172.16.1.12 做了一轮自动化回归。后续重点是继续和 agent / runtime 侧联调,补齐 Lite 模式下 Team 任务执行闭环和 Hermes Lite 会话稳定性,然后补充更完整的端到端验收测试。 diff --git a/docs/main/1.png b/docs/main/1.png new file mode 100644 index 0000000..8cb4035 Binary files /dev/null and b/docs/main/1.png differ diff --git a/docs/main/10.png b/docs/main/10.png new file mode 100644 index 0000000..8fc361c Binary files /dev/null and b/docs/main/10.png differ diff --git a/docs/main/11.png b/docs/main/11.png new file mode 100644 index 0000000..440fc2b Binary files /dev/null and b/docs/main/11.png differ diff --git a/docs/main/12.png b/docs/main/12.png new file mode 100644 index 0000000..1f5e657 Binary files /dev/null and b/docs/main/12.png differ diff --git a/docs/main/13.png b/docs/main/13.png new file mode 100644 index 0000000..622f096 Binary files /dev/null and b/docs/main/13.png differ diff --git a/docs/main/14.png b/docs/main/14.png new file mode 100644 index 0000000..ef88687 Binary files /dev/null and b/docs/main/14.png differ diff --git a/docs/main/15.png b/docs/main/15.png new file mode 100644 index 0000000..eb81779 Binary files /dev/null and b/docs/main/15.png differ diff --git a/docs/main/2.png b/docs/main/2.png new file mode 100644 index 0000000..6098f1f Binary files /dev/null and b/docs/main/2.png differ diff --git a/docs/main/3.png b/docs/main/3.png new file mode 100644 index 0000000..1c18a90 Binary files /dev/null and b/docs/main/3.png differ diff --git a/docs/main/4.png b/docs/main/4.png new file mode 100644 index 0000000..207401d Binary files /dev/null and b/docs/main/4.png differ diff --git a/docs/main/5.png b/docs/main/5.png new file mode 100644 index 0000000..26ef96f Binary files /dev/null and b/docs/main/5.png differ diff --git a/docs/main/6.png b/docs/main/6.png new file mode 100644 index 0000000..a32b663 Binary files /dev/null and b/docs/main/6.png differ diff --git a/docs/main/7.png b/docs/main/7.png new file mode 100644 index 0000000..7666378 Binary files /dev/null and b/docs/main/7.png differ diff --git a/docs/main/8.png b/docs/main/8.png new file mode 100644 index 0000000..92183ef Binary files /dev/null and b/docs/main/8.png differ diff --git a/docs/main/9.png b/docs/main/9.png new file mode 100644 index 0000000..1a430c8 Binary files /dev/null and b/docs/main/9.png differ diff --git a/docs/main/admin.png b/docs/main/admin.png new file mode 100644 index 0000000..f68c8f4 Binary files /dev/null and b/docs/main/admin.png differ diff --git a/docs/main/aigateway.png b/docs/main/aigateway.png new file mode 100644 index 0000000..6e87627 Binary files /dev/null and b/docs/main/aigateway.png differ diff --git a/docs/main/clawmanager_discord.jpg b/docs/main/clawmanager_discord.jpg new file mode 100644 index 0000000..be781b1 Binary files /dev/null and b/docs/main/clawmanager_discord.jpg differ diff --git a/docs/main/clawmanager_features.jpg b/docs/main/clawmanager_features.jpg new file mode 100644 index 0000000..28eebdc Binary files /dev/null and b/docs/main/clawmanager_features.jpg differ diff --git a/docs/main/clawmanager_group_chat.jpg b/docs/main/clawmanager_group_chat.jpg new file mode 100644 index 0000000..2b3c4f5 Binary files /dev/null and b/docs/main/clawmanager_group_chat.jpg differ diff --git a/docs/main/liteopenclaw.png b/docs/main/liteopenclaw.png new file mode 100644 index 0000000..e5e5d8e Binary files /dev/null and b/docs/main/liteopenclaw.png differ diff --git a/docs/main/portal.png b/docs/main/portal.png new file mode 100644 index 0000000..669c985 Binary files /dev/null and b/docs/main/portal.png differ diff --git a/docs/main/proopenclaw.png b/docs/main/proopenclaw.png new file mode 100644 index 0000000..f948336 Binary files /dev/null and b/docs/main/proopenclaw.png differ diff --git a/docs/main/team-create.png b/docs/main/team-create.png new file mode 100644 index 0000000..281ff48 Binary files /dev/null and b/docs/main/team-create.png differ diff --git a/docs/main/team-workflow.jpg b/docs/main/team-workflow.jpg new file mode 100644 index 0000000..5b876a4 Binary files /dev/null and b/docs/main/team-workflow.jpg differ diff --git a/docs/main/team-workspace.png b/docs/main/team-workspace.png new file mode 100644 index 0000000..b0639eb Binary files /dev/null and b/docs/main/team-workspace.png differ diff --git a/docs/openclaw-windows-node-exec.md b/docs/openclaw-windows-node-exec.md new file mode 100644 index 0000000..5cee925 --- /dev/null +++ b/docs/openclaw-windows-node-exec.md @@ -0,0 +1,233 @@ +# OpenClaw Gateway + Windows Node Exec Configuration + +本文说明一个常见 ClawManager 使用场景: + +```text +用户浏览器 + -> ClawManager Web / OpenClaw WebChat + -> 服务端 OpenClaw 实例里的 Gateway + -> 本机 Windows 上运行的 OpenClaw node + -> 在 Windows 本机执行 system.run / system.which +``` + +目标是:用户访问 ClawManager 创建出的 OpenClaw 实例 Web UI,在聊天中让 Agent 操作个人 Windows 电脑上的命令或自动化任务。 + +## 1. 基本判断 + +这个场景可行,但要区分两类授权: + +| 授权类型 | 作用 | 是否每次变化 | +| --- | --- | --- | +| Device pairing | 允许 Windows node 接入服务端 Gateway | 通常只需要一次 | +| Exec approval | 允许某条 `system.run` 在 Windows node 上执行 | ask 模式下每条命令都会生成新的 requestId | + +如果错误是: + +```json +{ + "status": "error", + "tool": "exec", + "error": "UNAVAILABLE: SYSTEM_RUN_DENIED: approval required" +} +``` + +说明链路已经打到 Windows node,问题不是网络,而是 `system.run` 被 Windows node 的 exec approval 策略拦住。 + +## 2. 前置条件 + +服务端 OpenClaw 实例里的 Gateway 必须满足: + +- Gateway WebSocket 端口能被 Windows 电脑访问,例如 `192.168.30.133:31889`。 +- Gateway 不能只监听容器内 `127.0.0.1`。 +- 如果使用明文 `ws://`,建议只在内网、VPN、Tailscale 或 SSH tunnel 内使用。 +- Windows node 需要拿到 Gateway 的认证 token 或 password。 + +Windows 本机需要满足: + +- 已安装 OpenClaw CLI。 +- 能访问服务端 Gateway 地址和端口。 +- Node 进程能持久运行,建议调通后改为 service 模式。 + +## 3. 启动 Windows Node + +在服务端 OpenClaw 实例里查看 Gateway token: + +```bash +openclaw config get gateway.auth.token +``` + +在 Windows PowerShell 中设置 token 并启动 node: + +```powershell +$env:OPENCLAW_GATEWAY_TOKEN="" +openclaw node run --host 192.168.30.133 --port 31889 --display-name "windows-node" +``` + +如果要后台常驻: + +```powershell +$env:OPENCLAW_GATEWAY_TOKEN="" +openclaw node install --host 192.168.30.133 --port 31889 --display-name "windows-node" --force +openclaw node restart +openclaw node status +``` + +如果 Gateway 使用 TLS,再加: + +```powershell +openclaw node run --host 192.168.30.133 --port 31889 --tls --display-name "windows-node" +``` + +## 4. 在 Gateway 侧完成 Node Pairing + +进入服务端 OpenClaw 实例终端,查看待审批设备: + +```bash +openclaw devices list +``` + +找到 role 为 `node` 的请求,批准最新的 `requestId`: + +```bash +openclaw devices approve +``` + +确认 node 已连接并 paired: + +```bash +openclaw nodes status +openclaw nodes describe --node "windows-node" +``` + +如果 `devices list` 每次看到新的 pairing `requestId`,通常说明 node 身份或认证信息变化了。优先检查: + +- Windows 上是否反复使用了不同的 `--node-id`。 +- Windows 上 `~/.openclaw/node.json` 是否被删除或重建。 +- Gateway token/password 是否变更。 +- Node 是否确实使用同一个 `OPENCLAW_GATEWAY_TOKEN` 启动。 + +## 5. 将 Exec 指向 Windows Node + +在服务端 OpenClaw 实例里配置默认 exec host: + +```bash +openclaw config set tools.exec.host node +openclaw config set tools.exec.node "windows-node" +``` + +如果有多个 Agent,也可以按 Agent 配置: + +```bash +openclaw config get agents.list +openclaw config set agents.list[0].tools.exec.node "windows-node" +``` + +## 6. Exec Approval 配置方案 + +### 方案 A:稳定自动化,免审批执行 + +仅建议在完全可信的内网或 VPN 环境中使用。 + +服务端 Gateway 请求策略: + +```bash +openclaw config set tools.exec.security full +openclaw config set tools.exec.ask off +openclaw gateway restart +``` + +同时把 Windows node 本地 approvals 策略也打开: + +```bash +openclaw approvals set --node "windows-node" --stdin <<'EOF' +{ + "version": 1, + "defaults": { + "security": "full", + "ask": "off", + "askFallback": "full" + } +} +EOF +``` + +注意:只改 `tools.exec.*` 不够。OpenClaw 会取 Gateway 请求策略和 node 本地 `~/.openclaw/exec-approvals.json` 中更严格的一方。如果 node 本地仍是 `ask: "always"` 或 `askFallback: "deny"`,还是会继续提示授权。 + +### 方案 B:更安全,使用 allowlist + +服务端 Gateway 请求策略: + +```bash +openclaw config set tools.exec.security allowlist +openclaw config set tools.exec.ask off +openclaw gateway restart +``` + +给 Windows node 添加允许执行的命令: + +```bash +openclaw approvals allowlist add --agent "*" --node "windows-node" "powershell.exe" +openclaw approvals allowlist add --agent "*" --node "windows-node" "pwsh.exe" +openclaw approvals allowlist add --agent "*" --node "windows-node" "cmd.exe" +``` + +更推荐只放行具体脚本或具体工具,例如: + +```bash +openclaw approvals allowlist add --agent "*" --node "windows-node" "C:\\Tools\\automation\\daily-task.ps1" +openclaw approvals allowlist add --agent "*" --node "windows-node" "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe" +``` + +Windows 上如果命令被包装成 `cmd.exe /c ...`,allowlist 模式下仍可能需要审批。这是 OpenClaw 的安全行为。若需要完全无提示自动化,使用方案 A。 + +## 7. 验证 + +在服务端 OpenClaw 实例里执行: + +```bash +openclaw nodes status +openclaw nodes describe --node "windows-node" +openclaw approvals get --node "windows-node" +``` + +然后在 WebChat 里让 Agent 执行一个简单命令: + +```text +请在 node 上执行 hostname,并告诉我输出。 +``` + +如果仍然报 `SYSTEM_RUN_DENIED`: + +1. 确认 `openclaw nodes status` 中 node 是 connected/paired。 +2. 确认 `openclaw nodes describe --node "windows-node"` 中包含 `system.run`。 +3. 确认 `openclaw approvals get --node "windows-node"` 的 effective policy 是预期值。 +4. 如果 effective policy 仍然是 ask 或 deny,说明 node 本地 approvals 文件没有被更新。 +5. 如果是 allowlist miss,说明命令没有命中 allowlist,改成 full 或补 allowlist。 + +## 8. requestId 每次不同怎么处理 + +如果 requestId 来自 `devices list`: + +- 这是 node pairing 请求。 +- 应该只需要批准一次。 +- 如果每次都变,说明 node 身份被重置或认证信息变化。 + +如果 requestId 来自 exec approval: + +- 这是命令执行审批。 +- 每条新命令都有自己的 requestId,变化是正常现象。 +- 要避免反复授权,需要配置 `tools.exec.ask off`,并同步配置 node 本地 approvals。 + +## 9. 安全建议 + +- 不要把 Gateway WebSocket 端口直接暴露公网。 +- 生产环境优先使用 VPN、Tailscale、WireGuard 或 SSH tunnel。 +- 如果只是做固定自动化,优先使用 allowlist 放行具体脚本。 +- `security full + ask off` 相当于允许 WebChat 控制 Windows 本机命令,只有在完全可信环境使用。 +- 定期检查 Windows node 的 `~/.openclaw/exec-approvals.json`。 +- 不再使用时,及时停掉 node: + +```powershell +openclaw node stop +openclaw node uninstall +``` diff --git a/docs/resource-management.md b/docs/resource-management.md new file mode 100644 index 0000000..db580a9 --- /dev/null +++ b/docs/resource-management.md @@ -0,0 +1,32 @@ +# Resource Management Guide + +Resource Management is the reusable asset layer for OpenClaw workspaces in ClawManager. It is centered on channels, skills, bundles, and the snapshots used to compile those assets into instance-ready configuration. + +## Main Resource Types + +- `Channels` for workspace connectivity and integration templates +- `Skills` for reusable uploaded packages that can be installed into runtime instances +- Config skills for bootstrap configuration delivered through runtime environment payloads +- `Bundles` for composing repeatable resource sets, including both config resources and uploaded skills +- injection snapshots for tracking the compiled result applied to an instance + +## Core Workflows + +1. Create or import channels and skills in the OpenClaw Config Center. +2. Organize selected config resources and uploaded skills into reusable bundles. +3. Review scan posture for skills through Security Center. +4. Apply resources or bundles to OpenClaw workspaces. +5. Inspect runtime state and instance-level resource results after injection. + +## How It Connects to the Platform + +- Resource Management defines what should be delivered to a workspace. +- Config resources are compiled into bootstrap environment payloads. Uploaded skills in a bundle are installed through the Agent Control Plane skill installation path. +- Agent Control Plane applies and tracks those changes at runtime. +- Security Center and `skill-scanner` help review the risk posture of reusable skills before broad rollout. + +## Related Guides + +- [Security / Skill Scanner Guide](./security-skill-scanner.md) +- [Agent Control Plane Guide](./agent-control-plane.md) +- [Admin and User Guide](./admin-user-guide.md) diff --git a/docs/runtime-agent-integration-guide.md b/docs/runtime-agent-integration-guide.md new file mode 100644 index 0000000..13c0053 --- /dev/null +++ b/docs/runtime-agent-integration-guide.md @@ -0,0 +1,492 @@ +# Runtime Agent 通用接入规范 + +本文定义任意新 runtime 接入 ClawManager Agent Control Plane 的通用方案。后续新增 OpenClaw、Hermes 以外的 runtime 时,应优先遵守本文,再补充该 runtime 自己的镜像构建细节。 + +## 接入目标 + +每个可托管 runtime 镜像内都应包含一个常驻 agent。Agent 不需要暴露端口,只需要向 ClawManager 发起出站 HTTP 请求,并完成: + +- 注册和 session token 续期。 +- 心跳和在线状态上报。 +- 运行时状态、CPU、内存、磁盘、网络、健康信息上报。 +- skill inventory 同步和 skill 包上传。 +- 平台命令轮询、执行、完成结果回传。 +- 使用 ClawManager 注入的 AI Gateway 环境变量访问 LLM。 + +Agent 应把 runtime 自身名称写入 `host_info.runtime_type`、`summary.runtime_type`、`system_info.runtime_type`。当前 v1 协议里仍保留 `openclaw_status`、`openclaw_pid`、`openclaw_version` 这几个历史字段名,新 runtime 在协议升级前需要复用这些字段承载自己的主进程状态、PID 和版本。 + +## 平台侧新增 runtime type 检查清单 + +新增 runtime 不只是做镜像。ClawManager 平台侧至少需要完成下面几项: + +| 模块 | 必做事项 | +| --- | --- | +| 数据库 | 在 `instances.type` 枚举迁移中加入新的 `runtime_type` | +| 后端 runtime 支持 | 在 runtime 类型校验、镜像解析、默认端口、持久化目录、代理路径逻辑中加入新类型 | +| 后端托管能力 | 在 `supportsManagedRuntimeIntegration` 加入新类型,否则不会注入 Agent 和 AI Gateway 环境变量,注册也会被拒绝 | +| Agent 注册 | 确认注册 allowlist、错误映射、测试覆盖都包含新类型 | +| Runtime Image Cards | 在系统镜像设置的支持类型、默认镜像、默认启用项中加入新类型 | +| 创建实例页 | 在前端 `INSTANCE_TYPES`、类型文案、图标、默认 env 模板中加入新类型 | +| 图标 | 提供 runtime 官方图标或明确授权图标,放入 `frontend/public` 或现有资产路径 | +| 测试 | 覆盖 env 注入、agent 注册、runtime 状态上报、创建实例表单、系统镜像卡片 | + +如果新 runtime 采用 Webtop/KasmVNC 基础镜像,默认约定通常是: + +- 桌面端口:`3001` +- 持久化目录:`/config` +- 代理路径:由 ClawManager 写入 `SUBFOLDER=/api/v1/instances/{instance_id}/proxy/` + +如果新 runtime 不基于 Webtop,必须在平台侧明确它的服务端口、健康检查、代理路径、持久化目录和用户数据目录,不能让镜像和平台各自猜测。 + +## 镜像和 Agent 运行要求 + +Runtime 镜像必须满足: + +- Agent 随容器启动自动运行,例如 systemd、s6 overlay、supervisord 或入口脚本。 +- Agent 不占用 runtime 的业务端口或桌面端口。 +- Agent 通过环境变量读取所有 ClawManager 配置,不写死地址、实例 ID、token、路径。 +- Agent 的本地状态必须写入持久化目录,例如 `${CLAWMANAGER_AGENT_PERSISTENT_DIR}/-agent/`。 +- Agent 日志中不能打印 bootstrap token、session token、AI Gateway API key。 +- Agent 可以在 ClawManager 暂时不可达时退避重试,不应退出导致容器不可用。 + +建议本地状态目录: + +```text +${CLAWMANAGER_AGENT_PERSISTENT_DIR}/-agent/ + session.json + state.json + logs/ + cache/ +``` + +## ClawManager 注入的环境变量 + +### Agent 控制面 + +| 变量 | 说明 | +| --- | --- | +| `CLAWMANAGER_AGENT_ENABLED` | 为 `true` 时启动 agent | +| `CLAWMANAGER_AGENT_BASE_URL` | ClawManager API 根地址,不带 `/api/v1/agent` 后缀 | +| `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | 首次注册使用的一次性 bootstrap token | +| `CLAWMANAGER_AGENT_INSTANCE_ID` | 当前实例 ID | +| `CLAWMANAGER_AGENT_PROTOCOL_VERSION` | 当前为 `v1` | +| `CLAWMANAGER_AGENT_PERSISTENT_DIR` | 当前实例持久化目录 | +| `CLAWMANAGER_AGENT_DISK_LIMIT_BYTES` | 实例磁盘配额字节数 | + +Agent 启动时如果 `CLAWMANAGER_AGENT_ENABLED` 不是 `true`,应进入空闲状态或直接不启动控制面逻辑。 + +### AI Gateway + +支持托管 runtime 的实例会被注入 OpenAI-compatible 网关变量: + +| 变量 | 说明 | +| --- | --- | +| `CLAWMANAGER_LLM_BASE_URL` | ClawManager AI Gateway OpenAI-compatible base URL | +| `CLAWMANAGER_LLM_API_KEY` | 当前实例专属 Gateway API key | +| `CLAWMANAGER_LLM_MODEL` | 平台注入的模型目录 JSON,首项通常包含 `auto` | +| `CLAWMANAGER_LLM_PROVIDER` | 当前为 `openai-compatible` | +| `CLAWMANAGER_INSTANCE_TOKEN` | 当前实例 token,和 Gateway API key 同源 | +| `OPENAI_BASE_URL` | OpenAI SDK 兼容别名 | +| `OPENAI_API_BASE` | OpenAI SDK 兼容别名 | +| `OPENAI_API_KEY` | OpenAI SDK 兼容别名 | +| `OPENAI_MODEL` | 默认模型,通常为 `auto` | + +Runtime 内的应用和 agent 如果需要调用模型,优先使用这些变量,不要让用户在镜像内手工写入 provider key。 + +## Agent 生命周期 + +推荐主循环: + +1. 读取环境变量,确认 agent 已启用。 +2. 生成稳定 `agent_id`,建议格式为 `--main`。 +3. 从持久化目录读取 session token;没有 token 时用 bootstrap token 注册。 +4. 发送一次完整 state report。 +5. 按注册响应里的 `heartbeat_interval_seconds` 发送 heartbeat。 +6. 按注册响应里的 `command_poll_interval_seconds` 拉取命令;heartbeat 返回 `has_pending_command=true` 时立即拉取。 +7. 按 5 到 10 秒间隔采样并发送 state report。 +8. 定期同步 skill inventory,或在 skill 变化后立即同步。 +9. 接口返回 401、session 过期或本地 token 丢失时,重新注册。 + +注册响应里的 session token 当前为滚动续期。每次成功 heartbeat 都会延长有效期。Agent 仍应能处理 401 并自动重新注册。 + +## API 约定 + +以下 `{base}` 都表示 `CLAWMANAGER_AGENT_BASE_URL`,认证头均为: + +```http +Authorization: Bearer +Content-Type: application/json +``` + +### 注册 + +```http +POST {base}/api/v1/agent/register +Authorization: Bearer {CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN} +``` + +```json +{ + "instance_id": 123, + "agent_id": "myruntime-123-main", + "agent_version": "0.1.0", + "protocol_version": "v1", + "capabilities": [ + "runtime.status", + "runtime.health", + "metrics.report", + "skills.inventory", + "skills.upload", + "commands.poll", + "llm.gateway" + ], + "host_info": { + "runtime_type": "myruntime", + "runtime_name": "My Runtime", + "image": "registry.example.com/myruntime:latest", + "desktop_base": "webtop", + "persistent_dir": "/config", + "port": 3001, + "arch": "amd64" + } +} +``` + +响应: + +```json +{ + "data": { + "session_token": "agt_sess_xxx", + "session_expires_at": "2026-04-28T10:00:00Z", + "heartbeat_interval_seconds": 15, + "command_poll_interval_seconds": 5, + "server_time": "2026-04-27T10:00:00Z" + } +} +``` + +Agent 必须缓存 `session_token`,后续接口都使用它认证。 + +### 心跳 + +```http +POST {base}/api/v1/agent/heartbeat +Authorization: Bearer {session_token} +``` + +```json +{ + "agent_id": "myruntime-123-main", + "timestamp": "2026-04-27T10:00:15Z", + "openclaw_status": "running", + "summary": { + "runtime_type": "myruntime", + "runtime_status": "running", + "runtime_pid": 245, + "runtime_version": "0.4.0", + "openclaw_pid": 245, + "skill_count": 8, + "disk_used_bytes": 2147483648, + "disk_limit_bytes": 10737418240 + } +} +``` + +兼容要求: + +- `openclaw_status` 当前仍是 v1 状态槽位。新 runtime 填自己的主进程状态。 +- `summary.openclaw_pid` 是当前后端识别 PID 的兼容别名;建议同时上报 `runtime_pid` 和 `openclaw_pid`,后续协议升级后再收敛。 +- 状态建议使用 `starting`、`running`、`stopped`、`error`、`unknown`。 +- 心跳按服务端响应间隔执行,默认约 15 秒。 +- ClawManager 45 秒内收到心跳显示 online,45 到 120 秒显示 stale,超过 120 秒显示 offline。 + +### 完整状态和监测数据上报 + +```http +POST {base}/api/v1/agent/state/report +Authorization: Bearer {session_token} +``` + +```json +{ + "agent_id": "myruntime-123-main", + "reported_at": "2026-04-27T10:00:20Z", + "runtime": { + "openclaw_status": "running", + "openclaw_pid": 245, + "openclaw_version": "myruntime-0.4.0", + "current_config_revision_id": null + }, + "system_info": { + "runtime_type": "myruntime", + "runtime_name": "My Runtime", + "sampled_at": "2026-04-27T10:00:20Z", + "cpu": { + "cores": 2, + "load": { + "1m": 0.64, + "5m": 0.52, + "15m": 0.40 + } + }, + "memory": { + "mem_total_bytes": 4294967296, + "mem_available_bytes": 2147483648 + }, + "disk": { + "mount_path": "/config", + "root_total_bytes": 10737418240, + "root_free_bytes": 8589934592 + }, + "network": { + "interfaces": [ + { + "name": "eth0", + "status": "up", + "addresses": ["10.42.0.12"], + "rx_bytes": 123456789, + "tx_bytes": 98765432 + } + ] + } + }, + "health": { + "runtime_process": "ok", + "desktop": "ok", + "agent": "ok", + "metrics_collector": "ok", + "metrics_sample_interval_seconds": 5 + } +} +``` + +后端会原样保存 `system_info` 和 `health`。前端实例详情页当前按以下字段绘制指标: + +| 路径 | 类型 | 单位 | 说明 | +| --- | --- | --- | --- | +| `system_info.cpu.cores` | number | 核数 | 容器可用 CPU 核数 | +| `system_info.cpu.load.1m` | number | load average | 1 分钟 load | +| `system_info.cpu.load.5m` | number | load average | 5 分钟 load | +| `system_info.cpu.load.15m` | number | load average | 15 分钟 load | +| `system_info.memory.mem_total_bytes` | number | bytes | 内存上限或总内存 | +| `system_info.memory.mem_available_bytes` | number | bytes | 可用内存 | +| `system_info.disk.root_total_bytes` | number | bytes | 持久化目录所在文件系统总容量 | +| `system_info.disk.root_free_bytes` | number | bytes | 持久化目录所在文件系统剩余容量 | +| `system_info.network.interfaces[].rx_bytes` | number | bytes | 网卡累计接收字节数 | +| `system_info.network.interfaces[].tx_bytes` | number | bytes | 网卡累计发送字节数 | + +CPU 百分比由前端按 `load.1m / cores * 100` 计算。内存百分比由 `(mem_total_bytes - mem_available_bytes) / mem_total_bytes * 100` 计算。磁盘百分比由 `(root_total_bytes - root_free_bytes) / root_total_bytes * 100` 计算。网络速率由相邻两次 `rx_bytes`、`tx_bytes` counter 差值计算,所以 agent 必须上报单调递增的累计 counter,不要把瞬时速率填进这两个字段。 + +推荐采样来源: + +- CPU load:`/proc/loadavg`。 +- CPU cores:优先 cgroup CPU quota,例如 cgroup v2 `/sys/fs/cgroup/cpu.max`;没有 quota 时使用 `/proc/cpuinfo` 或语言运行时 CPU 数。 +- 内存:优先 cgroup memory limit/current,例如 cgroup v2 `/sys/fs/cgroup/memory.max`、`/sys/fs/cgroup/memory.current`;没有限制时使用 `/proc/meminfo` 的 `MemTotal` 和 `MemAvailable`。 +- 磁盘:对 `CLAWMANAGER_AGENT_PERSISTENT_DIR` 调用 `statvfs`。 +- 网络:读取 `/proc/net/dev`,默认排除 `lo`。 + +上报频率: + +- 启动成功后立即发送一次。 +- 正常运行每 5 秒发送一次;资源敏感时可放宽到 10 秒。 +- 采样间隔不要短于 2 秒。 +- runtime 状态变化、skill inventory 变化、命令完成后立即补发一次。 + +### 命令轮询和执行 + +拉取命令: + +```http +GET {base}/api/v1/agent/commands/next +Authorization: Bearer {session_token} +``` + +如果没有命令,响应的 `data` 可能为空。拿到命令后,agent 必须先标记开始,再执行: + +```http +POST {base}/api/v1/agent/commands/{id}/start +Authorization: Bearer {session_token} +``` + +```json +{ + "agent_id": "myruntime-123-main", + "started_at": "2026-04-27T10:01:00Z" +} +``` + +完成后必须 finish,失败也要 finish: + +```http +POST {base}/api/v1/agent/commands/{id}/finish +Authorization: Bearer {session_token} +``` + +```json +{ + "agent_id": "myruntime-123-main", + "status": "succeeded", + "finished_at": "2026-04-27T10:01:05Z", + "result": { + "message": "system info collected" + }, + "error_message": "" +} +``` + +当前通用命令类型: + +| 命令 | Agent 行为 | +| --- | --- | +| `collect_system_info` | 立即采样,发送 state report,并在 finish result 中带上同一份摘要 | +| `health_check` | 检查主进程、桌面入口、agent、metrics collector,并发送 state report | +| `sync_skill_inventory` | 扫描 skill 目录并上报完整 inventory | +| `refresh_skill_inventory` | 重新扫描 skill 目录并上报完整 inventory | +| `collect_skill_package` | 打包指定 skill 并上传 | +| `install_skill` | 下载并安装平台指定 skill version | +| `update_skill` | 更新已安装 skill | +| `uninstall_skill` / `remove_skill` | 移除指定 skill | +| `disable_skill` | 禁用指定 skill | +| `quarantine_skill` | 隔离指定 skill | +| `handle_skill_risk` | 按平台风控 payload 处理 skill | +| `apply_config_revision` | 获取并应用配置 revision | +| `reload_config` | 重新加载 runtime 配置 | + +`start_openclaw`、`stop_openclaw`、`restart_openclaw` 是历史命名命令。非 OpenClaw runtime 不应误执行,除非平台侧明确把它们映射为该 runtime 的启动、停止、重启语义。 + +Agent 遇到未知命令时,应 finish 为 `failed`,`error_message` 写明 `unsupported command type: `。 + +### Skill inventory + +上报: + +```http +POST {base}/api/v1/agent/skills/inventory +Authorization: Bearer {session_token} +``` + +```json +{ + "agent_id": "myruntime-123-main", + "reported_at": "2026-04-27T10:02:00Z", + "mode": "full", + "trigger": "startup", + "skills": [ + { + "skill_id": "filesystem-name-or-manifest-id", + "skill_version": "1.0.0", + "identifier": "vendor.skill-name", + "install_path": "/config/myruntime/skills/vendor.skill-name", + "content_md5": "0123456789abcdef0123456789abcdef", + "source": "runtime", + "type": "agent-skill", + "size_bytes": 12345, + "file_count": 12, + "collected_at": "2026-04-27T10:02:00Z", + "metadata": { + "runtime_type": "myruntime" + } + } + ] +} +``` + +要求: + +- `identifier` 和 `content_md5` 必须稳定。 +- `mode=full` 表示这次 inventory 是全量结果,平台会用它对齐实例 skill 状态。 +- skill 内容变化后要重新计算 `content_md5` 并上报。 +- 如果支持上传 skill 包,使用 `POST {base}/api/v1/agent/skills/upload`,multipart 表单中带 `file`、`agent_id`、`skill_id`、`skill_version`、`identifier`、`content_md5`、`source`。 +- `content_md5` 必须按目录内容指纹计算,不是 zip 文件 MD5。完整算法见 [Skill Content MD5 Calculation Spec](skill-content-md5-spec.md)。 + +### 配置和安装包下载 + +应用配置 revision: + +```http +GET {base}/api/v1/agent/config/revisions/{id} +Authorization: Bearer {session_token} +``` + +下载 skill version: + +```http +GET {base}/api/v1/agent/skills/versions/{external_version_id}/download +Authorization: Bearer {session_token} +``` + +下载内容只能应用到当前实例,不应泄露到其他实例或日志。 + +## 错误处理和重试 + +Agent 应按下面规则处理错误: + +| 情况 | 处理 | +| --- | --- | +| HTTP 401 | 删除本地 session token,使用 bootstrap token 重新注册 | +| HTTP 403 | 停止重试当前认证路径,记录简短错误,等待配置修复 | +| HTTP 404 | 对命令或资源类请求标记失败,错误写入 finish | +| HTTP 429 / 5xx / 网络错误 | 指数退避重试,保留本地状态 | +| JSON 校验失败 | 修正 payload;命令执行场景下 finish 为 failed | +| 采样部分失败 | 上报可用字段,并在 `health.metrics_collector` 写 `degraded` 或 `error` | + +推荐退避:1s、2s、5s、10s、30s,最大 60s,并加入少量 jitter。 + +## 安全要求 + +- 不记录 token、API key、完整 Authorization header。 +- session token 只写入当前实例持久化目录,文件权限建议 `0600`。 +- 不接受来自用户输入的任意 URL 覆盖 `CLAWMANAGER_AGENT_BASE_URL`。 +- 上传 skill 包前需要限制目录边界,不能打包持久化目录以外的文件。 +- 解压平台下发的 skill 包时必须防止 zip slip,例如拒绝 `../` 和绝对路径。 +- 命令执行要做超时控制,超过 `timeout_seconds` 应主动终止并 finish 为 failed。 + +## 验收标准 + +新增 runtime agent 接入完成后,至少满足: + +1. 创建实例后容器环境中存在 Agent 和 AI Gateway 变量。 +2. Agent 能注册成功,实例详情页显示 agent online。 +3. 心跳持续发送,45 秒内不会变 stale。 +4. `GET /api/v1/instances/{instance_id}/runtime` 能看到 `runtime.system_info` 和 `runtime.health`。 +5. CPU、Memory、Disk、Network 指标在实例详情页 10 秒内开始刷新。 +6. `collect_system_info` 命令能成功完成,并更新 state report。 +7. `health_check` 命令能成功完成,并体现 runtime、agent、metrics collector 状态。 +8. Skill inventory 能上报全量结果,skill 变化后能重新同步。 +9. session token 过期或 401 后能自动重新注册。 +10. 日志中没有 bootstrap token、session token、AI Gateway API key。 + +## 最小实现伪代码 + +```text +load env +if CLAWMANAGER_AGENT_ENABLED != "true": + sleep forever + +agent_id = "-" + CLAWMANAGER_AGENT_INSTANCE_ID + "-main" +session = load session from persistent dir +if session missing: + session = register_with_bootstrap_token() + save session + +report_state(sample_system_info(), health_check()) +sync_skill_inventory(trigger="startup") + +loop: + every heartbeat_interval: + resp = heartbeat(summary) + if resp.has_pending_command: + poll_and_execute_commands() + + every command_poll_interval: + poll_and_execute_commands() + + every 5 seconds: + report_state(sample_system_info(), health_snapshot) + + on 401: + delete session + session = register_with_bootstrap_token() + save session +``` diff --git a/docs/security-skill-scanner.md b/docs/security-skill-scanner.md new file mode 100644 index 0000000..7f89131 --- /dev/null +++ b/docs/security-skill-scanner.md @@ -0,0 +1,30 @@ +# Security / Skill Scanner Guide + +Security Center is the review and scanning surface for skill assets in ClawManager. It works with `skill-scanner` to help teams understand asset coverage, risk posture, and scanning status before skills are reused across workspaces. + +## What It Covers + +- skill asset inventory across the platform +- scan status, coverage, and recent scan jobs +- risk-level distribution for discovered and uploaded skills +- scanner configuration, including external analysis integrations where configured + +## Main Workflows + +1. Review the asset inventory and identify high-risk or unscanned skills. +2. Start incremental or full scans from Security Center. +3. Inspect recent scan jobs and detailed outcomes. +4. Tune scanner configuration and analysis integrations. +5. Feed scanning results back into skill approval and workspace rollout decisions. + +## Why It Matters + +- keeps reusable skills visible and reviewable +- adds a security checkpoint to the resource supply chain +- supports scale by replacing ad hoc per-instance trust decisions with centralized scanning workflows + +## Related Guides + +- [Resource Management Guide](./resource-management.md) +- [Agent Control Plane Guide](./agent-control-plane.md) +- [AI Gateway Guide](./aigateway.md) diff --git a/docs/skill-content-md5-spec.md b/docs/skill-content-md5-spec.md new file mode 100644 index 0000000..82573aa --- /dev/null +++ b/docs/skill-content-md5-spec.md @@ -0,0 +1,134 @@ +# Skill Content MD5 Calculation Spec + +本文定义 ClawManager / OpenClaw / Hermes 之间统一使用的 `content_md5` 计算方式。Hermes agent 的 inventory 上报、`collect_skill_package` 上传、`install_skill` 安装后校验,都必须使用同一套算法。 + +## 结论 + +`content_md5` 不是 zip 文件本身的 MD5,也不包含 zip entry 顺序、压缩等级、mtime、权限等元数据。它是 skill 目录内容的规范化 MD5。 + +上传 zip 时,zip 必须包含且只包含一个顶层 skill 目录。ClawManager 会先剥掉这个顶层 skill 目录,再对目录内部内容计算 `content_md5`。 + +例如上传包结构为: + +```text +weather/ + skill.json + src/ + main.py +``` + +实际参与 MD5 的路径是: + +```text +skill.json +src +src/main.py +``` + +注意:只剥掉 zip 的顶层 skill 目录 `weather/` 一次,不要再剥掉 skill 内部的 `src/`、`lib/`、`dist/` 等目录。 + +## 规范化规则 + +1. 以 skill 根目录作为基准,收集所有普通文件。 +2. 路径统一使用 POSIX `/` 分隔符。 +3. 去掉路径开头的 `./`,并做 clean 处理。 +4. 跳过空路径、`.`、`..`、包含 `..` 越界语义的路径。 +5. 跳过任意路径段以 `.` 开头的文件和目录,例如 `.git/config`、`.cache/a`、`.DS_Store`。 +6. 目录项不直接从文件系统读取,而是由文件路径的父目录推导出来。 +7. 将目录项和文件项放在同一个列表中,按规范化路径字典序升序排序。 +8. 对每个目录项写入以下字节: + +```text +{relative_path}\n +dir\n +``` + +9. 对每个文件项写入以下字节: + +```text +{relative_path}\n +file\n +{raw_file_bytes} +\n +``` + +10. 对上述连续字节流计算 MD5,输出 32 位小写 hex 字符串。 + +不要改写文件内容。不要转换换行符,不要格式化 JSON,不要忽略空文件,不要把文件权限、mtime、owner、zip 压缩参数写入 digest。 + +## Hermes Agent Checklist + +Hermes agent 需要检查下面几个点: + +- inventory 上报的 `content_md5` 应该对 `/config/.hermes/skills/{skill_name}` 目录内部内容计算。 +- 上传 `collect_skill_package` zip 时,zip 内应该有一个顶层目录 `{skill_name}/`。 +- inventory 阶段和上传阶段必须使用同一份 skill 目录内容计算 MD5。 +- 如果本地目录是 `/config/.hermes/skills/weather/src/main.py`,参与 MD5 的路径必须是 `src/main.py`,不是 `weather/src/main.py`,也不是 `main.py`。 +- 如果 `content_md5` 和 ClawManager 返回的 expected 不一致,先检查是否多剥或少剥了顶层目录,其次检查是否把隐藏目录、文件元数据或 zip bytes 算进去了。 + +## Python Reference Implementation + +下面实现可直接给 Hermes agent 端对齐算法: + +```python +import hashlib +from pathlib import Path + + +def _is_hidden_relative_path(rel: str) -> bool: + return any(part.startswith(".") for part in rel.split("/")) + + +def skill_content_md5(skill_dir: str | Path) -> str: + root = Path(skill_dir).resolve() + files: dict[str, bytes] = {} + dirs: set[str] = set() + + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + + rel = path.relative_to(root).as_posix() + if rel.startswith("./"): + rel = rel[2:] + if not rel or rel == "." or rel.startswith("../") or _is_hidden_relative_path(rel): + continue + + files[rel] = path.read_bytes() + parts = rel.split("/") + for i in range(1, len(parts)): + parent = "/".join(parts[:i]) + if parent and not _is_hidden_relative_path(parent): + dirs.add(parent) + + entries: dict[str, str] = {rel: "file" for rel in files} + for rel in dirs: + entries[rel] = "dir" + + digest = hashlib.md5() + for rel in sorted(entries): + digest.update(rel.encode("utf-8")) + digest.update(b"\n") + if entries[rel] == "dir": + digest.update(b"dir\n") + else: + digest.update(b"file\n") + digest.update(files[rel]) + digest.update(b"\n") + + return digest.hexdigest() +``` + +## Zip Upload Reference + +上传给 ClawManager 的 zip 应保持一个顶层目录: + +```text +weather/ + skill.json + src/main.py +``` + +Hermes agent 在本地计算 MD5 时应对目录 `/config/.hermes/skills/weather` 调用 `skill_content_md5()`。不要对 zip 文件调用 MD5。 + +如果 agent 需要在上传前自检,可以先把 zip 解开,确认去掉 `weather/` 后得到的文件列表与本地计算使用的相对路径一致。 diff --git a/docs/superpowers/plans/2026-06-01-clawmanager-v2-runtime-pool.md b/docs/superpowers/plans/2026-06-01-clawmanager-v2-runtime-pool.md new file mode 100644 index 0000000..2ff15ef --- /dev/null +++ b/docs/superpowers/plans/2026-06-01-clawmanager-v2-runtime-pool.md @@ -0,0 +1,3080 @@ +# ClawManager V2 Runtime Pool Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build ClawManager V2 so OpenClaw and Hermes instances run as gateway processes inside shared runtime Pods, with workspace isolation, fast failover, admin runtime observability, and a simpler user UI. + +**Architecture:** The backend remains the source of truth in MySQL, uses Redis for hot locks/events/cache, and runs an embedded leader-elected runtime scheduler across multiple backend replicas. Runtime Pods are managed by Kubernetes Deployments and `clawmanager-agent`; each runtime Pod hosts up to 100 gateways and exposes gateway traffic by Pod IP plus assigned port. Workspaces live under a shared `/workspaces` mount and are accessed through a dedicated backend file manager API, not through the frame. + +**Tech Stack:** Go 1.26, Gin, MySQL via `upper/db`, Redis RESP client, Kubernetes client-go, React 19, Vite, TypeScript, Tailwind CSS, Kubernetes YAML. + +--- + +## Scope Check + +This release touches several subsystems, but they are tightly ordered: schema first, scheduler second, lifecycle/proxy/files third, UI and manifests last. Keep the work in this single master plan, but commit after each task so any subsystem can be reviewed independently. + +## File Structure + +Backend data and migrations: +- Modify: `backend/internal/db/migrations.go` - existing embedded migration runner remains unchanged unless ordering assumptions fail in tests. +- Create: `backend/internal/db/migrations/023_add_runtime_pool_v2.sql` - runtime Pod, gateway binding, rollout, and workspace audit schema. +- Modify: `backend/internal/db/migrations_test.go` - verify migration file ordering and SQL splitting still handles migration 023. +- Modify: `backend/internal/models/instance.go` - add workspace fields and V2 runtime metadata. +- Create: `backend/internal/models/runtime_pod.go` - runtime Pod state and metrics model. +- Create: `backend/internal/models/instance_runtime_binding.go` - active gateway binding model. +- Create: `backend/internal/models/runtime_rollout.go` - rollout state model for gray upgrades. +- Create: `backend/internal/models/workspace_file_audit.go` - upload/download/write audit model. +- Modify: `backend/internal/repository/instance_repository.go` - V2 lifecycle selectors and status update helpers. +- Create: `backend/internal/repository/runtime_pod_repository.go` - runtime Pod CRUD and capacity claims. +- Create: `backend/internal/repository/instance_runtime_binding_repository.go` - gateway binding CRUD and generation-safe updates. +- Create: `backend/internal/repository/runtime_rollout_repository.go` - rollout CRUD and selectors. +- Create: `backend/internal/repository/workspace_file_audit_repository.go` - append-only file audit writes. + +Backend runtime scheduling and agent integration: +- Create: `docs/clawmanager-agent-v2-contract.md` - exact HTTP dual-channel contract required from `clawmanager-agent`. +- Create: `backend/internal/services/redis_client.go` - platform Redis client reusing the existing RESP approach. +- Create: `backend/internal/services/runtime_agent_client.go` - backend-to-agent command client. +- Create: `backend/internal/services/runtime_scheduler.go` - leader loop, assignment, failover, and gray-drain orchestration. +- Create: `backend/internal/services/runtime_capacity.go` - capacity calculations and runtime type normalization. +- Create: `backend/internal/services/runtime_events.go` - Redis-backed runtime event fanout for all backend replicas. +- Create: `backend/internal/services/runtime_leader.go` - Kubernetes Lease leader election wrapper. +- Create: `backend/internal/services/k8s/runtime_deployment_service.go` - create, scale, and inspect OpenClaw/Hermes runtime Deployments. +- Modify: `backend/internal/services/instance_service.go` - route V2 create/start/stop/delete through scheduler state; keep legacy behavior for old instance types. +- Modify: `backend/internal/services/instance_proxy_service.go` - proxy V2 instances to binding Pod IP plus gateway port; keep legacy service proxy fallback. +- Modify: `backend/internal/services/sync_service.go` - stop assuming every running instance owns a Pod. +- Modify: `backend/cmd/server/main.go` - wire repositories, services, scheduler start, routes, and websocket runtime event bridge. + +Backend workspace file APIs: +- Create: `backend/internal/services/workspace_path_guard.go` - path clean, realpath, and symlink escape prevention. +- Create: `backend/internal/services/workspace_file_service.go` - list, preview, download, upload, mkdir, rename, delete. +- Create: `backend/internal/handlers/workspace_file_handler.go` - user-facing workspace file endpoints. +- Modify: `backend/internal/handlers/instance_handler.go` - attach workspace routes and simplify status response for V2. +- Modify: `backend/internal/utils/response.go` - add V2 file validation errors to client-safe bad request mapping. + +Backend admin APIs: +- Create: `backend/internal/handlers/runtime_pool_handler.go` - admin runtime Pod status, metrics, drain, and rollout endpoints. +- Modify: `backend/internal/services/websocket_service.go` - add admin runtime event subscriptions with user-role checks. +- Modify: `backend/internal/handlers/websocket_handler.go` if it exists in this repository; otherwise wire websocket changes in the existing handler file discovered during implementation. + +Frontend user and admin experience: +- Modify: `frontend/package.json` and lock file - add `lucide-react` if icons are not already available. +- Modify: `frontend/src/index.css` - neutral, simple design tokens; remove heavy gradients and oversized radii. +- Modify: `frontend/src/components/UserLayout.tsx` - simpler user shell. +- Modify: `frontend/src/components/AdminLayout.tsx` - simpler admin shell and add Runtime Pods navigation. +- Modify: `frontend/src/types/instance.ts` - V2 instance availability and workspace types. +- Create: `frontend/src/types/runtimePool.ts` - admin runtime Pod, gateway, metrics, rollout types. +- Create: `frontend/src/services/workspaceService.ts` - workspace API client. +- Create: `frontend/src/services/runtimePoolService.ts` - admin runtime API client. +- Modify: `frontend/src/hooks/useWebSocket.ts` - add admin runtime event hook. +- Modify: `frontend/src/services/instanceService.ts` - V2 status fields and workspace entry points. +- Modify: `frontend/src/pages/instances/CreateInstancePage.tsx` - only allow creating OpenClaw and Hermes for V2. +- Modify: `frontend/src/pages/instances/InstanceListPage.tsx` - hide Pod/service/resource details from ordinary users. +- Modify: `frontend/src/pages/instances/InstanceDetailPage.tsx` - show availability, service frame, workspace file manager, and simple controls. +- Create: `frontend/src/components/WorkspaceFileManager.tsx` - independent file list, preview, upload, download, rename, delete. +- Create: `frontend/src/components/InstanceServiceFrame.tsx` - OpenClaw/Hermes iframe wrapper with availability state. +- Create: `frontend/src/pages/admin/RuntimePodsPage.tsx` - admin-only runtime Pod metrics and gateway usage. +- Modify: `frontend/src/pages/admin/SystemSettingsPage.tsx` - add OpenClaw/Hermes runtime image and gray rollout controls. +- Modify: `frontend/src/router/index.tsx` - route `/admin/runtime-pods`. + +Deployment: +- Modify: `deployments/k8s/clawmanager.yaml` - one-YAML install for Redis, workspace store, backend replicas, runtime Deployments, RBAC, env. +- Modify: `deployments/k3s/clawmanager.yaml` - same defaults adapted to k3s/local path storage. +- Modify: `backend/deployments/k8s/clawreef-incluster.yaml` if this file is still shipped as an install path; otherwise leave it unchanged and document that `deployments/k8s/clawmanager.yaml` is canonical. + +## Agent Contract Summary + +The `clawmanager-agent` changes are documented but implemented outside this repository. The backend implementation must assume these endpoints and payloads: + +Agent-to-backend channel: + +```http +POST /api/v1/runtime-agent/register +POST /api/v1/runtime-agent/heartbeat +POST /api/v1/runtime-agent/gateways/report +POST /api/v1/runtime-agent/skills/report +POST /api/v1/runtime-agent/metrics/report +``` + +Backend-to-agent channel: + +```http +GET http://{pod_ip}:19090/v1/health +GET http://{pod_ip}:19090/v1/metrics +POST http://{pod_ip}:19090/v1/gateways +DELETE http://{pod_ip}:19090/v1/gateways/{gateway_id} +POST http://{pod_ip}:19090/v1/gateways/{gateway_id}/health +POST http://{pod_ip}:19090/v1/drain +``` + +Gateway create request: + +```json +{ + "instance_id": 123, + "user_id": 45, + "agent_type": "openclaw", + "workspace_path": "/workspaces/openclaw/user-45/instance-123", + "port_range": { "start": 20000, "end": 20099 }, + "uid": 200123, + "gid": 200123, + "cpu_cores": 2, + "memory_mb": 4096, + "disk_quota_mb": 20480, + "generation": 7 +} +``` + +Gateway create response: + +```json +{ + "gateway_id": "gw-123-7", + "port": 20017, + "pid": 8842, + "status": "running" +} +``` + +The agent must allocate a free port inside `20000-20099`, enforce one Linux UID/GID per instance using `200000 + instance_id`, apply cgroup CPU/memory limits, enforce workspace quota, reject symlink escapes inside workspaces, report gateway health, and survive backend replica changes. During gray upgrade, `POST /v1/drain` must reject new gateways and continue reporting existing gateway health until ClawManager migrates them away. + +## Task 1: Schema, Models, And Repository Interfaces + +**Files:** +- Create: `backend/internal/db/migrations/023_add_runtime_pool_v2.sql` +- Modify: `backend/internal/db/migrations_test.go` +- Modify: `backend/internal/models/instance.go` +- Create: `backend/internal/models/runtime_pod.go` +- Create: `backend/internal/models/instance_runtime_binding.go` +- Create: `backend/internal/models/runtime_rollout.go` +- Create: `backend/internal/models/workspace_file_audit.go` +- Modify: `backend/internal/repository/instance_repository.go` +- Create: `backend/internal/repository/runtime_pod_repository.go` +- Create: `backend/internal/repository/instance_runtime_binding_repository.go` +- Create: `backend/internal/repository/runtime_rollout_repository.go` +- Create: `backend/internal/repository/workspace_file_audit_repository.go` + +- [ ] **Step 1: Add failing migration order test** + +Append this test to `backend/internal/db/migrations_test.go`: + +```go +func TestMigration023IsEmbedded(t *testing.T) { + files, err := migrationFiles.ReadDir("migrations") + if err != nil { + t.Fatalf("read embedded migrations: %v", err) + } + + found := false + for _, file := range files { + if file.Name() == "023_add_runtime_pool_v2.sql" { + found = true + break + } + } + if !found { + t.Fatalf("migration 023_add_runtime_pool_v2.sql is not embedded") + } +} +``` + +- [ ] **Step 2: Run migration test and verify it fails** + +Run: + +```powershell +go test ./backend/internal/db -run TestMigration023IsEmbedded -count=1 +``` + +Expected: fail with `migration 023_add_runtime_pool_v2.sql is not embedded`. + +- [ ] **Step 3: Create migration 023** + +Create `backend/internal/db/migrations/023_add_runtime_pool_v2.sql` with: + +```sql +ALTER TABLE instances + ADD COLUMN workspace_path VARCHAR(1024) NULL AFTER mount_path, + ADD COLUMN workspace_usage_bytes BIGINT NOT NULL DEFAULT 0 AFTER workspace_path, + ADD COLUMN runtime_generation INT NOT NULL DEFAULT 1 AFTER workspace_usage_bytes, + ADD COLUMN runtime_error_message TEXT NULL AFTER runtime_generation; + +CREATE TABLE runtime_pods ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + runtime_type VARCHAR(32) NOT NULL, + namespace VARCHAR(128) NOT NULL, + pod_name VARCHAR(255) NOT NULL, + pod_uid VARCHAR(128) NULL, + pod_ip VARCHAR(64) NULL, + node_name VARCHAR(255) NULL, + deployment_name VARCHAR(255) NOT NULL, + image_ref VARCHAR(512) NOT NULL, + agent_endpoint VARCHAR(255) NULL, + state VARCHAR(32) NOT NULL DEFAULT 'pending', + capacity INT NOT NULL DEFAULT 100, + used_slots INT NOT NULL DEFAULT 0, + draining TINYINT(1) NOT NULL DEFAULT 0, + cpu_millis_used BIGINT NOT NULL DEFAULT 0, + memory_bytes_used BIGINT NOT NULL DEFAULT 0, + disk_bytes_used BIGINT NOT NULL DEFAULT 0, + network_rx_bytes BIGINT NOT NULL DEFAULT 0, + network_tx_bytes BIGINT NOT NULL DEFAULT 0, + metrics_json JSON NULL, + last_seen_at DATETIME NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_runtime_pod_namespace_name (namespace, pod_name), + KEY idx_runtime_pods_schedulable (runtime_type, state, draining, used_slots, capacity), + KEY idx_runtime_pods_last_seen (last_seen_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE instance_runtime_bindings ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + instance_id INT NOT NULL, + runtime_pod_id BIGINT NOT NULL, + runtime_type VARCHAR(32) NOT NULL, + gateway_id VARCHAR(128) NOT NULL, + gateway_port INT NOT NULL, + gateway_pid INT NULL, + workspace_path VARCHAR(1024) NOT NULL, + state VARCHAR(32) NOT NULL DEFAULT 'creating', + generation INT NOT NULL DEFAULT 1, + last_health_at DATETIME NULL, + error_message TEXT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_instance_runtime_binding_instance (instance_id), + UNIQUE KEY uk_instance_runtime_binding_gateway (runtime_pod_id, gateway_port), + KEY idx_instance_runtime_binding_pod_state (runtime_pod_id, state), + CONSTRAINT fk_instance_runtime_binding_instance + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + CONSTRAINT fk_instance_runtime_binding_pod + FOREIGN KEY (runtime_pod_id) REFERENCES runtime_pods(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE runtime_rollouts ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + runtime_type VARCHAR(32) NOT NULL, + target_image_ref VARCHAR(512) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + batch_size INT NOT NULL DEFAULT 1, + max_unavailable INT NOT NULL DEFAULT 1, + started_by INT NULL, + started_at DATETIME NULL, + finished_at DATETIME NULL, + error_message TEXT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + KEY idx_runtime_rollouts_type_status (runtime_type, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE workspace_file_audits ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + instance_id INT NOT NULL, + user_id INT NOT NULL, + action VARCHAR(32) NOT NULL, + relative_path VARCHAR(1024) NOT NULL, + bytes BIGINT NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + KEY idx_workspace_file_audits_instance_time (instance_id, created_at), + KEY idx_workspace_file_audits_user_time (user_id, created_at), + CONSTRAINT fk_workspace_file_audits_instance + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +``` + +- [ ] **Step 4: Extend instance model** + +Add fields to `backend/internal/models/instance.go` after `MountPath` and after `StoppedAt`: + +```go + WorkspacePath *string `db:"workspace_path" json:"workspace_path,omitempty"` + WorkspaceUsageBytes int64 `db:"workspace_usage_bytes" json:"workspace_usage_bytes"` + RuntimeGeneration int `db:"runtime_generation" json:"runtime_generation"` + RuntimeErrorMessage *string `db:"runtime_error_message" json:"runtime_error_message,omitempty"` +``` + +- [ ] **Step 5: Add runtime models** + +Create `backend/internal/models/runtime_pod.go`: + +```go +package models + +import "time" + +type RuntimePod struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + Namespace string `db:"namespace" json:"namespace"` + PodName string `db:"pod_name" json:"pod_name"` + PodUID *string `db:"pod_uid" json:"pod_uid,omitempty"` + PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"` + NodeName *string `db:"node_name" json:"node_name,omitempty"` + DeploymentName string `db:"deployment_name" json:"deployment_name"` + ImageRef string `db:"image_ref" json:"image_ref"` + AgentEndpoint *string `db:"agent_endpoint" json:"agent_endpoint,omitempty"` + State string `db:"state" json:"state"` + Capacity int `db:"capacity" json:"capacity"` + UsedSlots int `db:"used_slots" json:"used_slots"` + Draining bool `db:"draining" json:"draining"` + CPUMillisUsed int64 `db:"cpu_millis_used" json:"cpu_millis_used"` + MemoryBytesUsed int64 `db:"memory_bytes_used" json:"memory_bytes_used"` + DiskBytesUsed int64 `db:"disk_bytes_used" json:"disk_bytes_used"` + NetworkRXBytes int64 `db:"network_rx_bytes" json:"network_rx_bytes"` + NetworkTXBytes int64 `db:"network_tx_bytes" json:"network_tx_bytes"` + MetricsJSON *string `db:"metrics_json" json:"metrics_json,omitempty"` + LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (RuntimePod) TableName() string { + return "runtime_pods" +} +``` + +Create `backend/internal/models/instance_runtime_binding.go`: + +```go +package models + +import "time" + +type InstanceRuntimeBinding struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + RuntimePodID int64 `db:"runtime_pod_id" json:"runtime_pod_id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + GatewayID string `db:"gateway_id" json:"gateway_id"` + GatewayPort int `db:"gateway_port" json:"gateway_port"` + GatewayPID *int `db:"gateway_pid" json:"gateway_pid,omitempty"` + WorkspacePath string `db:"workspace_path" json:"workspace_path"` + State string `db:"state" json:"state"` + Generation int `db:"generation" json:"generation"` + LastHealthAt *time.Time `db:"last_health_at" json:"last_health_at,omitempty"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (InstanceRuntimeBinding) TableName() string { + return "instance_runtime_bindings" +} +``` + +Create `backend/internal/models/runtime_rollout.go`: + +```go +package models + +import "time" + +type RuntimeRollout struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + TargetImageRef string `db:"target_image_ref" json:"target_image_ref"` + Status string `db:"status" json:"status"` + BatchSize int `db:"batch_size" json:"batch_size"` + MaxUnavailable int `db:"max_unavailable" json:"max_unavailable"` + StartedBy *int `db:"started_by" json:"started_by,omitempty"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (RuntimeRollout) TableName() string { + return "runtime_rollouts" +} +``` + +Create `backend/internal/models/workspace_file_audit.go`: + +```go +package models + +import "time" + +type WorkspaceFileAudit struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + UserID int `db:"user_id" json:"user_id"` + Action string `db:"action" json:"action"` + RelativePath string `db:"relative_path" json:"relative_path"` + Bytes int64 `db:"bytes" json:"bytes"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +func (WorkspaceFileAudit) TableName() string { + return "workspace_file_audits" +} +``` + +- [ ] **Step 6: Add repository interfaces and implementations** + +Implement interfaces with these method signatures. Use `upper/db` collections for normal CRUD and raw SQL for capacity claims requiring atomic increments. + +`backend/internal/repository/runtime_pod_repository.go`: + +```go +package repository + +import ( + "context" + "fmt" + "time" + + "clawreef/internal/models" + "github.com/upper/db/v4" +) + +type RuntimePodRepository interface { + UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error + GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) + GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) + List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) + ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) + TryClaimSlot(ctx context.Context, podID int64) (bool, error) + ReleaseSlot(ctx context.Context, podID int64) error + MarkState(ctx context.Context, podID int64, state string, draining bool) error + MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error + UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error +} + +type RuntimePodMetricsUpdate struct { + CPUMillisUsed int64 + MemoryBytesUsed int64 + DiskBytesUsed int64 + NetworkRXBytes int64 + NetworkTXBytes int64 + MetricsJSON string + LastSeenAt time.Time +} + +type runtimePodRepository struct { + sess db.Session +} + +func NewRuntimePodRepository(sess db.Session) RuntimePodRepository { + return &runtimePodRepository{sess: sess} +} + +func (r *runtimePodRepository) collection() db.Collection { + return r.sess.Collection("runtime_pods") +} +``` + +Then add each method in the same file. The `TryClaimSlot` implementation must use this SQL shape: + +```go +func (r *runtimePodRepository) TryClaimSlot(ctx context.Context, podID int64) (bool, error) { + res, err := r.sess.SQL(). + ExecContext(ctx, `UPDATE runtime_pods + SET used_slots = used_slots + 1 + WHERE id = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity`, podID) + if err != nil { + return false, err + } + affected, err := res.RowsAffected() + if err != nil { + return false, err + } + return affected == 1, nil +} +``` + +`backend/internal/repository/instance_runtime_binding_repository.go` must include: + +```go +type InstanceRuntimeBindingRepository interface { + Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error + GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) + GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) + ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) + ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) + UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error + UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error + DeleteByInstanceID(ctx context.Context, instanceID int) error +} +``` + +Extend `backend/internal/repository/instance_repository.go` with: + +```go + GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) + GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) + UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error + SetWorkspacePath(ctx context.Context, id int, workspacePath string) error + UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error +``` + +- [ ] **Step 7: Run backend tests for schema compilation** + +Run: + +```powershell +go test ./backend/internal/db ./backend/internal/models ./backend/internal/repository -count=1 +``` + +Expected: pass, or report packages with no test files and no compile errors. + +- [ ] **Step 8: Commit schema and repository layer** + +Run: + +```powershell +git add backend/internal/db backend/internal/models backend/internal/repository +git commit -m "feat: add runtime pool schema" +``` + +## Task 2: Agent Contract Documentation And HTTP Client + +**Files:** +- Create: `docs/clawmanager-agent-v2-contract.md` +- Create: `backend/internal/services/runtime_agent_client.go` +- Create: `backend/internal/services/runtime_agent_client_test.go` + +- [ ] **Step 1: Write the agent contract document** + +Create `docs/clawmanager-agent-v2-contract.md` with these sections: + +```markdown +# clawmanager-agent V2 Contract + +## Purpose + +The agent runs inside each shared OpenClaw or Hermes runtime Pod. It manages gateway subprocesses, ports, cgroups, Linux users, workspace quota, health checks, skill/status reporting, and metrics. + +## Agent-To-Backend Reports + +All report requests use header `X-ClawManager-Agent-Token`. + +### POST /api/v1/runtime-agent/register + +Request body: + +```json +{ + "runtime_type": "openclaw", + "namespace": "clawmanager-system", + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "pod_uid": "pod-uid", + "pod_ip": "10.42.0.31", + "node_name": "node-a", + "deployment_name": "openclaw-runtime", + "image_ref": "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest", + "agent_endpoint": "http://10.42.0.31:19090", + "capacity": 100 +} +``` + +### POST /api/v1/runtime-agent/heartbeat + +Request body: + +```json +{ + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "runtime_type": "openclaw", + "state": "ready", + "used_slots": 37, + "draining": false +} +``` + +### POST /api/v1/runtime-agent/metrics/report + +Request body: + +```json +{ + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "cpu_millis_used": 13600, + "memory_bytes_used": 42949672960, + "disk_bytes_used": 214748364800, + "network_rx_bytes": 9223372, + "network_tx_bytes": 19223372, + "gateways": [ + { + "instance_id": 123, + "gateway_id": "gw-123-7", + "port": 20017, + "state": "running", + "last_health_at": "2026-06-01T10:00:00Z" + } + ] +} +``` + +## Backend-To-Agent Commands + +All command requests use header `X-ClawManager-Control-Token`. + +### POST /v1/gateways + +The agent creates a gateway subprocess and returns the bound port. If no port is free in the requested range, return HTTP 409. + +### DELETE /v1/gateways/{gateway_id} + +The agent stops the gateway and releases cgroup, process, and port resources. + +### POST /v1/drain + +The agent marks the Pod as draining and rejects new gateway creation. Existing gateways continue until ClawManager migrates them. + +## Isolation Requirements + +Every gateway runs as UID/GID `200000 + instance_id`. The agent rejects workspace paths outside `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` after resolving symlinks. CPU and memory limits are enforced with cgroups. Workspace quota is enforced before writes. +``` + +- [ ] **Step 2: Write failing client tests** + +Create `backend/internal/services/runtime_agent_client_test.go`: + +```go +package services + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestRuntimeAgentClientCreateGateway(t *testing.T) { + var gotToken string + var gotReq RuntimeAgentCreateGatewayRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/gateways" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + gotToken = r.Header.Get("X-ClawManager-Control-Token") + if err := json.NewDecoder(r.Body).Decode(&gotReq); err != nil { + t.Fatalf("decode request: %v", err) + } + _ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-7-3", + Port: 20017, + PID: intPtr(8842), + Status: "running", + }) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + resp, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{ + InstanceID: 7, + UserID: 8, + AgentType: "openclaw", + WorkspacePath: "/workspaces/openclaw/user-8/instance-7", + PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099}, + UID: 200007, + GID: 200007, + CPUCores: 2, + MemoryMB: 4096, + DiskQuotaMB: 20480, + Generation: 3, + }) + if err != nil { + t.Fatalf("CreateGateway returned error: %v", err) + } + if gotToken != "secret" { + t.Fatalf("unexpected token %q", gotToken) + } + if gotReq.InstanceID != 7 || gotReq.PortRange.Start != 20000 { + t.Fatalf("unexpected request %#v", gotReq) + } + if resp.GatewayID != "gw-7-3" || resp.Port != 20017 { + t.Fatalf("unexpected response %#v", resp) + } +} + +func TestRuntimeAgentClientConflict(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "no free port", http.StatusConflict) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + _, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{}) + if err == nil || err.Error() != "runtime agent conflict: no free port\n" { + t.Fatalf("unexpected error %v", err) + } +} + +func intPtr(v int) *int { + return &v +} +``` + +- [ ] **Step 3: Run client tests and verify they fail** + +Run: + +```powershell +go test ./backend/internal/services -run RuntimeAgentClient -count=1 +``` + +Expected: fail because `RuntimeAgentCreateGatewayRequest` and `NewRuntimeAgentClient` do not exist. + +- [ ] **Step 4: Implement runtime agent client** + +Create `backend/internal/services/runtime_agent_client.go`: + +```go +package services + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +type RuntimeAgentClient interface { + Health(ctx context.Context, endpoint string) error + CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) + DeleteGateway(ctx context.Context, endpoint, gatewayID string) error + Drain(ctx context.Context, endpoint string) error +} + +type RuntimeAgentPortRange struct { + Start int `json:"start"` + End int `json:"end"` +} + +type RuntimeAgentCreateGatewayRequest struct { + InstanceID int `json:"instance_id"` + UserID int `json:"user_id"` + AgentType string `json:"agent_type"` + WorkspacePath string `json:"workspace_path"` + PortRange RuntimeAgentPortRange `json:"port_range"` + UID int `json:"uid"` + GID int `json:"gid"` + CPUCores float64 `json:"cpu_cores"` + MemoryMB int `json:"memory_mb"` + DiskQuotaMB int `json:"disk_quota_mb"` + Generation int `json:"generation"` +} + +type RuntimeAgentCreateGatewayResponse struct { + GatewayID string `json:"gateway_id"` + Port int `json:"port"` + PID *int `json:"pid,omitempty"` + Status string `json:"status"` +} + +type runtimeAgentHTTPClient struct { + controlToken string + httpClient *http.Client +} + +func NewRuntimeAgentClient(controlToken string) RuntimeAgentClient { + return &runtimeAgentHTTPClient{ + controlToken: controlToken, + httpClient: &http.Client{ + Timeout: 5 * time.Second, + }, + } +} + +func (c *runtimeAgentHTTPClient) Health(ctx context.Context, endpoint string) error { + return c.do(ctx, http.MethodGet, endpoint, "/v1/health", nil, nil) +} + +func (c *runtimeAgentHTTPClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) { + var resp RuntimeAgentCreateGatewayResponse + if err := c.do(ctx, http.MethodPost, endpoint, "/v1/gateways", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *runtimeAgentHTTPClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error { + return c.do(ctx, http.MethodDelete, endpoint, "/v1/gateways/"+gatewayID, nil, nil) +} + +func (c *runtimeAgentHTTPClient) Drain(ctx context.Context, endpoint string) error { + return c.do(ctx, http.MethodPost, endpoint, "/v1/drain", map[string]bool{"draining": true}, nil) +} + +func (c *runtimeAgentHTTPClient) do(ctx context.Context, method, endpoint, path string, body any, out any) error { + endpoint = strings.TrimRight(endpoint, "/") + var reader io.Reader + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return err + } + reader = bytes.NewReader(payload) + } + req, err := http.NewRequestWithContext(ctx, method, endpoint+path, reader) + if err != nil { + return err + } + req.Header.Set("X-ClawManager-Control-Token", c.controlToken) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode == http.StatusConflict { + return fmt.Errorf("runtime agent conflict: %s", string(msg)) + } + return fmt.Errorf("runtime agent status %d: %s", resp.StatusCode, string(msg)) + } + if out == nil { + return nil + } + return json.NewDecoder(resp.Body).Decode(out) +} +``` + +- [ ] **Step 5: Run client tests** + +Run: + +```powershell +go test ./backend/internal/services -run RuntimeAgentClient -count=1 +``` + +Expected: pass. + +- [ ] **Step 6: Commit agent contract and client** + +Run: + +```powershell +git add docs/clawmanager-agent-v2-contract.md backend/internal/services/runtime_agent_client.go backend/internal/services/runtime_agent_client_test.go +git commit -m "feat: add runtime agent contract" +``` + +## Task 3: Runtime Capacity, Kubernetes Runtime Deployments, And Leader Election + +**Files:** +- Create: `backend/internal/services/runtime_capacity.go` +- Create: `backend/internal/services/runtime_capacity_test.go` +- Create: `backend/internal/services/runtime_leader.go` +- Create: `backend/internal/services/k8s/runtime_deployment_service.go` +- Create: `backend/internal/services/k8s/runtime_deployment_service_test.go` +- Modify: `backend/internal/config/config.go` + +- [ ] **Step 1: Write runtime capacity tests** + +Create `backend/internal/services/runtime_capacity_test.go`: + +```go +package services + +import "testing" + +func TestNormalizeV2RuntimeType(t *testing.T) { + tests := map[string]string{ + "openclaw": "openclaw", + "hermes": "hermes", + "webtop": "", + "ubuntu": "", + } + for input, want := range tests { + got, ok := NormalizeV2RuntimeType(input) + if want == "" { + if ok { + t.Fatalf("expected %s to be rejected", input) + } + continue + } + if !ok || got != want { + t.Fatalf("NormalizeV2RuntimeType(%q) = %q, %v", input, got, ok) + } + } +} + +func TestRuntimeWorkspacePath(t *testing.T) { + got := RuntimeWorkspacePath("openclaw", 45, 123) + want := "/workspaces/openclaw/user-45/instance-123" + if got != want { + t.Fatalf("workspace path mismatch: want %s got %s", want, got) + } +} + +func TestRuntimeLinuxID(t *testing.T) { + got := RuntimeLinuxID(123) + if got != 200123 { + t.Fatalf("runtime linux id mismatch: %d", got) + } +} +``` + +- [ ] **Step 2: Implement capacity helpers** + +Create `backend/internal/services/runtime_capacity.go`: + +```go +package services + +import "fmt" + +const ( + RuntimeTypeOpenClaw = "openclaw" + RuntimeTypeHermes = "hermes" + + RuntimeGatewayPortStart = 20000 + RuntimeGatewayPortEnd = 20099 + RuntimePodCapacity = 100 + RuntimeLinuxIDBase = 200000 +) + +func NormalizeV2RuntimeType(instanceType string) (string, bool) { + switch instanceType { + case RuntimeTypeOpenClaw: + return RuntimeTypeOpenClaw, true + case RuntimeTypeHermes: + return RuntimeTypeHermes, true + default: + return "", false + } +} + +func RuntimeWorkspacePath(runtimeType string, userID int, instanceID int) string { + return fmt.Sprintf("/workspaces/%s/user-%d/instance-%d", runtimeType, userID, instanceID) +} + +func RuntimeLinuxID(instanceID int) int { + return RuntimeLinuxIDBase + instanceID +} +``` + +- [ ] **Step 3: Run capacity tests** + +Run: + +```powershell +go test ./backend/internal/services -run 'NormalizeV2RuntimeType|RuntimeWorkspacePath|RuntimeLinuxID' -count=1 +``` + +Expected: pass. + +- [ ] **Step 4: Add runtime config fields** + +Modify `backend/internal/config/config.go` by adding fields to the main config struct: + +```go + Runtime RuntimeConfig +``` + +Add this struct and env parsing: + +```go +type RuntimeConfig struct { + Namespace string + WorkspaceRoot string + AgentControlToken string + AgentReportToken string + BackendReplicaID string + RedisURL string + SchedulerEnabled bool + HeartbeatTimeout time.Duration + SchedulerTick time.Duration + OpenClawImage string + HermesImage string + MaxGatewaysPerPod int + GatewayPortStart int + GatewayPortEnd int +} +``` + +Parse with defaults: + +```go +Runtime: RuntimeConfig{ + Namespace: getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", "clawmanager-system")), + WorkspaceRoot: getEnv("RUNTIME_WORKSPACE_ROOT", "/workspaces"), + AgentControlToken: getEnv("RUNTIME_AGENT_CONTROL_TOKEN", ""), + AgentReportToken: getEnv("RUNTIME_AGENT_REPORT_TOKEN", ""), + BackendReplicaID: getEnv("HOSTNAME", "clawmanager-backend-local"), + RedisURL: getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", "")), + SchedulerEnabled: getEnvBool("RUNTIME_SCHEDULER_ENABLED", true), + HeartbeatTimeout: getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", 10*time.Second), + SchedulerTick: getEnvDuration("RUNTIME_SCHEDULER_TICK", 2*time.Second), + OpenClawImage: getEnv("OPENCLAW_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest"), + HermesImage: getEnv("HERMES_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest"), + MaxGatewaysPerPod: getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", RuntimePodCapacity), + GatewayPortStart: getEnvInt("RUNTIME_GATEWAY_PORT_START", RuntimeGatewayPortStart), + GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", RuntimeGatewayPortEnd), +}, +``` + +If `getEnvDuration`, `getEnvBool`, or `getEnvInt` is missing, add them in `config.go` using `strconv` and `time.ParseDuration`. + +- [ ] **Step 5: Add Kubernetes runtime deployment service test** + +Create `backend/internal/services/k8s/runtime_deployment_service_test.go` with a unit test that constructs the Deployment and checks the shared workspace mount: + +```go +package k8s + +import "testing" + +func TestBuildRuntimeDeploymentUsesSharedWorkspaceAndAgentPort(t *testing.T) { + dep := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "openclaw-runtime", + Namespace: "clawmanager-system", + RuntimeType: "openclaw", + Image: "example/openclaw:latest", + Replicas: 2, + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/exports/workspaces", + AgentControlToken: "control", + AgentReportToken: "report", + }) + + if dep.Name != "openclaw-runtime" || *dep.Spec.Replicas != 2 { + t.Fatalf("unexpected deployment identity %#v", dep) + } + container := dep.Spec.Template.Spec.Containers[0] + if container.Ports[0].ContainerPort != 19090 { + t.Fatalf("agent port missing: %#v", container.Ports) + } + foundWorkspace := false + for _, mount := range container.VolumeMounts { + if mount.Name == "workspaces" && mount.MountPath == "/workspaces" { + foundWorkspace = true + } + } + if !foundWorkspace { + t.Fatalf("workspace mount missing: %#v", container.VolumeMounts) + } +} +``` + +- [ ] **Step 6: Implement Kubernetes runtime deployment service** + +Create `backend/internal/services/k8s/runtime_deployment_service.go` with: + +```go +package k8s + +import ( + "context" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/kubernetes" +) + +type RuntimeDeploymentSpec struct { + Name string + Namespace string + RuntimeType string + Image string + Replicas int32 + WorkspaceNFSServer string + WorkspaceNFSPath string + AgentControlToken string + AgentReportToken string +} + +type RuntimeDeploymentService interface { + Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error + Scale(ctx context.Context, namespace, name string, replicas int32) error +} + +type runtimeDeploymentService struct { + client kubernetes.Interface +} + +func NewRuntimeDeploymentService(client kubernetes.Interface) RuntimeDeploymentService { + return &runtimeDeploymentService{client: client} +} + +func int32Ptr(v int32) *int32 { + return &v +} + +func BuildRuntimeDeployment(spec RuntimeDeploymentSpec) *appsv1.Deployment { + labels := map[string]string{ + "app": spec.Name, + "clawmanager.io/runtime-type": spec.RuntimeType, + } + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: spec.Name, + Namespace: spec.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(spec.Replicas), + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "runtime", + Image: spec.Image, + Ports: []corev1.ContainerPort{{ + Name: "agent", + ContainerPort: 19090, + }}, + Env: []corev1.EnvVar{ + {Name: "CLAWMANAGER_RUNTIME_TYPE", Value: spec.RuntimeType}, + {Name: "CLAWMANAGER_AGENT_PORT", Value: "19090"}, + {Name: "CLAWMANAGER_GATEWAY_PORT_START", Value: "20000"}, + {Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: "20099"}, + {Name: "CLAWMANAGER_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken}, + {Name: "CLAWMANAGER_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "workspaces", + MountPath: "/workspaces", + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "workspaces", + VolumeSource: corev1.VolumeSource{ + NFS: &corev1.NFSVolumeSource{ + Server: spec.WorkspaceNFSServer, + Path: spec.WorkspaceNFSPath, + }, + }, + }}, + }, + }, + }, + } +} +``` + +Add `Ensure` and `Scale` using AppsV1 Deployments `Get`, `Create`, `Update`, and `UpdateScale`. + +- [ ] **Step 7: Implement Lease leader service** + +Create `backend/internal/services/runtime_leader.go`: + +```go +package services + +import ( + "context" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +type RuntimeLeaderService interface { + IsLeader(ctx context.Context) bool +} + +type runtimeLeaderService struct { + client kubernetes.Interface + namespace string + name string + holder string + ttl time.Duration +} + +func NewRuntimeLeaderService(client kubernetes.Interface, namespace, holder string) RuntimeLeaderService { + return &runtimeLeaderService{ + client: client, + namespace: namespace, + name: "clawmanager-runtime-scheduler", + holder: holder, + ttl: 15 * time.Second, + } +} + +func (s *runtimeLeaderService) IsLeader(ctx context.Context) bool { + now := metav1.Now() + leaseClient := s.client.CoordinationV1().Leases(s.namespace) + lease, err := leaseClient.Get(ctx, s.name, metav1.GetOptions{}) + if err != nil { + lease = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: s.name, Namespace: s.namespace}, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &s.holder, + RenewTime: &now, + LeaseDurationSeconds: int32Ptr32(int32(s.ttl.Seconds())), + }, + } + _, createErr := leaseClient.Create(ctx, lease, metav1.CreateOptions{}) + return createErr == nil + } + if lease.Spec.HolderIdentity != nil && *lease.Spec.HolderIdentity != s.holder && lease.Spec.RenewTime != nil { + expires := lease.Spec.RenewTime.Time.Add(s.ttl) + if time.Now().Before(expires) { + return false + } + } + lease.Spec.HolderIdentity = &s.holder + lease.Spec.RenewTime = &now + _, err = leaseClient.Update(ctx, lease, metav1.UpdateOptions{}) + return err == nil +} + +func int32Ptr32(v int32) *int32 { + return &v +} +``` + +- [ ] **Step 8: Run runtime deployment and leader tests** + +Run: + +```powershell +go test ./backend/internal/services ./backend/internal/services/k8s -run 'Runtime|Leader' -count=1 +``` + +Expected: pass. + +- [ ] **Step 9: Commit runtime capacity and Kubernetes services** + +Run: + +```powershell +git add backend/internal/config backend/internal/services backend/internal/services/k8s +git commit -m "feat: add runtime pool infrastructure services" +``` + +## Task 4: Runtime Scheduler And Redis Event Fanout + +**Files:** +- Create: `backend/internal/services/redis_client.go` +- Create: `backend/internal/services/runtime_events.go` +- Create: `backend/internal/services/runtime_scheduler.go` +- Create: `backend/internal/services/runtime_scheduler_test.go` +- Modify: `backend/cmd/server/main.go` + +- [ ] **Step 1: Extract a platform Redis client** + +Create `backend/internal/services/redis_client.go` by reusing the existing RESP parsing pattern from `team_redis.go`. The exported interface must be: + +```go +type PlatformRedisClient interface { + SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) + Del(ctx context.Context, key string) error + XAdd(ctx context.Context, key string, fields map[string]string) (string, error) + XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) +} +``` + +Use the same URL parsing behavior as `newRedisBus`, and add: + +```go +func NewPlatformRedisClient(rawURL string) (PlatformRedisClient, error) { + return newRedisBus(rawURL) +} +``` + +Extend `redisBus` in `team_redis.go` only if needed so the same type satisfies `SetNX` and `Del`: + +```go +func (b *redisBus) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) { + reply, err := b.do(ctx, "SET", key, value, "NX", "PX", fmt.Sprintf("%d", ttl.Milliseconds())) + if err != nil { + return false, err + } + return reply == "OK", nil +} + +func (b *redisBus) Del(ctx context.Context, key string) error { + _, err := b.do(ctx, "DEL", key) + return err +} +``` + +- [ ] **Step 2: Add runtime event service** + +Create `backend/internal/services/runtime_events.go`: + +```go +package services + +import ( + "context" + "encoding/json" + "time" +) + +const runtimeEventStreamKey = "clawmanager:runtime-events" + +type RuntimeEvent struct { + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` + CreatedAt time.Time `json:"created_at"` +} + +type RuntimeEventService interface { + Publish(ctx context.Context, eventType string, payload any) error + Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) +} + +type runtimeEventService struct { + redis PlatformRedisClient +} + +func NewRuntimeEventService(redis PlatformRedisClient) RuntimeEventService { + return &runtimeEventService{redis: redis} +} + +func (s *runtimeEventService) Publish(ctx context.Context, eventType string, payload any) error { + raw, err := json.Marshal(payload) + if err != nil { + return err + } + event := RuntimeEvent{Type: eventType, Payload: raw, CreatedAt: time.Now().UTC()} + eventRaw, err := json.Marshal(event) + if err != nil { + return err + } + _, err = s.redis.XAdd(ctx, runtimeEventStreamKey, map[string]string{"event": string(eventRaw)}) + return err +} + +func (s *runtimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) { + return s.redis.XRead(ctx, runtimeEventStreamKey, lastID, block) +} +``` + +- [ ] **Step 3: Write scheduler assignment test with fakes** + +Create `backend/internal/services/runtime_scheduler_test.go` with fake repositories and this test: + +```go +func TestRuntimeSchedulerAssignsCreatingInstanceToReadyPod(t *testing.T) { + instance := models.Instance{ID: 123, UserID: 45, Type: "openclaw", CPUCores: 2, MemoryGB: 4, DiskGB: 20, RuntimeGeneration: 1} + pod := models.RuntimePod{ID: 9, RuntimeType: "openclaw", PodIP: stringPtr("10.42.0.31"), AgentEndpoint: stringPtr("http://10.42.0.31:19090"), State: "ready", Capacity: 100, UsedSlots: 0} + s := newRuntimeSchedulerForTest(instance, pod) + + if err := s.assignInstance(context.Background(), instance); err != nil { + t.Fatalf("assignInstance returned error: %v", err) + } + if s.fakeAgent.createGatewayCalls != 1 { + t.Fatalf("expected one create gateway call, got %d", s.fakeAgent.createGatewayCalls) + } + binding := s.fakeBindings.byInstance[123] + if binding.GatewayPort != 20017 || binding.RuntimePodID != 9 { + t.Fatalf("unexpected binding %#v", binding) + } +} +``` + +The fake agent must return: + +```go +&RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-123-1", + Port: 20017, + PID: intPtr(8842), + Status: "running", +} +``` + +- [ ] **Step 4: Implement scheduler** + +Create `backend/internal/services/runtime_scheduler.go` with: + +```go +package services + +import ( + "context" + "fmt" + "log" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + k8ssvc "clawreef/internal/services/k8s" +) + +type RuntimeScheduler struct { + instanceRepo repository.InstanceRepository + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + rolloutRepo repository.RuntimeRolloutRepository + agentClient RuntimeAgentClient + events RuntimeEventService + leader RuntimeLeaderService + deployments k8ssvc.RuntimeDeploymentService + tick time.Duration +} + +func NewRuntimeScheduler( + instanceRepo repository.InstanceRepository, + podRepo repository.RuntimePodRepository, + bindingRepo repository.InstanceRuntimeBindingRepository, + rolloutRepo repository.RuntimeRolloutRepository, + agentClient RuntimeAgentClient, + events RuntimeEventService, + leader RuntimeLeaderService, + deployments k8ssvc.RuntimeDeploymentService, + tick time.Duration, +) *RuntimeScheduler { + return &RuntimeScheduler{ + instanceRepo: instanceRepo, + podRepo: podRepo, + bindingRepo: bindingRepo, + rolloutRepo: rolloutRepo, + agentClient: agentClient, + events: events, + leader: leader, + deployments: deployments, + tick: tick, + } +} + +func (s *RuntimeScheduler) Start(ctx context.Context) { + go func() { + ticker := time.NewTicker(s.tick) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !s.leader.IsLeader(ctx) { + continue + } + if err := s.reconcile(ctx); err != nil { + log.Printf("runtime scheduler reconcile failed: %v", err) + } + } + } + }() +} + +func (s *RuntimeScheduler) reconcile(ctx context.Context) error { + creating, err := s.instanceRepo.GetV2Creating(ctx, 100) + if err != nil { + return err + } + for _, instance := range creating { + if err := s.assignInstance(ctx, instance); err != nil { + msg := err.Error() + _ = s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &msg) + } + } + running, err := s.instanceRepo.GetV2DesiredRunning(ctx, 200) + if err != nil { + return err + } + for _, instance := range running { + if _, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID); err != nil { + if assignErr := s.assignInstance(ctx, instance); assignErr != nil { + msg := assignErr.Error() + _ = s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &msg) + } + } + } + return nil +} + +func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.Instance) error { + runtimeType, ok := NormalizeV2RuntimeType(instance.Type) + if !ok { + return fmt.Errorf("unsupported V2 runtime type %s", instance.Type) + } + pods, err := s.podRepo.ListSchedulable(ctx, runtimeType) + if err != nil { + return err + } + for _, pod := range pods { + if pod.AgentEndpoint == nil { + continue + } + claimed, err := s.podRepo.TryClaimSlot(ctx, pod.ID) + if err != nil { + return err + } + if !claimed { + continue + } + if err := s.createGatewayOnPod(ctx, instance, pod); err != nil { + _ = s.podRepo.ReleaseSlot(ctx, pod.ID) + continue + } + return nil + } + return fmt.Errorf("no schedulable runtime pod for %s", runtimeType) +} + +func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance models.Instance, pod models.RuntimePod) error { + workspacePath := RuntimeWorkspacePath(pod.RuntimeType, instance.UserID, instance.ID) + linuxID := RuntimeLinuxID(instance.ID) + resp, err := s.agentClient.CreateGateway(ctx, *pod.AgentEndpoint, RuntimeAgentCreateGatewayRequest{ + InstanceID: instance.ID, + UserID: instance.UserID, + AgentType: pod.RuntimeType, + WorkspacePath: workspacePath, + PortRange: RuntimeAgentPortRange{Start: RuntimeGatewayPortStart, End: RuntimeGatewayPortEnd}, + UID: linuxID, + GID: linuxID, + CPUCores: instance.CPUCores, + MemoryMB: instance.MemoryGB * 1024, + DiskQuotaMB: instance.DiskGB * 1024, + Generation: instance.RuntimeGeneration, + }) + if err != nil { + return err + } + binding := &models.InstanceRuntimeBinding{ + InstanceID: instance.ID, + RuntimePodID: pod.ID, + RuntimeType: pod.RuntimeType, + GatewayID: resp.GatewayID, + GatewayPort: resp.Port, + GatewayPID: resp.PID, + WorkspacePath: workspacePath, + State: "running", + Generation: instance.RuntimeGeneration, + } + if err := s.bindingRepo.Create(ctx, binding); err != nil { + return err + } + if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil { + return err + } + return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "running", instance.RuntimeGeneration, nil) +} +``` + +Add failover and rollout in the same file with these public methods: + +```go +func (s *RuntimeScheduler) DrainPod(ctx context.Context, podID int64) error +func (s *RuntimeScheduler) FailoverPod(ctx context.Context, podID int64, reason string) error +func (s *RuntimeScheduler) StartRollout(ctx context.Context, rolloutID int64) error +``` + +`FailoverPod` must mark the Pod `unhealthy`, list its bindings, delete each binding, release the old slot, increment instance generation, set instance status `creating`, and let the next scheduler tick recreate the gateway on another Pod. + +- [ ] **Step 5: Wire scheduler in `main.go`** + +In `backend/cmd/server/main.go`, create the new repositories after the existing repositories: + +```go +runtimePodRepo := repository.NewRuntimePodRepository(sess) +bindingRepo := repository.NewInstanceRuntimeBindingRepository(sess) +rolloutRepo := repository.NewRuntimeRolloutRepository(sess) +``` + +Create Redis and runtime services: + +```go +platformRedis, err := services.NewPlatformRedisClient(cfg.Runtime.RedisURL) +if err != nil { + log.Printf("platform redis disabled: %v", err) +} +runtimeEvents := services.NewRuntimeEventService(platformRedis) +agentClient := services.NewRuntimeAgentClient(cfg.Runtime.AgentControlToken) +leader := services.NewRuntimeLeaderService(k8sClient, cfg.Runtime.Namespace, cfg.Runtime.BackendReplicaID) +runtimeDeployments := k8s.NewRuntimeDeploymentService(k8sClient) +runtimeScheduler := services.NewRuntimeScheduler(instanceRepo, runtimePodRepo, bindingRepo, rolloutRepo, agentClient, runtimeEvents, leader, runtimeDeployments, cfg.Runtime.SchedulerTick) +if cfg.Runtime.SchedulerEnabled { + runtimeScheduler.Start(ctx) +} +``` + +If `platformRedis` can be nil in local development, make `NewRuntimeEventService` accept nil and return a no-op service that never panics. + +- [ ] **Step 6: Run scheduler tests** + +Run: + +```powershell +go test ./backend/internal/services -run RuntimeScheduler -count=1 +``` + +Expected: pass. + +- [ ] **Step 7: Commit scheduler** + +Run: + +```powershell +git add backend/internal/services backend/cmd/server/main.go +git commit -m "feat: add runtime scheduler" +``` + +## Task 5: V2 Instance Lifecycle And Pod-IP Proxy + +**Files:** +- Modify: `backend/internal/services/instance_service.go` +- Modify: `backend/internal/services/instance_proxy_service.go` +- Modify: `backend/internal/services/sync_service.go` +- Modify: `backend/internal/handlers/instance_handler.go` +- Modify: `backend/cmd/server/main.go` +- Create: `backend/internal/services/instance_proxy_service_v2_test.go` + +- [ ] **Step 1: Add proxy test for binding target** + +Create `backend/internal/services/instance_proxy_service_v2_test.go` with a test that builds an instance, a binding, and a runtime Pod: + +```go +func TestInstanceProxyServiceUsesRuntimeBindingForV2(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/apps/openclaw" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + })) + defer target.Close() + + podIP, port := splitHostPortForTest(t, target.URL) + instance := models.Instance{ID: 123, UserID: 45, Type: "openclaw", Status: "running"} + binding := models.InstanceRuntimeBinding{InstanceID: 123, RuntimePodID: 9, GatewayPort: port, State: "running"} + pod := models.RuntimePod{ID: 9, PodIP: &podIP, State: "ready"} + + service := newProxyServiceForV2Test(instance, binding, pod) + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/123/proxy/apps/openclaw", nil) + rec := httptest.NewRecorder() + + service.Proxy(rec, req, 123, "/apps/openclaw") + if rec.Code != http.StatusOK || rec.Body.String() != "ok" { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } +} +``` + +- [ ] **Step 2: Modify instance creation for V2** + +In `backend/internal/services/instance_service.go`, keep legacy behavior for non-V2 types and add this branch near the start of create: + +```go +runtimeType, isV2 := NormalizeV2RuntimeType(req.Type) +if isV2 { + instance := &models.Instance{ + UserID: userID, + Name: strings.TrimSpace(req.Name), + Description: normalizeStringPtr(req.Description), + Type: runtimeType, + RuntimeType: "gateway", + Status: "creating", + CPUCores: req.CPUCores, + MemoryGB: req.MemoryGB, + DiskGB: req.DiskGB, + StorageClass: strings.TrimSpace(req.StorageClass), + MountPath: "/workspaces", + RuntimeGeneration: 1, + } + if err := s.instanceRepo.Create(ctx, instance); err != nil { + return nil, err + } + workspacePath := RuntimeWorkspacePath(runtimeType, userID, instance.ID) + if err := os.MkdirAll(workspacePath, 0750); err != nil { + return nil, err + } + if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil { + return nil, err + } + instance.WorkspacePath = &workspacePath + return instance, nil +} +``` + +Keep validation so new creation accepts only `openclaw` and `hermes`; old records with `ubuntu`, `webtop`, `debian`, `centos`, or `custom` remain readable and manageable through legacy paths. + +- [ ] **Step 3: Modify start, stop, restart, delete** + +For V2 instances in `instance_service.go`: + +```go +func (s *instanceService) Start(ctx context.Context, userID int, instanceID int) error { + instance, err := s.mustOwnInstance(ctx, userID, instanceID) + if err != nil { + return err + } + if _, ok := NormalizeV2RuntimeType(instance.Type); ok { + return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "creating", instance.RuntimeGeneration+1, nil) + } + return s.startLegacy(ctx, instance) +} + +func (s *instanceService) Stop(ctx context.Context, userID int, instanceID int) error { + instance, err := s.mustOwnInstance(ctx, userID, instanceID) + if err != nil { + return err + } + if _, ok := NormalizeV2RuntimeType(instance.Type); ok { + binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID) + if err == nil { + pod, podErr := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if podErr == nil && pod.AgentEndpoint != nil { + _ = s.agentClient.DeleteGateway(ctx, *pod.AgentEndpoint, binding.GatewayID) + } + _ = s.bindingRepo.DeleteByInstanceID(ctx, instance.ID) + _ = s.runtimePodRepo.ReleaseSlot(ctx, binding.RuntimePodID) + } + return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "stopped", instance.RuntimeGeneration, nil) + } + return s.stopLegacy(ctx, instance) +} +``` + +Use the existing legacy code by moving old logic into private helpers `startLegacy`, `stopLegacy`, and `deleteLegacy`. + +- [ ] **Step 4: Modify status for ordinary users** + +In `backend/internal/handlers/instance_handler.go`, make `/instances/:id/status` return user-safe availability for V2: + +```json +{ + "instance_status": { + "instance_id": 123, + "status": "running", + "availability": "available", + "agent_type": "openclaw", + "workspace_usage_bytes": 123456 + } +} +``` + +Do not include Pod name, namespace, Pod IP, service name, port, capacity, node, or runtime scheduling fields for normal users. + +- [ ] **Step 5: Modify proxy for V2** + +In `backend/internal/services/instance_proxy_service.go`, resolve target for V2: + +```go +func (s *instanceProxyService) resolveV2Target(ctx context.Context, instanceID int) (*url.URL, error) { + binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID) + if err != nil { + return nil, fmt.Errorf("instance gateway is not available") + } + pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if err != nil { + return nil, err + } + if pod.PodIP == nil || *pod.PodIP == "" { + return nil, fmt.Errorf("runtime pod ip is not available") + } + return &url.URL{Scheme: "http", Host: net.JoinHostPort(*pod.PodIP, strconv.Itoa(binding.GatewayPort))}, nil +} +``` + +Use this target when `NormalizeV2RuntimeType(instance.Type)` succeeds, otherwise execute the existing Service-based proxy path. + +- [ ] **Step 6: Run lifecycle and proxy tests** + +Run: + +```powershell +go test ./backend/internal/services -run 'Instance|Proxy|RuntimeScheduler' -count=1 +``` + +Expected: pass. + +- [ ] **Step 7: Commit lifecycle and proxy** + +Run: + +```powershell +git add backend/internal/services backend/internal/handlers backend/cmd/server/main.go +git commit -m "feat: route V2 instances through runtime gateways" +``` + +## Task 6: Workspace File Manager Backend + +**Files:** +- Create: `backend/internal/services/workspace_path_guard.go` +- Create: `backend/internal/services/workspace_path_guard_test.go` +- Create: `backend/internal/services/workspace_file_service.go` +- Create: `backend/internal/services/workspace_file_service_test.go` +- Create: `backend/internal/handlers/workspace_file_handler.go` +- Modify: `backend/internal/utils/response.go` +- Modify: `backend/cmd/server/main.go` + +- [ ] **Step 1: Write path guard tests** + +Create `backend/internal/services/workspace_path_guard_test.go`: + +```go +package services + +import ( + "os" + "path/filepath" + "testing" +) + +func TestResolveWorkspacePathRejectsTraversal(t *testing.T) { + root := t.TempDir() + _, err := ResolveWorkspacePath(root, "../outside.txt") + if err == nil || err.Error() != "workspace path escapes instance workspace" { + t.Fatalf("unexpected error %v", err) + } +} + +func TestResolveWorkspacePathRejectsSymlinkEscape(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + _, err := ResolveWorkspacePath(root, "escape/file.txt") + if err == nil || err.Error() != "workspace path escapes instance workspace" { + t.Fatalf("unexpected error %v", err) + } +} + +func TestResolveWorkspacePathAllowsNestedFile(t *testing.T) { + root := t.TempDir() + got, err := ResolveWorkspacePath(root, "a/b.txt") + if err != nil { + t.Fatalf("ResolveWorkspacePath returned error: %v", err) + } + want := filepath.Join(root, "a", "b.txt") + if got != want { + t.Fatalf("want %s got %s", want, got) + } +} +``` + +- [ ] **Step 2: Implement path guard** + +Create `backend/internal/services/workspace_path_guard.go`: + +```go +package services + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func ResolveWorkspacePath(workspaceRoot, relativePath string) (string, error) { + workspaceRoot = filepath.Clean(workspaceRoot) + cleanRelative := filepath.Clean("/" + strings.TrimSpace(relativePath)) + cleanRelative = strings.TrimPrefix(cleanRelative, string(filepath.Separator)) + fullPath := filepath.Join(workspaceRoot, cleanRelative) + + rootReal, err := filepath.EvalSymlinks(workspaceRoot) + if err != nil { + return "", err + } + parent := fullPath + if info, statErr := os.Lstat(fullPath); statErr == nil && !info.IsDir() { + parent = filepath.Dir(fullPath) + } + if _, statErr := os.Lstat(parent); statErr == nil { + realParent, evalErr := filepath.EvalSymlinks(parent) + if evalErr != nil { + return "", evalErr + } + if !isPathInside(rootReal, realParent) { + return "", fmt.Errorf("workspace path escapes instance workspace") + } + } + if !isPathInside(workspaceRoot, fullPath) { + return "", fmt.Errorf("workspace path escapes instance workspace") + } + return fullPath, nil +} + +func isPathInside(root, candidate string) bool { + rel, err := filepath.Rel(root, candidate) + if err != nil { + return false + } + return rel == "." || (!strings.HasPrefix(rel, ".."+string(filepath.Separator)) && rel != "..") +} +``` + +- [ ] **Step 3: Write workspace service tests** + +Create `backend/internal/services/workspace_file_service_test.go`: + +```go +func TestWorkspaceFileServicePreviewTextLimit(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "small.txt"), []byte("hello"), 0640); err != nil { + t.Fatal(err) + } + service := NewWorkspaceFileService(fakeAuditRepo{}) + preview, err := service.Preview(context.Background(), WorkspaceFileScope{InstanceID: 1, UserID: 2, WorkspacePath: root}, "small.txt") + if err != nil { + t.Fatalf("Preview returned error: %v", err) + } + if preview.Kind != "text" || preview.Text != "hello" { + t.Fatalf("unexpected preview %#v", preview) + } +} + +func TestWorkspaceFileServiceRejectsLargeTextPreview(t *testing.T) { + root := t.TempDir() + large := bytes.Repeat([]byte("a"), 1024*1024+1) + if err := os.WriteFile(filepath.Join(root, "large.log"), large, 0640); err != nil { + t.Fatal(err) + } + service := NewWorkspaceFileService(fakeAuditRepo{}) + _, err := service.Preview(context.Background(), WorkspaceFileScope{InstanceID: 1, UserID: 2, WorkspacePath: root}, "large.log") + if err == nil || err.Error() != "text preview exceeds 1 MiB" { + t.Fatalf("unexpected error %v", err) + } +} +``` + +- [ ] **Step 4: Implement workspace file service** + +Create `backend/internal/services/workspace_file_service.go` with these exported types: + +```go +type WorkspaceFileScope struct { + InstanceID int + UserID int + WorkspacePath string +} + +type WorkspaceEntry struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` + ModifiedAt time.Time `json:"modified_at"` + Previewable bool `json:"previewable"` + Downloadable bool `json:"downloadable"` +} + +type WorkspacePreview struct { + Kind string `json:"kind"` + ContentType string `json:"content_type"` + Text string `json:"text,omitempty"` + URL string `json:"url,omitempty"` +} + +type WorkspaceFileService interface { + List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) + Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) + OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) + Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) error + Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) error + Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) error + Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error +} +``` + +Rules to implement: +- Text preview extensions: `.txt`, `.md`, `.json`, `.yaml`, `.yml`, `.log`, `.py`, `.js`, `.ts`, `.go`, `.sh`; max 1 MiB. +- Image preview extensions: `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.svg`; max 10 MiB. +- PDF preview uses iframe URL and content type `application/pdf`. +- Unknown binary files are download-only. +- Upload max is read from env `WORKSPACE_UPLOAD_MAX_BYTES`, default `524288000`. +- Audit upload, mkdir, rename, delete, and download. Do not audit preview. + +- [ ] **Step 5: Implement handler routes** + +Create `backend/internal/handlers/workspace_file_handler.go` with endpoints: + +```go +GET /api/v1/instances/:id/workspace/files +GET /api/v1/instances/:id/workspace/preview +GET /api/v1/instances/:id/workspace/download +POST /api/v1/instances/:id/workspace/upload +POST /api/v1/instances/:id/workspace/folders +PATCH /api/v1/instances/:id/workspace/entries +DELETE /api/v1/instances/:id/workspace/entries +``` + +Each handler must load the instance, confirm ownership from auth context, require `WorkspacePath != nil`, and create: + +```go +scope := services.WorkspaceFileScope{ + InstanceID: instance.ID, + UserID: userID, + WorkspacePath: *instance.WorkspacePath, +} +``` + +- [ ] **Step 6: Wire workspace routes** + +In `backend/cmd/server/main.go`: + +```go +workspaceAuditRepo := repository.NewWorkspaceFileAuditRepository(sess) +workspaceFileService := services.NewWorkspaceFileService(workspaceAuditRepo) +workspaceFileHandler := handlers.NewWorkspaceFileHandler(instanceService, workspaceFileService) + +instances := api.Group("/instances") +instances.GET("/:id/workspace/files", workspaceFileHandler.List) +instances.GET("/:id/workspace/preview", workspaceFileHandler.Preview) +instances.GET("/:id/workspace/download", workspaceFileHandler.Download) +instances.POST("/:id/workspace/upload", workspaceFileHandler.Upload) +instances.POST("/:id/workspace/folders", workspaceFileHandler.Mkdir) +instances.PATCH("/:id/workspace/entries", workspaceFileHandler.Rename) +instances.DELETE("/:id/workspace/entries", workspaceFileHandler.Delete) +``` + +- [ ] **Step 7: Run workspace backend tests** + +Run: + +```powershell +go test ./backend/internal/services ./backend/internal/handlers -run 'Workspace|ResolveWorkspacePath' -count=1 +``` + +Expected: pass. + +- [ ] **Step 8: Commit workspace backend** + +Run: + +```powershell +git add backend/internal/services backend/internal/handlers backend/internal/repository backend/internal/utils backend/cmd/server/main.go +git commit -m "feat: add workspace file APIs" +``` + +## Task 7: Runtime Agent Report And Admin APIs + +**Files:** +- Create: `backend/internal/handlers/runtime_agent_handler.go` +- Create: `backend/internal/handlers/runtime_pool_handler.go` +- Create: `backend/internal/handlers/runtime_pool_handler_test.go` +- Modify: `backend/internal/services/websocket_service.go` +- Modify: `backend/cmd/server/main.go` + +- [ ] **Step 1: Implement runtime agent report handler** + +Create `backend/internal/handlers/runtime_agent_handler.go` with: + +```go +type RuntimeAgentHandler struct { + cfg config.RuntimeConfig + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + events services.RuntimeEventService +} + +func (h *RuntimeAgentHandler) requireAgentToken(c *gin.Context) bool { + if h.cfg.AgentReportToken == "" || c.GetHeader("X-ClawManager-Agent-Token") != h.cfg.AgentReportToken { + utils.Unauthorized(c, "invalid runtime agent token") + return false + } + return true +} +``` + +Add handlers: +- `Register` upserts `runtime_pods`. +- `Heartbeat` updates state, used slots, draining, and last seen. +- `ReportMetrics` updates metrics and binding health, then publishes `runtime_pod_metrics` to Redis events. +- `ReportGateways` reconciles binding state from agent. +- `ReportSkills` stores skill reports using existing skill service hooks if available; if no hook exists, accept the request and publish a `runtime_agent_skills_reported` event. + +- [ ] **Step 2: Implement admin runtime handler** + +Create `backend/internal/handlers/runtime_pool_handler.go`: + +```go +type RuntimePoolHandler struct { + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + rolloutRepo repository.RuntimeRolloutRepository + scheduler *services.RuntimeScheduler +} + +func (h *RuntimePoolHandler) ListPods(c *gin.Context) +func (h *RuntimePoolHandler) GetPodGateways(c *gin.Context) +func (h *RuntimePoolHandler) DrainPod(c *gin.Context) +func (h *RuntimePoolHandler) StartRollout(c *gin.Context) +``` + +`ListPods` returns: + +```json +{ + "pods": [ + { + "id": 9, + "runtime_type": "openclaw", + "pod_name": "openclaw-runtime-abcde", + "pod_ip": "10.42.0.31", + "node_name": "node-a", + "state": "ready", + "used_slots": 37, + "capacity": 100, + "draining": false, + "cpu_millis_used": 13600, + "memory_bytes_used": 42949672960, + "disk_bytes_used": 214748364800, + "network_rx_bytes": 9223372, + "network_tx_bytes": 19223372, + "last_seen_at": "2026-06-01T10:00:00Z" + } + ] +} +``` + +- [ ] **Step 3: Add admin handler tests** + +Create `backend/internal/handlers/runtime_pool_handler_test.go` and verify non-admin access is rejected and admin access returns metrics. Use existing auth middleware test helpers if present; otherwise call the handler directly with `gin.CreateTestContext`. + +- [ ] **Step 4: Extend websocket service** + +Modify `backend/internal/services/websocket_service.go`: + +```go +type WebSocketTopic string + +const ( + WebSocketTopicUser WebSocketTopic = "user" + WebSocketTopicRuntimeAdmin WebSocketTopic = "runtime_admin" +) +``` + +Add subscription filtering: + +```go +type Client struct { + UserID int + Role string + Topic WebSocketTopic + Send chan []byte +} + +func (h *Hub) BroadcastRuntimeAdmin(message []byte) { + h.mu.RLock() + defer h.mu.RUnlock() + for client := range h.clients { + if client.Role == "admin" && client.Topic == WebSocketTopicRuntimeAdmin { + select { + case client.Send <- message: + default: + } + } + } +} +``` + +Start one goroutine per backend replica to `XREAD` `clawmanager:runtime-events` and broadcast runtime admin messages to local websocket clients. + +- [ ] **Step 5: Wire routes** + +In `backend/cmd/server/main.go`: + +```go +runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, runtimeEvents) +api.POST("/runtime-agent/register", runtimeAgentHandler.Register) +api.POST("/runtime-agent/heartbeat", runtimeAgentHandler.Heartbeat) +api.POST("/runtime-agent/gateways/report", runtimeAgentHandler.ReportGateways) +api.POST("/runtime-agent/skills/report", runtimeAgentHandler.ReportSkills) +api.POST("/runtime-agent/metrics/report", runtimeAgentHandler.ReportMetrics) + +runtimePoolHandler := handlers.NewRuntimePoolHandler(runtimePodRepo, bindingRepo, rolloutRepo, runtimeScheduler) +admin.GET("/runtime-pods", runtimePoolHandler.ListPods) +admin.GET("/runtime-pods/:id/gateways", runtimePoolHandler.GetPodGateways) +admin.POST("/runtime-pods/:id/drain", runtimePoolHandler.DrainPod) +admin.POST("/runtime-rollouts", runtimePoolHandler.StartRollout) +``` + +- [ ] **Step 6: Run admin API tests** + +Run: + +```powershell +go test ./backend/internal/handlers ./backend/internal/services -run 'RuntimePool|RuntimeAgent|WebSocket' -count=1 +``` + +Expected: pass. + +- [ ] **Step 7: Commit admin runtime APIs** + +Run: + +```powershell +git add backend/internal/handlers backend/internal/services backend/cmd/server/main.go +git commit -m "feat: add runtime admin APIs" +``` + +## Task 8: Frontend Simple UI Foundation + +**Files:** +- Modify: `frontend/package.json` +- Modify: package lock file if present after install. +- Modify: `frontend/src/index.css` +- Modify: `frontend/src/components/UserLayout.tsx` +- Modify: `frontend/src/components/AdminLayout.tsx` +- Modify: `frontend/src/App.css` if it still injects decorative gradients. + +- [ ] **Step 1: Add icon dependency** + +Run: + +```powershell +cd frontend +npm install lucide-react +cd .. +``` + +Expected: `frontend/package.json` and the lock file include `lucide-react`. + +- [ ] **Step 2: Replace global visual tokens** + +Modify `frontend/src/index.css` so the design uses a neutral, simple base: + +```css +:root { + font-family: + Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", + sans-serif; + color: #111827; + background: #f7f8fa; + font-synthesis: none; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +body { + margin: 0; + min-width: 320px; + min-height: 100vh; + background: #f7f8fa; +} + +button, +input, +select, +textarea { + font: inherit; +} + +.cm-surface { + background: #ffffff; + border: 1px solid #e5e7eb; + border-radius: 8px; +} + +.cm-button { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 6px; + min-height: 36px; + border-radius: 6px; + border: 1px solid #d1d5db; + background: #ffffff; + color: #111827; +} + +.cm-button-primary { + border-color: #2563eb; + background: #2563eb; + color: #ffffff; +} +``` + +Remove decorative gradient or orb classes from `App.css`. + +- [ ] **Step 3: Simplify user layout** + +Modify `frontend/src/components/UserLayout.tsx`: +- Use a plain top bar and left nav. +- Keep border radius at `rounded-lg` or smaller. +- Use `lucide-react` icons for nav and command buttons. +- Remove runtime details from ordinary user navigation labels. +- Keep language switcher and account actions. + +Use this layout shape: + +```tsx +
+ +
+
+ ... +
+
+ +
+
+
+``` + +- [ ] **Step 4: Simplify admin layout** + +Modify `frontend/src/components/AdminLayout.tsx` with the same visual system and add a nav entry: + +```tsx +{ + to: "/admin/runtime-pods", + label: "Runtime Pods", + icon: Server, +} +``` + +Do not expose admin-only runtime metrics in `UserLayout`. + +- [ ] **Step 5: Run frontend lint** + +Run: + +```powershell +cd frontend +npm run lint +cd .. +``` + +Expected: pass. + +- [ ] **Step 6: Commit UI foundation** + +Run: + +```powershell +git add frontend/package.json frontend/package-lock.json frontend/src/index.css frontend/src/App.css frontend/src/components/UserLayout.tsx frontend/src/components/AdminLayout.tsx +git commit -m "style: simplify ClawManager UI shell" +``` + +## Task 9: Frontend Workspace Manager And V2 User Pages + +**Files:** +- Modify: `frontend/src/types/instance.ts` +- Create: `frontend/src/types/workspace.ts` +- Create: `frontend/src/services/workspaceService.ts` +- Create: `frontend/src/components/WorkspaceFileManager.tsx` +- Create: `frontend/src/components/InstanceServiceFrame.tsx` +- Modify: `frontend/src/pages/instances/CreateInstancePage.tsx` +- Modify: `frontend/src/pages/instances/InstanceListPage.tsx` +- Modify: `frontend/src/pages/instances/InstanceDetailPage.tsx` +- Modify: `frontend/src/services/instanceService.ts` + +- [ ] **Step 1: Add TypeScript types** + +Create `frontend/src/types/workspace.ts`: + +```ts +export interface WorkspaceEntry { + name: string; + path: string; + is_dir: boolean; + size: number; + modified_at: string; + previewable: boolean; + downloadable: boolean; +} + +export interface WorkspacePreview { + kind: "text" | "image" | "pdf" | "download"; + content_type: string; + text?: string; + url?: string; +} +``` + +Modify `frontend/src/types/instance.ts`: + +```ts +export type V2InstanceType = "openclaw" | "hermes"; +export type InstanceAvailability = "available" | "starting" | "unavailable"; + +export interface InstanceStatus { + instance_id: number; + status: string; + availability?: InstanceAvailability; + agent_type?: V2InstanceType; + workspace_usage_bytes?: number; + created_at?: string; + started_at?: string; +} +``` + +Keep legacy optional Pod fields only if existing pages still need them for admin legacy screens. + +- [ ] **Step 2: Add workspace service** + +Create `frontend/src/services/workspaceService.ts`: + +```ts +import api from "./api"; +import type { WorkspaceEntry, WorkspacePreview } from "../types/workspace"; + +export const workspaceService = { + async list(instanceId: number, path = ""): Promise { + const response = await api.get(`/instances/${instanceId}/workspace/files`, { + params: { path }, + }); + return response.data.data.entries; + }, + + async preview(instanceId: number, path: string): Promise { + const response = await api.get(`/instances/${instanceId}/workspace/preview`, { + params: { path }, + }); + return response.data.data.preview; + }, + + downloadUrl(instanceId: number, path: string): string { + return `/api/v1/instances/${instanceId}/workspace/download?path=${encodeURIComponent(path)}`; + }, + + async upload(instanceId: number, path: string, file: File): Promise { + const formData = new FormData(); + formData.append("file", file); + await api.post(`/instances/${instanceId}/workspace/upload`, formData, { + params: { path }, + headers: { "Content-Type": "multipart/form-data" }, + }); + }, + + async mkdir(instanceId: number, path: string): Promise { + await api.post(`/instances/${instanceId}/workspace/folders`, { path }); + }, + + async rename(instanceId: number, oldPath: string, newPath: string): Promise { + await api.patch(`/instances/${instanceId}/workspace/entries`, { + old_path: oldPath, + new_path: newPath, + }); + }, + + async remove(instanceId: number, path: string): Promise { + await api.delete(`/instances/${instanceId}/workspace/entries`, { + params: { path }, + }); + }, +}; +``` + +- [ ] **Step 3: Build independent file manager** + +Create `frontend/src/components/WorkspaceFileManager.tsx`. It must: +- Fetch entries with React Query. +- Show breadcrumb path. +- Upload via hidden file input. +- Provide icon buttons for preview, download, rename, delete, and new folder. +- Show text preview in a compact preformatted pane. +- Show image/PDF preview in a modal or right panel. +- Never render an arbitrary path input that can bypass the breadcrumb. + +Core state: + +```tsx +const [currentPath, setCurrentPath] = useState(""); +const [previewPath, setPreviewPath] = useState(null); +const entriesQuery = useQuery({ + queryKey: ["workspace", instanceId, currentPath], + queryFn: () => workspaceService.list(instanceId, currentPath), +}); +``` + +- [ ] **Step 4: Build service frame** + +Create `frontend/src/components/InstanceServiceFrame.tsx`: + +```tsx +interface InstanceServiceFrameProps { + instanceId: number; + availability: "available" | "starting" | "unavailable"; +} + +export function InstanceServiceFrame({ instanceId, availability }: InstanceServiceFrameProps) { + if (availability === "starting") { + return
Starting
; + } + if (availability === "unavailable") { + return
Unavailable
; + } + return ( +