// Copyright 2026 Alibaba Group Holding Ltd. //go:build !windows // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package runtime import ( "bufio" "context" "fmt" "io" "os" "path/filepath" "strconv" "strings" "syscall" "time" "github.com/google/uuid" "github.com/alibaba/opensandbox/execd/pkg/isolation" "github.com/alibaba/opensandbox/execd/pkg/log" "github.com/alibaba/opensandbox/execd/pkg/telemetry" "github.com/alibaba/opensandbox/execd/pkg/vfs" ) const isolatedRunEndMarkerPrefix = "__ISOLATED_RUN_END__" // IsolatedRunner is the concrete isolated session runner. type IsolatedRunner struct { ctrl *Controller isolator isolation.Isolator upperMgr *isolation.UpperManager allowedWritable []string stopGC chan struct{} } // NewIsolatedRunner creates the isolated session runner. func NewIsolatedRunner(ctrl *Controller, iso isolation.Isolator, cfg isolation.Config) (*IsolatedRunner, error) { mgr, err := isolation.NewUpperManager(cfg.UpperRoot, cfg.UpperMaxBytes) if err != nil { return nil, fmt.Errorf("isolated runner: upper manager: %w", err) } r := &IsolatedRunner{ ctrl: ctrl, isolator: iso, upperMgr: mgr, allowedWritable: cfg.AllowedWritable, stopGC: make(chan struct{}), } go r.gcLoop() // Register with telemetry so gauges can read session/upper stats. telemetry.SetIsolationStatsProvider(r.statsSnapshot) return r, nil } // statsSnapshot returns current isolation stats for telemetry gauges. func (r *IsolatedRunner) statsSnapshot() telemetry.IsolationStats { sessionCount := int64(0) r.ctrl.isolatedSessionMap.Range(func(_, _ any) bool { sessionCount++ return true }) usage, _ := r.upperMgr.Usage() return telemetry.IsolationStats{ ActiveSessions: sessionCount, UpperUsageBytes: usage, } } // startGC begins periodic idle session cleanup. func (r *IsolatedRunner) gcLoop() { ticker := time.NewTicker(60 * time.Second) defer ticker.Stop() for { select { case <-r.stopGC: return case <-ticker.C: r.CollectIdle() } } } // CollectIdle scans sessions and deletes those past their idle timeout // or whose bwrap process has died. func (r *IsolatedRunner) CollectIdle() { now := time.Now() r.ctrl.isolatedSessionMap.Range(func(key, value any) bool { s, ok := value.(*isolatedSession) if !ok { return true } sessionID := s.id if s.dead() { log.Info("idle GC: cleaning up dead session %s", sessionID) if err := r.DeleteIsolatedSession(sessionID); err != nil { log.Warn("idle GC: delete dead session %s: %v", sessionID, err) } return true } s.mu.RLock() timeout := time.Duration(s.opts.IdleTimeoutSeconds) * time.Second idle := now.Sub(s.lastRunAt) s.mu.RUnlock() if timeout > 0 && idle > timeout { if !s.runMu.TryLock() { return true } s.runMu.Unlock() log.Info("idle GC: deleting session %s (idle %v > timeout %v)", sessionID, idle, timeout) if err := r.DeleteIsolatedSession(sessionID); err != nil { log.Warn("idle GC: delete session %s: %v", sessionID, err) } } return true }) } // StopGC stops the background GC goroutine. func (r *IsolatedRunner) StopGC() { close(r.stopGC) } // Available reports whether the isolator is ready. func (r *IsolatedRunner) Available() bool { return r.isolator.Available() } // CreateIsolatedSession starts a new bwrap + bash session. func (r *IsolatedRunner) CreateIsolatedSession(opts *IsolatedSessionOptions) (string, error) { if err := r.validateExtraWritable(opts.ExtraWritable); err != nil { return "", err } if err := r.validateBinds(opts.Binds); err != nil { return "", err } if err := os.MkdirAll(opts.WorkspacePath, 0o755); err != nil { return "", fmt.Errorf("create workspace: %w", err) } id := uuid.New().String() session := newIsolatedSession(id, opts, r.isolator) // Allocate upper directory for overlay mode. if opts.WorkspaceMode == string(isolation.WorkspaceOverlay) || opts.WorkspaceMode == "" { upperID, upperDir, workDir, err := r.upperMgr.Allocate() if err != nil { return "", fmt.Errorf("allocate upper: %w", err) } session.upperID = upperID session.upperDir = upperDir session.workDir = workDir } if err := session.start(); err != nil { if session.upperID != "" { _ = r.upperMgr.Remove(session.upperID) } return "", fmt.Errorf("start bwrap: %w", err) } r.ctrl.isolatedSessionMap.Store(id, session) log.Info("created isolated session %s (profile=%s, mode=%s)", id, opts.Profile, opts.WorkspaceMode) return id, nil } // GetIsolatedSession returns session state. func (r *IsolatedRunner) GetIsolatedSession(id string) (*IsolatedSessionState, error) { s := r.lookup(id) if s == nil { return nil, ErrContextNotFound } s.mu.RLock() defer s.mu.RUnlock() status := SessionStatusActive if s.dead() { status = SessionStatusDead } state := &IsolatedSessionState{ Status: status, CreatedAt: s.createdAt, LastRunAt: s.lastRunAt, } if s.opts.IdleTimeoutSeconds > 0 { remaining := s.opts.IdleTimeoutSeconds - int(time.Since(s.lastRunAt).Seconds()) if remaining < 0 { remaining = 0 } state.IdleRemainingSeconds = &remaining } return state, nil } // Session status values. const ( SessionStatusActive = "active" SessionStatusDead = "dead" ) // IsolatedSessionState is returned by GetIsolatedSession. type IsolatedSessionState struct { Status string CreatedAt time.Time LastRunAt time.Time IdleRemainingSeconds *int } // IsolatedSessionSummary describes a single session in a list response. type IsolatedSessionSummary struct { SessionID string IsolatedSessionState } // ListIsolatedSessions returns a summary of all active isolated sessions. func (r *IsolatedRunner) ListIsolatedSessions() []IsolatedSessionSummary { summaries := make([]IsolatedSessionSummary, 0) r.ctrl.isolatedSessionMap.Range(func(key, value any) bool { s, ok := value.(*isolatedSession) if !ok { return true } s.mu.RLock() status := SessionStatusActive if s.dead() { status = SessionStatusDead } summary := IsolatedSessionSummary{ SessionID: s.id, IsolatedSessionState: IsolatedSessionState{ Status: status, CreatedAt: s.createdAt, LastRunAt: s.lastRunAt, }, } if s.opts.IdleTimeoutSeconds > 0 { remaining := s.opts.IdleTimeoutSeconds - int(time.Since(s.lastRunAt).Seconds()) if remaining < 0 { remaining = 0 } summary.IdleRemainingSeconds = &remaining } s.mu.RUnlock() summaries = append(summaries, summary) return true }) return summaries } // StdoutCallback is called for each line of stdout output during Run. type StdoutCallback func(line string) // RunInIsolatedSession executes code in the session. // Runs are serialized per session via s.runMu. // envs are exported in the bash session before code runs. func (r *IsolatedRunner) RunInIsolatedSession(ctx context.Context, id string, code string, envs map[string]string, onStdout StdoutCallback) error { s := r.lookup(id) if s == nil { return ErrContextNotFound } // Serialize concurrent runs on the same session. s.runMu.Lock() defer s.runMu.Unlock() if s.dead() { return fmt.Errorf("session process has exited") } s.mu.RLock() stdin := s.stdin stdout := s.stdout s.mu.RUnlock() if stdin == nil || stdout == nil { return fmt.Errorf("session not started") } // Prepend env exports before user code. runMarker := fmt.Sprintf("%s_%s", isolatedRunEndMarkerPrefix, uuid.New().String()) var script string if len(envs) > 0 { script += "(\n" for k, v := range envs { script += fmt.Sprintf("export %s=%s\n", shellescape(k), shellescape(v)) } script += code if !strings.HasSuffix(script, "\n") { script += "\n" } script += ")\n" } else { script += code if !strings.HasSuffix(script, "\n") { script += "\n" } } script += fmt.Sprintf("echo %s $?\n", runMarker) // On timeout/cancel, send SIGINT to interrupt the running command // without killing the persistent bash session. Closing stdin would // terminate bash entirely. done := make(chan struct{}) defer close(done) go func() { select { case <-ctx.Done(): if s.cmd != nil && s.cmd.Process != nil { _ = syscall.Kill(-s.cmd.Process.Pid, syscall.SIGINT) } case <-done: } }() if _, err := io.WriteString(stdin, script); err != nil { return fmt.Errorf("write stdin: %w", err) } exitCode, err := scanUntilMarker(ctx, stdout, runMarker, onStdout) if err != nil { return err } s.mu.Lock() s.lastRunAt = time.Now() s.mu.Unlock() if exitCode != 0 { return fmt.Errorf("command exited with code %d", exitCode) } return nil } // scanUntilMarker reads stdout lines until the end marker is found. // Returns the exit code from the marker line. func scanUntilMarker(ctx context.Context, stdout io.ReadCloser, runMarker string, onStdout StdoutCallback) (int, error) { scanner := bufio.NewScanner(stdout) scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) var exitCode int markerSeen := false scanDone := make(chan struct{}) go func() { defer close(scanDone) for scanner.Scan() { line := scanner.Text() // The marker may appear mid-line if the previous command's // output didn't end with a newline (e.g. cat of a file // without trailing newline). if idx := strings.Index(line, runMarker); idx >= 0 { markerSeen = true if idx > 0 && onStdout != nil { onStdout(line[:idx]) } markerPart := line[idx:] parts := strings.Fields(markerPart) if len(parts) >= 2 { if code, convErr := strconv.Atoi(parts[1]); convErr == nil { exitCode = code } } return } if onStdout != nil { onStdout(line) } } }() select { case <-scanDone: case <-ctx.Done(): // Wait for scanner goroutine to finish so it doesn't consume the // next run's output on the shared stdout pipe. <-scanDone return 0, ctx.Err() } if err := scanner.Err(); err != nil { return 0, fmt.Errorf("read stdout: %w", err) } if !markerSeen { return 1, fmt.Errorf("session process exited without end marker (process may have died or called exit)") } return exitCode, nil } // DeleteIsolatedSession destroys the session. func (r *IsolatedRunner) DeleteIsolatedSession(id string) error { s := r.lookup(id) if s == nil { return ErrContextNotFound } s.mu.Lock() defer s.mu.Unlock() if err := s.stop(); err != nil { log.Warn("stop isolated session %s: %v", id, err) } if s.upperID != "" { if err := r.upperMgr.Remove(s.upperID); err != nil { log.Warn("remove upper dir for session %s: %v", id, err) } } r.ctrl.isolatedSessionMap.Delete(id) log.Info("deleted isolated session %s", id) return nil } // DiffUpper returns an error (Phase 2). func (r *IsolatedRunner) DiffUpper(id string, w io.Writer) error { return fmt.Errorf("diff not implemented yet") } // CommitUpper returns an error (Phase 2). func (r *IsolatedRunner) CommitUpper(id string) error { return fmt.Errorf("commit not implemented yet") } // GetMergedView returns a VFS for the session's filesystem. func (r *IsolatedRunner) GetMergedView(id string) (vfs.FS, error) { s := r.lookup(id) if s == nil { return nil, ErrContextNotFound } s.mu.RLock() defer s.mu.RUnlock() // MergedView chowns files on the host side (execd's namespace). // In setpriv mode the requested uid/gid are real host IDs, so use them. // In userns mode the requested uid/gid are in-namespace IDs mapped to // execd's real host uid/gid, so host-side files must use execd's own // host uid/gid — chowning to the in-namespace ID would fail with EPERM // (unprivileged execd) or create files that appear as nobody/overflow // inside the sandbox. uid := uint32(os.Getuid()) gid := uint32(os.Getgid()) if isolation.UidMode(s.opts.UidMode) != isolation.UidModeUserns { if s.opts.Uid != nil { uid = *s.opts.Uid } if s.opts.Gid != nil { gid = *s.opts.Gid } } mode := isolation.WorkspaceOverlay upper := s.upperDir switch isolation.WorkspaceMode(s.opts.WorkspaceMode) { case isolation.WorkspaceRW: mode = isolation.WorkspaceRW upper = s.opts.WorkspacePath // writes go directly to workspace case isolation.WorkspaceRO: mode = isolation.WorkspaceRO } return isolation.NewMergedView(s.opts.WorkspacePath, upper, mode, uid, gid), nil } // Capabilities returns the current isolator capabilities. func (r *IsolatedRunner) Capabilities() isolation.Capabilities { return r.isolator.Capabilities() } func (r *IsolatedRunner) lookup(id string) *isolatedSession { v, ok := r.ctrl.isolatedSessionMap.Load(id) if !ok { return nil } s, ok := v.(*isolatedSession) if !ok { return nil } return s } func (r *IsolatedRunner) validateExtraWritable(paths []string) error { if len(paths) == 0 { return nil } if len(r.allowedWritable) == 0 { return fmt.Errorf("extra_writable not allowed: no paths in allowlist") } for i := range paths { resolved, err := r.resolveAllowedSource(paths[i]) if err != nil { return fmt.Errorf("extra_writable path %q: %w", paths[i], err) } // Mount the fully-resolved path so validation and mount target agree. paths[i] = resolved } return nil } // resolveAllowedSource requires src to exist, fully resolves symlinks, checks // the resolved real path against the writable allowlist, and returns it. It is // shared by extra_writable and binds so both enforce identical semantics: // - the source must already exist (bwrap --bind requires this anyway), and // - the allowlist is enforced against the fully-resolved real path, leaving // no unresolved suffix that could be swapped to an out-of-allowlist symlink // between validation and bwrap start. func (r *IsolatedRunner) resolveAllowedSource(src string) (string, error) { if src == "" { return "", fmt.Errorf("source is required") } resolved, err := filepath.EvalSymlinks(filepath.Clean(src)) if err != nil { return "", fmt.Errorf("must be an existing path: %w", err) } if !r.pathAllowedResolved(resolved) { return "", fmt.Errorf("not in allowlist") } return resolved, nil } // validateBinds checks that every bind's source path falls within the writable // allowlist. Read-only binds are validated too, so read access to arbitrary host // paths outside the allowlist is not possible. // // The source of each bind must already exist and is fully resolved via // filepath.EvalSymlinks; the resolved real path is written back in place. This // enforces the allowlist against the real target and closes the TOCTOU window: // bwrap is handed a fully-resolved path with no unresolved suffix, so a symlink // created or swapped between validation and bwrap start cannot redirect the // mount outside the allowlist. (bwrap's --bind requires the source to exist, so // this adds no functional restriction.) func (r *IsolatedRunner) validateBinds(binds []isolation.BindMount) error { if len(binds) == 0 { return nil } if len(r.allowedWritable) == 0 { return fmt.Errorf("binds not allowed: no paths in allowlist") } for i := range binds { resolved, err := r.resolveAllowedSource(binds[i].Source) if err != nil { return fmt.Errorf("binds source %q: %w", binds[i].Source, err) } // Mount the fully-resolved path so validation and mount target agree. binds[i].Source = resolved } return nil } // pathAllowedResolved reports whether an already symlink-resolved path is equal // to, or nested under, any allowlist entry. Allowlist entries are themselves // symlink-resolved so the comparison is between real paths on both sides. func (r *IsolatedRunner) pathAllowedResolved(resolved string) bool { for _, allowed := range r.allowedWritable { allowedClean := resolveSymlinks(filepath.Clean(allowed)) if resolved == allowedClean || strings.HasPrefix(resolved, allowedClean+"/") { return true } } return false } // resolveSymlinks returns the real path of p with symlinks resolved. Because p // (or a leading component) may not exist yet, it resolves the longest existing // prefix and re-appends the remaining components, so a symlinked ancestor is // still followed while a not-yet-created leaf is preserved. func resolveSymlinks(p string) string { if p == "" { return p } remaining := "" cur := p for { if resolved, err := filepath.EvalSymlinks(cur); err == nil { return filepath.Clean(filepath.Join(resolved, remaining)) } parent := filepath.Dir(cur) if parent == cur { // Reached the root without an existing prefix; fall back to lexical. return p } remaining = filepath.Join(filepath.Base(cur), remaining) cur = parent } } // shellescape wraps s in single quotes, escaping embedded single quotes. func shellescape(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" }