feat: preview node media artifacts in dashboard

2026-04-15 06:17:28 +08:00 · 2026-03-09 01:26:49 +08:00
parent 2d5a384342
commit 94cd67b487
6 changed files with 402 additions and 1 deletions
--- a/cmd/clawgo/cmd_node.go
+++ b/cmd/clawgo/cmd_node.go
@@ -81,7 +81,9 @@ var (
 	nodeAgentLoopFactory     = agent.NewAgentLoop
 	nodeLocalExecutorFactory = newNodeLocalExecutor
 	nodeCameraSnapFunc       = captureNodeCameraSnapshot
+	nodeCameraClipFunc       = captureNodeCameraClip
 	nodeScreenSnapFunc       = captureNodeScreenSnapshot
+	nodeScreenRecordFunc     = captureNodeScreenRecord
 )

 const nodeArtifactInlineLimit = 512 * 1024
@@ -742,6 +744,26 @@ func executeNodeRequest(ctx context.Context, client *http.Client, info nodes.Nod
 			resp.Code = "local_runtime_error"
 			return resp
 		}
+	case "camera_clip":
+		execResp, err := executeNodeCameraClip(ctx, info, next)
+		if err == nil {
+			return execResp
+		}
+		if strings.TrimSpace(opts.Endpoint) == "" {
+			resp.Error = err.Error()
+			resp.Code = "local_runtime_error"
+			return resp
+		}
+	case "screen_record":
+		execResp, err := executeNodeScreenRecord(ctx, info, next)
+		if err == nil {
+			return execResp
+		}
+		if strings.TrimSpace(opts.Endpoint) == "" {
+			resp.Error = err.Error()
+			resp.Code = "local_runtime_error"
+			return resp
+		}
 	}
 	if strings.TrimSpace(opts.Endpoint) == "" {
 		resp.Error = "node endpoint not configured"
@@ -894,6 +916,67 @@ func executeNodeScreenSnapshot(ctx context.Context, info nodes.NodeInfo, req nod
 	}, nil
 }

+func executeNodeCameraClip(ctx context.Context, info nodes.NodeInfo, req nodes.Request) (nodes.Response, error) {
+	executor, err := getNodeLocalExecutor()
+	if err != nil {
+		return nodes.Response{}, err
+	}
+	durationMs := durationArg(req.Args, "duration_ms", 3000)
+	outputPath, err := nodeCameraClipFunc(ctx, executor.workspace, req.Args)
+	if err != nil {
+		return nodes.Response{}, err
+	}
+	artifact, err := buildNodeArtifact(executor.workspace, outputPath)
+	if err != nil {
+		return nodes.Response{}, err
+	}
+	return nodes.Response{
+		OK:     true,
+		Code:   "ok",
+		Node:   info.ID,
+		Action: req.Action,
+		Payload: map[string]interface{}{
+			"transport":   "clawgo-local",
+			"media_type":  "video",
+			"storage":     artifact["storage"],
+			"duration_ms": durationMs,
+			"artifacts":   []map[string]interface{}{artifact},
+			"meta": map[string]interface{}{
+				"facing": stringArg(req.Args, "facing"),
+			},
+		},
+	}, nil
+}
+
+func executeNodeScreenRecord(ctx context.Context, info nodes.NodeInfo, req nodes.Request) (nodes.Response, error) {
+	executor, err := getNodeLocalExecutor()
+	if err != nil {
+		return nodes.Response{}, err
+	}
+	durationMs := durationArg(req.Args, "duration_ms", 3000)
+	outputPath, err := nodeScreenRecordFunc(ctx, executor.workspace, req.Args)
+	if err != nil {
+		return nodes.Response{}, err
+	}
+	artifact, err := buildNodeArtifact(executor.workspace, outputPath)
+	if err != nil {
+		return nodes.Response{}, err
+	}
+	return nodes.Response{
+		OK:     true,
+		Code:   "ok",
+		Node:   info.ID,
+		Action: req.Action,
+		Payload: map[string]interface{}{
+			"transport":   "clawgo-local",
+			"media_type":  "video",
+			"storage":     artifact["storage"],
+			"duration_ms": durationMs,
+			"artifacts":   []map[string]interface{}{artifact},
+		},
+	}, nil
+}
+
 func getNodeLocalExecutor() (*nodeLocalExecutor, error) {
 	key := strings.TrimSpace(getConfigPath())
 	if key == "" {
@@ -1176,6 +1259,83 @@ func captureNodeScreenSnapshot(ctx context.Context, workspace string, args map[s
 	}
 }

+func captureNodeCameraClip(ctx context.Context, workspace string, args map[string]interface{}) (string, error) {
+	outputPath, err := nodeMediaOutputPath(workspace, "camera", ".mp4", stringArg(args, "filename"))
+	if err != nil {
+		return "", err
+	}
+	durationSec := fmt.Sprintf("%.3f", float64(durationArg(args, "duration_ms", 3000))/1000.0)
+	switch runtime.GOOS {
+	case "linux":
+		if _, err := os.Stat("/dev/video0"); err != nil {
+			return "", fmt.Errorf("camera device /dev/video0 not found")
+		}
+		if _, err := exec.LookPath("ffmpeg"); err != nil {
+			return "", fmt.Errorf("ffmpeg not installed")
+		}
+		cmd := exec.CommandContext(ctx, "ffmpeg", "-y", "-f", "video4linux2", "-t", durationSec, "-i", "/dev/video0", "-pix_fmt", "yuv420p", outputPath)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return "", fmt.Errorf("camera clip failed: %v, output=%s", err, strings.TrimSpace(string(out)))
+		}
+		return outputPath, nil
+	case "darwin":
+		if _, err := exec.LookPath("ffmpeg"); err != nil {
+			return "", fmt.Errorf("ffmpeg not installed")
+		}
+		cmd := exec.CommandContext(ctx, "ffmpeg", "-y", "-f", "avfoundation", "-t", durationSec, "-i", "0:none", "-pix_fmt", "yuv420p", outputPath)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return "", fmt.Errorf("camera clip failed: %v, output=%s", err, strings.TrimSpace(string(out)))
+		}
+		return outputPath, nil
+	default:
+		return "", fmt.Errorf("camera_clip not supported on %s", runtime.GOOS)
+	}
+}
+
+func captureNodeScreenRecord(ctx context.Context, workspace string, args map[string]interface{}) (string, error) {
+	outputPath, err := nodeMediaOutputPath(workspace, "screen", ".mp4", stringArg(args, "filename"))
+	if err != nil {
+		return "", err
+	}
+	durationMs := durationArg(args, "duration_ms", 3000)
+	durationSec := fmt.Sprintf("%.3f", float64(durationMs)/1000.0)
+	durationWholeSec := strconv.Itoa((durationMs + 999) / 1000)
+	switch runtime.GOOS {
+	case "darwin":
+		if _, err := exec.LookPath("ffmpeg"); err == nil {
+			cmd := exec.CommandContext(ctx, "ffmpeg", "-y", "-f", "avfoundation", "-t", durationSec, "-i", "1:none", "-pix_fmt", "yuv420p", outputPath)
+			if out, err := cmd.CombinedOutput(); err == nil {
+				return outputPath, nil
+			} else if strings.TrimSpace(string(out)) != "" {
+				return "", fmt.Errorf("screen record failed: %v, output=%s", err, strings.TrimSpace(string(out)))
+			}
+		}
+		return "", fmt.Errorf("ffmpeg not installed")
+	case "linux":
+		candidates := [][]string{
+			{"ffmpeg", "-y", "-f", "x11grab", "-t", durationSec, "-i", os.Getenv("DISPLAY"), "-pix_fmt", "yuv420p", outputPath},
+			{"wf-recorder", "-f", outputPath, "-d", durationWholeSec},
+		}
+		for _, candidate := range candidates {
+			if candidate[0] == "ffmpeg" && strings.TrimSpace(os.Getenv("DISPLAY")) == "" {
+				continue
+			}
+			if _, err := exec.LookPath(candidate[0]); err != nil {
+				continue
+			}
+			cmd := exec.CommandContext(ctx, candidate[0], candidate[1:]...)
+			if out, err := cmd.CombinedOutput(); err == nil {
+				return outputPath, nil
+			} else if strings.TrimSpace(string(out)) != "" && candidate[0] == "ffmpeg" {
+				continue
+			}
+		}
+		return "", fmt.Errorf("no supported screen recorder found (ffmpeg x11grab or wf-recorder)")
+	default:
+		return "", fmt.Errorf("screen_record not supported on %s", runtime.GOOS)
+	}
+}
+
 func nodeMediaOutputPath(workspace, kind, ext, requested string) (string, error) {
 	root := strings.TrimSpace(workspace)
 	if root == "" {
@@ -1205,6 +1365,31 @@ func nodeMediaOutputPath(workspace, kind, ext, requested string) (string, error)
 	return fullPath, nil
 }

+func durationArg(args map[string]interface{}, key string, fallback int) int {
+	if len(args) == 0 {
+		return fallback
+	}
+	switch v := args[key].(type) {
+	case int:
+		if v > 0 {
+			return v
+		}
+	case int64:
+		if v > 0 {
+			return int(v)
+		}
+	case float64:
+		if v > 0 {
+			return int(v)
+		}
+	case json.Number:
+		if n, err := v.Int64(); err == nil && n > 0 {
+			return int(n)
+		}
+	}
+	return fallback
+}
+
 func structToWirePayload(v interface{}) map[string]interface{} {
 	b, _ := json.Marshal(v)
 	var out map[string]interface{}
--- a/cmd/clawgo/cmd_node_test.go
+++ b/cmd/clawgo/cmd_node_test.go
@@ -386,3 +386,99 @@ func TestExecuteNodeRequestRunsLocalScreenSnapshot(t *testing.T) {
 		t.Fatalf("unexpected artifact: %+v", artifacts[0])
 	}
 }
+
+func TestExecuteNodeRequestRunsLocalCameraClip(t *testing.T) {
+	prevCfg := globalConfigPathOverride
+	prevExecutors := nodeLocalExecutors
+	prevClip := nodeCameraClipFunc
+	globalConfigPathOverride = filepath.Join(t.TempDir(), "config.json")
+	nodeLocalExecutors = map[string]*nodeLocalExecutor{}
+	defer func() {
+		globalConfigPathOverride = prevCfg
+		nodeLocalExecutors = prevExecutors
+		nodeCameraClipFunc = prevClip
+	}()
+
+	cfg := config.DefaultConfig()
+	cfg.Agents.Defaults.Workspace = filepath.Join(t.TempDir(), "workspace")
+	if err := config.SaveConfig(globalConfigPathOverride, cfg); err != nil {
+		t.Fatalf("save config: %v", err)
+	}
+	nodeCameraClipFunc = func(ctx context.Context, workspace string, args map[string]interface{}) (string, error) {
+		out := filepath.Join(workspace, "artifacts", "node", "camera-test.mp4")
+		if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
+			return "", err
+		}
+		if err := os.WriteFile(out, []byte("video-bytes"), 0644); err != nil {
+			return "", err
+		}
+		return out, nil
+	}
+
+	info := nodes.NodeInfo{ID: "edge-clip", Name: "Edge Clip"}
+	resp := executeNodeRequest(context.Background(), &http.Client{Timeout: time.Second}, info, nodeRegisterOptions{}, &nodes.Request{
+		Action: "camera_clip",
+		Args:   map[string]interface{}{"duration_ms": 2500},
+	})
+	if !resp.OK {
+		t.Fatalf("expected ok response, got %+v", resp)
+	}
+	if got, _ := resp.Payload["duration_ms"].(int); got != 2500 {
+		t.Fatalf("unexpected duration payload: %+v", resp.Payload)
+	}
+	artifacts, ok := resp.Payload["artifacts"].([]map[string]interface{})
+	if !ok || len(artifacts) != 1 {
+		t.Fatalf("expected one artifact, got %+v", resp.Payload["artifacts"])
+	}
+	if artifacts[0]["name"] != "camera-test.mp4" {
+		t.Fatalf("unexpected artifact: %+v", artifacts[0])
+	}
+}
+
+func TestExecuteNodeRequestRunsLocalScreenRecord(t *testing.T) {
+	prevCfg := globalConfigPathOverride
+	prevExecutors := nodeLocalExecutors
+	prevRecord := nodeScreenRecordFunc
+	globalConfigPathOverride = filepath.Join(t.TempDir(), "config.json")
+	nodeLocalExecutors = map[string]*nodeLocalExecutor{}
+	defer func() {
+		globalConfigPathOverride = prevCfg
+		nodeLocalExecutors = prevExecutors
+		nodeScreenRecordFunc = prevRecord
+	}()
+
+	cfg := config.DefaultConfig()
+	cfg.Agents.Defaults.Workspace = filepath.Join(t.TempDir(), "workspace")
+	if err := config.SaveConfig(globalConfigPathOverride, cfg); err != nil {
+		t.Fatalf("save config: %v", err)
+	}
+	nodeScreenRecordFunc = func(ctx context.Context, workspace string, args map[string]interface{}) (string, error) {
+		out := filepath.Join(workspace, "artifacts", "node", "screen-test.mp4")
+		if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
+			return "", err
+		}
+		if err := os.WriteFile(out, []byte("screen-video"), 0644); err != nil {
+			return "", err
+		}
+		return out, nil
+	}
+
+	info := nodes.NodeInfo{ID: "edge-record", Name: "Edge Record"}
+	resp := executeNodeRequest(context.Background(), &http.Client{Timeout: time.Second}, info, nodeRegisterOptions{}, &nodes.Request{
+		Action: "screen_record",
+		Args:   map[string]interface{}{"duration_ms": 1800},
+	})
+	if !resp.OK {
+		t.Fatalf("expected ok response, got %+v", resp)
+	}
+	if got, _ := resp.Payload["duration_ms"].(int); got != 1800 {
+		t.Fatalf("unexpected duration payload: %+v", resp.Payload)
+	}
+	artifacts, ok := resp.Payload["artifacts"].([]map[string]interface{})
+	if !ok || len(artifacts) != 1 {
+		t.Fatalf("expected one artifact, got %+v", resp.Payload["artifacts"])
+	}
+	if artifacts[0]["name"] != "screen-test.mp4" {
+		t.Fatalf("unexpected artifact: %+v", artifacts[0])
+	}
+}