mirror of
https://github.com/YspCoder/clawgo.git
synced 2026-04-14 02:37:29 +08:00
633 lines
16 KiB
Go
633 lines
16 KiB
Go
package api
|
|
|
|
import (
|
|
"archive/tar"
|
|
"archive/zip"
|
|
"compress/gzip"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
rpcpkg "github.com/YspCoder/clawgo/pkg/rpc"
|
|
"github.com/YspCoder/clawgo/pkg/tools"
|
|
)
|
|
|
|
const (
|
|
skillArchiveUploadLimit int64 = 32 << 20
|
|
skillArchiveMaxFiles = 256
|
|
skillArchiveMaxSingleFile int64 = 8 << 20
|
|
skillArchiveMaxExpanded int64 = 32 << 20
|
|
)
|
|
|
|
type archiveExtractLimits struct {
|
|
maxFiles int
|
|
maxSingleFile int64
|
|
maxExpanded int64
|
|
fileCount int
|
|
totalExpanded int64
|
|
}
|
|
|
|
func (l *archiveExtractLimits) addFile(size int64) error {
|
|
if size < 0 {
|
|
return fmt.Errorf("invalid archive entry size")
|
|
}
|
|
l.fileCount++
|
|
if l.maxFiles > 0 && l.fileCount > l.maxFiles {
|
|
return fmt.Errorf("archive contains too many files")
|
|
}
|
|
if l.maxSingleFile > 0 && size > l.maxSingleFile {
|
|
return fmt.Errorf("archive entry exceeds size limit")
|
|
}
|
|
l.totalExpanded += size
|
|
if l.maxExpanded > 0 && l.totalExpanded > l.maxExpanded {
|
|
return fmt.Errorf("archive exceeds expanded size limit")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func mustMap(v interface{}) map[string]interface{} {
|
|
if v == nil {
|
|
return map[string]interface{}{}
|
|
}
|
|
data, err := json.Marshal(v)
|
|
if err != nil {
|
|
return map[string]interface{}{}
|
|
}
|
|
out := map[string]interface{}{}
|
|
if err := json.Unmarshal(data, &out); err != nil {
|
|
return map[string]interface{}{}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (s *Server) handleWebUISkills(w http.ResponseWriter, r *http.Request) {
|
|
if !s.checkAuth(r) {
|
|
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
svc := s.skillsRPCService()
|
|
skillsDir := filepath.Join(s.workspacePath, "skills")
|
|
if strings.TrimSpace(skillsDir) == "" {
|
|
http.Error(w, "workspace not configured", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
_ = os.MkdirAll(skillsDir, 0755)
|
|
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
resp, rpcErr := svc.View(r.Context(), rpcpkg.SkillsViewRequest{
|
|
ID: strings.TrimSpace(r.URL.Query().Get("id")),
|
|
File: strings.TrimSpace(r.URL.Query().Get("file")),
|
|
Files: strings.TrimSpace(r.URL.Query().Get("files")) == "1",
|
|
CheckUpdates: strings.TrimSpace(r.URL.Query().Get("check_updates")) == "1",
|
|
})
|
|
if rpcErr != nil {
|
|
http.Error(w, rpcErr.Message, rpcHTTPStatus(rpcErr))
|
|
return
|
|
}
|
|
writeJSON(w, mergeJSONMap(map[string]interface{}{"ok": true}, mustMap(resp)))
|
|
|
|
case http.MethodPost:
|
|
ct := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Type")))
|
|
if strings.Contains(ct, "multipart/form-data") {
|
|
imported, err := importSkillArchiveFromMultipart(r, skillsDir)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
writeJSON(w, map[string]interface{}{"ok": true, "imported": imported})
|
|
return
|
|
}
|
|
|
|
var body map[string]interface{}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
http.Error(w, "invalid json", http.StatusBadRequest)
|
|
return
|
|
}
|
|
ignoreSuspicious, _ := tools.MapBoolArg(body, "ignore_suspicious")
|
|
resp, rpcErr := svc.Mutate(r.Context(), rpcpkg.SkillsMutateRequest{
|
|
Action: stringFromMap(body, "action"),
|
|
ID: stringFromMap(body, "id"),
|
|
Name: stringFromMap(body, "name"),
|
|
Description: rawStringFromMap(body, "description"),
|
|
SystemPrompt: rawStringFromMap(body, "system_prompt"),
|
|
Tools: stringListFromMap(body, "tools"),
|
|
IgnoreSuspicious: ignoreSuspicious,
|
|
File: stringFromMap(body, "file"),
|
|
Content: rawStringFromMap(body, "content"),
|
|
})
|
|
if rpcErr != nil {
|
|
http.Error(w, rpcErr.Message, rpcHTTPStatus(rpcErr))
|
|
return
|
|
}
|
|
writeJSON(w, mergeJSONMap(map[string]interface{}{"ok": true}, mustMap(resp)))
|
|
|
|
case http.MethodDelete:
|
|
resp, rpcErr := svc.Mutate(r.Context(), rpcpkg.SkillsMutateRequest{
|
|
Action: "delete",
|
|
ID: strings.TrimSpace(r.URL.Query().Get("id")),
|
|
})
|
|
if rpcErr != nil {
|
|
http.Error(w, rpcErr.Message, rpcHTTPStatus(rpcErr))
|
|
return
|
|
}
|
|
writeJSON(w, mergeJSONMap(map[string]interface{}{"ok": true}, mustMap(resp)))
|
|
|
|
default:
|
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
|
}
|
|
}
|
|
|
|
func buildSkillMarkdown(name, desc string, toolsList []string, systemPrompt string) string {
|
|
if desc == "" {
|
|
desc = "No description provided."
|
|
}
|
|
if len(toolsList) == 0 {
|
|
toolsList = []string{""}
|
|
}
|
|
toolLines := make([]string, 0, len(toolsList))
|
|
for _, t := range toolsList {
|
|
if t == "" {
|
|
continue
|
|
}
|
|
toolLines = append(toolLines, "- "+t)
|
|
}
|
|
if len(toolLines) == 0 {
|
|
toolLines = []string{"- (none)"}
|
|
}
|
|
return fmt.Sprintf(`---
|
|
name: %s
|
|
description: %s
|
|
---
|
|
|
|
# %s
|
|
|
|
%s
|
|
|
|
## Tools
|
|
%s
|
|
|
|
## System Prompt
|
|
%s
|
|
`, name, desc, name, desc, strings.Join(toolLines, "\n"), systemPrompt)
|
|
}
|
|
|
|
func readSkillMeta(path string) (desc string, toolsList []string, systemPrompt string) {
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return "", []string{}, ""
|
|
}
|
|
s := string(b)
|
|
reDesc := regexp.MustCompile(`(?m)^description:\s*(.+)$`)
|
|
reTools := regexp.MustCompile(`(?m)^##\s*Tools\s*$`)
|
|
rePrompt := regexp.MustCompile(`(?m)^##\s*System Prompt\s*$`)
|
|
if m := reDesc.FindStringSubmatch(s); len(m) > 1 {
|
|
desc = m[1]
|
|
}
|
|
if loc := reTools.FindStringIndex(s); loc != nil {
|
|
block := s[loc[1]:]
|
|
if p := rePrompt.FindStringIndex(block); p != nil {
|
|
block = block[:p[0]]
|
|
}
|
|
for _, line := range strings.Split(block, "\n") {
|
|
line = strings.TrimPrefix(line, "-")
|
|
if line != "" {
|
|
toolsList = append(toolsList, line)
|
|
}
|
|
}
|
|
}
|
|
if toolsList == nil {
|
|
toolsList = []string{}
|
|
}
|
|
if loc := rePrompt.FindStringIndex(s); loc != nil {
|
|
systemPrompt = s[loc[1]:]
|
|
}
|
|
return
|
|
}
|
|
|
|
func queryClawHubSkillVersion(ctx context.Context, skill string) (found bool, version string, err error) {
|
|
if skill == "" {
|
|
return false, "", fmt.Errorf("skill empty")
|
|
}
|
|
clawhubPath := strings.TrimSpace(resolveClawHubBinary(ctx))
|
|
if clawhubPath == "" {
|
|
return false, "", fmt.Errorf("clawhub not installed")
|
|
}
|
|
cctx, cancel := context.WithTimeout(ctx, 8*time.Second)
|
|
defer cancel()
|
|
cmd := exec.CommandContext(cctx, clawhubPath, "search", skill, "--json")
|
|
out, runErr := cmd.Output()
|
|
if runErr != nil {
|
|
return false, "", runErr
|
|
}
|
|
var payload interface{}
|
|
if err := json.Unmarshal(out, &payload); err != nil {
|
|
return false, "", err
|
|
}
|
|
lowerSkill := strings.ToLower(skill)
|
|
var walk func(v interface{}) (bool, string)
|
|
walk = func(v interface{}) (bool, string) {
|
|
switch t := v.(type) {
|
|
case map[string]interface{}:
|
|
name := strings.ToLower(strings.TrimSpace(anyToString(t["name"])))
|
|
if name == "" {
|
|
name = strings.ToLower(strings.TrimSpace(anyToString(t["id"])))
|
|
}
|
|
if name == lowerSkill || strings.Contains(name, lowerSkill) {
|
|
ver := anyToString(t["version"])
|
|
if ver == "" {
|
|
ver = anyToString(t["latest_version"])
|
|
}
|
|
return true, ver
|
|
}
|
|
for _, vv := range t {
|
|
if ok, ver := walk(vv); ok {
|
|
return ok, ver
|
|
}
|
|
}
|
|
case []interface{}:
|
|
for _, vv := range t {
|
|
if ok, ver := walk(vv); ok {
|
|
return ok, ver
|
|
}
|
|
}
|
|
}
|
|
return false, ""
|
|
}
|
|
ok, ver := walk(payload)
|
|
return ok, ver, nil
|
|
}
|
|
|
|
func ensureClawHubReady(ctx context.Context) (string, error) {
|
|
outs := make([]string, 0, 4)
|
|
if p := resolveClawHubBinary(ctx); p != "" {
|
|
return "clawhub already installed at: " + p, nil
|
|
}
|
|
nodeOut, err := ensureNodeRuntime(ctx)
|
|
if nodeOut != "" {
|
|
outs = append(outs, nodeOut)
|
|
}
|
|
if err != nil {
|
|
return strings.Join(outs, "\n"), err
|
|
}
|
|
clawOut, err := runInstallCommand(ctx, "npm i -g clawhub")
|
|
if clawOut != "" {
|
|
outs = append(outs, clawOut)
|
|
}
|
|
if err != nil {
|
|
return strings.Join(outs, "\n"), err
|
|
}
|
|
if p := resolveClawHubBinary(ctx); p != "" {
|
|
outs = append(outs, "clawhub installed at: "+p)
|
|
return strings.Join(outs, "\n"), nil
|
|
}
|
|
return strings.Join(outs, "\n"), fmt.Errorf("installed clawhub but executable still not found in PATH")
|
|
}
|
|
|
|
func importSkillArchiveFromMultipart(r *http.Request, skillsDir string) ([]string, error) {
|
|
if r.ContentLength > skillArchiveUploadLimit {
|
|
return nil, fmt.Errorf("archive upload exceeds size limit")
|
|
}
|
|
if r.Body != nil {
|
|
r.Body = http.MaxBytesReader(nil, r.Body, skillArchiveUploadLimit)
|
|
}
|
|
if err := r.ParseMultipartForm(8 << 20); err != nil {
|
|
return nil, err
|
|
}
|
|
f, h, err := r.FormFile("file")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("file required")
|
|
}
|
|
defer f.Close()
|
|
|
|
uploadDir := filepath.Join(os.TempDir(), "clawgo_skill_uploads")
|
|
_ = os.MkdirAll(uploadDir, 0755)
|
|
archivePath := filepath.Join(uploadDir, fmt.Sprintf("%d_%s", time.Now().UnixNano(), filepath.Base(h.Filename)))
|
|
out, err := os.Create(archivePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := io.Copy(out, f); err != nil {
|
|
_ = out.Close()
|
|
_ = os.Remove(archivePath)
|
|
return nil, err
|
|
}
|
|
_ = out.Close()
|
|
defer os.Remove(archivePath)
|
|
|
|
extractDir, err := os.MkdirTemp("", "clawgo_skill_extract_*")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer os.RemoveAll(extractDir)
|
|
|
|
limits := archiveExtractLimits{
|
|
maxFiles: skillArchiveMaxFiles,
|
|
maxSingleFile: skillArchiveMaxSingleFile,
|
|
maxExpanded: skillArchiveMaxExpanded,
|
|
}
|
|
if err := extractArchive(archivePath, extractDir, &limits); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
type candidate struct {
|
|
name string
|
|
dir string
|
|
}
|
|
candidates := make([]candidate, 0)
|
|
seen := map[string]struct{}{}
|
|
err = filepath.WalkDir(extractDir, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
if d.IsDir() {
|
|
return nil
|
|
}
|
|
if strings.EqualFold(d.Name(), "SKILL.md") {
|
|
dir := filepath.Dir(path)
|
|
rel, relErr := filepath.Rel(extractDir, dir)
|
|
if relErr != nil {
|
|
return nil
|
|
}
|
|
rel = filepath.ToSlash(strings.TrimSpace(rel))
|
|
if rel == "" {
|
|
rel = "."
|
|
}
|
|
name := filepath.Base(rel)
|
|
if rel == "." {
|
|
name = archiveBaseName(h.Filename)
|
|
}
|
|
name = sanitizeSkillName(name)
|
|
if name == "" {
|
|
return nil
|
|
}
|
|
if _, ok := seen[name]; ok {
|
|
return nil
|
|
}
|
|
seen[name] = struct{}{}
|
|
candidates = append(candidates, candidate{name: name, dir: dir})
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(candidates) == 0 {
|
|
return nil, fmt.Errorf("no SKILL.md found in archive")
|
|
}
|
|
|
|
imported := make([]string, 0, len(candidates))
|
|
for _, c := range candidates {
|
|
dst := filepath.Join(skillsDir, c.name)
|
|
if _, err := os.Stat(dst); err == nil {
|
|
return nil, fmt.Errorf("skill already exists: %s", c.name)
|
|
}
|
|
if _, err := os.Stat(dst + ".disabled"); err == nil {
|
|
return nil, fmt.Errorf("disabled skill already exists: %s", c.name)
|
|
}
|
|
if err := copyDir(c.dir, dst); err != nil {
|
|
return nil, err
|
|
}
|
|
imported = append(imported, c.name)
|
|
}
|
|
sort.Strings(imported)
|
|
return imported, nil
|
|
}
|
|
|
|
func archiveBaseName(filename string) string {
|
|
name := filepath.Base(strings.TrimSpace(filename))
|
|
lower := strings.ToLower(name)
|
|
switch {
|
|
case strings.HasSuffix(lower, ".tar.gz"):
|
|
return name[:len(name)-len(".tar.gz")]
|
|
case strings.HasSuffix(lower, ".tgz"):
|
|
return name[:len(name)-len(".tgz")]
|
|
case strings.HasSuffix(lower, ".zip"):
|
|
return name[:len(name)-len(".zip")]
|
|
case strings.HasSuffix(lower, ".tar"):
|
|
return name[:len(name)-len(".tar")]
|
|
default:
|
|
ext := filepath.Ext(name)
|
|
return strings.TrimSuffix(name, ext)
|
|
}
|
|
}
|
|
|
|
func sanitizeSkillName(name string) string {
|
|
name = strings.TrimSpace(name)
|
|
if name == "" {
|
|
return ""
|
|
}
|
|
var b strings.Builder
|
|
lastDash := false
|
|
for _, ch := range strings.ToLower(name) {
|
|
if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' {
|
|
b.WriteRune(ch)
|
|
lastDash = false
|
|
continue
|
|
}
|
|
if !lastDash {
|
|
b.WriteRune('-')
|
|
lastDash = true
|
|
}
|
|
}
|
|
out := strings.Trim(b.String(), "-")
|
|
if out == "" || out == "." {
|
|
return ""
|
|
}
|
|
return out
|
|
}
|
|
|
|
func extractArchive(archivePath, targetDir string, limits *archiveExtractLimits) error {
|
|
lower := strings.ToLower(archivePath)
|
|
switch {
|
|
case strings.HasSuffix(lower, ".zip"):
|
|
return extractZip(archivePath, targetDir, limits)
|
|
case strings.HasSuffix(lower, ".tar.gz"), strings.HasSuffix(lower, ".tgz"):
|
|
return extractTarGz(archivePath, targetDir, limits)
|
|
case strings.HasSuffix(lower, ".tar"):
|
|
return extractTar(archivePath, targetDir, limits)
|
|
default:
|
|
return fmt.Errorf("unsupported archive format: %s", filepath.Base(archivePath))
|
|
}
|
|
}
|
|
|
|
func extractZip(archivePath, targetDir string, limits *archiveExtractLimits) error {
|
|
zr, err := zip.OpenReader(archivePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer zr.Close()
|
|
|
|
for _, f := range zr.File {
|
|
if !f.FileInfo().IsDir() && limits != nil {
|
|
if err := limits.addFile(int64(f.UncompressedSize64)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := writeArchivedEntry(targetDir, f.Name, f.FileInfo().IsDir(), func() (io.ReadCloser, error) {
|
|
return f.Open()
|
|
}, limits.maxSingleFile); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func extractTarGz(archivePath, targetDir string, limits *archiveExtractLimits) error {
|
|
f, err := os.Open(archivePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
gz, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer gz.Close()
|
|
return extractTarReader(tar.NewReader(gz), targetDir, limits)
|
|
}
|
|
|
|
func extractTar(archivePath, targetDir string, limits *archiveExtractLimits) error {
|
|
f, err := os.Open(archivePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
return extractTarReader(tar.NewReader(f), targetDir, limits)
|
|
}
|
|
|
|
func extractTarReader(tr *tar.Reader, targetDir string, limits *archiveExtractLimits) error {
|
|
for {
|
|
hdr, err := tr.Next()
|
|
if errors.Is(err, io.EOF) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch hdr.Typeflag {
|
|
case tar.TypeDir:
|
|
maxEntryBytes := int64(0)
|
|
if limits != nil {
|
|
maxEntryBytes = limits.maxSingleFile
|
|
}
|
|
if err := writeArchivedEntry(targetDir, hdr.Name, true, nil, maxEntryBytes); err != nil {
|
|
return err
|
|
}
|
|
case tar.TypeReg, tar.TypeRegA:
|
|
if limits != nil {
|
|
if err := limits.addFile(hdr.Size); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
name := hdr.Name
|
|
maxEntryBytes := int64(0)
|
|
if limits != nil {
|
|
maxEntryBytes = limits.maxSingleFile
|
|
}
|
|
if err := writeArchivedEntry(targetDir, name, false, func() (io.ReadCloser, error) {
|
|
return io.NopCloser(tr), nil
|
|
}, maxEntryBytes); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func writeArchivedEntry(targetDir, name string, isDir bool, opener func() (io.ReadCloser, error), maxBytes int64) error {
|
|
clean := filepath.Clean(strings.TrimSpace(name))
|
|
clean = strings.TrimPrefix(clean, string(filepath.Separator))
|
|
clean = strings.TrimPrefix(clean, "/")
|
|
for strings.HasPrefix(clean, "../") {
|
|
clean = strings.TrimPrefix(clean, "../")
|
|
}
|
|
if clean == "." || clean == "" {
|
|
return nil
|
|
}
|
|
dst := filepath.Join(targetDir, clean)
|
|
absTarget, _ := filepath.Abs(targetDir)
|
|
absDst, _ := filepath.Abs(dst)
|
|
if !strings.HasPrefix(absDst, absTarget+string(filepath.Separator)) && absDst != absTarget {
|
|
return fmt.Errorf("invalid archive entry path: %s", name)
|
|
}
|
|
if isDir {
|
|
return os.MkdirAll(dst, 0755)
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
rc, err := opener()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rc.Close()
|
|
out, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close()
|
|
reader := io.Reader(rc)
|
|
if maxBytes > 0 {
|
|
reader = io.LimitReader(rc, maxBytes+1)
|
|
}
|
|
written, err := io.Copy(out, reader)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if maxBytes > 0 && written > maxBytes {
|
|
return fmt.Errorf("archive entry exceeds size limit")
|
|
}
|
|
return err
|
|
}
|
|
|
|
func copyDir(src, dst string) error {
|
|
entries, err := os.ReadDir(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(dst, 0755); err != nil {
|
|
return err
|
|
}
|
|
for _, e := range entries {
|
|
srcPath := filepath.Join(src, e.Name())
|
|
dstPath := filepath.Join(dst, e.Name())
|
|
info, err := e.Info()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
if err := copyDir(srcPath, dstPath); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
in, err := os.Open(srcPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
out, err := os.Create(dstPath)
|
|
if err != nil {
|
|
_ = in.Close()
|
|
return err
|
|
}
|
|
if _, err := io.Copy(out, in); err != nil {
|
|
_ = out.Close()
|
|
_ = in.Close()
|
|
return err
|
|
}
|
|
_ = out.Close()
|
|
_ = in.Close()
|
|
}
|
|
return nil
|
|
}
|