package api import ( "archive/tar" "archive/zip" "compress/gzip" "context" "encoding/json" "errors" "fmt" "io" "net/http" "os" "os/exec" "path/filepath" "regexp" "sort" "strings" "time" rpcpkg "github.com/YspCoder/clawgo/pkg/rpc" "github.com/YspCoder/clawgo/pkg/tools" ) const ( skillArchiveUploadLimit int64 = 32 << 20 skillArchiveMaxFiles = 256 skillArchiveMaxSingleFile int64 = 8 << 20 skillArchiveMaxExpanded int64 = 32 << 20 ) type archiveExtractLimits struct { maxFiles int maxSingleFile int64 maxExpanded int64 fileCount int totalExpanded int64 } func (l *archiveExtractLimits) addFile(size int64) error { if size < 0 { return fmt.Errorf("invalid archive entry size") } l.fileCount++ if l.maxFiles > 0 && l.fileCount > l.maxFiles { return fmt.Errorf("archive contains too many files") } if l.maxSingleFile > 0 && size > l.maxSingleFile { return fmt.Errorf("archive entry exceeds size limit") } l.totalExpanded += size if l.maxExpanded > 0 && l.totalExpanded > l.maxExpanded { return fmt.Errorf("archive exceeds expanded size limit") } return nil } func mustMap(v interface{}) map[string]interface{} { if v == nil { return map[string]interface{}{} } data, err := json.Marshal(v) if err != nil { return map[string]interface{}{} } out := map[string]interface{}{} if err := json.Unmarshal(data, &out); err != nil { return map[string]interface{}{} } return out } func (s *Server) handleWebUISkills(w http.ResponseWriter, r *http.Request) { if !s.checkAuth(r) { http.Error(w, "unauthorized", http.StatusUnauthorized) return } svc := s.skillsRPCService() skillsDir := filepath.Join(s.workspacePath, "skills") if strings.TrimSpace(skillsDir) == "" { http.Error(w, "workspace not configured", http.StatusInternalServerError) return } _ = os.MkdirAll(skillsDir, 0755) switch r.Method { case http.MethodGet: resp, rpcErr := svc.View(r.Context(), rpcpkg.SkillsViewRequest{ ID: strings.TrimSpace(r.URL.Query().Get("id")), File: strings.TrimSpace(r.URL.Query().Get("file")), Files: strings.TrimSpace(r.URL.Query().Get("files")) == "1", CheckUpdates: strings.TrimSpace(r.URL.Query().Get("check_updates")) == "1", }) if rpcErr != nil { http.Error(w, rpcErr.Message, rpcHTTPStatus(rpcErr)) return } writeJSON(w, mergeJSONMap(map[string]interface{}{"ok": true}, mustMap(resp))) case http.MethodPost: ct := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Type"))) if strings.Contains(ct, "multipart/form-data") { imported, err := importSkillArchiveFromMultipart(r, skillsDir) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } writeJSON(w, map[string]interface{}{"ok": true, "imported": imported}) return } var body map[string]interface{} if err := json.NewDecoder(r.Body).Decode(&body); err != nil { http.Error(w, "invalid json", http.StatusBadRequest) return } ignoreSuspicious, _ := tools.MapBoolArg(body, "ignore_suspicious") resp, rpcErr := svc.Mutate(r.Context(), rpcpkg.SkillsMutateRequest{ Action: stringFromMap(body, "action"), ID: stringFromMap(body, "id"), Name: stringFromMap(body, "name"), Description: rawStringFromMap(body, "description"), SystemPrompt: rawStringFromMap(body, "system_prompt"), Tools: stringListFromMap(body, "tools"), IgnoreSuspicious: ignoreSuspicious, File: stringFromMap(body, "file"), Content: rawStringFromMap(body, "content"), }) if rpcErr != nil { http.Error(w, rpcErr.Message, rpcHTTPStatus(rpcErr)) return } writeJSON(w, mergeJSONMap(map[string]interface{}{"ok": true}, mustMap(resp))) case http.MethodDelete: resp, rpcErr := svc.Mutate(r.Context(), rpcpkg.SkillsMutateRequest{ Action: "delete", ID: strings.TrimSpace(r.URL.Query().Get("id")), }) if rpcErr != nil { http.Error(w, rpcErr.Message, rpcHTTPStatus(rpcErr)) return } writeJSON(w, mergeJSONMap(map[string]interface{}{"ok": true}, mustMap(resp))) default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } } func buildSkillMarkdown(name, desc string, toolsList []string, systemPrompt string) string { if desc == "" { desc = "No description provided." } if len(toolsList) == 0 { toolsList = []string{""} } toolLines := make([]string, 0, len(toolsList)) for _, t := range toolsList { if t == "" { continue } toolLines = append(toolLines, "- "+t) } if len(toolLines) == 0 { toolLines = []string{"- (none)"} } return fmt.Sprintf(`--- name: %s description: %s --- # %s %s ## Tools %s ## System Prompt %s `, name, desc, name, desc, strings.Join(toolLines, "\n"), systemPrompt) } func readSkillMeta(path string) (desc string, toolsList []string, systemPrompt string) { b, err := os.ReadFile(path) if err != nil { return "", []string{}, "" } s := string(b) reDesc := regexp.MustCompile(`(?m)^description:\s*(.+)$`) reTools := regexp.MustCompile(`(?m)^##\s*Tools\s*$`) rePrompt := regexp.MustCompile(`(?m)^##\s*System Prompt\s*$`) if m := reDesc.FindStringSubmatch(s); len(m) > 1 { desc = m[1] } if loc := reTools.FindStringIndex(s); loc != nil { block := s[loc[1]:] if p := rePrompt.FindStringIndex(block); p != nil { block = block[:p[0]] } for _, line := range strings.Split(block, "\n") { line = strings.TrimPrefix(line, "-") if line != "" { toolsList = append(toolsList, line) } } } if toolsList == nil { toolsList = []string{} } if loc := rePrompt.FindStringIndex(s); loc != nil { systemPrompt = s[loc[1]:] } return } func queryClawHubSkillVersion(ctx context.Context, skill string) (found bool, version string, err error) { if skill == "" { return false, "", fmt.Errorf("skill empty") } clawhubPath := strings.TrimSpace(resolveClawHubBinary(ctx)) if clawhubPath == "" { return false, "", fmt.Errorf("clawhub not installed") } cctx, cancel := context.WithTimeout(ctx, 8*time.Second) defer cancel() cmd := exec.CommandContext(cctx, clawhubPath, "search", skill, "--json") out, runErr := cmd.Output() if runErr != nil { return false, "", runErr } var payload interface{} if err := json.Unmarshal(out, &payload); err != nil { return false, "", err } lowerSkill := strings.ToLower(skill) var walk func(v interface{}) (bool, string) walk = func(v interface{}) (bool, string) { switch t := v.(type) { case map[string]interface{}: name := strings.ToLower(strings.TrimSpace(anyToString(t["name"]))) if name == "" { name = strings.ToLower(strings.TrimSpace(anyToString(t["id"]))) } if name == lowerSkill || strings.Contains(name, lowerSkill) { ver := anyToString(t["version"]) if ver == "" { ver = anyToString(t["latest_version"]) } return true, ver } for _, vv := range t { if ok, ver := walk(vv); ok { return ok, ver } } case []interface{}: for _, vv := range t { if ok, ver := walk(vv); ok { return ok, ver } } } return false, "" } ok, ver := walk(payload) return ok, ver, nil } func ensureClawHubReady(ctx context.Context) (string, error) { outs := make([]string, 0, 4) if p := resolveClawHubBinary(ctx); p != "" { return "clawhub already installed at: " + p, nil } nodeOut, err := ensureNodeRuntime(ctx) if nodeOut != "" { outs = append(outs, nodeOut) } if err != nil { return strings.Join(outs, "\n"), err } clawOut, err := runInstallCommand(ctx, "npm i -g clawhub") if clawOut != "" { outs = append(outs, clawOut) } if err != nil { return strings.Join(outs, "\n"), err } if p := resolveClawHubBinary(ctx); p != "" { outs = append(outs, "clawhub installed at: "+p) return strings.Join(outs, "\n"), nil } return strings.Join(outs, "\n"), fmt.Errorf("installed clawhub but executable still not found in PATH") } func importSkillArchiveFromMultipart(r *http.Request, skillsDir string) ([]string, error) { if r.ContentLength > skillArchiveUploadLimit { return nil, fmt.Errorf("archive upload exceeds size limit") } if r.Body != nil { r.Body = http.MaxBytesReader(nil, r.Body, skillArchiveUploadLimit) } if err := r.ParseMultipartForm(8 << 20); err != nil { return nil, err } f, h, err := r.FormFile("file") if err != nil { return nil, fmt.Errorf("file required") } defer f.Close() uploadDir := filepath.Join(os.TempDir(), "clawgo_skill_uploads") _ = os.MkdirAll(uploadDir, 0755) archivePath := filepath.Join(uploadDir, fmt.Sprintf("%d_%s", time.Now().UnixNano(), filepath.Base(h.Filename))) out, err := os.Create(archivePath) if err != nil { return nil, err } if _, err := io.Copy(out, f); err != nil { _ = out.Close() _ = os.Remove(archivePath) return nil, err } _ = out.Close() defer os.Remove(archivePath) extractDir, err := os.MkdirTemp("", "clawgo_skill_extract_*") if err != nil { return nil, err } defer os.RemoveAll(extractDir) limits := archiveExtractLimits{ maxFiles: skillArchiveMaxFiles, maxSingleFile: skillArchiveMaxSingleFile, maxExpanded: skillArchiveMaxExpanded, } if err := extractArchive(archivePath, extractDir, &limits); err != nil { return nil, err } type candidate struct { name string dir string } candidates := make([]candidate, 0) seen := map[string]struct{}{} err = filepath.WalkDir(extractDir, func(path string, d os.DirEntry, err error) error { if err != nil { return nil } if d.IsDir() { return nil } if strings.EqualFold(d.Name(), "SKILL.md") { dir := filepath.Dir(path) rel, relErr := filepath.Rel(extractDir, dir) if relErr != nil { return nil } rel = filepath.ToSlash(strings.TrimSpace(rel)) if rel == "" { rel = "." } name := filepath.Base(rel) if rel == "." { name = archiveBaseName(h.Filename) } name = sanitizeSkillName(name) if name == "" { return nil } if _, ok := seen[name]; ok { return nil } seen[name] = struct{}{} candidates = append(candidates, candidate{name: name, dir: dir}) } return nil }) if err != nil { return nil, err } if len(candidates) == 0 { return nil, fmt.Errorf("no SKILL.md found in archive") } imported := make([]string, 0, len(candidates)) for _, c := range candidates { dst := filepath.Join(skillsDir, c.name) if _, err := os.Stat(dst); err == nil { return nil, fmt.Errorf("skill already exists: %s", c.name) } if _, err := os.Stat(dst + ".disabled"); err == nil { return nil, fmt.Errorf("disabled skill already exists: %s", c.name) } if err := copyDir(c.dir, dst); err != nil { return nil, err } imported = append(imported, c.name) } sort.Strings(imported) return imported, nil } func archiveBaseName(filename string) string { name := filepath.Base(strings.TrimSpace(filename)) lower := strings.ToLower(name) switch { case strings.HasSuffix(lower, ".tar.gz"): return name[:len(name)-len(".tar.gz")] case strings.HasSuffix(lower, ".tgz"): return name[:len(name)-len(".tgz")] case strings.HasSuffix(lower, ".zip"): return name[:len(name)-len(".zip")] case strings.HasSuffix(lower, ".tar"): return name[:len(name)-len(".tar")] default: ext := filepath.Ext(name) return strings.TrimSuffix(name, ext) } } func sanitizeSkillName(name string) string { name = strings.TrimSpace(name) if name == "" { return "" } var b strings.Builder lastDash := false for _, ch := range strings.ToLower(name) { if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' { b.WriteRune(ch) lastDash = false continue } if !lastDash { b.WriteRune('-') lastDash = true } } out := strings.Trim(b.String(), "-") if out == "" || out == "." { return "" } return out } func extractArchive(archivePath, targetDir string, limits *archiveExtractLimits) error { lower := strings.ToLower(archivePath) switch { case strings.HasSuffix(lower, ".zip"): return extractZip(archivePath, targetDir, limits) case strings.HasSuffix(lower, ".tar.gz"), strings.HasSuffix(lower, ".tgz"): return extractTarGz(archivePath, targetDir, limits) case strings.HasSuffix(lower, ".tar"): return extractTar(archivePath, targetDir, limits) default: return fmt.Errorf("unsupported archive format: %s", filepath.Base(archivePath)) } } func extractZip(archivePath, targetDir string, limits *archiveExtractLimits) error { zr, err := zip.OpenReader(archivePath) if err != nil { return err } defer zr.Close() for _, f := range zr.File { if !f.FileInfo().IsDir() && limits != nil { if err := limits.addFile(int64(f.UncompressedSize64)); err != nil { return err } } if err := writeArchivedEntry(targetDir, f.Name, f.FileInfo().IsDir(), func() (io.ReadCloser, error) { return f.Open() }, limits.maxSingleFile); err != nil { return err } } return nil } func extractTarGz(archivePath, targetDir string, limits *archiveExtractLimits) error { f, err := os.Open(archivePath) if err != nil { return err } defer f.Close() gz, err := gzip.NewReader(f) if err != nil { return err } defer gz.Close() return extractTarReader(tar.NewReader(gz), targetDir, limits) } func extractTar(archivePath, targetDir string, limits *archiveExtractLimits) error { f, err := os.Open(archivePath) if err != nil { return err } defer f.Close() return extractTarReader(tar.NewReader(f), targetDir, limits) } func extractTarReader(tr *tar.Reader, targetDir string, limits *archiveExtractLimits) error { for { hdr, err := tr.Next() if errors.Is(err, io.EOF) { return nil } if err != nil { return err } switch hdr.Typeflag { case tar.TypeDir: maxEntryBytes := int64(0) if limits != nil { maxEntryBytes = limits.maxSingleFile } if err := writeArchivedEntry(targetDir, hdr.Name, true, nil, maxEntryBytes); err != nil { return err } case tar.TypeReg, tar.TypeRegA: if limits != nil { if err := limits.addFile(hdr.Size); err != nil { return err } } name := hdr.Name maxEntryBytes := int64(0) if limits != nil { maxEntryBytes = limits.maxSingleFile } if err := writeArchivedEntry(targetDir, name, false, func() (io.ReadCloser, error) { return io.NopCloser(tr), nil }, maxEntryBytes); err != nil { return err } } } } func writeArchivedEntry(targetDir, name string, isDir bool, opener func() (io.ReadCloser, error), maxBytes int64) error { clean := filepath.Clean(strings.TrimSpace(name)) clean = strings.TrimPrefix(clean, string(filepath.Separator)) clean = strings.TrimPrefix(clean, "/") for strings.HasPrefix(clean, "../") { clean = strings.TrimPrefix(clean, "../") } if clean == "." || clean == "" { return nil } dst := filepath.Join(targetDir, clean) absTarget, _ := filepath.Abs(targetDir) absDst, _ := filepath.Abs(dst) if !strings.HasPrefix(absDst, absTarget+string(filepath.Separator)) && absDst != absTarget { return fmt.Errorf("invalid archive entry path: %s", name) } if isDir { return os.MkdirAll(dst, 0755) } if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { return err } rc, err := opener() if err != nil { return err } defer rc.Close() out, err := os.Create(dst) if err != nil { return err } defer out.Close() reader := io.Reader(rc) if maxBytes > 0 { reader = io.LimitReader(rc, maxBytes+1) } written, err := io.Copy(out, reader) if err != nil { return err } if maxBytes > 0 && written > maxBytes { return fmt.Errorf("archive entry exceeds size limit") } return err } func copyDir(src, dst string) error { entries, err := os.ReadDir(src) if err != nil { return err } if err := os.MkdirAll(dst, 0755); err != nil { return err } for _, e := range entries { srcPath := filepath.Join(src, e.Name()) dstPath := filepath.Join(dst, e.Name()) info, err := e.Info() if err != nil { return err } if info.IsDir() { if err := copyDir(srcPath, dstPath); err != nil { return err } continue } in, err := os.Open(srcPath) if err != nil { return err } out, err := os.Create(dstPath) if err != nil { _ = in.Close() return err } if _, err := io.Copy(out, in); err != nil { _ = out.Close() _ = in.Close() return err } _ = out.Close() _ = in.Close() } return nil }