DBT/clawgo
1
0
Fork 0
mirror of https://github.com/YspCoder/clawgo.git synced 2026-04-13 05:37:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

add skill selection reason audit and status coverage metric

Browse Source
This commit is contained in:
DBT
2026-02-24 05:57:25 +00:00
parent 4e15f06d31
commit 141b6a2ccb
2 changed files with 40 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
cmd/clawgo/cmd_status.go
View File

@@ -116,8 +116,8 @@ func statusCmd() {
fmt.Printf(" %s: %d\n", trigger, agg[trigger])
}
}
if total, okCnt, failCnt, top, err := collectSkillExecStats(filepath.Join(workspace, "memory", "skill-audit.jsonl")); err == nil && total > 0 {
fmt.Printf("Skill Exec: total=%d ok=%d fail=%d\n", total, okCnt, failCnt)
if total, okCnt, failCnt, reasonCov, top, err := collectSkillExecStats(filepath.Join(workspace, "memory", "skill-audit.jsonl")); err == nil && total > 0 {
fmt.Printf("Skill Exec: total=%d ok=%d fail=%d reason_coverage=%.2f\n", total, okCnt, failCnt, reasonCov)
if top != "" {
fmt.Printf("Skill Exec Top: %s\n", top)
}
@@ -395,13 +395,14 @@ func collectAutonomyTaskSummary(path string) (map[string]int, map[string]int, ma
return summary, priorities, reasons, nextRetry, totalDedupe, nil
}
func collectSkillExecStats(path string) (int, int, int, string, error) {
func collectSkillExecStats(path string) (int, int, int, float64, string, error) {
data, err := os.ReadFile(path)
if err != nil {
return 0, 0, 0, "", err
return 0, 0, 0, 0, "", err
}
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
total, okCnt, failCnt := 0, 0, 0
reasonCnt := 0
skillCounts := map[string]int{}
for _, line := range lines {
line = strings.TrimSpace(line)
@@ -409,8 +410,9 @@ func collectSkillExecStats(path string) (int, int, int, string, error) {
continue
}
var row struct {
Skill string `json:"skill"`
OK bool `json:"ok"`
Skill string `json:"skill"`
Reason string `json:"reason"`
OK bool `json:"ok"`
}
if err := json.Unmarshal([]byte(line), &row); err != nil {
continue
@@ -421,6 +423,10 @@ func collectSkillExecStats(path string) (int, int, int, string, error) {
} else {
failCnt++
}
r := strings.TrimSpace(strings.ToLower(row.Reason))
if r != "" && r != "unspecified" {
reasonCnt++
}
s := strings.TrimSpace(row.Skill)
if s == "" {
s = "unknown"
@@ -438,7 +444,11 @@ func collectSkillExecStats(path string) (int, int, int, string, error) {
if topSkill != "" {
topSkill = fmt.Sprintf("%s(%d)", topSkill, topN)
}
return total, okCnt, failCnt, topSkill, nil
reasonCoverage := 0.0
if total > 0 {
reasonCoverage = float64(reasonCnt) / float64(total)
}
return total, okCnt, failCnt, reasonCoverage, topSkill, nil
}
func collectRecentSubagentSessions(sessionsDir string, limit int) ([]string, error) {

34
pkg/tools/skill_exec.go
View File

@@ -48,6 +48,10 @@ func (t *SkillExecTool) Parameters() map[string]interface{} {
"default": 60,
"description": "Execution timeout in seconds",
},
"reason": map[string]interface{}{
"type": "string",
"description": "Why this skill/script was selected for the current user request",
},
},
"required": []string{"skill", "script"},
}
@@ -56,8 +60,15 @@ func (t *SkillExecTool) Parameters() map[string]interface{} {
func (t *SkillExecTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) {
skill, _ := args["skill"].(string)
script, _ := args["script"].(string)
reason, _ := args["reason"].(string)
reason = strings.TrimSpace(reason)
if reason == "" {
reason = "unspecified"
}
if strings.TrimSpace(skill) == "" || strings.TrimSpace(script) == "" {
return "", fmt.Errorf("skill and script are required")
err := fmt.Errorf("skill and script are required")
t.writeAudit(skill, script, reason, false, err.Error())
return "", err
}
timeoutSec := 60
@@ -67,31 +78,31 @@ func (t *SkillExecTool) Execute(ctx context.Context, args map[string]interface{}
skillDir, err := t.resolveSkillDir(skill)
if err != nil {
t.writeAudit(skill, script, false, err.Error())
t.writeAudit(skill, script, reason, false, err.Error())
return "", err
}
if _, err := os.Stat(filepath.Join(skillDir, "SKILL.md")); err != nil {
err = fmt.Errorf("SKILL.md missing for skill: %s", skill)
t.writeAudit(skill, script, false, err.Error())
t.writeAudit(skill, script, reason, false, err.Error())
return "", err
}
relScript := filepath.Clean(script)
if strings.Contains(relScript, "..") || filepath.IsAbs(relScript) {
err := fmt.Errorf("script must be relative path inside skill directory")
t.writeAudit(skill, script, false, err.Error())
t.writeAudit(skill, script, reason, false, err.Error())
return "", err
}
if !strings.HasPrefix(relScript, "scripts"+string(os.PathSeparator)) {
err := fmt.Errorf("script must be under scripts/ directory")
t.writeAudit(skill, script, false, err.Error())
t.writeAudit(skill, script, reason, false, err.Error())
return "", err
}
scriptPath := filepath.Join(skillDir, relScript)
if _, err := os.Stat(scriptPath); err != nil {
err = fmt.Errorf("script not found: %s", scriptPath)
t.writeAudit(skill, script, false, err.Error())
t.writeAudit(skill, script, reason, false, err.Error())
return "", err
}
@@ -109,13 +120,13 @@ func (t *SkillExecTool) Execute(ctx context.Context, args map[string]interface{}
cmd, err := buildSkillCommand(runCtx, scriptPath, cmdArgs)
if err != nil {
t.writeAudit(skill, script, false, err.Error())
t.writeAudit(skill, script, reason, false, err.Error())
return "", err
}
cmd.Dir = skillDir
output, err := cmd.CombinedOutput()
if err != nil {
t.writeAudit(skill, script, false, err.Error())
t.writeAudit(skill, script, reason, false, err.Error())
return "", fmt.Errorf("skill execution failed: %w\n%s", err, string(output))
}
@@ -123,20 +134,21 @@ func (t *SkillExecTool) Execute(ctx context.Context, args map[string]interface{}
if out == "" {
out = "(no output)"
}
t.writeAudit(skill, script, true, "")
t.writeAudit(skill, script, reason, true, "")
return out, nil
}
func (t *SkillExecTool) writeAudit(skill, script string, ok bool, errText string) {
func (t *SkillExecTool) writeAudit(skill, script, reason string, ok bool, errText string) {
if strings.TrimSpace(t.workspace) == "" {
return
}
memDir := filepath.Join(t.workspace, "memory")
_ = os.MkdirAll(memDir, 0755)
row := fmt.Sprintf("{\"time\":%q,\"skill\":%q,\"script\":%q,\"ok\":%t,\"error\":%q}\n",
row := fmt.Sprintf("{\"time\":%q,\"skill\":%q,\"script\":%q,\"reason\":%q,\"ok\":%t,\"error\":%q}\n",
time.Now().UTC().Format(time.RFC3339),
strings.TrimSpace(skill),
strings.TrimSpace(script),
strings.TrimSpace(reason),
ok,
strings.TrimSpace(errText),
)