feat: harden jsonl runtime reliability

pkg/providers/execution.go

@@ -4,8 +4,10 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
+	"time"
 )
 
 func newProviderExecutionError(code, message, stage string, retryable bool, source string) *ProviderExecutionError {
@@ -18,6 +20,18 @@ func newProviderExecutionError(code, message, stage string, retryable bool, sour
 	}
 }
 
+func NewProviderExecutionError(code, message, stage string, retryable bool, source string) *ProviderExecutionError {
+	return newProviderExecutionError(code, message, stage, retryable, source)
+}
+
+func ExecutionErrorCode(err error) string {
+	var execErr *ProviderExecutionError
+	if errors.As(err, &execErr) && execErr != nil {
+		return execErr.Code
+	}
+	return ""
+}
+
 func (p *HTTPProvider) executeJSONAttempts(ctx context.Context, endpoint string, payload interface{}, mutate func(*http.Request, authAttempt), classify func(int, []byte) (oauthFailureReason, bool)) (ProviderExecutionResult, error) {
 	jsonData, err := json.Marshal(payload)
 	if err != nil {
@@ -42,12 +56,13 @@ func (p *HTTPProvider) executeJSONAttempts(ctx context.Context, endpoint string,
 		}
 		body, status, ctype, err := p.doJSONAttempt(req, attempt)
 		if err != nil {
+			execErr := newProviderExecutionError("request_failed", err.Error(), "request", false, p.providerName)
 			return ProviderExecutionResult{
 				StatusCode:  status,
 				ContentType: ctype,
 				AttemptKind: attempt.kind,
-				Error:       newProviderExecutionError("request_failed", err.Error(), "request", false, p.providerName),
-			}, err
+				Error:       execErr,
+			}, execErr
 		}
 		reason, retry := classify(status, body)
 		last = ProviderExecutionResult{
@@ -78,8 +93,10 @@ func (p *HTTPProvider) executeStreamAttempts(ctx context.Context, endpoint strin
 	}
 	var last ProviderExecutionResult
 	for _, attempt := range attempts {
-		req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(jsonData))
+		attemptCtx, cancel := context.WithCancel(ctx)
+		req, err := http.NewRequestWithContext(attemptCtx, http.MethodPost, endpoint, bytes.NewReader(jsonData))
 		if err != nil {
+			cancel()
 			return ProviderExecutionResult{Error: newProviderExecutionError("request_build_failed", err.Error(), "request", false, p.providerName)}, fmt.Errorf("failed to create request: %w", err)
 		}
 		req.Header.Set("Content-Type", "application/json")
@@ -89,14 +106,21 @@ func (p *HTTPProvider) executeStreamAttempts(ctx context.Context, endpoint strin
 		if mutate != nil {
 			mutate(req, attempt)
 		}
-		body, status, ctype, quotaHit, err := p.doStreamAttempt(req, attempt, onEvent)
+		streamOptions := streamAttemptTimeouts(ctx)
+		body, status, ctype, quotaHit, err := p.doStreamAttempt(req, attempt, onEvent, streamOptions, cancel)
+		cancel()
 		if err != nil {
+			code := "stream_failed"
+			if streamOptions.staleTriggered {
+				code = "stream_stale"
+			}
+			execErr := newProviderExecutionError(code, err.Error(), "request", true, p.providerName)
 			return ProviderExecutionResult{
 				StatusCode:  status,
 				ContentType: ctype,
 				AttemptKind: attempt.kind,
-				Error:       newProviderExecutionError("stream_failed", err.Error(), "request", false, p.providerName),
-			}, err
+				Error:       execErr,
+			}, execErr
 		}
 		reason, _ := classifyOAuthFailure(status, body)
 		last = ProviderExecutionResult{
@@ -115,3 +139,47 @@ func (p *HTTPProvider) executeStreamAttempts(ctx context.Context, endpoint strin
 	}
 	return last, nil
 }
+
+type streamAttemptOptions struct {
+	firstDeltaTimeout time.Duration
+	idleDeltaTimeout  time.Duration
+	staleTriggered    bool
+}
+
+func streamAttemptTimeouts(ctx context.Context) *streamAttemptOptions {
+	opts := &streamAttemptOptions{
+		firstDeltaTimeout: 20 * time.Second,
+		idleDeltaTimeout:  45 * time.Second,
+	}
+	if deadline, ok := ctx.Deadline(); ok {
+		remaining := time.Until(deadline)
+		if remaining > 0 {
+			first := remaining / 4
+			idle := remaining / 3
+			if remaining < 5*time.Second {
+				first = maxDuration(100*time.Millisecond, first)
+			} else if first < 5*time.Second {
+				first = 5 * time.Second
+			}
+			if remaining < 15*time.Second {
+				idle = maxDuration(250*time.Millisecond, idle)
+			} else if idle < 15*time.Second {
+				idle = 15 * time.Second
+			}
+			if first < opts.firstDeltaTimeout {
+				opts.firstDeltaTimeout = first
+			}
+			if idle < opts.idleDeltaTimeout {
+				opts.idleDeltaTimeout = idle
+			}
+		}
+	}
+	return opts
+}
+
+func maxDuration(a, b time.Duration) time.Duration {
+	if a > b {
+		return a
+	}
+	return b
+}