hobokenchicken
eb67287b56
30-second resty client timeout was killing long streaming responses mid-generation. Models with large output windows (e.g. deepseek-v4-pro at 384K max_tokens) routinely exceed 30s. Raised all providers to 10 minutes (Ollama already at 15min, unchanged). Circuit breaker recovery timeout raised from 30s to 5min.
67 lines
1.9 KiB
Go
67 lines
1.9 KiB
Go
package providers
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/sony/gobreaker"
|
|
"gophergate/internal/models"
|
|
)
|
|
|
|
type CircuitBreakerProvider struct {
|
|
provider Provider
|
|
cb *gobreaker.CircuitBreaker
|
|
}
|
|
|
|
func NewCircuitBreakerProvider(p Provider) Provider {
|
|
name := p.Name()
|
|
var maxRequests uint32 = 5
|
|
var interval = 60 * time.Second
|
|
var timeout = 5 * time.Minute
|
|
|
|
settings := gobreaker.Settings{
|
|
Name: name,
|
|
MaxRequests: maxRequests,
|
|
Interval: interval,
|
|
Timeout: timeout,
|
|
ReadyToTrip: func(counts gobreaker.Counts) bool {
|
|
// Trip after 3 consecutive failures
|
|
return counts.ConsecutiveFailures > 3
|
|
},
|
|
}
|
|
return &CircuitBreakerProvider{
|
|
provider: p,
|
|
cb: gobreaker.NewCircuitBreaker(settings),
|
|
}
|
|
}
|
|
|
|
func (cbp *CircuitBreakerProvider) Name() string {
|
|
return cbp.provider.Name()
|
|
}
|
|
|
|
func (cbp *CircuitBreakerProvider) ChatCompletion(ctx context.Context, req *models.UnifiedRequest) (*models.ChatCompletionResponse, error) {
|
|
result, err := cbp.cb.Execute(func() (interface{}, error) {
|
|
return cbp.provider.ChatCompletion(ctx, req)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return result.(*models.ChatCompletionResponse), nil
|
|
}
|
|
|
|
func (cbp *CircuitBreakerProvider) ChatCompletionStream(ctx context.Context, req *models.UnifiedRequest) (<-chan *models.ChatCompletionStreamResponse, error) {
|
|
// Circuit breaker for streaming is tricky. We'll just call the provider directly.
|
|
// Future: Implement a way to track stream failures in the circuit breaker.
|
|
return cbp.provider.ChatCompletionStream(ctx, req)
|
|
}
|
|
|
|
func (cbp *CircuitBreakerProvider) ImageGeneration(ctx context.Context, req *models.ImageGenerationRequest) (*models.ImageGenerationResponse, error) {
|
|
result, err := cbp.cb.Execute(func() (interface{}, error) {
|
|
return cbp.provider.ImageGeneration(ctx, req)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return result.(*models.ImageGenerationResponse), nil
|
|
}
|