From 654f6ab6d105ed44ebd34c546833e042f6315e9c Mon Sep 17 00:00:00 2001 From: hobokenchicken Date: Tue, 21 Jul 2026 17:40:18 +0000 Subject: [PATCH] perf: add HTTP connection pooling across providers and disable proxy buffering on SSE streams --- internal/providers/deepseek.go | 2 +- internal/providers/gemini.go | 2 +- internal/providers/grok.go | 2 +- internal/providers/helpers.go | 33 +++++++++++++++++++++++++++++++++ internal/providers/moonshot.go | 2 +- internal/providers/ollama.go | 3 +-- internal/providers/openai.go | 2 +- internal/providers/xiaomi.go | 2 +- internal/server/server.go | 2 ++ 9 files changed, 42 insertions(+), 8 deletions(-) diff --git a/internal/providers/deepseek.go b/internal/providers/deepseek.go index 9240baba..b7743d8d 100644 --- a/internal/providers/deepseek.go +++ b/internal/providers/deepseek.go @@ -22,7 +22,7 @@ type DeepSeekProvider struct { func NewDeepSeekProvider(cfg config.DeepSeekConfig, apiKey string) *DeepSeekProvider { return &DeepSeekProvider{ - client: resty.New().SetTimeout(10 * time.Minute), + client: NewOptimizedRestyClient(10 * time.Minute), config: cfg, apiKey: apiKey, } diff --git a/internal/providers/gemini.go b/internal/providers/gemini.go index 47d8b6ee..87a5ad7e 100644 --- a/internal/providers/gemini.go +++ b/internal/providers/gemini.go @@ -21,7 +21,7 @@ type GeminiProvider struct { func NewGeminiProvider(cfg config.GeminiConfig, apiKey string) *GeminiProvider { return &GeminiProvider{ - client: resty.New().SetTimeout(10 * time.Minute), + client: NewOptimizedRestyClient(10 * time.Minute), config: cfg, apiKey: apiKey, } diff --git a/internal/providers/grok.go b/internal/providers/grok.go index 12933a22..3e0cf13a 100644 --- a/internal/providers/grok.go +++ b/internal/providers/grok.go @@ -20,7 +20,7 @@ type GrokProvider struct { func NewGrokProvider(cfg config.GrokConfig, apiKey string) *GrokProvider { return &GrokProvider{ - client: resty.New().SetTimeout(10 * time.Minute), + client: NewOptimizedRestyClient(10 * time.Minute), config: cfg, apiKey: apiKey, } diff --git a/internal/providers/helpers.go b/internal/providers/helpers.go index 159c054e..6b0d35d3 100644 --- a/internal/providers/helpers.go +++ b/internal/providers/helpers.go @@ -5,10 +5,15 @@ import ( "encoding/json" "fmt" "io" + "net" + "net/http" "regexp" "strings" + "time" "gophergate/internal/models" + + "github.com/go-resty/resty/v2" ) var keySanitizeRegex = regexp.MustCompile(`(?i)(key|api_key|secret)=[^&]+`) @@ -18,6 +23,34 @@ func SanitizeURL(rawURL string) string { return keySanitizeRegex.ReplaceAllString(rawURL, "$1=REDACTED") } +// Shared HTTP transport configured with high connection pooling, TCP keep-alive, +// and HTTP/2 multiplexing to minimize latency when connecting to upstream LLM providers. +var sharedHTTPTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 200, + MaxIdleConnsPerHost: 50, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, +} + +// NewOptimizedRestyClient creates a resty client equipped with HTTP connection pooling. +func NewOptimizedRestyClient(timeout time.Duration) *resty.Client { + httpClient := &http.Client{ + Transport: sharedHTTPTransport, + } + client := resty.NewWithClient(httpClient) + if timeout > 0 { + client.SetTimeout(timeout) + } + return client +} + func sanitizeFunctionName(name string) string { var sb strings.Builder diff --git a/internal/providers/moonshot.go b/internal/providers/moonshot.go index 401d19c2..261d3f00 100644 --- a/internal/providers/moonshot.go +++ b/internal/providers/moonshot.go @@ -21,7 +21,7 @@ type MoonshotProvider struct { func NewMoonshotProvider(cfg config.MoonshotConfig, apiKey string) *MoonshotProvider { return &MoonshotProvider{ - client: resty.New().SetTimeout(10 * time.Minute), + client: NewOptimizedRestyClient(10 * time.Minute), config: cfg, apiKey: strings.TrimSpace(apiKey), } diff --git a/internal/providers/ollama.go b/internal/providers/ollama.go index 5151171b..433e7da0 100644 --- a/internal/providers/ollama.go +++ b/internal/providers/ollama.go @@ -20,10 +20,9 @@ type OllamaProvider struct { } func NewOllamaProvider(cfg config.OllamaConfig) *OllamaProvider { - client := resty.New() + client := NewOptimizedRestyClient(15 * time.Minute) // Set reasonable timeouts for local Ollama server (longer for larger models) // For streaming, we want a very long timeout or none at all to handle generation time - client.SetTimeout(15 * time.Minute) client.SetRetryCount(2) client.SetRetryWaitTime(1 * time.Second) diff --git a/internal/providers/openai.go b/internal/providers/openai.go index 6941b4da..dcaff0a0 100644 --- a/internal/providers/openai.go +++ b/internal/providers/openai.go @@ -22,7 +22,7 @@ type OpenAIProvider struct { func NewOpenAIProvider(cfg config.OpenAIConfig, apiKey string) *OpenAIProvider { return &OpenAIProvider{ - client: resty.New().SetTimeout(10 * time.Minute), + client: NewOptimizedRestyClient(10 * time.Minute), config: cfg, apiKey: apiKey, } diff --git a/internal/providers/xiaomi.go b/internal/providers/xiaomi.go index 10e39025..1b0e3f50 100644 --- a/internal/providers/xiaomi.go +++ b/internal/providers/xiaomi.go @@ -21,7 +21,7 @@ type XiaomiProvider struct { func NewXiaomiProvider(cfg config.XiaomiConfig, apiKey string) *XiaomiProvider { return &XiaomiProvider{ - client: resty.New().SetTimeout(10 * time.Minute), + client: NewOptimizedRestyClient(10 * time.Minute), config: cfg, apiKey: strings.TrimSpace(apiKey), } diff --git a/internal/server/server.go b/internal/server/server.go index c36b6407..b851d7a5 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -403,6 +403,7 @@ func (s *Server) handleResponses(c *gin.Context) { c.Header("Content-Type", "text/event-stream") c.Header("Cache-Control", "no-cache") c.Header("Connection", "keep-alive") + c.Header("X-Accel-Buffering", "no") var lastUsage *models.ResponsesUsage c.Stream(func(w io.Writer) bool { @@ -765,6 +766,7 @@ func (s *Server) handleChatCompletions(c *gin.Context) { c.Header("Content-Type", "text/event-stream") c.Header("Cache-Control", "no-cache") c.Header("Connection", "keep-alive") + c.Header("X-Accel-Buffering", "no") var lastUsage *models.Usage c.Stream(func(w io.Writer) bool {