perf: add HTTP connection pooling across providers and disable proxy buffering on SSE streams

2026-07-21 17:40:18 +00:00
parent 0decc63e8c
commit 654f6ab6d1
9 changed files with 42 additions and 8 deletions
@@ -403,6 +403,7 @@ func (s *Server) handleResponses(c *gin.Context) {
 		c.Header("Content-Type", "text/event-stream")
 		c.Header("Cache-Control", "no-cache")
 		c.Header("Connection", "keep-alive")
+		c.Header("X-Accel-Buffering", "no")

 		var lastUsage *models.ResponsesUsage
 		c.Stream(func(w io.Writer) bool {
@@ -765,6 +766,7 @@ func (s *Server) handleChatCompletions(c *gin.Context) {
 		c.Header("Content-Type", "text/event-stream")
 		c.Header("Cache-Control", "no-cache")
 		c.Header("Connection", "keep-alive")
+		c.Header("X-Accel-Buffering", "no")

 		var lastUsage *models.Usage
 		c.Stream(func(w io.Writer) bool {