feat: add cache token tracking and cache-aware cost calculation
Track cache_read_tokens and cache_write_tokens end-to-end: parse from provider responses (OpenAI, DeepSeek, Grok, Gemini), persist to SQLite, apply cache-aware pricing from the model registry, and surface in API responses and the dashboard. - Add cache fields to ProviderResponse, StreamUsage, RequestLog structs - Parse cached_tokens (OpenAI/Grok), prompt_cache_hit/miss (DeepSeek), cachedContentTokenCount (Gemini) from provider responses - Send stream_options.include_usage for streaming; capture real usage from final SSE chunk in AggregatingStream - ALTER TABLE migration for cache_read_tokens/cache_write_tokens columns - Cache-aware cost formula using registry cache_read/cache_write rates - Update Provider trait calculate_cost signature across all providers - Add cache_read_tokens/cache_write_tokens to Usage API response - Dashboard: cache hit rate card, cache columns in pricing and usage tables, cache token aggregation in SQL queries - Remove API debug panel and verbose console logging from api.js - Bump static asset cache-bust to v5
This commit is contained in:
@@ -97,18 +97,21 @@ async fn get_model_cost(
|
||||
model: &str,
|
||||
prompt_tokens: u32,
|
||||
completion_tokens: u32,
|
||||
cache_read_tokens: u32,
|
||||
cache_write_tokens: u32,
|
||||
provider: &Arc<dyn crate::providers::Provider>,
|
||||
state: &AppState,
|
||||
) -> f64 {
|
||||
// Check in-memory cache for cost overrides (no SQLite hit)
|
||||
if let Some(cached) = state.model_config_cache.get(model).await {
|
||||
if let (Some(p), Some(c)) = (cached.prompt_cost_per_m, cached.completion_cost_per_m) {
|
||||
// Manual overrides don't have cache-specific rates, so use simple formula
|
||||
return (prompt_tokens as f64 * p / 1_000_000.0) + (completion_tokens as f64 * c / 1_000_000.0);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to provider's registry-based calculation
|
||||
provider.calculate_cost(model, prompt_tokens, completion_tokens, &state.model_registry)
|
||||
// Fallback to provider's registry-based calculation (cache-aware)
|
||||
provider.calculate_cost(model, prompt_tokens, completion_tokens, cache_read_tokens, cache_write_tokens, &state.model_registry)
|
||||
}
|
||||
|
||||
async fn chat_completions(
|
||||
@@ -281,6 +284,8 @@ async fn chat_completions(
|
||||
&response.model,
|
||||
response.prompt_tokens,
|
||||
response.completion_tokens,
|
||||
response.cache_read_tokens,
|
||||
response.cache_write_tokens,
|
||||
&provider,
|
||||
&state,
|
||||
)
|
||||
@@ -294,6 +299,8 @@ async fn chat_completions(
|
||||
prompt_tokens: response.prompt_tokens,
|
||||
completion_tokens: response.completion_tokens,
|
||||
total_tokens: response.total_tokens,
|
||||
cache_read_tokens: response.cache_read_tokens,
|
||||
cache_write_tokens: response.cache_write_tokens,
|
||||
cost,
|
||||
has_images,
|
||||
status: "success".to_string(),
|
||||
@@ -340,6 +347,8 @@ async fn chat_completions(
|
||||
prompt_tokens: response.prompt_tokens,
|
||||
completion_tokens: response.completion_tokens,
|
||||
total_tokens: response.total_tokens,
|
||||
cache_read_tokens: if response.cache_read_tokens > 0 { Some(response.cache_read_tokens) } else { None },
|
||||
cache_write_tokens: if response.cache_write_tokens > 0 { Some(response.cache_write_tokens) } else { None },
|
||||
}),
|
||||
};
|
||||
|
||||
@@ -368,6 +377,8 @@ async fn chat_completions(
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
total_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
cost: 0.0,
|
||||
has_images: false,
|
||||
status: "error".to_string(),
|
||||
|
||||
Reference in New Issue
Block a user