llm_optimizer

config.classifier_caller = ->(prompt) {
  RubyLLM.chat(model: "amazon.nova-micro-v1:0", provider: :bedrock, assume_model_exists: true)
    .ask(prompt).content.strip.downcase
}

gem "llm_optimizer"

LlmOptimizer.configure do |config|
  config.compress_prompt    = true
  config.use_semantic_cache = true
  config.redis_url          = ENV["REDIS_URL"]

  # Wire up your app's LLM client
  config.llm_caller = ->(prompt, model:) {
    # Use whatever LLM client your app already has
    MyLlmService.chat(prompt, model: model)
  }

  # Wire up your embeddings provider (required if use_semantic_cache: true)
  config.embedding_caller = ->(text) {
    MyEmbeddingService.embed(text)
  }
end

result = LlmOptimizer.optimize("What is Redis?")

puts result.response          # => "Redis is an in-memory data store..."
puts result.cache_status      # => :hit or :miss
puts result.model_tier        # => :simple or :complex
puts result.model             # => "gemini-2.5-flash-lite"
puts result.original_tokens   # => 5
puts result.compressed_tokens # => 4
puts result.latency_ms        # => 12.4

# config/initializers/llm_optimizer.rb
require "llm_optimizer"

LlmOptimizer.configure do |config|
  # --- Feature flags (all off by default) ---
  config.compress_prompt    = true   # strip stop words before sending to LLM
  config.use_semantic_cache = true   # cache responses by vector similarity
  config.manage_history     = true   # summarize old messages when over token budget

  # --- Model routing ---
  config.route_to      = :auto                        # :auto, :simple, or :complex
  config.simple_model  = "gemini-2.5-flash-lite" # used for simple prompts
  config.complex_model = "claude-haiku-4-5-20251001" # used for complex prompts

  # --- Redis (required if use_semantic_cache: true) ---
  config.redis_url = ENV["REDIS_URL"]

  # --- Token / cache settings ---
  config.similarity_threshold = 0.96   # cosine similarity cutoff for cache hit
  config.token_budget         = 4000   # max tokens before history summarization
  config.cache_ttl            = 86400  # cache TTL in seconds (24h)
  config.timeout_seconds      = 5      # timeout for external API calls

  # --- Logging ---
  config.logger        = Rails.logger
  config.debug_logging = Rails.env.development? # logs full prompt+response in dev

  # --- Wire up your app's LLM client ---
  # Replace the body with however your app calls the LLM
  config.llm_caller = ->(prompt, model:) {
    model ||= "claude-haiku-4-5-20251001"
    provider = if model.include?("claude") then :anthropic
               elsif model.include?("gpt") then :openai
               elsif model.include?("gemini") then :gemini
               else :ollama
               end
    chat = RubyLLM.chat(model: model, provider: provider, assume_model_exists: true)
    chat.ask(prompt).content
  }

  # Embeddings caller — wire to your embeddings provider (required if use_semantic_cache: true)
  config.embedding_caller = ->(text) {
    response = RubyLLM.embed(text, provider: :gemini, model: 'gemini-embedding-001')
    response.vectors
  }

  # Classifier caller — optional, improves routing accuracy for ambiguous prompts
  # Falls back to word-count heuristic if not set or if the call fails
  config.classifier_caller = ->(prompt) {
    RubyLLM.chat(model: "amazon.nova-micro-v1:0", provider: :bedrock, assume_model_exists: true)
      .ask(prompt).content.strip.downcase
  }

  # Messages caller - optional, handles converation summary and hostiry manager.
  config.system_prompt = "You are a sarcastic comic person who gives witty responses in a non harmful way. If any serious question is asked, handle it in a calm way."

  config.messages_caller = ->(messages, model:) {
    chat = RubyLLM.chat(model: model)
    messages[0..-2].each { |m| chat.add_message(role: m[:role], content: m[:content]) }
    response = chat.ask(messages.last[:content])
    response.content
  }
end

result = LlmOptimizer.optimize(prompt) do |config|
  config.route_to      = :simple
  config.compress_prompt = false
end