Braintrust Ruby SDK

gem "braintrust", require: "braintrust/setup"

require "braintrust"

Braintrust.init

Braintrust.init(
  default_project: "my-project",
  auto_instrument: { only: [:openai] }
)

require "braintrust"
require "openai"

Braintrust.init(auto_instrument: false) # Or BRAINTRUST_AUTO_INSTRUMENT=false

# Instrument all OpenAI clients
Braintrust.instrument!(:openai)

# OR instrument a single client
client = OpenAI::Client.new
Braintrust.instrument!(:openai, target: client)

tracer = OpenTelemetry.tracer_provider.tracer("my-app")

tracer.in_span("process-request") do |span|
  span.set_attribute("user.id", user_id)

  # LLM calls inside here are automatically nested under this span
  response = client.chat.completions.create(...)
end

require "braintrust/trace/attachment"

att = Braintrust::Trace::Attachment.from_file("image/png", "./photo.png")

# Use in messages (OpenAI/Anthropic format)
messages = [
  {
    role: "user",
    content: [
      {type: "text", text: "What's in this image?"},
      att.to_h
    ]
  }
]

# Log to span
span.set_attribute("braintrust.input_json", JSON.generate(messages))

Braintrust::Trace::Attachment.from_bytes("image/jpeg", image_data)
Braintrust::Trace::Attachment.from_file("application/pdf", "./doc.pdf")
Braintrust::Trace::Attachment.from_url("https://example.com/image.png")

tracer = OpenTelemetry.tracer_provider.tracer("my-app")

tracer.in_span("my-operation") do |span|
  # your code here
  puts "View trace at: #{Braintrust::Trace.permalink(span)}"
end

require "braintrust"

Braintrust.init

Braintrust::Eval.run(
  project: "my-project",
  experiment: "classifier-v1",
  cases: [
    {input: "apple", expected: "fruit"},
    {input: "carrot", expected: "vegetable"}
  ],
  task: ->(input:) { classify(input) },
  scorers: [
    ->(expected:, output:) { output == expected ? 1.0 : 0.0 }
  ]
)

Braintrust::Eval.run(
  project: "my-project",
  dataset: "my-dataset",
  task: ->(input:) { classify(input) },
  scorers: [...]
)

Braintrust::Eval.run(
  project: "my-project",
  experiment: "classifier-v1",
  cases: [
    {input: "apple", expected: "fruit", tags: ["produce"], metadata: {difficulty: "easy"}},
    {input: "salmon", expected: "protein", tags: ["seafood"], metadata: {difficulty: "medium"}}
  ],
  task: ->(input:) { classify(input) },
  scorers: [...]
)

Braintrust::Eval.run(
  project: "my-project",
  cases: [...],
  task: ->(input:) { ... },
  scorers: ["accuracy-scorer"]
)

Braintrust::Eval.run(
  project: "my-project",
  cases: [...],
  task: ->(input:) { ... },
  scorers: [
    Braintrust::Scorer.new("exact_match") do |expected:, output:|
      output == expected ? 1.0 : 0.0
    end
  ]
)

Braintrust::Scorer.new("translation") do |expected:, output:|
  common_words = output.downcase.split & expected.downcase.split
  overlap = common_words.size.to_f / expected.split.size
  {
    score: overlap,
    metadata: {word_overlap: common_words.size, missing_words: expected.downcase.split - output.downcase.split}
  }
end

Braintrust::Eval.run(
  project: "my-project",
  cases: [{input: "What is 2+2?", expected: "4"}],
  task: Braintrust::Task.new { |input:| my_llm_pipeline(input) },
  scorers: [
    # Access the full trace to inspect LLM spans
    Braintrust::Scorer.new("uses_system_prompt") do |output:, trace:|
      messages = trace.thread  # reconstructed message thread from LLM spans
      messages.any? { |m| m["role"] == "system" } ? 1.0 : 0.0
    end,

    # Filter spans by type
    Braintrust::Scorer.new("single_llm_call") do |output:, trace:|
      trace.spans(span_type: "llm").length == 1 ? 1.0 : 0.0
    end,

    # Scorers without trace: still work — the parameter is filtered out automatically
    Braintrust::Scorer.new("exact_match") do |output:, expected:|
      output == expected ? 1.0 : 0.0
    end
  ]
)

# eval_server.ru
require "braintrust/eval"
require "braintrust/server"

# Define evaluators — these can reference your application code (models, services, etc.)
food_classifier = Braintrust::Eval::Evaluator.new(
  task: ->(input:) { FoodClassifier.classify(input) },
  scorers: [
    Braintrust::Scorer.new("exact_match") { |expected:, output:| output == expected ? 1.0 : 0.0 }
  ]
)

# Initialize Braintrust (requires BRAINTRUST_API_KEY)
Braintrust.init(blocking_login: true)

# Start the server
run Braintrust::Server::Rack.app(
  evaluators: {
    "food-classifier" => food_classifier
  }
)

class FoodClassifier < Braintrust::Eval::Evaluator
  def task
    ->(input:) { classify(input) }
  end

  def scorers
    [Braintrust::Scorer.new("exact_match") { |expected:, output:| output == expected ? 1.0 : 0.0 }]
  end
end

gem "rack"
gem "puma" # recommended

require 'bundler/setup'
Bundler.require