💎🔗 LangSmith for Ruby

gem 'langsmithrb'

require 'langsmith'

Langsmith.api_key = "your_api_key"
Langsmith.api_url = "https://api.smith.langchain.com"  # Optional

require 'langsmith'

client = Langsmith::Client.new(
  api_key: "your_api_key",
  api_url: "https://api.smith.langchain.com"  # Optional
)

require 'langsmith'

# Initialize the client
client = Langsmith::Client.new(api_key: "your_api_key")

# Create a new project
project = client.create_project(name: "my-project", description: "My awesome LLM project")

# List all projects
projects = client.list_projects

# Create a new run
run = client.create_run(
  name: "llm-call",
  run_type: "llm",
  project_name: "my-project",
  inputs: { prompt: "Tell me a joke about Ruby programming" }
)

# Update a run with outputs
run.end(outputs: { response: "Why did the Ruby developer go broke? Because he used all his gems!" })

# Or update with an error
run.end(error: "API connection failed")

# Get a run by ID
run = client.get_run(run_id: "run_123")

# List runs in a project
runs = client.list_runs(project_name: "my-project", limit: 10)

# Create a new project
project = client.create_project(name: "my-project", description: "My awesome LLM project")

# Get a project by name
project = client.get_project(name: "my-project")

# List all projects
projects = client.list_projects(limit: 10)

# Create a run in a specific project
run = project.create_run(
  name: "llm-call",
  run_type: "llm",
  inputs: { prompt: "Tell me a joke about Ruby programming" }
)

# List runs in a project
runs = project.list_runs(limit: 10)

# Create a new dataset
dataset = client.create_dataset(name: "qa-dataset", description: "Question-answering examples")

# Add examples to the dataset
dataset.create_example(
  inputs: { question: "What is Ruby?" },
  outputs: { answer: "Ruby is a dynamic, open source programming language with a focus on simplicity and productivity." }
)

# List examples in the dataset
examples = dataset.list_examples(limit: 10)

# Create an evaluation run
evaluation = dataset.create_evaluation_run(
  evaluator_name: "correctness",
  run_ids: ["run_123", "run_456"]
)

# Check if the evaluation is complete
evaluation.completed?

# Wait for the evaluation to complete
evaluation.wait_for_completion(timeout: 300)

# Get the evaluation results
results = evaluation.results

# Add feedback to a run
feedback = client.create_feedback(
  run_id: "run_123",
  key: "helpfulness",
  score: 0.95,
  comment: "Very helpful response!"
)

# Get feedback for a run
feedback_list = client.get_feedback(run_id: "run_123")

# Add feedback directly to a run
run.add_feedback(key: "correctness", score: 0.8, comment: "Mostly correct")

# Get feedback for a run
feedback_list = run.get_feedback

# Create a new tracer session
tracer_session = client.create_tracer_session(
  name: "my-tracer-session",
  description: "Tracing my LLM application"
)

# Get a tracer session by ID
tracer_session = client.get_tracer_session(tracer_session_id: "ts_123")

# List tracer sessions
tracer_sessions = client.list_tracer_sessions(limit: 10)

# Update a tracer session
updated_session = client.update_tracer_session(
  tracer_session_id: "ts_123",
  name: "updated-tracer-session",
  description: "Updated description"
)

# Delete a tracer session
client.delete_tracer_session(tracer_session_id: "ts_123")

# Get the current organization
org = client.get_current_organization

# List organizations
orgs = client.list_organizations

# Get an organization by ID
org = client.get_organization(organization_id: "org_123")

# List API keys
keys = client.list_api_keys

# Create a new API key
new_key = client.create_api_key(name: "my-api-key")

# Delete an API key
client.delete_api_key(api_key_id: "key_123")

# Get a tenant by ID
tenant = client.get_tenant(tenant_id: "tenant_123")

# List tenants
tenants = client.list_tenants(limit: 10)

# Create a new tenant
new_tenant = client.create_tenant(
  name: "my-tenant",
  description: "My team's tenant"
)

# Update a tenant
updated_tenant = client.update_tenant(
  tenant_id: "tenant_123",
  name: "updated-tenant",
  description: "Updated description"
)

# Create a new tag
tag = client.create_tag(name: "important")

# Get a tag by ID
tag = client.get_tag(tag_id: "tag_123")

# List tags
tags = client.list_tags(limit: 10)

# Delete a tag
client.delete_tag(tag_id: "tag_123")

# Create a new prompt
prompt = client.create_prompt(
  name: "qa-prompt",
  prompt_template: "Answer the following question: {{question}}",
  description: "A simple Q&A prompt template"
)

# Get a prompt by ID
prompt = client.get_prompt(prompt_id: "prompt_123")

# List prompts
prompts = client.list_prompts(limit: 10)

# Update a prompt
updated_prompt = client.update_prompt(
  prompt_id: "prompt_123",
  name: "updated-prompt",
  prompt_template: "Please answer this question: {{question}}",
  description: "Updated description"
)

# Delete a prompt
client.delete_prompt(prompt_id: "prompt_123")

# Create a new annotation queue
queue = client.create_annotation_queue(
  name: "feedback-queue",
  description: "Queue for human feedback"
)

# Get an annotation queue by ID
queue = client.get_annotation_queue(annotation_queue_id: "queue_123")

# List annotation queues
queues = client.list_annotation_queues(limit: 10)

# Update an annotation queue
updated_queue = client.update_annotation_queue(
  annotation_queue_id: "queue_123",
  name: "updated-queue",
  description: "Updated description"
)

# Delete an annotation queue
client.delete_annotation_queue(annotation_queue_id: "queue_123")

# Create a new feedback configuration
config = client.create_feedback_configuration(
  name: "helpfulness",
  description: "Measure helpfulness of responses",
  criteria: "How helpful was this response?"
)

# Get a feedback configuration by ID
config = client.get_feedback_configuration(feedback_configuration_id: "config_123")

# List feedback configurations
configs = client.list_feedback_configurations(limit: 10)

# Update a feedback configuration
updated_config = client.update_feedback_configuration(
  feedback_configuration_id: "config_123",
  name: "updated-config",
  description: "Updated description",
  criteria: "Updated criteria"
)

# Delete a feedback configuration
client.delete_feedback_configuration(feedback_configuration_id: "config_123")

# Get project analytics
analytics = client.get_project_analytics(
  project_id: "proj_123",
  start_time: Time.now - 86400, # 24 hours ago
  end_time: Time.now,
  group_by: "run_type",
  metrics: ["latency", "token_usage"]
)

# Get dataset analytics
analytics = client.get_dataset_analytics(
  dataset_id: "ds_123",
  start_time: Time.now - 86400, # 24 hours ago
  end_time: Time.now,
  group_by: "example_id"
)

# Create a comment on a run
comment = client.create_run_comment(
  run_id: "run_123",
  comment: "This run looks promising",
  metadata: { source: "code review" }
)

# List comments for a run
comments = client.list_run_comments(run_id: "run_123")

# Delete a comment
client.delete_run_comment(run_id: "run_123", comment_id: "comment_123")

# Create an event
event = client.create_event(
  event_type: "user_feedback",
  payload: { score: 0.9, comment: "Great response!" }
)

# List events
events = client.list_events(
  event_type: "user_feedback",
  start_time: Time.now - 86400, # 24 hours ago
  end_time: Time.now
)

# Get settings
settings = client.get_settings

# Update settings
updated_settings = client.update_settings(
  default_project_id: "proj_123",
  default_dataset_id: "ds_123"
)

# Bulk tag runs
result = client.bulk_tag_runs(
  run_ids: ["run_123", "run_456", "run_789"],
  tag_ids: ["tag_123", "tag_456"]
)

# Bulk untag runs
result = client.bulk_untag_runs(
  run_ids: ["run_123", "run_456"],
  tag_ids: ["tag_123"]
)

# Bulk delete runs
result = client.bulk_delete_runs(
  run_ids: ["run_123", "run_456", "run_789"]
)

# Create a webhook
webhook = client.create_webhook(
  url: "https://example.com/webhook",
  event_types: ["run.created", "run.updated"],
  metadata: { description: "Notify our system when runs are created or updated" }
)

# Get a webhook by ID
webhook = client.get_webhook(webhook_id: "webhook_123")

# List webhooks
webhooks = client.list_webhooks(limit: 10)

# Update a webhook
updated_webhook = client.update_webhook(
  webhook_id: "webhook_123",
  url: "https://updated-example.com/webhook",
  event_types: ["run.created", "run.updated", "run.deleted"],
  metadata: { description: "Updated webhook configuration" }
)

# Delete a webhook
client.delete_webhook(webhook_id: "webhook_123")

# List team members
members = client.list_team_members(organization_id: "org_123", limit: 10)

# Invite a team member
invite = client.invite_team_member(
  organization_id: "org_123",
  email: "new-member@example.com",
  role: "member"
)

# Remove a team member
client.remove_team_member(
  organization_id: "org_123",
  user_id: "user_123"
)

require 'langsmith'

begin
  client = Langsmith::Client.new(api_key: "invalid_key")
  client.list_projects
rescue Langsmith::Errors::AuthenticationError => e
  puts "Authentication failed: #{e.message}"
rescue Langsmith::Errors::ResourceNotFoundError => e
  puts "Resource not found: #{e.message}"
rescue Langsmith::Errors::APIError => e
  puts "API error: #{e.message}"
rescue StandardError => e
  puts "Unexpected error: #{e.message}"
end