Quickstart

from benchflow import BaseAgent

class YourAgent(BaseAgent):
  def call_api(self, task_step_inputs):
    ...

from benchflow import load_benchmark
from YourAgent import YourAgent

benchmark_name="organization/selected_benchmark", 
bf_token=os.getenv("BF_TOKEN")
bench = load_benchmark(benchmark_name, bf_token)

your_agent = YourAgent()

run_ids = bench.run(
    task_ids=[0],                   # Run the task with IDs
    agents=your_agents,             # Your agent
    api={                           # Your provider API configuration
        "provider": "",
        "model": "",
        "OPENAI_API_KEY": "",
    },
    requirements_txt="requirements.txt",  # Extra dependencies
    args={}                         # Arguments for your benchmarks
)

results = bench.get_results(run_ids)

from benchflow import BenchClient

class YourClient(BenchClient):
  def prepare_input(self, raw_step_inputs: Dict[str, Any]) -> dict:
    # Each benchmark should supply a dataset for task step as a dict. 
    # For example, if your benchmark is a Q&A dataset, your returned dictionary should include at least a "question" key. 
    # You can also include any additional fields contained in your dataset.
    ...
  
  def parse_response(self, raw_response: str) -> Dict[str, Any]:
    ...
    
# Then you can use your Client in your evaluation script
# For example:
client = YourClient(intelligence_url)

for steps in your_task_steps:
  env = {"question": "question text", "hint": "hint"}
  response = client.run_bench(env)

score = your_eval_method(response)

from benchflow import BaseBench
from benchflow.schemas import BenchArgs, BenchmarkResult

class YourBench(BaseBench):
  def get_args(task_id) -> BenchArgs:
     ...
  def get_image_name(self) -> str:
     ...
  def get_results_dir_in_container(self) -> str:
     ...
  def get_log_files_dir_in_container(self) -> str:
     ...
  def get_result(self, task_id: str) -> BenchmarkResult:
     ...
  def get_all_tasks(self, split: str) -> Dict[str, Any]:
     ...