pip install praisonaiagents

from praisonaiagents import Agent
from praisonaiagents.eval import AccuracyEvaluator

# Create agent
agent = Agent(instructions="You are a math tutor. Answer concisely.")

# Create evaluator
evaluator = AccuracyEvaluator(
    agent=agent,
    input_text="What is 2 + 2?",
    expected_output="4",
    num_iterations=3,  # Run multiple times for statistical significance
    
)

# Run evaluation
result = evaluator.run(print_summary=True)

print(f"Average Score: {result.avg_score}/10")
print(f"Passed: {result.passed}")

from praisonaiagents import Agent
from praisonaiagents.eval import PerformanceEvaluator

agent = Agent(instructions="You are a helpful assistant.")

evaluator = PerformanceEvaluator(
    agent=agent,
    input_text="What is the capital of France?",
    num_iterations=10,  # Number of benchmark runs
    warmup_runs=2,      # Warmup runs before measurement
    track_memory=True,  # Track memory usage
    
)

result = evaluator.run(print_summary=True)

print(f"Average Time: {result.avg_run_time:.4f}s")
print(f"Min Time: {result.min_run_time:.4f}s")
print(f"Max Time: {result.max_run_time:.4f}s")
print(f"P95 Time: {result.p95_run_time:.4f}s")
print(f"Avg Memory: {result.avg_memory:.2f} MB")

from praisonaiagents import Agent
from praisonaiagents.eval import ReliabilityEvaluator

def search_web(query: str) -> str:
    """Search the web."""
    return f"Results for: {query}"

def calculate(expression: str) -> str:
    """Calculate expression."""
    return str(eval(expression))

agent = Agent(
    instructions="You have search and calculator tools.",
    tools=[search_web, calculate]
)

evaluator = ReliabilityEvaluator(
    agent=agent,
    input_text="Search for weather and calculate 25 * 4",
    expected_tools=["search_web", "calculate"],
    forbidden_tools=["delete_file"],  # Should NOT be called
    
)

result = evaluator.run(print_summary=True)

print(f"Passed: {result.passed}")
print(f"Pass Rate: {result.pass_rate:.1%}")

from praisonaiagents import Agent
from praisonaiagents.eval import CriteriaEvaluator

agent = Agent(instructions="You are a customer service agent.")

# Numeric scoring (1-10)
evaluator = CriteriaEvaluator(
    criteria="Response is helpful, empathetic, and provides a clear solution",
    agent=agent,
    input_text="My order hasn't arrived yet.",
    scoring_type="numeric",  # Score 1-10
    threshold=7.0,           # Pass if score >= 7
    num_iterations=2,
    
)

result = evaluator.run(print_summary=True)

print(f"Average Score: {result.avg_score}/10")
print(f"Pass Rate: {result.pass_rate:.1%}")

# Binary scoring (pass/fail)
binary_evaluator = CriteriaEvaluator(
    criteria="Response does not contain offensive language",
    agent=agent,
    input_text="Tell me a joke",
    scoring_type="binary",
    
)

binary_result = binary_evaluator.run(print_summary=True)

from praisonaiagents.eval import CriteriaEvaluator

def handle_failure(score):
    print(f"ALERT: Evaluation failed with score {score.score}")
    print(f"Reasoning: {score.reasoning}")
    # Send alert, log to monitoring system, etc.

evaluator = CriteriaEvaluator(
    criteria="Response is professional",
    agent=agent,
    input_text="Help me",
    on_fail=handle_failure,
    threshold=8.0
)

evaluator.run()

from praisonaiagents.eval import AccuracyEvaluator, CriteriaEvaluator

# Accuracy evaluation of pre-generated output
accuracy_eval = AccuracyEvaluator(
    func=lambda x: "unused",  # Placeholder
    input_text="What is 2+2?",
    expected_output="4"
)

result = accuracy_eval.evaluate_output("The answer is 4")

# Criteria evaluation of pre-generated output
criteria_eval = CriteriaEvaluator(
    criteria="Response is helpful and accurate",
    func=lambda x: "unused"
)

result = criteria_eval.evaluate_output("Here's how to solve that...")

evaluator = AccuracyEvaluator(
    agent=agent,
    input_text="Test input",
    expected_output="Expected output",
    save_results_path="results/{name}_{eval_id}.json"  # Supports placeholders
)

result = evaluator.run()
# Results automatically saved to file

praisonai eval accuracy \
    --prompt "What is 2+2?" \  # Direct prompt (no agents.yaml needed)
    --expected "4"

# Or with agents.yaml:
praisonai eval accuracy \
    --agent agents.yaml \
    --input "What is 2+2?" \
    --expected "4" \
    --iterations 3 \
    --output results.json \
    --verbose

praisonai eval performance \
    --agent agents.yaml \
    --input "Hello" \
    --iterations 10 \
    --warmup 2 \
    --memory \
    --output perf_results.json

praisonai eval reliability \
    --agent agents.yaml \
    --input "Search for weather" \
    --expected-tools "search_web,calculate" \
    --forbidden-tools "delete_file" \
    --output reliability.json

praisonai eval criteria \
    --agent agents.yaml \
    --input "Help me with my order" \
    --criteria "Response is helpful and professional" \
    --scoring numeric \
    --threshold 7.0 \
    --iterations 2 \
    --output criteria.json

praisonai eval batch \
    --agent agents.yaml \
    --test-file tests.json \
    --batch-type accuracy \
    --output batch_results.json

[
    {
        "input": "What is 2+2?",
        "expected": "4"
    },
    {
        "input": "What is the capital of France?",
        "expected": "Paris"
    }
]

result.evaluations  # List of individual scores
result.avg_score    # Average score (0-10)
result.min_score    # Minimum score
result.max_score    # Maximum score
result.std_dev      # Standard deviation
result.passed       # True if avg_score >= 7
result.to_dict()    # Convert to dictionary
result.to_json()    # Convert to JSON string

result.metrics          # List of PerformanceMetrics
result.avg_run_time     # Average runtime in seconds
result.min_run_time     # Minimum runtime
result.max_run_time     # Maximum runtime
result.median_run_time  # Median runtime
result.p95_run_time     # 95th percentile runtime
result.avg_memory       # Average memory usage (MB)
result.max_memory       # Peak memory usage (MB)

result.tool_results  # List of ToolCallResult
result.passed_calls  # Tools that passed
result.failed_calls  # Tools that failed
result.pass_rate     # Pass rate (0-1)
result.passed        # True if all checks passed
result.status        # "PASSED" or "FAILED"

result.evaluations  # List of CriteriaScore
result.criteria     # The evaluation criteria
result.scoring_type # "numeric" or "binary"
result.threshold    # Pass threshold
result.avg_score    # Average score
result.pass_rate    # Pass rate (0-1)
result.passed       # True if passed threshold

id,name,prompts,judge_rubric,judge_threshold,judge_model
test_01,Helpful Response,"Explain Python decorators",Response is clear and accurate,7.0,gpt-4o-mini
test_02,Code Quality,"Create a function to sort a list",Code is correct and well-documented,8.0,gpt-4o-mini

# Run with judge evaluation
praisonai test interactive --csv tests.csv

# Skip judge even if rubric is present
praisonai test interactive --csv tests.csv --no-judge

# Use a different judge model
praisonai test interactive --csv tests.csv --judge-model gpt-4o

{
  "score": 8.5,
  "passed": true,
  "reasoning": "SCORE: 8.5\nREASONING: The response clearly explains...",
  "threshold": 7.0,
  "model": "gpt-4o-mini"
}

# Good rubrics
"Response contains working Python code with proper error handling"
"Explanation covers syntax, use cases, and at least one example"
"File was created with correct content and proper formatting"

# Avoid vague rubrics
"Response is helpful"
"Code is good"
"Answer is correct"

from tests.live.interactive.github_advanced.judge_rubric import (
    PR_QUALITY_RUBRIC,
    evaluate_with_rubric,
)

# Get evaluation prompt
prompt = PR_QUALITY_RUBRIC.get_prompt()

# Evaluate with context
result = evaluate_with_rubric(
    rubric=PR_QUALITY_RUBRIC,
    context={
        "pr_title": "Fix subtract sign bug",
        "pr_body": "Closes #1. Fixed the subtract function.",
        "branch": "fix/subtract-sign",
    },
    judge_model="gpt-4o-mini",
)

print(result["overall_score"])  # 0-10
print(result["passed"])  # True/False