Function main

scripts/eval_simpleqa_benchmark.py:543–588

Main entry point

()

def main():
    """Main entry point"""
    args = parse_args()
    
    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)
    
    # Create evaluator
    evaluator = SimpleQAEvaluator(
        model=args.model,
        approach=args.approach,
        base_url=args.base_url,
        grader_model=args.grader_model,
        timeout=args.timeout,
        cache_dir=args.cache_dir,
        output_dir=args.output_dir,
        use_verified=args.verified
    )
    
    try:
        # Run evaluation
        metrics = evaluator.run_evaluation(
            num_samples=args.num_samples,
            start_index=args.start_index
        )
        
        print("\n" + "="*50)
        print("EVALUATION SUMMARY")
        print("="*50)
        print(f"Model: {args.model}")
        print(f"Approach: {args.approach}")
        print(f"Questions: {metrics['total_questions']}")
        print(f"Accuracy: {metrics['accuracy']:.1f}%")
        print(f"F1 Score: {metrics['f1_score']:.3f}")
        print(f"Correct: {metrics['correct']}")
        print(f"Incorrect: {metrics['incorrect']}")
        print(f"Not Attempted: {metrics['not_attempted']}")
        
        if metrics['errors'] > 0:
            print(f"Errors: {metrics['errors']}")
        
    except KeyboardInterrupt:
        print("\nEvaluation interrupted by user")
    except Exception as e:
        logger.error(f"Evaluation failed: {e}")
        raise


if __name__ == "__main__":