Parse command line arguments
()
| 489 | |
| 490 | |
| 491 | def parse_args(): |
| 492 | """Parse command line arguments""" |
| 493 | parser = argparse.ArgumentParser( |
| 494 | description="Evaluate OptILLM on SimpleQA factuality benchmark" |
| 495 | ) |
| 496 | |
| 497 | # Model and approach |
| 498 | parser.add_argument("--model", type=str, default="gpt-4o-mini", |
| 499 | help="Model to evaluate (default: gpt-4o-mini)") |
| 500 | parser.add_argument("--approach", type=str, default="none", |
| 501 | choices=["none", "web_search", "deep_research"], |
| 502 | help="Approach to use (default: none)") |
| 503 | |
| 504 | # Server configuration |
| 505 | parser.add_argument("--base-url", type=str, default=DEFAULT_BASE_URL, |
| 506 | help=f"OptILLM base URL (default: {DEFAULT_BASE_URL})") |
| 507 | parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, |
| 508 | help=f"Request timeout in seconds (default: {DEFAULT_TIMEOUT})") |
| 509 | |
| 510 | # Grading configuration |
| 511 | parser.add_argument("--grader-model", type=str, default=DEFAULT_GRADER_MODEL, |
| 512 | help=f"Model for grading responses (default: {DEFAULT_GRADER_MODEL})") |
| 513 | |
| 514 | # Evaluation parameters |
| 515 | parser.add_argument("--num-samples", type=int, default=None, |
| 516 | help="Number of questions to evaluate (default: all)") |
| 517 | parser.add_argument("--start-index", type=int, default=0, |
| 518 | help="Start from specific question index (default: 0)") |
| 519 | |
| 520 | # Search-specific parameters |
| 521 | parser.add_argument("--num-search-results", type=int, default=10, |
| 522 | help="Number of search results per query (default: 10)") |
| 523 | parser.add_argument("--headless", action="store_true", |
| 524 | help="Run browser in headless mode for web search") |
| 525 | |
| 526 | # Output configuration |
| 527 | parser.add_argument("--cache-dir", type=str, default="cache", |
| 528 | help="Directory for caching dataset (default: cache)") |
| 529 | parser.add_argument("--output-dir", type=str, default="results", |
| 530 | help="Directory for saving results (default: results)") |
| 531 | |
| 532 | # Dataset selection |
| 533 | parser.add_argument("--verified", action="store_true", |
| 534 | help="Use SimpleQA-Verified dataset (1k verified questions) instead of original SimpleQA") |
| 535 | |
| 536 | # Debugging |
| 537 | parser.add_argument("--verbose", action="store_true", |
| 538 | help="Enable verbose logging") |
| 539 | |
| 540 | return parser.parse_args() |
| 541 | |
| 542 | |
| 543 | def main(): |