(model: str)
| 96 | return {"decision": decision, "explanation": explanation} |
| 97 | |
| 98 | def main(model: str): |
| 99 | # Load the dataset |
| 100 | dataset = load_dataset("google/frames-benchmark", split="test") |
| 101 | |
| 102 | filename = f"evaluation_results_{model.replace('/', '_')}.json" |
| 103 | existing_results = load_existing_results(filename) |
| 104 | last_processed_index = get_last_processed_index(existing_results) |
| 105 | |
| 106 | for item in tqdm(dataset, desc="Processing samples"): |
| 107 | index = int(item['Unnamed: 0']) |
| 108 | if index <= last_processed_index: |
| 109 | continue |
| 110 | |
| 111 | prompt = generate_llm_prompt(item['Prompt'], item['wiki_links']) |
| 112 | llm_response = get_llm_response(prompt, model) |
| 113 | evaluation = evaluate_response(item['Prompt'], llm_response, item['Answer'], model) |
| 114 | |
| 115 | result = { |
| 116 | "index": index, |
| 117 | "prompt": item['Prompt'], |
| 118 | "ground_truth": item['Answer'], |
| 119 | "llm_response": llm_response, |
| 120 | "evaluation_decision": evaluation['decision'], |
| 121 | "evaluation_explanation": evaluation['explanation'], |
| 122 | "reasoning_type": item['reasoning_types'] |
| 123 | } |
| 124 | |
| 125 | save_result(filename, result) |
| 126 | # print(f"Index: {index}, Decision: {result['evaluation_decision']}") |
| 127 | # time.sleep(SLEEP_INTERVAL) |
| 128 | |
| 129 | # Calculate and print summary statistics |
| 130 | results = load_existing_results(filename) |
| 131 | total_samples = len(results) |
| 132 | correct_answers = sum(1 for r in results if r['evaluation_decision'] == 'TRUE') |
| 133 | accuracy = correct_answers / total_samples |
| 134 | |
| 135 | print(f"Model: {model}") |
| 136 | print(f"Total samples: {total_samples}") |
| 137 | print(f"Correct answers: {correct_answers}") |
| 138 | print(f"Accuracy: {accuracy:.2%}") |
| 139 | |
| 140 | # Print accuracy by reasoning type |
| 141 | reasoning_types = set(r['reasoning_type'] for r in results) |
| 142 | for rt in reasoning_types: |
| 143 | rt_samples = [r for r in results if r['reasoning_type'] == rt] |
| 144 | rt_correct = sum(1 for r in rt_samples if r['evaluation_decision'] == 'TRUE') |
| 145 | rt_accuracy = rt_correct / len(rt_samples) |
| 146 | print(f"Accuracy for {rt}: {rt_accuracy:.2%}") |
| 147 | |
| 148 | if __name__ == "__main__": |
| 149 | parser = argparse.ArgumentParser(description="Evaluate LLM performance on google/frames-benchmark") |
no test coverage detected