(question_list)
| 38 | |
| 39 | |
| 40 | def generate_sample_batch(question_list): |
| 41 | llm = LLM( |
| 42 | model=args.model, |
| 43 | trust_remote_code=True, |
| 44 | tensor_parallel_size=torch.cuda.device_count(), |
| 45 | gpu_memory_utilization=0.90, |
| 46 | ) |
| 47 | sampling_params = SamplingParams(max_tokens=8192, |
| 48 | temperature=0, |
| 49 | stop=["\n###\nProblem: ", "<|eot_id|>"],) |
| 50 | outputs = llm.generate(question_list, sampling_params, use_tqdm=True) |
| 51 | completions = [output.outputs[0].text for output in outputs] |
| 52 | return completions |
| 53 | |
| 54 | |
| 55 | def remove_boxed(s): |