(self)
| 77 | ) |
| 78 | |
| 79 | def test_vision_encoder(self): |
| 80 | # Load and process test image |
| 81 | x = self.hf_image_proc( |
| 82 | images=[Image.open("assets/dog.jpeg")], return_tensors="np" |
| 83 | ).pixel_values |
| 84 | |
| 85 | # Infer with HuggingFace model |
| 86 | with torch.inference_mode(): |
| 87 | # Get expected |
| 88 | x_tc = torch.tensor(x) |
| 89 | expected_out = self.hf_clip.vision_model(x_tc, output_hidden_states=True) |
| 90 | expected_last_hidden = expected_out.last_hidden_state.numpy() |
| 91 | expected_pooler_output = expected_out.pooler_output.numpy() |
| 92 | expected_hidden_states = [hs.numpy() for hs in expected_out.hidden_states] |
| 93 | # Test MLX vision encoder |
| 94 | out = self.mx_clip.vision_model( |
| 95 | mx.array(x.transpose(0, 2, 3, 1)), output_hidden_states=True |
| 96 | ) |
| 97 | self.assertTrue( |
| 98 | np.allclose( |
| 99 | out.last_hidden_state, expected_last_hidden, rtol=1e-4, atol=1e-3 |
| 100 | ), |
| 101 | ) |
| 102 | self.assertTrue( |
| 103 | np.allclose( |
| 104 | out.pooler_output, expected_pooler_output, rtol=1e-4, atol=1e-3 |
| 105 | ), |
| 106 | ) |
| 107 | for expected_hs, out_hs in zip(expected_hidden_states, out.hidden_states): |
| 108 | self.assertTrue( |
| 109 | np.allclose(expected_hs, out_hs, rtol=1e-4, atol=1e-3), |
| 110 | ) |
| 111 | |
| 112 | def test_clip_model(self): |
| 113 | image_input = self.hf_image_proc( |
nothing calls this directly
no outgoing calls
no test coverage detected