| 73 | } |
| 74 | |
| 75 | void validate_quantized_input( |
| 76 | std::string_view tag, |
| 77 | const array& w, |
| 78 | const array& scales, |
| 79 | int group_size, |
| 80 | int bits, |
| 81 | const std::optional<array>& biases = std::nullopt) { |
| 82 | if (w.dtype() != uint32) { |
| 83 | std::ostringstream msg; |
| 84 | msg << "[" << tag << "] The weight matrix should be uint32 " |
| 85 | << "but received " << w.dtype(); |
| 86 | throw std::invalid_argument(msg.str()); |
| 87 | } |
| 88 | |
| 89 | if (biases && scales.shape() != biases->shape()) { |
| 90 | std::ostringstream msg; |
| 91 | msg << "[" << tag << "] Scales and biases should have the same shape. " |
| 92 | << "Received scales with shape " << scales.shape() |
| 93 | << " and biases with " << biases->shape(); |
| 94 | throw std::invalid_argument(msg.str()); |
| 95 | } |
| 96 | |
| 97 | if (!std::equal( |
| 98 | w.shape().begin(), w.shape().end() - 2, scales.shape().begin())) { |
| 99 | std::ostringstream msg; |
| 100 | msg << "[" << tag |
| 101 | << "] Weight and scales should have the same batch shape. " |
| 102 | << "Received weight with shape " << w.shape() << ", scales with " |
| 103 | << scales.shape() << "."; |
| 104 | throw std::invalid_argument(msg.str()); |
| 105 | } |
| 106 | |
| 107 | if (w.shape(-1) * 32 / bits != scales.shape(-1) * group_size) { |
| 108 | std::ostringstream msg; |
| 109 | msg << "[" << tag << "] The shapes of the weight and scales are " |
| 110 | << "incompatible based on bits and group_size. w.shape() == " |
| 111 | << w.shape() << " and scales.shape() == " << scales.shape() |
| 112 | << " with group_size=" << group_size << " and bits=" << bits; |
| 113 | throw std::invalid_argument(msg.str()); |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | std::pair<int, int> extract_quantized_matmul_dims( |
| 118 | std::string_view tag, |
no test coverage detected