(
A: T.Tensor(A_shape, in_dtype),
B: T.Tensor(B_shape, in_dtype),
Bias: T.Tensor(Bias_shape, out_dtype),
C: T.Tensor(C_shape, out_dtype),
)
| 49 | |
| 50 | @T.prim_func |
| 51 | def main( |
| 52 | A: T.Tensor(A_shape, in_dtype), |
| 53 | B: T.Tensor(B_shape, in_dtype), |
| 54 | Bias: T.Tensor(Bias_shape, out_dtype), |
| 55 | C: T.Tensor(C_shape, out_dtype), |
| 56 | ): |
| 57 | with T.Kernel(T.ceildiv(N, n_partition), M, threads=(reduce_thread, n_partition)) as ( |
| 58 | bx, |
| 59 | by, |
| 60 | ): |
| 61 | A_local = T.alloc_local((micro_size_k,), in_dtype) |
| 62 | B_local = T.alloc_local((micro_size_k,), in_dtype) |
| 63 | accum_res = T.alloc_local((1,), accum_dtype) |
| 64 | reduced_accum_res = T.alloc_local((1,), accum_dtype) |
| 65 | |
| 66 | kr = T.get_thread_binding(0) |
| 67 | ni = T.get_thread_binding(1) |
| 68 | |
| 69 | T.clear(accum_res) |
| 70 | for ko in T.serial(T.ceildiv(K, block_K)): |
| 71 | for v in T.vectorized(micro_size_k): |
| 72 | A_local[v] = A[by, ko * block_K + kr * micro_size_k + v] |
| 73 | |
| 74 | for v in T.vectorized(micro_size_k): |
| 75 | B_local[v] = B[ |
| 76 | bx * n_partition + ni, |
| 77 | ko * block_K + kr * micro_size_k + v, |
| 78 | ] |
| 79 | |
| 80 | if use_dp4a: |
| 81 | for ki in T.serial(micro_size_k // dp4a_size): |
| 82 | T.dp4a( |
| 83 | A_local[ki * dp4a_size], |
| 84 | B_local[ki * dp4a_size], |
| 85 | accum_res[0], |
| 86 | ) |
| 87 | else: |
| 88 | for ki in T.serial(micro_size_k): |
| 89 | accum_res[0] += A_local[ki].astype(accum_dtype) * B_local[ki].astype(accum_dtype) |
| 90 | |
| 91 | with T.attr( |
| 92 | T.comm_reducer(lambda x, y: x + y, [T.cast(0, accum_dtype)]), |
| 93 | "reduce_scope", |
| 94 | T.reinterpret(T.uint64(0), dtype="handle"), |
| 95 | ): |
| 96 | T.evaluate( |
| 97 | T.tvm_thread_allreduce( |
| 98 | T.uint32(1), |
| 99 | accum_res[0], |
| 100 | True, |
| 101 | reduced_accum_res[0], |
| 102 | kr, |
| 103 | dtype="handle", |
| 104 | ) |
| 105 | ) |
| 106 | if kr == 0: |
| 107 | if with_bias: |
| 108 | C[by, bx * n_partition + ni] = reduced_accum_res[0] + Bias[bx * n_partition + ni] |
nothing calls this directly
no test coverage detected
searching dependent graphs…