Functionparallel_attn_fwd_kernel(
q,
k,
v,
o,
g_cumsum,
sink_bias,
lse,
scale,
cu_seqlens,
chunk_indic
fla/ops/attn/parallel.py:28
Functionparallel_based_bwd_kernel(
q,
k,
v,
do,
dz,
dq,
dk,
dv,
scale,
T,
B: tl.constexpr,
H: t
fla/ops/based/parallel.py:269
Functionparallel_based_fwd_kernel(
q,
k,
v,
o,
z,
scale,
T,
B: tl.constexpr,
H: tl.constexpr,
K: tl.con
fla/ops/based/parallel.py:19
Functionparallel_delta_ruler""" Args: q (torch.Tensor): queries of shape `[B, T, H, K]`. k (torch.Tensor): keys of shape `[B, T, H, K
fla/ops/delta_rule/parallel.py:305
Functionparallel_delta_rule_fwd_kernel(
q,
k,
k2, # original k
v,
beta,
o,
o_new,
attn,
T,
K: tl.constexpr,
fla/ops/delta_rule/parallel.py:151
Functionparallel_deltaformer_bwd_kernel_qk(
grad_q_ptr,
grad_k_ptr,
q_ptr,
k_ptr,
grad_v_ptr,
u_ptr,
lse_ptr,
beta_ptr,
fla/ops/deltaformer/parallel.py:459
Functionparallel_deltaformer_bwd_kernel_row_sum(
row_dot_ptr,
q_ptr,
k_ptr,
grad_v_ptr,
u_ptr,
lse_ptr,
H,
T,
D: tl.const
fla/ops/deltaformer/parallel.py:365
Functionparallel_deltaformer_bwd_kernel_u(
o_ptr,
q_ptr,
k_ptr,
v_ptr,
lse_ptr,
beta_ptr,
H,
T,
C,
D: tl.conste
fla/ops/deltaformer/parallel.py:278
Functionparallel_deltaformer_fwd_kernel(
q_ptr,
k_ptr,
v_ptr,
u_ptr,
w_ptr,
lse_ptr,
beta_ptr,
H,
T,
C,
D
fla/ops/deltaformer/parallel.py:137
Functionparallel_nsa_bwd_kernel_dkv(
q,
k,
v,
lse,
delta,
do,
dk,
dv,
block_mask,
cu_seqlens,
chunk_i
fla/ops/nsa/parallel.py:418
Functionparallel_nsa_bwd_kernel_dq(
q,
k,
v,
lse,
delta,
do,
dq,
scale,
block_indices,
block_counts,
fla/ops/nsa/parallel.py:309
Functionparallel_nsa_compression_bwd_kernel_dkv(
q,
k,
v,
lse,
delta,
do,
dk,
dv,
cu_seqlens,
chunk_indices,
chun
fla/ops/nsa/compression.py:237
Functionparallel_nsa_compression_bwd_kernel_dq(
q,
k,
v,
lse,
delta,
do,
dq,
scale,
cu_seqlens,
token_indices,
c
fla/ops/nsa/compression.py:133
Functionparallel_nsa_compression_fwd_kernel(
q,
k,
v,
o,
lse,
scale,
cu_seqlens,
token_indices,
chunk_offsets,
T,
fla/ops/nsa/compression.py:30
Functionparallel_nsa_fwd_kernel(
q,
k,
v,
o,
lse,
scale,
block_indices,
block_counts,
cu_seqlens,
tok
fla/ops/nsa/parallel.py:182
Functionparallel_nsa_kernel_mask(
block_indices,
block_counts,
block_mask,
T,
H: tl.constexpr,
S: tl.constexpr,
BS
fla/ops/nsa/parallel.py:272
Functionparallel_nsa_kernel_topk(
q,
k,
lse,
scale,
block_indices,
cu_seqlens,
token_indices,
chunk_offsets,
fla/ops/nsa/parallel.py:44
Functionparallel_parallax_bwd_kernel_dkv(
q,
r,
k,
v,
d1,
bart,
m,
delta_t,
delta_b,
grad_o,
grad_k,
g
fla/ops/parallax/parallel.py:427
Functionparallel_parallax_bwd_kernel_dqr(
q,
r,
k,
v,
d1,
bart,
m,
delta_t,
delta_b,
grad_o,
grad_q,
g
fla/ops/parallax/parallel.py:255
Functionparallel_parallax_bwd_kernel_preprocess(
grad_o,
o,
barv,
delta_t,
delta_b,
cu_seqlens,
chunk_indices,
T,
HQ: tl.
fla/ops/parallax/parallel.py:208
Functionparallel_parallax_fwd_kernel(
q,
r,
k,
v,
o,
barv,
d1,
bart,
m,
scale,
cu_seqlens,
chunk_i
fla/ops/parallax/parallel.py:31
Functionparallel_path_bwd_dkv_kernel(
q,
k,
v,
g_cumsum,
hc_whole,
scale,
L,
D,
dk,
dv,
do,
dg_cum
fla/ops/path_attn/parallel_path_bwd_inter_dkv.py:20
Functionparallel_path_bwd_dq_kernel(
q,
k,
v,
g_cumsum,
hc_whole,
scale,
L,
D,
dq,
do,
dhc_whole,
fla/ops/path_attn/parallel_path_bwd_inter_dqh.py:22
Functionparallel_path_bwd_intra_chunk_kernel(
q, k, v, g_cumsum, w1, w2,
L, D,
dq, dq_new, dk, dv, dw1, dw2, do, dg_cumsum,
offsets, indic
fla/ops/path_attn/parallel_path_bwd_intra.py:20
Functionparallel_path_fwd_kernel(
q,
k,
v,
o,
o_new,
g_cumsum,
w1,
w2,
scale,
L,
L_new,
M,
fla/ops/path_attn/parallel_path_fwd.py:20
Functionparallel_path_fwd_kernel_prepare_k_cache(
k, k_new, w1, w2,
offsets, indices,
T,
H: tl.constexpr,
K: tl.constexpr,
BT: tl.cons
fla/ops/path_attn/prepare_k_cache.py:19
Functionparallel_rebased_bwd_kernel(
q,
k,
v,
do,
dz,
dq,
dk,
dv,
scale,
T,
B: tl.constexpr,
H: t
fla/ops/rebased/parallel.py:274
Functionparallel_rebased_fwd_kernel(
q,
k,
v,
o,
z,
scale,
T,
B: tl.constexpr,
H: tl.constexpr,
K: tl.con
fla/ops/rebased/parallel.py:19
Functionparallel_simple_gla_bwd_kernel(
q,
k,
v,
g,
do,
dq,
dk,
dv,
dg,
scale,
cu_seqlens,
chunk_ind
fla/ops/simple_gla/parallel.py:381
Functionparallel_simple_gla_fwd_kernel(
q,
k,
v,
g,
o,
attn,
scale,
cu_seqlens,
chunk_indices,
T,
B: tl.
fla/ops/simple_gla/parallel.py:47
Functionparallel_wall_attn_bwd_kernel_dkv(
q,
k,
v,
g_cumsum,
g_scalar_cumsum,
lse,
delta,
do,
dk,
dv,
dg_c
fla/ops/wall_attn/parallel.py:400
Functionparallel_wall_attn_bwd_kernel_dq(
q,
k,
v,
g_cumsum,
g_scalar_cumsum,
lse,
delta,
do,
dq,
dg_cumsum,
fla/ops/wall_attn/parallel.py:227
Functionparallel_wall_attn_decode_kernel(
q, # [B, T_q, HQ, K]
k_tilde, # [B, T_kv, HQ, K] pre-rescaled keys (per-Q-h
fla/ops/wall_attn/decode.py:36
Functionparallel_wall_attn_fwd_kernel(
q,
k,
v,
o,
g_cumsum,
g_scalar_cumsum,
sink_bias,
lse,
scale,
cu_seq
fla/ops/wall_attn/parallel.py:61
Functionpowglu_fwd_kernel(
x, y, z,
stride_x_row,
stride_y_row,
stride_z_row,
m,
T,
D: tl.constexpr,
B:
fla/modules/activations.py:968
Functionpowglu_fwdbwd_kernel(
x, y, g, dx, dy, z,
stride_x_row,
stride_y_row,
stride_g_row,
stride_dx_row,
stride_
fla/modules/activations.py:1010
Functionpre_process_fwd_kernel_merged(
k,
v,
w,
g,
gk,
bg,
u,
hm,
cu_seqlens,
T,
H: tl.constexpr,
H
fla/ops/cp/chunk_delta_h.py:42
Functionprepare_wy_repr_bwd_kda_kernel(
k,
v,
beta,
gk,
A,
dA,
dw,
du,
dk,
dk2,
dv,
db,
dg,
fla/ops/kda/wy_fast.py:136
Functionprepare_wy_repr_bwd_kernel(
A_ab_inv,
A_ak,
ag,
v,
dw,
du,
dv,
dv0,
dag,
dAak,
dAab,
cu_
fla/ops/generalized_delta_rule/dplr/wy_fast_bwd.py:33
Functionprepare_wy_repr_bwd_kernel(
k,
v,
beta,
g,
A,
dw,
du,
dk,
dv,
db,
dg,
cu_seqlens,
ch
fla/ops/gated_delta_rule/wy_fast.py:104
Functionprepare_wy_repr_bwd_kernel(
k,
v,
beta,
gv,
A,
dA,
dw,
du,
dk,
dv,
db,
dgv,
cu_seqle
fla/ops/gated_oja_rule/wy_fast.py:109
Functionprepare_wy_repr_bwd_kernel(
k,
v,
p,
beta,
g0,
g,
A,
dw,
du,
dk,
dv,
dp,
dbeta,
fla/ops/comba/wy_fast.py:159
Functionprepare_wy_repr_bwd_kernel(
k,
v,
beta,
A,
dw,
du,
dk,
dv,
dbeta,
cu_seqlens,
chunk_indices,
fla/ops/delta_rule/wy_fast.py:96
Functionprepare_wy_repr_fwd_kernel_chunk32(
a,
b,
A,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
K: tl.constexpr,
fla/ops/generalized_delta_rule/iplr/wy_fast.py:30
Functionprepare_wy_repr_fwd_kernel_chunk32(
A_ab,
A_ab_inv,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
BT: tl.constexpr,
fla/ops/generalized_delta_rule/dplr/wy_fast_fwd.py:30
Functionprepare_wy_repr_fwd_kernel_chunk64(
a,
b,
A,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
K: tl.constexpr,
fla/ops/generalized_delta_rule/iplr/wy_fast.py:85
Functionprepare_wy_repr_fwd_kernel_chunk64(
A_ab,
A_ab_inv,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
BT: tl.constexpr,
fla/ops/generalized_delta_rule/dplr/wy_fast_fwd.py:76
Functionrecompute_w_u_fwd_gdn2_kernel(
q,
k,
qg,
kg,
v,
b, # channel-wise erase gate, [B, T, H, K]
w_gate,
fla/ops/gdn2/wy_fast.py:51
Functionrecompute_w_u_fwd_kda_kernel(
q,
k,
qg,
kg,
v,
beta,
w,
u,
A,
gk,
cu_seqlens,
chunk_indice
fla/ops/kda/wy_fast.py:33
Functionrecompute_w_u_fwd_kernel(
k,
v,
beta,
w,
u,
A,
g,
cu_seqlens,
chunk_indices,
T,
H: tl.cons
fla/ops/gated_delta_rule/wy_fast.py:32
Functionrecompute_w_u_fwd_kernel(
k,
v,
vg,
beta,
w,
u,
A,
gv,
cu_seqlens,
chunk_indices,
T,
H
fla/ops/gated_oja_rule/wy_fast.py:30
Functionrecompute_w_u_fwd_kernel(
k,
v,
beta,
w,
u,
A,
g,
cu_seqlens,
chunk_indices,
T,
H: tl.cons
fla/ops/comba/wy_fast.py:288
Functionrecompute_w_u_fwd_kernel(
k,
v,
beta,
w,
u,
A,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
fla/ops/delta_rule/wy_fast.py:34
Functionrms_norm_ref(x, weight, bias, z=None, eps=1e-6, group_size=None, norm_before_gate=True, upcast=True)
fla/modules/layernorm_gated.py:27
Methodrotary_embedding_fwdbwd(
self,
x,
cos,
sin,
seqlen_offsets=0,
cu_seqlens=None,
fla/modules/backends/triton_ascend/__init__.py:28
Functionrotary_embedding_kernel(
x,
cos,
sin,
y,
cu_seqlens,
chunk_indices,
seq_offsets,
T,
B: tl.constex
fla/modules/rotary.py:48
Functionrotary_embedding_kernel(
x,
cos,
sin,
y,
cu_seqlens,
chunk_indices,
seq_offsets,
T,
B: tl.constex
fla/modules/backends/triton_ascend/rotary.py:32
Functionrwkv_mix_bwd_kenel(
dk1_ptr0,
xk_ptr,
dx_ptr,
dx_prev_ptr,
batch_size,
token_length,
hidden_dim: tl.
fla/ops/rwkv7/channel_mixing.py:206
Functionrwkv_seq_mix_kernel(
x_ptr,
x_prev_ptr,
mix_k_ptr,
output_ptr,
batch_size: tl.constexpr,
token_length,
fla/ops/rwkv7/channel_mixing.py:39