Functionchunk_fwd_h(
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
gk: torch.Tensor,
gv: torch.Tensor,
fla/ops/common/chunk_h_split.py:429
Functionchunk_fwd_kernel_h(
k,
v,
h,
g,
g_gamma,
gk,
gv,
h0,
ht,
cu_seqlens,
split_offsets,
fla/ops/common/chunk_h.py:36
Functionchunk_fwd_kernel_h_parallel(
k,
v,
h,
g,
gk,
gv,
h0,
ht,
cu_seqlens,
chunk_indices,
T,
H:
fla/ops/common/chunk_h_parallel.py:39
Functionchunk_fwd_kernel_h_reduction(
h,
g,
gk,
gv,
kvt,
ht,
cu_seqlens,
chunk_offsets,
T,
H: tl.constexpr
fla/ops/common/chunk_h_parallel.py:154
Functionchunk_fwd_kernel_h_reduction(
g,
gk,
gv,
hs,
hr,
ht,
cu_seqlens,
split_offsets,
T,
S: tl.constexpr
fla/ops/common/chunk_h_split.py:153
Functionchunk_fwd_kernel_h_split(
k,
v,
g,
gk,
gv,
hs,
hr,
h0,
ht,
cu_seqlens,
split_indices,
fla/ops/common/chunk_h_split.py:33
Functionchunk_fwd_kernel_o(
q,
k,
v,
h,
g,
o,
cu_seqlens,
chunk_indices,
scale,
T,
num_house
fla/ops/gated_delta_product/chunk_deltaproduct_o.py:36
Functionchunk_fwd_kernel_o(
q,
k,
v,
h,
g,
g_gamma,
o,
cu_seqlens,
chunk_indices,
scale,
T,
fla/ops/common/chunk_o.py:37
Functionchunk_fwd_mesa_cg_dim64_kernel(
dq,
dq_final,
k,
h,
g,
beta,
lamb,
cu_seqlens,
chunk_indices,
T,
fla/ops/mesa_net/chunk_cg_solver_bwd.py:37
Functionchunk_fwd_mesa_cg_dim64_kernel(
q,
q_final,
k,
h,
o,
v,
h_kv,
g,
beta,
lamb,
cu_seqlens,
chu
fla/ops/mesa_net/chunk_cg_solver_fwd.py:37
Functionchunk_gated_delta_product_bwd_dhu(
q: torch.Tensor,
k: torch.Tensor,
w: torch.Tensor,
g: torch.Tensor,
h0: torch.Tensor,
fla/ops/gated_delta_product/chunk_deltaproduct_h.py:458
Functionchunk_gated_delta_product_bwd_kernel_dhu_blockdim64(
q,
k,
w,
g,
dht,
dh0,
do,
dh,
dv,
dv2,
cu_seqlens,
chunk_off
fla/ops/gated_delta_product/chunk_deltaproduct_h.py:217
Functionchunk_gated_delta_product_fwd_kernel_h_blockdim64(
k,
v,
w,
v_new,
g,
h,
h0,
ht,
cu_seqlens,
chunk_offsets,
T,
fla/ops/gated_delta_product/chunk_deltaproduct_h.py:38
Functionchunk_gated_delta_rule_bwd_kernel_dhu_blockdim64(
q,
k,
w,
g,
gk,
dht,
dh0,
do,
dh,
dv,
dv2,
cu_seqlens,
c
fla/ops/common/chunk_delta_h.py:316
Functionchunk_gated_delta_rule_fwd_kernel_h_blockdim64(
k,
v,
w,
v_new,
g,
gk,
h,
h0,
ht,
cu_seqlens,
chunk_offsets,
fla/ops/common/chunk_delta_h.py:41
Functionchunk_gdn2_bwd_kernel_intra(
q,
k,
g,
b,
dAqk,
dAkk,
dq,
dq2,
dk,
dk2,
dg,
dg2,
db,
fla/ops/gdn2/chunk_intra.py:428
Functionchunk_gdn2_bwd_kernel_wy_dqkg_fused(
q,
k,
v,
v_new,
g,
b,
w_gate,
A,
h,
do,
dh,
dq,
dk,
fla/ops/gdn2/chunk_bwd.py:67
Functionchunk_gdn2_fwd_kernel_inter_solve_fused(
q,
k,
g,
b,
Aqk,
Akkd,
Akk,
scale,
cu_seqlens,
chunk_indices,
T,
fla/ops/gdn2/chunk_intra.py:162
Functionchunk_gdn2_fwd_kernel_intra_sub_chunk(
q,
k,
g,
b,
Aqk,
Akk,
scale,
cu_seqlens,
chunk_indices,
T,
H: tl
fla/ops/gdn2/chunk_intra.py:53
Functionchunk_gdn2_fwd_kernel_intra_token_parallel(
q,
k,
g,
b,
Aqk,
Akk,
scale,
cu_seqlens,
N,
T,
H: tl.constexpr,
fla/ops/gdn2/chunk_intra_token_parallel.py:42
Functionchunk_generalized_iplr_delta_rule_fwd_kernel_h(
k,
v,
d,
b,
u,
v_new,
h,
h0,
ht,
cu_seqlens,
chunk_offsets,
fla/ops/generalized_delta_rule/iplr/chunk.py:41
Functionchunk_generalized_iplr_delta_rule_fwd_kernel_o(
q,
k,
v,
u,
b,
h,
o,
cu_seqlens,
chunk_indices,
scale,
T,
H:
fla/ops/generalized_delta_rule/iplr/chunk.py:126
Functionchunk_gla_bwd_kernel_dA(
v,
do,
dA,
cu_seqlens,
chunk_indices,
scale,
T,
H: tl.constexpr,
V: tl.c
fla/ops/gla/chunk.py:539
Functionchunk_gla_bwd_kernel_dv(
k,
g,
A,
do,
dh,
dv,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
fla/ops/gla/chunk.py:592
Functionchunk_gla_bwd_kernel_inter(
q,
k,
v,
g,
h,
do,
dh,
dq,
dk,
dq2,
dk2,
dg,
cu_seqlens,
fla/ops/gla/chunk.py:675
Functionchunk_gla_bwd_kernel_intra(
q,
k,
g,
dA,
dq,
dk,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
fla/ops/gla/chunk.py:404
Functionchunk_gla_fwd_A_kernel_intra_sub_inter(
q,
k,
g,
A,
cu_seqlens,
chunk_indices,
scale,
T,
H: tl.constexpr,
K:
fla/ops/gla/chunk.py:38
Functionchunk_gla_fwd_A_kernel_intra_sub_intra(
q,
k,
g,
A,
cu_seqlens,
chunk_indices,
scale,
T,
H: tl.constexpr,
K:
fla/ops/gla/chunk.py:111
Functionchunk_gla_fwd_A_kernel_intra_sub_intra_split(
q,
k,
g,
A,
cu_seqlens,
chunk_indices,
scale,
T,
B: tl.constexpr,
H:
fla/ops/gla/chunk.py:184
Functionchunk_gla_fwd_kernel_o(
q,
v,
g,
h,
o,
A,
cu_seqlens,
chunk_indices,
scale,
T,
H: tl.con
fla/ops/gla/chunk.py:313
Functionchunk_global_cumsum_scalar_kernel(
s,
o,
scale,
cu_seqlens,
T,
B: tl.constexpr,
H: tl.constexpr,
BT: tl.constex
fla/ops/utils/cumsum.py:145
Functionchunk_global_cumsum_vector_kernel(
s,
o,
scale,
cu_seqlens,
T,
B: tl.constexpr,
H: tl.constexpr,
S: tl.constexp
fla/ops/utils/cumsum.py:205
Functionchunk_gsa_bwd_k_kernel_dA(
v,
g,
do,
dA,
chunk_indices,
cu_seqlens,
scale,
T,
B: tl.constexpr,
fla/ops/gsa/chunk.py:200
Functionchunk_gsa_bwd_k_kernel_dqkvg(
q,
k,
v,
h,
g,
A,
do,
dh,
dq,
dk,
dv,
dg,
dgv,
dA,
fla/ops/gated_oja_rule/chunk_h.py:528
Functionchunk_gsa_bwd_k_kernel_dqkvg(
q,
k,
v,
h,
g,
A,
do,
dh,
dq,
dk,
dv,
dg,
dgv,
dA,
fla/ops/gsa/chunk.py:301
Functionchunk_gsa_bwd_k_kernel_intra_dvg(
v,
g,
o,
A,
do,
dv,
dg,
cu_seqlens,
chunk_indices,
T,
HQ: tl.con
fla/ops/gsa/chunk.py:427
Functionchunk_gsa_fwd_k_kernel_inter(
q,
k,
h,
g,
o,
A,
cu_seqlens,
chunk_indices,
scale,
T,
HQ: tl.co
fla/ops/gsa/chunk.py:38
Functionchunk_gsa_fwd_k_kernel_intra(
v,
g,
o,
A,
cu_seqlens,
chunk_indices,
T,
HQ: tl.constexpr,
H: tl.conste
fla/ops/gsa/chunk.py:112
Functionchunk_hgrn_bwd_kernel_h(
g,
gc,
dx,
do,
T,
D: tl.constexpr,
BT: tl.constexpr,
BD: tl.constexpr,
)
fla/ops/hgrn/chunk.py:133
Functionchunk_hgrn_bwd_kernel_o(
g,
gc,
o,
dx,
dg,
s_b,
s_t,
s_d,
T,
D: tl.constexpr,
BT: tl.cons
fla/ops/hgrn/chunk.py:179
Functionchunk_hgrn_fwd_kernel_h(
x,
g,
gc,
o,
h0,
T,
D: tl.constexpr,
BT: tl.constexpr,
BD: tl.constexpr,
fla/ops/hgrn/chunk.py:53
Functionchunk_hgrn_fwd_kernel_o(
gc,
o,
s_b,
s_t,
s_d,
T,
D: tl.constexpr,
BT: tl.constexpr,
BD: tl.const
fla/ops/hgrn/chunk.py:95
Functionchunk_iplr_delta_rule_ref(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
tests/ops/test_iplr_delta.py:18
Functionchunk_kda_bwd_kernel_dAv(
q,
k,
v,
A,
do,
dv,
dA,
cu_seqlens,
chunk_indices,
scale,
T,
fla/ops/kda/chunk_bwd.py:43
Functionchunk_kda_bwd_kernel_intra(
q,
k,
g,
beta,
dAqk,
dAkk,
dq,
dq2,
dk,
dk2,
dg,
dg2,
db
fla/ops/kda/chunk_intra.py:381
Functionchunk_kda_bwd_kernel_wy_dqkg_fused(
q,
k,
v,
v_new,
g,
beta,
A,
h,
do,
dh,
dq,
dk,
dv,
d
fla/ops/kda/chunk_bwd.py:126
Functionchunk_kda_fwd_kernel_intra_sub_chunk(
q,
k,
g,
beta,
Aqk,
Akk,
scale,
cu_seqlens,
chunk_indices,
T,
H:
fla/ops/kda/chunk_intra.py:659
Functionchunk_kda_fwd_kernel_intra_token_parallel(
q,
k,
g,
beta,
Aqk,
Akk,
scale,
cu_seqlens,
N,
T,
H: tl.constexp
fla/ops/kda/chunk_intra_token_parallel.py:32
Functionchunk_lightning_attnr""" Args: q (torch.Tensor): queries of shape `[B, T, H, K]`. k (torch.Tensor): keys of shape `[B, T, H, K
fla/ops/lightning_attn/chunk.py:14
Functionchunk_local_cumsum_scalar_kernel(
s,
o,
scale,
cu_seqlens,
chunk_indices,
T,
B: tl.constexpr,
H: tl.constexpr,
fla/ops/utils/cumsum.py:32
Functionchunk_local_cumsum_vector_kernel(
s,
o,
scale,
cu_seqlens,
chunk_indices,
T,
B: tl.constexpr,
H: tl.constexpr,
fla/ops/utils/cumsum.py:87
Functionchunk_mesa_net_fwd_kernel_h(
k,
v,
beta,
g,
h,
h_kv,
h_init,
h_kv_init,
h_final,
h_kv_final,
fla/ops/mesa_net/chunk_h_fwd.py:32
Functionchunk_mesa_net_h_kk_bwd_intra_kernel(
k,
beta,
h,
dh,
g,
q_star,
dq,
dk,
dg,
dbeta,
dk_beta,
dlamb
fla/ops/mesa_net/chunk_h_kk_intra_bwd.py:20
Functionchunk_mesa_net_h_kv_bwd_intra_kernel(
q_star,
k,
v,
beta,
h_kv,
g,
do,
dh_kv,
dq,
dk_beta,
dg,
dv,
fla/ops/mesa_net/chunk_h_kv_intra_bwd.py:33
Functionchunk_mesa_net_h_kv_bwd_intra_kernel_dkv(
q_star,
k,
v,
beta,
h_kv,
g,
do,
dh_kv,
dk_beta,
dg,
dv,
cu_
fla/ops/mesa_net/chunk_h_kv_intra_bwd_separate.py:32
Functionchunk_mesa_net_h_kv_bwd_intra_kernel_dq(
q_star,
k,
v,
beta,
h_kv,
g,
do,
dq,
dg_prev,
dg,
cu_seqlens,
fla/ops/mesa_net/chunk_h_kv_intra_bwd_separate.py:151
Functionchunk_oja_bwd_kernel_dA(
v,
gv,
do,
dA,
chunk_indices,
cu_seqlens,
scale,
T,
B: tl.constexpr,
fla/ops/gated_oja_rule/chunk_o.py:263
Functionchunk_oja_bwd_kernel_dhu_blockdim64(
q,
vg,
w,
gv,
dht,
dh0,
do,
dh,
dk,
dk2,
cu_seqlens,
chunk_o
fla/ops/gated_oja_rule/chunk_h.py:265
Functionchunk_oja_bwd_kernel_dqk(
q,
k,
h,
gv,
A,
dq,
dk,
dA,
do,
scale,
cu_seqlens,
chunk_ind
fla/ops/gated_oja_rule/chunk_o.py:405
Functionchunk_oja_bwd_kernel_dv_o(
v,
g,
o,
A,
do,
dv,
dv2,
dg,
cu_seqlens,
chunk_indices,
T,
H
fla/ops/gated_oja_rule/chunk_o.py:543
Functionchunk_oja_bwd_kernel_dvwg_h(
k,
v,
gv,
h,
dh,
dk,
dw,
dv,
dgv_last,
dgk,
cu_seqlens,
chun
fla/ops/gated_oja_rule/chunk_h.py:664
Functionchunk_oja_fwd_inter(
q,
k,
h,
gv,
o,
A,
cu_seqlens,
chunk_indices,
scale,
T,
HQ: tl.c
fla/ops/gated_oja_rule/chunk_o.py:34
Functionchunk_oja_fwd_intra(
v,
gv,
o,
A,
cu_seqlens,
chunk_indices,
T,
HQ: tl.constexpr,
H: tl.const
fla/ops/gated_oja_rule/chunk_o.py:108
Functionchunk_oja_fwd_kernel_h_blockdim64(
v,
u,
w,
k_new,
gv,
h,
h0,
ht,
cu_seqlens,
chunk_offsets,
T,
fla/ops/gated_oja_rule/chunk_h.py:38
Functionchunk_rwkv6_bwd_kernel_dh(
q,
gi,
ge,
do,
dh,
dht,
dh0,
cu_seqlens,
chunk_offsets,
scale,
T
fla/ops/rwkv6/chunk.py:420
Functionchunk_rwkv6_bwd_kernel_inter(
q,
k,
v,
h,
gi,
ge,
u,
do,
dh,
dA,
dq,
dk,
dq2,
dk2,
fla/ops/rwkv6/chunk.py:642
Functionchunk_rwkv6_bwd_kernel_intra(
q,
k,
gi,
ge,
dA,
dq,
dk,
cu_seqlens,
chunk_indices,
T,
H: tl.co
fla/ops/rwkv6/chunk.py:501
Functionchunk_rwkv6_fwd_A_kernel_intra_sub_inter(
q,
k,
gi, # cumulative decay inclusive
ge, # cumulative decay exclusive
A,
cu_seql
fla/ops/rwkv6/chunk.py:135
Functionchunk_rwkv6_fwd_A_kernel_intra_sub_intra(
q,
k,
gi,
ge,
u,
A,
cu_seqlens,
chunk_indices,
scale,
T,
H: tl.c
fla/ops/rwkv6/chunk.py:211
Functionchunk_rwkv6_fwd_A_kernel_intra_sub_intra_merge(
A,
A2,
cu_seqlens,
chunk_indices,
T,
B: tl.constexpr,
H: tl.constexpr,
BT: t
fla/ops/rwkv6/chunk.py:367
Functionchunk_rwkv6_fwd_A_kernel_intra_sub_intra_split(
q,
k,
gi,
ge,
u,
A,
cu_seqlens,
chunk_indices,
scale,
B: tl.constexp
fla/ops/rwkv6/chunk.py:286
Functionchunk_rwkv6_fwd_cumsum_kernel(
s,
oi,
oe,
scale,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
S: tl.c
fla/ops/rwkv6/chunk.py:46
Functionchunk_scaled_dot_comba_pkt_fwd_kernel(
k,
p,
beta,
g0,
g,
A,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
fla/ops/comba/wy_fast.py:32
Functionchunk_scaled_dot_kkt_bwd_kernel_gk(
k,
g,
beta,
dA,
dk,
dg,
db,
cu_seqlens,
chunk_indices,
B,
T,
fla/ops/gated_oja_rule/chunk_kkt.py:231
Functionchunk_scaled_dot_kkt_fwd_kernel(
k,
g,
beta,
A,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
HV: tl.con
fla/ops/common/chunk_scaled_dot_kkt.py:32
Functionchunk_scaled_dot_kkt_fwd_kernel(
k,
g,
beta,
A,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
K: tl.cons
fla/ops/gated_oja_rule/chunk_kkt.py:30
Functionchunk_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
k,
g,
beta,
A,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
K: tl.cons
fla/ops/gated_oja_rule/chunk_kkt.py:91
Functionchunk_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
k,
g,
beta,
A,
cu_seqlens,
chunk_indices,
T,
H: tl.constexpr,
K: tl.cons
fla/ops/gated_oja_rule/chunk_kkt.py:167
Functionchunk_transform_qk_bwd_kernel_prepare(
q,
k,
v,
w,
beta,
g_cumsum,
L,
D,
h,
q_new,
k_new,
AT,
d
fla/ops/path_attn/intra_chunk_preprocess_bwd_prepare.py:20
Functionchunk_transform_qk_fwd_kernel(
q,
k,
v,
beta,
o,
A,
q_new,
k_new,
A_local,
scale,
T,
K: tl.
fla/ops/delta_rule/parallel.py:26
Functionchunk_ttt_linear_bwd_kernel_dv_local(
q,
k,
eta,
do,
dv,
cu_seqlens,
chunk_indices,
scale,
T,
H: tl.conste
fla/ops/ttt/chunk.py:333
Functionchunk_ttt_linear_bwd_kernel_h(
k,
v,
v_new,
eta,
w,
b,
eps,
h,
h0,
hb0,
x,
y,
r,
cu
fla/ops/ttt/chunk.py:231
Functionchunk_ttt_linear_bwd_kernel_norm(
q,
k,
v,
v_new,
x,
y,
r,
w,
b,
eta,
h,
dht,
dhbt,
dh
fla/ops/ttt/chunk.py:408
Functionchunk_ttt_linear_bwd_norm_ref(
q: torch.Tensor, # [B, H, L, D]
k: torch.Tensor, # [B, H, L, D]
v: torch.Tensor, # [B, H, L,
fla/ops/ttt/chunk.py:996
Functionchunk_ttt_linear_fwd_kernel_h(
k,
v,
v_new,
eta,
w,
b,
eps,
h,
hb,
h0,
hb0,
ht,
hbt,
fla/ops/ttt/chunk.py:33
Functionchunk_ttt_linear_fwd_kernel_o(
q,
k,
v,
eta,
h,
hb,
o,
cu_seqlens,
chunk_indices,
scale,
T,
fla/ops/ttt/chunk.py:138
Functionchunkwise_bwd_kernel_dhg(
do,
q,
g,
l,
h_l,
dh,
dg_last,
ell,
T,
cu_seqlens,
H: tl.constex
fla/ops/log_linear_attn/chunk.py:931
Functionchunkwise_bwd_kernel_diag(
do,
q,
k,
v,
g,
l,
llut,
mask,
dq,
dk,
dv,
dg,
dl,
c
fla/ops/log_linear_attn/chunk.py:1352
Functionchunkwise_bwd_kernel_dkg(
dh,
k,
v,
g,
dg_last,
dk,
dg,
cu_seqlens,
T,
H: tl.constexpr,
K:
fla/ops/log_linear_attn/chunk.py:1196
Functionchunkwise_bwd_kernel_dv(
dh,
k,
g,
dv,
T,
cu_seqlens,
H: tl.constexpr,
K: tl.constexpr,
V: tl.con
fla/ops/log_linear_attn/chunk.py:1286
Functionchunkwise_bwd_kernel_hdqgl(
do,
q,
k,
v,
g,
l,
h_l,
dq,
dg,
dl,
ell,
T,
cu_seqlens,
fla/ops/log_linear_attn/chunk.py:1048