i@#>UddlZddlmZmZmZddlmZddlZddlm Z edZ edZ iZ e ejjefed<ehdZd ed ed eee e fgee e fffd Zed d d4dddddej,dej,dej,deej,dededed ej,fdZdeeded efdZdej,deded ej,fdZdej,d ej,d!ed"edeed#ed ej,fd$Zd!ed"ed dfd%Zdej,d ej,d!ed"edeed ej,f d&Zed'd d5dddddd(dd)dej,d ej,d*ej,d+eej,d,eej,d-eej,d.ed/ed0ed#edeed1ed2eed eej,ej,ej,ej,ffd3Z y)6N)CallableOptionalTypeVar) ParamSpec)_dtype_mappings_P_RONNX_ATEN_DECOMP_TABLE> op_type opset_versionreturncddtttfdtttfffd }|S)zDDecorator to register an ONNX operator with a custom implementation.funcrcd}tjjdd|d|}|tt t tj j |<|j||S)Nopsetzonnx::.) mutates_args)torchlibrary custom_opr getattropsonnx register_fake)roverloadtorch_oprrs N/var/www/html/engine/venv/lib/python3.12/site-packages/torch/onnx/ops/_impl.py decoratorz_onnx_op..decorators|=/*==**WIQxj )+   wwuyy~~w'GRS t$)rrr )rrr#s`` r"_onnx_opr%s0 R( Xb"f-=  r$RotaryEmbeddingF) interleaved num_headsrotary_embedding_dimx cos_cache sin_cache position_idsr(r)r*cV|jt}ddtjj dk(fdtjjdk(fdtjjdk(fdtjj dk(xrj dk(fd n@tjj d k(xrj d k(fd |d k(rtj |d }nG|d k(rBtj|dk7fdd}||z} || g} tj || }tjt|jd k(d|jd } |dk(r| }|ddddddd|f} |dddddd|df} |dz ntjjdk(xrjdk(fdtjjdk(xrjdk(fdtjjdk(fdtjjdk(fdtjdtjd|r%| dddddddddf} | dddddddddf}ntj| dd\} }| z|zz }| z|zz}|rftj|d}tj|d}tj||fd}tj || j} ntj||fd} tj| | fd}|d k(rtj |Stj |d S)z_RotaryEmbedding-23 https://onnx.ai/onnx/operators/onnx__RotaryEmbedding.html#rotaryembedding-23rNc"djS)Nz6position_ids must be 2D when provided. Received shape shape)r.sr"z%rotary_embedding_23..FsL\M_M_L`ar$c.ddjdS)Nz6position_ids first dim (batch) must match x.shape[0] ( ). Received rr3) batch_sizer.sr"r5z%rotary_embedding_23..Js$LZLXdeqewewxyezd{|r$r c.ddjdS)Nz;position_ids second dim (sequence) must match x.shape[-2] (r7r r3)r.sequence_lengthsr"r5z%rotary_embedding_23..Ns=QRaQbbno{pBpBCDpEoFGr$c<djdjS)NzWcos_cache/sin_cache must be 2D when position_ids is provided. Received cos_cache shape , sin_cache shape r3r,r-sr"r5z%rotary_embedding_23..R&((1'88J9??J[]r$c<djdjS)Nz[cos_cache/sin_cache must be 3D when position_ids is not provided. Received cos_cache shape r<r3r=sr"r5z%rotary_embedding_23..Xr>r$)rr1r r?cdS)NzKnum_heads must be provided for 3D inputs. Received input tensor with shape r) input_shapesr"r5z%rotary_embedding_23..esabmanor$cy)Nzx should be a 4D tensor by nowrrr$r"r5z%rotary_embedding_23..lr$c0djdddS)Nzcos has shape  but expected (batch=, seq=, ...)r3)r8cosr:sr"r5z%rotary_embedding_23..$. +@ FSbRccijr$c0djdddS)Nzsin has shape rGrHrIr3)r8r:sinsr"r5z%rotary_embedding_23..rKr$c0djdddS)NzLast dimension of cos cache (rN') should match rotary_embedding_dim/2 ().r3)rJrotary_embedding_dim_halfsr"r5z%rotary_embedding_23.../ " >effAACDr$c0djdddS)NzLast dimension of sin cache (rNrPrQr3)rRrMsr"r5z%rotary_embedding_23..rSr$dim) r4lenr_checkrVpermutereshape unsqueezechunkcat)r+r,r-r.r(r)r* input_rank hidden_size head_size new_shapex_rotate x_not_rotatex1x2realimagx_rotate_concatoutputr8rJrCrRr:rMs ``` @@@@@@r"rotary_embedding_23rj/s''K[!JQJ!"oO    ! # a     q !Z / |     q !_ 4 G   MMOq 9Y]]_%9 ]  MMOq 9Y]]_%9 ] Q MM!\ * q N o "!n 9, )YG MM!Y ' LLQWW"$LM Iq (Aq////0HQ12334L 4 9     LL ! "Fsyy|'Fj LL ! "Fsyy|'Fj LL " 22 D LL " 22 D // Q C // Q C  aAqt!tm $ aAqt!tm $Xqb1B 8cBh D 8cBh DtR(tR())T4Lb9==(..A99dD\r2 YY,/R 8FQ}}V[11 == ..r$scaler`c:||Sdtj|z S)z/Get the scale factor for attention computation.g?)mathsqrt)rkr`s r"_get_scale_factorros %5GC$))I2F,FGr$tensorr8c|jd|jd}}||z}|j||||jddjS)z1Reshape 3D tensor to 4D for multi-head attention.r r1)r4view transpose contiguous)rpr8r)r:r_r`s r"_reshape_3d_to_4drusP$*<<?FLLO[Oy(I JIF 1a r$QKcurrent_q_num_headscurrent_kv_num_headsqk_matmul_output_modec |dk(rt|||||Stjtj||j ddS)z1Get QK output tensor based on the specified mode.rr0rN)_compute_qk_output_for_mode_0r zeros_likematmulrs)rvrwrxryrkrzs r"_get_qk_output_for_aten_spdarsO!, q%';U   Q B0C DEEr$cHtjzdk(fdy)z-Validate Group Query Attention configuration.rcdddS)Nz q_num_heads (z%) must be divisible by kv_num_heads (z ) for GQAr)ryrxsr"r5z-_validate_gqa_configuration..s- 344YZnYooxyr$N)rrX)rxrys``r"_validate_gqa_configurationrs" LL22a7yr$c|}||k7r||z}|j|d}t||jd}tj|}||z} ||z} t j | | jddS)zDHelper function to compute QK output for qk_matmul_output_mode == 0.r rUr?r0rN)repeat_interleaveror4rmrnrr~rs) rvrwrxryrkK_for_qk repeat_factor scale_factor sqrt_scaleQ_scaledK_scaleds r"r|r|sH22+/CC &&}!&<$UAGGAJ7L<(J:~H*$H <<("4"4R"< ==r$ Attention) is_causal kv_num_heads q_num_headsrzrksoftcapsoftmax_precisionV attn_maskpast_key past_valuerrrrrc  d\} }}t|j}|jd}t|jdk(rWtj|dk7xr|dk7d|jd}t |||}t |||}t |||}tjt|jdk(xr2t|jdk(xrt|jdk(d|j|}t | |} |tj ||g| n|j}|tj ||g| n|j}||}}|j| }|j| }|j|}|j|}| d k(xr0| dk(xr)| duxr#|duxs|jtjk(}t|||rud}|"|jtjk(r|n|}tjjj||||d || t||k7 }t||||| | }nY||k7r+||z}|j|| }|j|| }tj |||j|j" }|rytj|dud tj$tj&||tj|j" }|j)|t+d}|?|jtjk(r|j)|t+d}n||z}t | |jd} t-j.| }!||!z}"||!z}#tj0|"|#j3dd}$|$}|$|z}%| dk(r|%}| d kDr| tj4|%| z z}%| dk(r|%}| w| t6vrW|%j}&|%j9t:j<| }%tj>|%d }'|'j9|&}'n/tj>|%d }'ntj>|%d }'| dk(r|'}tj0|'|}|dk(r1|j3ddjAjC||d}||||fS)zMAttention-23 https://onnx.ai/onnx/operators/onnx__Attention.html#attention-23)r r1r?rr?cy)Nz;q_num_heads and kv_num_heads must be provided for 3D inputsrrr$r"r5zattention_23..rEr$r rAcy)Nz'Q, K, and V should be 4D tensors by nowrrr$r"r5zattention_23..!rEr$NrUr)r dropout_prrk enable_gqa)dtypedevicecy)Nz'Cannot use both is_causal and attn_maskrrr$r"r5zattention_23..vrEr$z-infr0rNr1)"rWr4rrXruror]clonerboolrnn functionalscaled_dot_product_attentionrrzerosrtrilones masked_fillfloatrmrnr~rstanh-_ATTENTION_23_ALLOWED_INTERMEDIATE_PRECISIONStorONNX_DTYPE_TO_TORCH_DTYPEsoftmaxrtrr)(rvrwrrrrrrrrzrkrr num_head_dim sequence_dimhead_diminput_shape_lenr8q_sequence_length q_head_size present_key present_valuerxrykv_sequence_length can_use_sdpasdpa_attn_maskri qk_outputr attn_bias causal_maskrrrrqk_matmul_output qk_with_biasoriginal_dtype qk_softmaxs( r" attention_23rs &,3(L,!''lOJ 177|q 1  2!2 Q GGAJ a[ 9 a\ : a\ : LL AGG Ec!''la/ECLA4E9 ''(#K e[ 1E    8Q-\2 WWY  !  :q/|4 WWY  qA'',/77<0 -. 3 A !Q & A  % A$  ?)//UZZ"?  35IJ  +4??ejj+HiZiN$$AA $#';;B  1   !   "6 6/3GGM##M|#DA##M|#DAKK 1  LLT!#T  ** %&**88 K"--{lE&MJI  %**,%119*eFmL & 1 ) ; YY|, z>z>!<<(2D2DR2LM% ()3 A %$I S="UZZ w0F%GGL A %$I  ( $QQ!-!3!3+#==>OP  #]]: "]]rs| ..' *t_ T]ACUZZ22H<=B091- !$ xB (2r6"223* R ,0 D/  !D/ ||D/||D/||D/5<<( D/  D/D/D/ \\D/!D/NHXe_HHH  LL &) 69  \\ F ||F ||FF F E? F  F \\F$47 > ||> ||>> > E? >  \\ >, +r )-'+)- F9!"!'+F9 ||F9 ||F9 ||F9 % F9 u||$ F9 & F9F9F9F9F9 E?F9F9 }F9 5<<u||U\\ ABF9F9r$