a hcP @sdZddlZddlmZddlmZddlmZddlm Z m Z m Z m Z ddl mZmZmZmZe rzddlZddlmZe eZgd gd gd d d gd gdgd d ddgd gd gd d d gd gd gd d d dZdddddddddddddddddddddddddZddZd-edddZdd Zd!d"Zd#d$Zd%d&Z d'd(Z!d)d*Z"d+d,Z#dS).z;AWQ (Activation aware Weight Quantization) integration fileN)version)ACT2FN)PreTrainedModel)is_auto_awq_availableis_ipex_availableis_torch_availablelogging)AwqBackendPackingMethod AwqConfigAWQLinearVersionExllamaVersion)q_projk_projv_projo_proj) gate_projup_proj down_proj)Zinput_layernormZpost_attention_layernormZnormF) attentionmlp layernorm use_alibi)Zw1Zw3Zw2g.A)rrrr rope_theta)ZmistralZmixtralllamaZllavaactZc_fc)rlayer_before_actZ dense_h_to_4hrZfc_inZ gelu_impl)Z starcoder2ZRefinedWebModelZfalconZmptZgptjZgpt_neoxZ gpt_bigcodeZbloomc Csddlm}|tvr|S|D]n\}}t|d}t|d}||krt||rt|t|d}|j}t|} ||| |j |<t ||} q |S)Nr)ScaledActivationrr) Zawq.modules.actrAWQ_SCALES_MAPPINGSnamed_childrenhasattrgetattr out_featurestorchZones_modulesreplace_quantization_scales) model model_typernamemoduleZact_nameZlayer_before_act_namersizeZ scale_like_r,Y/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/integrations/awq.pyr%Ms     r%)returnc s|dur g}|j}ts td|tjkr|jtjkrHddlm }|}n|jtj krfddl m }|}n|jtj kr|jdtjkrddlm} | }q|jdtjkrddlm} | }qtd |jdn.|jtjkrdd lm} | }ntd |jndd lm} | }|D]\} }dur*g| t|tjr| |vrt fd d|Ds|j!}|j"}||j#|j$|||j%du|j&j'd|j(| <d}|j(| )dt*t+|,dkrt-||||d\}}.dq||fS)a Public method that recursively replaces the Linear layers of the given model with AWQ quantized layers. `accelerate` is needed to use this method. Returns the converted model and a boolean that indicates if the conversion has been successful or not. During the module replacement, we also infer the backend to use through the `quantization_config` object. Args: model (`torch.nn.Module`): The model to convert, can be any `torch.nn.Module` instance. quantization_config (`AwqConfig`): The quantization config object that contains the quantization parameters. modules_to_not_convert (`list`, *optional*): A list of modules to not convert. If a module name is in the list (e.g. `lm_head`), it will not be converted. current_key_name (`list`, *optional*): A list that contains the current key name. This is used for recursion and should not be passed by the user. has_been_replaced (`bool`, *optional*): A boolean that indicates if the conversion has been successful or not. This is used for recursion and should not be passed by the user. NzAWQ (either `autoawq` or `llmawq`) is not available. Please install it with `pip install autoawq` or check out the installation guide in https://github.com/mit-han-lab/llm-awqr) WQLinear_GEMM) WQLinear_GEMVr)WQLinear_Exllama)WQLinear_ExllamaV2Unrecognized Exllama version:  WQLinear_IPEXzUnrecognized AWQ version: )WQLinearc3s|]}|dvVqdS).N)join).0keycurrent_key_namer,r- z*replace_with_awq_linear..)w_bit group_size in_featuresr"biasdevTF)modules_to_not_convertr<quantization_confighas_been_replaced)/backendr ValueErrorr AUTOAWQrr ZGEMMZawq.modules.linear.gemmr/ZGEMVZawq.modules.linear.gemvr0ZEXLLAMAexllama_configr ONEawq.modules.linear.exllamar1TWOawq.modules.linear.exllamav2r2ZIPEXawq.modules.linear.gemm_ipexr5Zawq.quantize.qmoduler6rappend isinstancennZLinearanyrAr"bitsr@rBweightdevicer$Zrequires_grad_lenlistchildrenreplace_with_awq_linearpop)r&rDrEr<rFrHr/ target_clsr0r1r2r5r6r(r)rAr"r+r,r;r-r[^sp               r[cCst|tstd|jj|jdur8|j}|j|d<nj|jjt vrt |jj}|jj dd}|j }|j }t |d|}||d<||d<||d<|j|d<ntd |S) af Returns the fusing mapping given the quantization config and the model Args: model (`~PreTrainedModel`): The model to fuse - note this model should have been converted into AWQ format beforehand. quantization_config (`~transformers.quantization_config.AWQConfig`): The quantization configuration to use. z:The model should be an instance of `PreTrainedModel`, got N max_seq_lenTdecodernum_key_value_heads hidden_sizenum_attention_headsaFusing mapping not found either on the quantization config or the supported `AWQ_FUSED_MAPPINGS`. Please pass a `fused_mapping` argument in the `quantization_config` or raise an issue on transformers https://github.com/huggingface/transformers to add its support.)rRr TypeError __class____name__modules_to_fuseZfuse_max_seq_lenconfigr'AWQ_FUSED_MAPPINGSget_text_configrbrcr!rI)r&rEZcurrent_fused_mappingrhrbrcrar,r,r-get_modules_to_fuses&       rkc sbt|trt|}|j}t||}t|dd}|tjkr`ddl m }ddl m }ddl m}ntdg|D]\}|durtfdd |Drqtt|d |||jd krt||d ||n td t||||} | rtddqttdkr^|D]F\}tfdd Drt|drt|jdrd|j_q|S)aJ Optionally fuse some modules in the model to speedup inference. Args: model (`~PreTrainedModel`): The model to fuse - note this model should have been converted into AWQ format beforehand. quantization_config (`Union[AwqConfig, dict]`): The quantization configuration to use. rDNr)QuantAttentionFused) QuantFusedMLP)FasterTransformerRMSNormz0Fusing is only supported for the AutoAWQ backendc3s|]}|vVqdSNr,)r9Zmodule_name_to_not_convert)r(r,r-r=r>z#fuse_awq_modules..rZipexrz7The IPEX version AWQ does not support fuse mlp for now.r7c3s|]}vVqdSror,)r9Zfused_attention_parent_module)fused_attention_modules module_namer,r-r=(srh_attn_implementationZcustom)rRdictr from_dictrHrkr!r rJZawq.modules.fused.attnrlZawq.modules.fused.mlprmZawq.modules.fused.normrnrIZ named_modulesrT_fuse_awq_layernormr _fuse_awq_mlploggerinfo_fuse_awq_attention_layersrQsplitrXr rhrr) r&rErHrgrDrlrmrnr)Zattention_has_been_fusedr,)rprqr(r-fuse_awq_moduless@            r{cCsB|D]8}t||rt||}||j|j|jj|j|<~qdS)a Fuse the LayerNorm layers into a target class using autoawq Args: fuse_module_names (`list[str]`): The list of module names to fuse module (`nn.Module`): The pytorch parent module that has layernorm modules to fuse target_cls (`~autoawq.FasterTransformerRMSNorm`): The `FasterTransformerRMSNorm` class as it only supports that class for now. N)r r!rVZvariance_epsilontorWr$)fuse_module_namesr)r]rqZ old_moduler,r,r-ru0s    rucCst|dkrdSt||drt||d}t||d}t||d}|jj}|jjdd} | j} t| } ||||| } | dd\} }| | }t ||| |~~~dS)a Fuse the MLP layers into a target class using autoawq Args: model (`~PreTrainedModel`): The input pretrained model current_module_name (`str`): The current submodule name fuse_module_names (`list[str]`): The list of module names to fuse. For the MLP layers it has to be an array of length 3 that consists of the 3 MLP layers in the order (gate (dense layer post-attention) / up / down layers) module (`nn.Module`): The pytorch parent module that has layernorm modules to fuse target_cls (`~autoawq.QuantFusedMLP`): The `QuantFusedMLP` class as it only supports that class for now. rNrTr_r7) rXr r!qweightrWrhrj hidden_actrrsplit get_submodulesetattrr|)r&current_module_namer}r)r]rrrprevious_devicerhrZ activation_fnZ new_module parent_name child_nameparentr,r,r-rvGs  rvc CsLddlm}m}d}t|ddkr(|St||ddrHt||dd}t||rb|} d} n^t||rv|} d} nJtrt t j dt dkrddlm } t|| r| } d} nt d |jj} t||dd} t||dd }t||dd }|jd ur$tj|j| j|jgdd nd }| |j|j|j|j| j|j|jd utt|j}tj|j| j|jg| d |_tj|j| j|jg| d |_tj|j| j|jg| d |_t||r|j|_||_||d|d|d||| |d|d|ddd }d|_| dd\}}|!|}t"|||#| ~~ ~~d}|S)a Fuse the Attention layers into a target class using autoawq Args: model (`~PreTrainedModel`): The input pretrained model module (`nn.Module`): The pytorch parent module that has layernorm modules to fuse modules_to_fuse (`list[str]`): The module fusing mapping. The dictionary has to contain a field `attention` with attention module names in the correct order: q, k, v, o layer current_module_name (`str`): The current submodule name target_cls (`~autoawq.QuantAttentionFused`): The `QuantAttentionFused` class as it only supports that class for now. r)r/r0Frr~Zautoawqz0.2.6r4z'Unsupported q_proj type: {type(q_proj)}rN)dimrbrcrar^rrg@)rrTr7)$Zawq.modules.linearr/r0rXr r!rRrrparse importlibmetadatar5rIrrWrBr#catr?r@rAr"nextiterZ state_dictvaluesZqzerosscalesZ split_k_itersgetZis_hf_transformersrrrr|)r&r)rgrr]r/r0Zmodule_has_been_fusedrZlinear_target_clsZcat_dimr5rrrrrBZ qkv_layerZfused_attention_layerrrrr,r,r-rypsn  "  *     rycCsl|dtjkr$ddlm}||}nD|dtjkrVddlm}|||d|dd}ntd|d|S) z Runs post init for Exllama layers which performs: - Weights unpacking, reordering and repacking - Devices scratch space allocation rr)exllama_post_init)exllamav2_post_init max_input_lenmax_batch_size)rrr3)r rLrMrrNrOrrI)r&rKrrr,r,r-post_init_awq_exllama_moduless   rcCsddlm}||}|S)zl Runs post init for IPEX layers which performs: - Weights packing, reordering and repacking r)ipex_post_init)rPr)r&rr,r,r-post_init_awq_ipex_moduless r)NNNF)$__doc__r packagingrZ activationsrZmodeling_utilsrutilsrrrr Zutils.quantization_configr r r r r#Ztorch.nnrSZ get_loggerrfrwrirr%boolr[rkr{rurvryrrr,r,r,r-sp       i)@)_