a
    ¾Àhò  ã                   @   s†  d Z ddlZddlmZ ddlmZmZmZ ddlZ	ddl
Z
ddlZ
ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZ ddlmZmZ ddlm Z  e !e"¡Z#eeddG dd„ deƒƒƒZ$eeddG dd„ deƒƒƒZ%e
j&j'dd„ ƒZ(dFdd„Z)dd „ Z*G d!d"„ d"e
jj+ƒZ,G d#d$„ d$ej+ƒZ-G d%d&„ d&ej+ƒZ.G d'd(„ d(ej+ƒZ/G d)d*„ d*ej+ƒZ0G d+d,„ d,ej+ƒZ1G d-d.„ d.ej+ƒZ2G d/d0„ d0ej+ƒZ3G d1d2„ d2ej+ƒZ4G d3d4„ d4ej+ƒZ5G d5d6„ d6ej+ƒZ6G d7d8„ d8ej+ƒZ7G d9d:„ d:ej+ƒZ8G d;d<„ d<eƒZ9G d=d>„ d>ej+ƒZ:G d?d@„ d@ej+ƒZ;eG dAdB„ dBeƒƒZ<edCdG dDdE„ dEe<ƒƒZ=dEdBgZ>dS )GzPyTorch VITS model.é    N)Ú	dataclass)ÚAnyÚOptionalÚUnion)Únné   )ÚACT2FN)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)Ú_prepare_4d_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚModelOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )Ú
VitsConfigz`
    Describes the outputs for the VITS model, with potential hidden states and attentions.
    )Zcustom_introc                   @   sx   e Zd ZU dZdZeej ed< dZ	eej ed< dZ
eeej  ed< dZeeej  ed< dZeeej  ed< dS )ÚVitsModelOutputa"  
    waveform (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
        The final audio waveform predicted by the model.
    sequence_lengths (`torch.FloatTensor` of shape `(batch_size,)`):
        The length in samples of each element in the `waveform` batch.
    spectrogram (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_bins)`):
        The log-mel spectrogram predicted at the output of the flow model. This spectrogram is passed to the Hi-Fi
        GAN decoder model to obtain the final audio waveform.
    NÚwaveformÚsequence_lengthsÚspectrogramÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtorchÚFloatTensorÚ__annotations__r   r   Útupler   r   © r"   r"   úb/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/models/vits/modeling_vits.pyr   (   s   

r   zm
    Describes the outputs for the VITS text encoder model, with potential hidden states and attentions.
    c                   @   st   e Zd ZU dZdZeej ed< dZ	eej ed< dZ
eej ed< dZeeej  ed< dZeeej  ed< dS )ÚVitsTextEncoderOutputa…  
    prior_means (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        The predicted mean values of the prior distribution for the latent text variables.
    prior_log_variances (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        The predicted log-variance values of the prior distribution for the latent text variables.
    NÚlast_hidden_stateÚprior_meansÚprior_log_variancesr   r   )r   r   r   r   r%   r   r   r   r    r&   r'   r   r!   r   r"   r"   r"   r#   r$   @   s   
r$   c                 C   sT   | | }t  |d d …d |…d d …f ¡}t  |d d …|d …d d …f ¡}|| }|S ©N)r   ÚtanhÚsigmoid)Zinput_aZinput_bZnum_channelsZin_actZt_actZs_actÚactsr"   r"   r#   Úfused_add_tanh_sigmoid_multiplyU   s
      r,   Fç      @çü©ñÒMbP?c	                 C   sÎ   | | k| |k@ }	|	 }
t  | ¡}t  | ¡}t t d| ¡d ¡}tjj|dd}||d< ||d< | |
 ||
< d||
< t| |	 ||	dd…f ||	dd…f ||	dd…f |||||d	\||	< ||	< ||fS )	aô	  
    This transformation represents a monotonically increasing piecewise rational quadratic function. Outside of the
    `tail_bound`, the transform behaves as an identity function.

    Args:
        inputs (`torch.FloatTensor` of shape `(batch_size, channels, seq_len)`:
            Second half of the hidden-states input to the Vits convolutional flow module.
        unnormalized_widths (`torch.FloatTensor` of shape `(batch_size, channels, seq_len, duration_predictor_flow_bins)`):
            First `duration_predictor_flow_bins` of the hidden-states from the output of the convolution projection
            layer in the convolutional flow module
        unnormalized_heights (`torch.FloatTensor` of shape `(batch_size, channels, seq_len, duration_predictor_flow_bins)`):
            Second `duration_predictor_flow_bins` of the hidden-states from the output of the convolution projection
            layer in the convolutional flow module
        unnormalized_derivatives (`torch.FloatTensor` of shape `(batch_size, channels, seq_len, duration_predictor_flow_bins)`):
            Third `duration_predictor_flow_bins` of the hidden-states from the output of the convolution projection
            layer in the convolutional flow module
        reverse (`bool`, *optional*, defaults to `False`):
            Whether the model is being run in reverse mode.
        tail_bound (`float`, *optional* defaults to 5):
            Upper and lower limit bound for the rational quadratic function. Outside of this `tail_bound`, the
            transform behaves as an identity function.
        min_bin_width (`float`, *optional*, defaults to 1e-3):
            Minimum bin value across the width dimension for the piecewise rational quadratic function.
        min_bin_height (`float`, *optional*, defaults to 1e-3):
            Minimum bin value across the height dimension for the piecewise rational quadratic function.
        min_derivative (`float`, *optional*, defaults to 1e-3):
            Minimum bin value across the derivatives for the piecewise rational quadratic function.
    Returns:
        outputs (`torch.FloatTensor` of shape `(batch_size, channels, seq_len)`:
            Hidden-states as transformed by the piecewise rational quadratic function with the `tail_bound` limits
            applied.
        log_abs_det (`torch.FloatTensor` of shape `(batch_size, channels, seq_len)`:
            Logarithm of the absolute value of the determinants corresponding to the `outputs` with the `tail_bound`
            limits applied.
    r   )r   r   )Úpad©.r   ©.éÿÿÿÿç        N)	ÚinputsÚunnormalized_widthsÚunnormalized_heightsÚunnormalized_derivativesÚreverseÚ
tail_boundÚmin_bin_widthÚmin_bin_heightÚmin_derivative)	r   Ú
zeros_likeÚnpÚlogÚexpr   Ú
functionalr/   Ú_rational_quadratic_spline)r4   r5   r6   r7   r8   r9   r:   r;   r<   Zinside_interval_maskZoutside_interval_maskÚoutputsÚlog_abs_detÚconstantr"   r"   r#   Ú(_unconstrained_rational_quadratic_spline^   s,    .

÷rF   c	           *      C   s  |}	| }
t  | ¡|
k s&t  | ¡|	kr.tdƒ‚|jd }|| dkrXtd|› d|› ƒ‚|| dkrxtd|› d|› ƒ‚tjj|dd}|d||  |  }t j|dd}tjj	|d	d
dd}|	|
 | |
 }|
|d< |	|d< |ddd…f |ddd…f  }|tj 
|¡ }tjj|dd}|d||  |  }t j|dd}tjj	|d	d
dd}|	|
 | |
 }|
|d< |	|d< |ddd…f |ddd…f  }|rž|n|}|d  d7  < t j| d |kddd }|d }| d|¡d }| d|¡d }| d|¡d }|| }| d|¡d }| d|¡d }|ddd…f  d|¡d }| d|¡d }|| d|  }|s| | | }|d|  }||| d¡ ||   }|||  }|||  } | d¡|| d¡ d| |  |d|  d¡   }!t  |!¡dt  |¡  }"| |"fS | | }#|#| }$|||  |$ }%|| |$ }&| |# }'|& d¡d|% |'  }(|(dk ¡ svtd|(› ƒ‚d|' |& t  |(¡  })|)| | } |)d|)  }|||  }| d¡||) d¡ d| |  |d|)  d¡   }!t  |!¡dt  |¡  }"| |" fS dS )a(	  
    This transformation represents a monotonically increasing piecewise rational quadratic function. Unlike the
    function `_unconstrained_rational_quadratic_spline`, the function behaves the same across the `tail_bound`.

    Args:
        inputs (`torch.FloatTensor` of shape `(batch_size, channels, seq_len)`:
            Second half of the hidden-states input to the Vits convolutional flow module.
        unnormalized_widths (`torch.FloatTensor` of shape `(batch_size, channels, seq_len, duration_predictor_flow_bins)`):
            First `duration_predictor_flow_bins` of the hidden-states from the output of the convolution projection
            layer in the convolutional flow module
        unnormalized_heights (`torch.FloatTensor` of shape `(batch_size, channels, seq_len, duration_predictor_flow_bins)`):
            Second `duration_predictor_flow_bins` of the hidden-states from the output of the convolution projection
            layer in the convolutional flow module
        unnormalized_derivatives (`torch.FloatTensor` of shape `(batch_size, channels, seq_len, duration_predictor_flow_bins)`):
            Third `duration_predictor_flow_bins` of the hidden-states from the output of the convolution projection
            layer in the convolutional flow module
        reverse (`bool`):
            Whether the model is being run in reverse mode.
        tail_bound (`float`):
            Upper and lower limit bound for the rational quadratic function. Outside of this `tail_bound`, the
            transform behaves as an identity function.
        min_bin_width (`float`):
            Minimum bin value across the width dimension for the piecewise rational quadratic function.
        min_bin_height (`float`):
            Minimum bin value across the height dimension for the piecewise rational quadratic function.
        min_derivative (`float`):
            Minimum bin value across the derivatives for the piecewise rational quadratic function.
    Returns:
        outputs (`torch.FloatTensor` of shape `(batch_size, channels, seq_len)`:
            Hidden-states as transformed by the piecewise rational quadratic function.
        log_abs_det (`torch.FloatTensor` of shape `(batch_size, channels, seq_len)`:
            Logarithm of the absolute value of the determinants corresponding to the `outputs`.
    z-Input to a transform is not within its domainr2   ç      ð?zMinimal bin width z" too large for the number of bins zMinimal bin height ©Údimr   )r   r   rE   r3   )r/   ÚmodeÚvaluer0   r1   .Ngíµ ÷Æ°>).Né   é   r   zinvalid discriminant )r   ÚminÚmaxÚ
ValueErrorÚshaper   rA   ÚsoftmaxÚcumsumr/   ZsoftplusÚsumZgatherÚpowr?   ÚallÚRuntimeErrorÚsqrt)*r4   r5   r6   r7   r8   r9   r:   r;   r<   Úupper_boundÚlower_boundÚnum_binsÚwidthsZ	cumwidthsZderivativesZheightsZ
cumheightsZbin_locationsZbin_idxZinput_cumwidthsZinput_bin_widthsZinput_cumheightsÚdeltaZinput_deltaZinput_derivativesZinput_derivatives_plus_oneZinput_heightsZintermediate1ÚthetaZtheta_one_minus_thetaÚ	numeratorÚdenominatorrC   Zderivative_numeratorrD   Zintermediate2Zintermediate3ÚaÚbÚcZdiscriminantÚrootr"   r"   r#   rB   ¨   s–    ,
  
ÿþÿ

ÿþÿrB   c                       s6   e Zd Zeedœ‡ fdd„Zd	dd„Zdd„ Z‡  ZS )
ÚVitsWaveNet)ÚconfigÚ
num_layersc                    sD  t ƒ  ¡  |j| _|| _tj ¡ | _tj ¡ | _t 	|j
¡| _ttjjdƒrXtjjj}ntjj}|jdkr”tj |jd|j | d¡}||dd| _t|ƒD ]¢}|j| }|j| | d }tjj|jd|j |j||d}||dd}| j |¡ ||d k rd|j }	n|j}	tj |j|	d¡}
||
dd}
| j |
¡ qœd S )NÚweight_normr   rL   r   Úweight)Úname)Úin_channelsÚout_channelsÚkernel_sizeÚdilationÚpadding)ÚsuperÚ__init__Úhidden_sizerg   r   r   Ú
ModuleListÚ	in_layersÚres_skip_layersÚDropoutZwavenet_dropoutÚdropoutÚhasattrÚutilsÚparametrizationsrh   Úspeaker_embedding_sizeÚConv1dÚ
cond_layerÚrangeZwavenet_dilation_rateZwavenet_kernel_sizeÚappend)Úselfrf   rg   rh   r}   Úirn   ro   Zin_layerZres_skip_channelsZres_skip_layer©Ú	__class__r"   r#   rq   1  s<    


ûzVitsWaveNet.__init__Nc                 C   s   t  |¡}t  | jg¡}|d ur*|  |¡}t| jƒD ]â}| j| |ƒ}|d ur‚|d | j }|d d …||d| j  …d d …f }	n
t  |¡}	t||	|d ƒ}
|  	|
¡}
| j
| |
ƒ}|| jd k r|d d …d | j…d d …f }|| | }||d d …| jd …d d …f  }q4|| }q4|| S )NrL   r   r   )r   r=   Z	IntTensorrr   r}   r~   rg   rt   r,   rw   ru   )r€   r4   Úpadding_maskÚglobal_conditioningrC   Znum_channels_tensorr   r   Zcond_offsetZglobal_statesr+   Zres_skip_actsZres_actsr"   r"   r#   ÚforwardZ  s&    

&

"
zVitsWaveNet.forwardc                 C   sR   | j dkrtjj | j¡ | jD ]}tjj |¡ q | jD ]}tjj |¡ q:d S )Nr   )r{   r   r   ry   Úremove_weight_normr}   rt   ru   ©r€   Úlayerr"   r"   r#   r‡   w  s    


zVitsWaveNet.remove_weight_norm)N)	r   r   r   r   Úintrq   r†   r‡   Ú__classcell__r"   r"   r‚   r#   re   0  s   )
re   c                       s,   e Zd Zedœ‡ fdd„Zddd„Z‡  ZS )ÚVitsPosteriorEncoder©rf   c                    sR   t ƒ  ¡  |j| _t |j|jd¡| _t	||j
d| _t |j| jd d¡| _d S )Nr   ©rg   rL   )rp   rq   Ú	flow_sizerl   r   r|   Zspectrogram_binsrr   Úconv_prere   Z$posterior_encoder_num_wavenet_layersÚwavenetÚ	conv_proj©r€   rf   r‚   r"   r#   rq     s
    
zVitsPosteriorEncoder.__init__Nc                 C   sf   |   |¡| }|  |||¡}|  |¡| }tj|| jdd\}}|t |¡t |¡  | }|||fS )Nr   rH   )r   r‘   r’   r   Úsplitrl   Ú
randn_liker@   )r€   r4   r„   r…   ÚstatsÚmeanÚ
log_stddevZsampledr"   r"   r#   r†   ‰  s    zVitsPosteriorEncoder.forward)N©r   r   r   r   rq   r†   r‹   r"   r"   r‚   r#   rŒ   €  s   rŒ   c                       s@   e Zd Zd‡ fdd„	Zddd„Zd	d
„ Zdd„ Zdd„ Z‡  ZS )ÚHifiGanResidualBlockr   ©r   r   é   çš™™™™™¹?c                    sb   t ƒ  ¡  |ˆ_t ‡ ‡‡‡fdd„ttˆƒƒD ƒ¡ˆ_t ‡ ‡‡fdd„ttˆƒƒD ƒ¡ˆ_d S )Nc                    s2   g | ]*}t jˆ ˆ ˆd ˆ| ˆ ˆˆ| ¡d‘qS ©r   )Ústridern   ro   ©r   r|   Úget_padding)Ú.0r   ©Úchannelsrn   rm   r€   r"   r#   Ú
<listcomp>™  s   	øúz1HifiGanResidualBlock.__init__.<locals>.<listcomp>c                    s*   g | ]"}t jˆ ˆ ˆd d ˆ ˆd ¡d‘qS rž   r    ©r¢   Ú_)r¤   rm   r€   r"   r#   r¥   ¦  s   	ø
ú)	rp   rq   Úleaky_relu_sloper   rs   r~   ÚlenÚconvs1Úconvs2)r€   r¤   rm   rn   r¨   r‚   r£   r#   rq   ”  s    
	
÷ÿ	
÷ÿzHifiGanResidualBlock.__init__r   c                 C   s   || | d S )NrL   r"   )r€   rm   rn   r"   r"   r#   r¡   ³  s    z HifiGanResidualBlock.get_paddingc                 C   sL   t jj}tt jjdƒr t jjj}| jD ]}||ƒ q&| jD ]}||ƒ q:d S ©Nrh   )r   ry   rh   rx   rz   rª   r«   ©r€   rh   r‰   r"   r"   r#   Úapply_weight_norm¶  s    



z&HifiGanResidualBlock.apply_weight_normc                 C   s4   | j D ]}tj |¡ q| jD ]}tj |¡ qd S r(   )rª   r   ry   r‡   r«   rˆ   r"   r"   r#   r‡   À  s    

z'HifiGanResidualBlock.remove_weight_normc                 C   sX   t | j| jƒD ]D\}}|}tj || j¡}||ƒ}tj || j¡}||ƒ}|| }q|S r(   )Úziprª   r«   r   rA   Ú
leaky_relur¨   )r€   r   Zconv1Zconv2Úresidualr"   r"   r#   r†   Æ  s    
zHifiGanResidualBlock.forward)r   r›   r   )r   )	r   r   r   rq   r¡   r®   r‡   r†   r‹   r"   r"   r‚   r#   rš   “  s
   

rš   c                       sP   e Zd Zedœ‡ fdd„Zdd„ Zdd„ Zdeje	ej ejd	œd
d„Z
‡  ZS )ÚVitsHifiGanr   c              
      sD  t ƒ  ¡  || _t|jƒ| _t|jƒ| _tj	|j
|jdddd| _t ¡ | _tt|j|jƒƒD ]H\}\}}| j tj|jd|  |jd|d   |||| d d¡ q^t ¡ | _tt| jƒƒD ]F}|jd|d   }t|j|jƒD ] \}}| j t||||jƒ¡ qäqÀtj	|dddddd| _|jdkr@t 	|j|jd¡| _d S )	Né   r   r   )rm   rŸ   ro   rL   F)rm   rŸ   ro   Úbiasr   )rp   rq   rf   r©   Zresblock_kernel_sizesÚnum_kernelsÚupsample_ratesÚnum_upsamplesr   r|   r   Zupsample_initial_channelr   rs   Ú	upsamplerÚ	enumerater¯   Zupsample_kernel_sizesr   ÚConvTranspose1dÚ	resblocksr~   Zresblock_dilation_sizesrš   r¨   Ú	conv_postr{   Úcond)r€   rf   r   Zupsample_raterm   r¤   rn   r‚   r"   r#   rq   Ò  s<    
û

ûÿ

zVitsHifiGan.__init__c                 C   sL   t jj}tt jjdƒr t jjj}| jD ]}||ƒ q&| jD ]}| ¡  q:d S r¬   )r   ry   rh   rx   rz   r¸   r»   r®   r­   r"   r"   r#   r®   ö  s    



zVitsHifiGan.apply_weight_normc                 C   s0   | j D ]}tj |¡ q| jD ]}| ¡  qd S r(   )r¸   r   ry   r‡   r»   rˆ   r"   r"   r#   r‡      s    

zVitsHifiGan.remove_weight_normN)r   r…   Úreturnc                 C   sÀ   |   |¡}|dur ||  |¡ }t| jƒD ]p}tj || jj¡}| j	| |ƒ}| j
|| j  |ƒ}td| jƒD ] }|| j
|| j |  |ƒ7 }qn|| j }q*tj |¡}|  |¡}t |¡}|S )aG  
        Converts a spectrogram into a speech waveform.

        Args:
            spectrogram (`torch.FloatTensor` of shape `(batch_size, config.spectrogram_bins, sequence_length)`):
                Tensor containing the spectrograms.
            global_conditioning (`torch.FloatTensor` of shape `(batch_size, config.speaker_embedding_size, 1)`, *optional*):
                Tensor containing speaker embeddings, for multispeaker models.

        Returns:
            `torch.FloatTensor`: Tensor of shape shape `(batch_size, 1, num_frames)` containing the speech waveform.
        Nr   )r   r½   r~   r·   r   rA   r°   rf   r¨   r¸   r»   rµ   r¼   r   r)   )r€   r   r…   r   r   Z	res_stateÚjr   r"   r"   r#   r†     s    


zVitsHifiGan.forward)N)r   r   r   r   rq   r®   r‡   r   r   r   r†   r‹   r"   r"   r‚   r#   r²   Ñ  s   $
 ÿþr²   c                       s,   e Zd Zedœ‡ fdd„Zddd„Z‡  ZS )	ÚVitsResidualCouplingLayerr   c                    sR   t ƒ  ¡  |jd | _t | j|jd¡| _t||j	d| _
t |j| jd¡| _d S )NrL   r   rŽ   )rp   rq   r   Úhalf_channelsr   r|   rr   r   re   Z prior_encoder_num_wavenet_layersr‘   r¼   r“   r‚   r"   r#   rq   *  s
    
z"VitsResidualCouplingLayer.__init__NFc                 C   sÊ   t j|| jgd dd\}}|  |¡| }|  |||¡}|  |¡| }t  |¡}	|s”||t  |	¡ |  }t j||gdd}
t  	|	ddg¡}|
|fS || t  |	 ¡ | }t j||gdd}
|
d fS d S )NrL   r   rH   )
r   r”   rÁ   r   r‘   r¼   r=   r@   ÚcatrT   )r€   r4   r„   r…   r8   Ú
first_halfÚsecond_halfr   r—   r˜   rC   Úlog_determinantr"   r"   r#   r†   2  s    
z!VitsResidualCouplingLayer.forward)NFr™   r"   r"   r‚   r#   rÀ   )  s   rÀ   c                       s,   e Zd Zedœ‡ fdd„Zddd„Z‡  ZS )	ÚVitsResidualCouplingBlockr   c                    s8   t ƒ  ¡  t ¡ | _t|jƒD ]}| j t|ƒ¡ qd S r(   )	rp   rq   r   rs   Úflowsr~   Zprior_encoder_num_flowsr   rÀ   )r€   rf   r§   r‚   r"   r#   rq   E  s    

z"VitsResidualCouplingBlock.__init__NFc                 C   sf   |s0| j D ]"}||||ƒ\}}t |dg¡}q
n2t| j ƒD ]&}t |dg¡}||||dd\}}q:|S )Nr   T©r8   )rÇ   r   ÚflipÚreversed)r€   r4   r„   r…   r8   Úflowr§   r"   r"   r#   r†   K  s    
z!VitsResidualCouplingBlock.forward)NFr™   r"   r"   r‚   r#   rÆ   D  s   rÆ   c                       s.   e Zd Zdedœ‡ fdd„Zd	dd„Z‡  ZS )
ÚVitsDilatedDepthSeparableConvr3   r   c                    sÖ   t ƒ  ¡  |j}|j}|j| _t |¡| _t 	¡ | _
t 	¡ | _t 	¡ | _t 	¡ | _t| jƒD ]t}|| }|| | d }| j
 tj||||||d¡ | j t ||d¡¡ | j t |¡¡ | j t |¡¡ q\d S )NrL   )rk   rl   rm   Úgroupsrn   ro   r   )rp   rq   Úduration_predictor_kernel_sizerr   Zdepth_separable_num_layersrg   r   rv   rw   rs   Úconvs_dilatedÚconvs_pointwiseÚnorms_1Únorms_2r~   r   r|   Ú	LayerNorm)r€   rf   Údropout_raterm   r¤   r   rn   ro   r‚   r"   r#   rq   X  s2    




úÿ
z&VitsDilatedDepthSeparableConv.__init__Nc                 C   s®   |d ur|| }t | jƒD ]Š}| j| || ƒ}| j| | dd¡ƒ dd¡}tj |¡}| j| |ƒ}| j	| | dd¡ƒ dd¡}tj |¡}|  
|¡}|| }q|| S ©Nr   r2   )r~   rg   rÏ   rÑ   Ú	transposer   rA   ZgelurÐ   rÒ   rw   )r€   r4   r„   r…   r   r   r"   r"   r#   r†   t  s    

z%VitsDilatedDepthSeparableConv.forward)r3   )Nr™   r"   r"   r‚   r#   rÌ   W  s   rÌ   c                       s,   e Zd Zedœ‡ fdd„Zddd„Z‡  ZS )	ÚVitsConvFlowr   c                    sr   t ƒ  ¡  |j| _|jd | _|j| _|j| _	t
 | j| jd¡| _t|ƒ| _t
 | j| j| jd d  d¡| _d S )NrL   r   r   )rp   rq   rr   Úfilter_channelsÚdepth_separable_channelsrÁ   Zduration_predictor_flow_binsr[   Zduration_predictor_tail_boundr9   r   r|   r   rÌ   Úconv_ddsr’   r“   r‚   r"   r#   rq   †  s    

zVitsConvFlow.__init__NFc                 C   s   t j|| jgd dd\}}|  |¡}|  |||¡}|  |¡| }|j\}}	}
| ||	d|
¡ dddd¡}|dd | j	…f t
 | j¡ }|d| j	d| j	 …f t
 | j¡ }|dd| j	 d …f }t|||||| jd\}}t j||gdd| }|st  || ddg¡}||fS |d fS d S )	NrL   r   rH   r2   r   r   .)r8   r9   )r   r”   rÁ   r   rÚ   r’   rQ   ÚreshapeÚpermuter[   ÚmathrX   rØ   rF   r9   rÂ   rT   )r€   r4   r„   r…   r8   rÃ   rÄ   r   Ú
batch_sizer¤   Úlengthr5   r6   r7   rD   rC   rÅ   r"   r"   r#   r†   ‘  s,    
$ú
	zVitsConvFlow.forward)NFr™   r"   r"   r‚   r#   r×   …  s   r×   c                       s,   e Zd Zedœ‡ fdd„Zddd„Z‡  ZS )	ÚVitsElementwiseAffiner   c                    sB   t ƒ  ¡  |j| _t t | jd¡¡| _t t | jd¡¡| _	d S ©Nr   )
rp   rq   rÙ   r¤   r   Ú	Parameterr   ZzerosÚ	translateÚ	log_scaler“   r‚   r"   r#   rq   ±  s    
zVitsElementwiseAffine.__init__NFc                 C   sh   |s@| j t | j¡|  }|| }t | j| ddg¡}||fS || j  t | j ¡ | }|d fS d S ©Nr   rL   )rã   r   r@   rä   rT   )r€   r4   r„   r…   r8   rC   rÅ   r"   r"   r#   r†   ·  s    zVitsElementwiseAffine.forward)NFr™   r"   r"   r‚   r#   rà   °  s   rà   c                       s&   e Zd Z‡ fdd„Zddd„Z‡  ZS )	ÚVitsStochasticDurationPredictorc                    s  t ƒ  ¡  |j}|j}t ||d¡| _t ||d¡| _t||j	d| _
|dkr^t ||d¡| _t ¡ | _| j t|ƒ¡ t|jƒD ]}| j t|ƒ¡ q‚t d|d¡| _t ||d¡| _t||j	d| _t ¡ | _| j t|ƒ¡ t|jƒD ]}| j t|ƒ¡ qìd S )Nr   )rÔ   r   )rp   rq   r{   rr   r   r|   r   r’   rÌ   Úduration_predictor_dropoutrÚ   r½   rs   rÇ   r   rà   r~   Zduration_predictor_num_flowsr×   Úpost_conv_preÚpost_conv_projÚpost_conv_ddsÚ
post_flows)r€   rf   Ú	embed_dimrØ   r§   r‚   r"   r#   rq   Ã  s2    
þ
þ
z(VitsStochasticDurationPredictor.__init__NFrG   c                 C   s²  t  |¡}|  |¡}|d ur4t  |¡}||  |¡ }|  ||¡}|  |¡| }|s|  |¡}|  ||¡}|  |¡| }t  	| 
d¡d| 
d¡¡j|j|jd| }d}	|}
| jD ]0}||
||| d\}
}t  |
dg¡}
|	|7 }	q²t j|
ddgdd\}}|	t  tj |¡tj | ¡ | ddg¡7 }	t  dt dtj ¡|d   | ddg¡|	 }|t  |¡ | }t  t  |d¡¡| }t  | ddg¡}t j||gdd}| jD ].}||||d\}}t  |dg¡}||7 }q¬t  d	t dtj ¡|d   | ddg¡| }|| S tt| jƒƒ}|d d
… |d g }t  	| 
d¡d| 
d¡¡j|j|jd| }|D ](}t  |dg¡}||||dd\}}qht j|ddgdd\}}|S d S )Nr   rL   )ÚdeviceÚdtype)r…   r   rH   ç      à¿gñhãˆµøä>g      à?éþÿÿÿr2   T)r…   r8   )r   Údetachr   r½   rÚ   r’   rè   rê   ré   ÚrandnÚsizeÚtorí   rî   rë   rÉ   r”   rT   r   rA   Z
logsigmoidrÝ   r?   Úpir*   Ú	clamp_minrÂ   rÇ   ÚlistrÊ   )r€   r4   r„   r…   Z	durationsr8   Únoise_scaler   Zrandom_posteriorZlog_determinant_posterior_sumZlatents_posteriorrË   rÅ   rÃ   rÄ   ZlogqZlog_determinant_sumÚlatentsZnllrÇ   r§   Úlog_durationr"   r"   r#   r†   ã  sh    



&ÿÿ

ÿ

"ÿ*ÿÿ
0&ÿÿz'VitsStochasticDurationPredictor.forward)NNFrG   ©r   r   r   rq   r†   r‹   r"   r"   r‚   r#   ræ   Â  s    ræ   c                       s&   e Zd Z‡ fdd„Zddd„Z‡  ZS )ÚVitsDurationPredictorc                    s¬   t ƒ  ¡  |j}|j}t |j¡| _tj|j	|||d d| _
tj||jd| _tj||||d d| _tj||jd| _t |dd¡| _|jdkr¨t |j|j	d¡| _d S )NrL   )ro   ©Úepsr   r   )rp   rq   rÎ   Z"duration_predictor_filter_channelsr   rv   rç   rw   r|   rr   Úconv_1rÓ   Úlayer_norm_epsÚnorm_1Úconv_2Únorm_2Úprojr{   r½   )r€   rf   rm   rØ   r‚   r"   r#   rq   '  s    

zVitsDurationPredictor.__init__Nc                 C   s¸   t  |¡}|d ur*t  |¡}||  |¡ }|  || ¡}t  |¡}|  | dd¡¡ dd¡}|  |¡}|  || ¡}t  |¡}|  	| dd¡¡ dd¡}|  |¡}|  
|| ¡}|| S rÕ   )r   rñ   r½   rÿ   Zrelur  rÖ   rw   r  r  r  )r€   r4   r„   r…   r"   r"   r#   r†   6  s    





zVitsDurationPredictor.forward)Nrû   r"   r"   r‚   r#   rü   &  s   rü   c                
       s   e Zd ZdZedœ‡ fdd„Zejeedœdd„Z	deje
ej e
ej e
ej eeeje
ej f d
œdd„Zdd„ Zdd„ Zdd„ Z‡  ZS )ÚVitsAttentionz?Multi-headed attention with relative positional representation.r   c                    s,  t ƒ  ¡  |j| _|j| _|j| _|j| _| j| j | _	| j	d | _
| j	| j | jkrptd| j› d| j› dƒ‚tj| j| j|jd| _tj| j| j|jd| _tj| j| j|jd| _tj| j| j|jd| _| jr(t t d| jd d | j	¡| j
 ¡| _t t d| jd d | j	¡| j
 ¡| _d S )Nrï   zIhidden_size must be divisible by num_attention_heads (got `hidden_size`: z and `num_attention_heads`: z).)r´   r   rL   )rp   rq   rr   rì   Únum_attention_headsÚ	num_headsZattention_dropoutrw   Úwindow_sizeÚhead_dimÚscalingrP   r   ÚLinearZuse_biasÚk_projÚv_projÚq_projÚout_projrâ   r   rò   Ú	emb_rel_kÚ	emb_rel_vr“   r‚   r"   r#   rq   N  s(    

ÿÿ(zVitsAttention.__init__)ÚtensorÚseq_lenÚbszc                 C   s    |  ||| j| j¡ dd¡ ¡ S rå   )Úviewr  r	  rÖ   Ú
contiguous)r€   r  r  r  r"   r"   r#   Ú_shapeg  s    zVitsAttention._shapeNF)r   Úkey_value_statesÚattention_maskÚlayer_head_maskÚoutput_attentionsr¾   c                 C   s  |  ¡ \}}}|  |¡| j }	|  |  |¡d|¡}
|  |  |¡d|¡}|| j d| jf}|  |	||¡j|Ž }	|
j|Ž }
|j|Ž }|
  d¡}t	 
|	|
 dd¡¡}|  ¡ || j ||fkrÚtd|| j ||f› d|  ¡ › ƒ‚| jdur|  | j|¡}t	 |	| dd¡¡}|  |¡}||7 }|dur„|  ¡ |d||fkrZtd|d||f› d|  ¡ › ƒ‚| || j||¡| }| || j ||¡}tjj|dd	}|dur|  ¡ | jfkrÌtd
| jf› d|  ¡ › ƒ‚| dddd¡| || j||¡ }| || j ||¡}|r0| || j||¡}| || j ||¡}nd}tjj|| j| jd}t	 
||¡}|  ¡ || j || jfkr–td|| j|| jf› d|  ¡ › ƒ‚| jdurÎ|  | j|¡}|  |¡}t	 ||¡}||7 }| || j|| j¡}| dd¡}| ||| j¡}|  |¡}||fS )z#Input shape: Batch x Time x Channelr2   r   rL   z$Attention weights should be of size z	, but is Nrð   z!Attention mask should be of size rH   z/Head mask for a single layer should be of size )ÚpÚtrainingz `attn_output` should be of size )ró   r  r
  r  r  r  r  r	  r  r   ZbmmrÖ   rP   r  Ú_get_relative_embeddingsr  ÚmatmulÚ'_relative_position_to_absolute_positionr   rA   rR   rw   r  r  Ú'_absolute_position_to_relative_positionrÛ   rì   r  )r€   r   r  r  r  r  r  Ztgt_lenr§   Zquery_statesZ
key_statesZvalue_statesZ
proj_shapeZsrc_lenÚattn_weightsZkey_relative_embeddingsZrelative_logitsZrel_pos_biasZattn_weights_reshapedZ
attn_probsZattn_outputZvalue_relative_embeddingsZrelative_weightsr"   r"   r#   r†   j  sx    


ÿÿ

ÿ
ÿÿ"ÿÿ

zVitsAttention.forwardc              	   C   sn   t || jd  dƒ}|dkr6tj |dd||ddg¡}t | jd | dƒ}|d|  d }|d d …||…f S )Nr   r   rL   )rO   r  r   rA   r/   )r€   Zrelative_embeddingsrß   Z
pad_lengthZslice_start_positionZslice_end_positionr"   r"   r#   r  Ì  s    z&VitsAttention._get_relative_embeddingsc                 C   sŽ   |  ¡ \}}}tj |g d¢¡}| ||d | g¡}tj |d|d ddg¡}| ||d d| d g¡}|d d …d |…|d d …f }|S )N)r   r   r   r   r   r   rL   r   r   ©ró   r   rA   r/   r  ©r€   ÚxZbatch_headsrß   r§   Zx_flatZx_finalr"   r"   r#   r   Õ  s    z5VitsAttention._relative_position_to_absolute_positionc              	   C   sŠ   |  ¡ \}}}tj |d|d ddddg¡}| ||d| d  g¡}tj ||dddg¡}| ||d| g¡d d …d d …dd …f }|S )Nr   r   rL   r#  r$  r"   r"   r#   r!  ä  s    *z5VitsAttention._absolute_position_to_relative_position)NNNF)r   r   r   r   r   rq   r   ÚTensorrŠ   r  r   Úboolr!   r†   r  r   r!  r‹   r"   r"   r‚   r#   r  K  s$       úùb	r  c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚVitsFeedForwardc                    s¦   t ƒ  ¡  t |j|j|j¡| _t |j|j|j¡| _t 	|j
¡| _t|jtƒr^t|j | _n|j| _|jdkrœ|jd d }|jd }||ddddg| _nd | _d S )Nr   rL   r   )rp   rq   r   r|   rr   Zffn_dimZffn_kernel_sizerÿ   r  rv   Zactivation_dropoutrw   Ú
isinstanceZ
hidden_actÚstrr   Úact_fnro   )r€   rf   Úpad_leftÚ	pad_rightr‚   r"   r#   rq   ò  s    


zVitsFeedForward.__init__c                 C   s¢   |  ddd¡}|  ddd¡}|| }| jd ur>tj || j¡}|  |¡}|  |¡}|  |¡}|| }| jd ur~tj || j¡}|  |¡}|| }|  ddd¡}|S )Nr   rL   r   )	rÜ   ro   r   rA   r/   rÿ   r+  rw   r  )r€   r   r„   r"   r"   r#   r†     s    





zVitsFeedForward.forwardrû   r"   r"   r‚   r#   r(  ñ  s   r(  c                       sB   e Zd Zedœ‡ fdd„Zd	ejejeej e	dœdd„Z
‡  ZS )
ÚVitsEncoderLayerr   c                    sX   t ƒ  ¡  t|ƒ| _t |j¡| _tj|j	|j
d| _t|ƒ| _tj|j	|j
d| _d S )Nrý   )rp   rq   r  Ú	attentionr   rv   Zhidden_dropoutrw   rÓ   rr   r   Ú
layer_normr(  Úfeed_forwardÚfinal_layer_normr“   r‚   r"   r#   rq     s    


zVitsEncoderLayer.__init__NF)r   r„   r  r  c                 C   sp   |}| j |||d\}}|  |¡}|  || ¡}|}|  ||¡}|  |¡}|  || ¡}|f}|rl||f7 }|S )N)r   r  r  )r/  rw   r0  r1  r2  )r€   r   r„   r  r  r±   r"  rC   r"   r"   r#   r†   $  s     ý



zVitsEncoderLayer.forward)NF)r   r   r   r   rq   r   r&  r   r   r'  r†   r‹   r"   r"   r‚   r#   r.    s     ûûr.  c                
       s\   e Zd Zedœ‡ fdd„Zdejejeej ee	 ee	 ee	 e
eef dœdd„Z‡  ZS )	ÚVitsEncoderr   c                    sB   t ƒ  ¡  ˆ | _t ‡ fdd„tˆ jƒD ƒ¡| _d| _ˆ j	| _	d S )Nc                    s   g | ]}t ˆ ƒ‘qS r"   )r.  r¦   r   r"   r#   r¥   F  ó    z(VitsEncoder.__init__.<locals>.<listcomp>F)
rp   rq   rf   r   rs   r~   Znum_hidden_layersÚlayersZgradient_checkpointingÚ	layerdropr“   r‚   r   r#   rq   C  s
    
 zVitsEncoder.__init__N)r   r„   r  r  Úoutput_hidden_statesÚreturn_dictr¾   c                 C   sô   |rdnd }|rdnd }|d ur,t ||jƒ}|| }tƒ p@t| ƒ}	| jD ]j}
|rZ||f }tj dd¡}| jov|| j	k }|r€|	r˜|
||||d}|d }|r d}|rH||d f }qH|| }|rÊ||f }|sæt
dd„ |||fD ƒƒS t|||dS )	Nr"   r   r   )r  r„   r  )NNc                 s   s   | ]}|d ur|V  qd S r(   r"   )r¢   Úvr"   r"   r#   Ú	<genexpr>}  r4  z&VitsEncoder.forward.<locals>.<genexpr>)r%   r   r   )r   rî   r	   r
   r5  r>   ÚrandomÚuniformr  r6  r!   r   )r€   r   r„   r  r  r7  r8  Zall_hidden_statesZall_self_attentionsZsynced_gpusZencoder_layerZdropout_probabilityZskip_the_layerZlayer_outputsr"   r"   r#   r†   J  sB    	

ü
ýzVitsEncoder.forward)NNNN)r   r   r   r   rq   r   r   r   r&  r'  r   r!   r   r†   r‹   r"   r"   r‚   r#   r3  B  s       ù
ør3  c                
       sf   e Zd ZdZedœ‡ fdd„Zd
ejeje	ej e	e
 e	e
 e	e
 eeej ef dœdd	„Z‡  ZS )ÚVitsTextEncoderzs
    Transformer encoder that uses relative positional representation instead of absolute positional encoding.
    r   c                    sN   t ƒ  ¡  || _t |j|j|j¡| _t	|ƒ| _
tj|j|jd dd| _d S )NrL   r   )rm   )rp   rq   rf   r   Ú	EmbeddingZ
vocab_sizerr   Zpad_token_idÚembed_tokensr3  Úencoderr|   r   Úprojectr“   r‚   r"   r#   rq   ‹  s
    

zVitsTextEncoder.__init__NT)Ú	input_idsr„   r  r  r7  r8  r¾   c                 C   sª   |   |¡t | jj¡ }| j||||||d}|s:|d n|j}	|  |	 dd¡¡ dd¡| }
t	j
|
| jjdd\}}|s”|	||f|dd …  }|S t|	|||j|jdS )N)r   r„   r  r  r7  r8  r   r   rL   rH   )r%   r&   r'   r   r   )r?  rÝ   rX   rf   rr   r@  r%   rA  rÖ   r   r”   r   r$   r   r   )r€   rB  r„   r  r  r7  r8  r   Zencoder_outputsr%   r–   r&   r'   rC   r"   r"   r#   r†   ’  s,    	ú	ûzVitsTextEncoder.forward)NNNT)r   r   r   r   r   rq   r   r&  r   r   r'  r   r!   r$   r†   r‹   r"   r"   r‚   r#   r=  †  s       ùør=  c                   @   s2   e Zd ZU eed< dZdZdZej	dœdd„Z
dS )	ÚVitsPreTrainedModelrf   ZvitsrB  T)Úmodulec                 C   s„  | j j}t|tjƒr@|jjjd|d |jdur<|jj 	¡  n@t|tj
ƒrj|jj 	¡  |jj d¡ nt|tjtjfƒrÈtj |j¡ |jdurÆt |j|j|jd   ¡}tjj|j| |d n¸t|tjƒr|jjjd|d |jdur€|jj|j  	¡  nxt|tƒr\| j jr€| j j| j j }tjj|j|d d tjj|j|d d n$t|tƒr€|jj 	¡  |j j 	¡  dS )	zInitialize the weightsr3   )r—   ÚstdNrG   r   )ra   rb   rï   )rE  )!rf   Zinitializer_ranger)  r   r  ri   ÚdataZnormal_r´   Zzero_rÓ   Zfill_r|   rº   ÚinitZkaiming_normal_rÝ   rX   rÍ   rk   rm   Zuniform_r>  Zpadding_idxr  r  rr   r  r  r  rà   rã   rä   )r€   rD  rE  Úkr	  r"   r"   r#   Ú_init_weights¿  s2    


z!VitsPreTrainedModel._init_weightsN)r   r   r   r   r    Zbase_model_prefixZmain_input_nameZsupports_gradient_checkpointingr   ÚModulerI  r"   r"   r"   r#   rC  ¸  s
   
rC  z@
    The complete VITS model, for text-to-speech synthesis.
    c                       sz   e Zd Zedœ‡ fdd„Zdd„ Zed
eej	 eej	 ee
 ee ee ee eej eee ef dœdd	„ƒZ‡  ZS )Ú	VitsModelr   c                    s–   t ƒ  |¡ || _t|ƒ| _t|ƒ| _t|ƒ| _|j	rBt
|ƒ| _n
t|ƒ| _|jdkrht |j|j¡| _t|ƒ| _|j| _|j| _|j| _|  ¡  d S rá   )rp   rq   rf   r=  Útext_encoderrÆ   rË   r²   ÚdecoderÚ"use_stochastic_duration_predictionræ   Úduration_predictorrü   Únum_speakersr   r>  r{   Úembed_speakerrŒ   Zposterior_encoderÚspeaking_raterø   Únoise_scale_durationZ	post_initr“   r‚   r"   r#   rq   â  s    





zVitsModel.__init__c                 C   s   | j S r(   )rL  )r€   r"   r"   r#   Úget_encoderü  s    zVitsModel.get_encoderN)rB  r  Ú
speaker_idr  r7  r8  Úlabelsr¾   c           $      C   s’  |dur|n| j j}|dur |n| j j}|dur4|n| j j}|durLtdƒ‚| jjjj}|durr| 	d¡ 
|¡}	nt |¡ 	d¡ 
|¡}	| j jdkrü|durüd|  kr´| j jk sÎn td| j jd › dƒ‚t|tƒrêtjd|| jd	}|  |¡ 	d¡}
nd}
| j||	||||d
}|s$|d n|j}| dd¡}|	 dd¡}	|sP|d n|j}|sd|d n|j}| j jrŒ| j||	|
d| jd}n|  ||	|
¡}d| j }t t |¡|	 | ¡}t t |ddg¡d¡  ¡ }tj!| "¡ |j|jd}| 	d¡| 	d¡k }| 	d¡ 
|	j¡}t 	|	d¡t 	|d¡ }|j#\}}}}t $|d¡ %|| d¡}tj!||j|jd}| 	d¡|k }| 
|j¡ %|||¡}|t&j' (|g d¢¡dd…dd…f  }| 	d¡ dd¡| }t )| *d¡|¡ dd¡}t )| *d¡|¡ dd¡}|t +|¡t |¡ | j,  }| j-|||
dd}|| } |  .| |
¡}!|! *d¡}!|t/ 0| j j1¡ }"|s||!|"| f|dd…  }#|#S t2|!|"| |j3|j4dS )a¿  
        speaker_id (`int`, *optional*):
            Which speaker embedding to use. Only used for multispeaker models.
        labels (`torch.FloatTensor` of shape `(batch_size, config.spectrogram_bins, sequence_length)`, *optional*):
            Float values of target spectrogram. Timesteps set to `-100.0` are ignored (masked) for the loss
            computation.

        Example:

        ```python
        >>> from transformers import VitsTokenizer, VitsModel, set_seed
        >>> import torch

        >>> tokenizer = VitsTokenizer.from_pretrained("facebook/mms-tts-eng")
        >>> model = VitsModel.from_pretrained("facebook/mms-tts-eng")

        >>> inputs = tokenizer(text="Hello - my dog is cute", return_tensors="pt")

        >>> set_seed(555)  # make deterministic

        >>> with torch.no_grad():
        ...     outputs = model(inputs["input_ids"])
        >>> outputs.waveform.shape
        torch.Size([1, 45824])
        ```
        Nz&Training of VITS is not supported yet.r2   r   r   z Set `speaker_id` in the range 0-Ú.)r   )ró   Z
fill_valuerí   )rB  r„   r  r  r7  r8  rL   T)r8   rø   rG   )rî   rí   )r   r   r   r   r   r   r   rÈ   )r   r   r   r   r   )5rf   r  r7  Zuse_return_dictÚNotImplementedErrorrL  r?  ri   rî   Z	unsqueezerô   r   Z	ones_likerP  rP   r)  rŠ   Úfullrí   rQ  r%   rÖ   r&   r'   rN  rO  rS  rR  Úceilr@   rö   rT   ÚlongZarangerO   rQ   rS   r  r   rA   r/   r  Zsqueezer•   rø   rË   rM  r>   Úprodr¶   r   r   r   )$r€   rB  r  rU  r  r7  r8  rV  Z
mask_dtypeZinput_padding_maskZspeaker_embeddingsZtext_encoder_outputr   r&   r'   rú   Zlength_scaleÚdurationZpredicted_lengthsÚindicesZoutput_padding_maskZ	attn_maskrÞ   r§   Zoutput_lengthZinput_lengthZcum_durationZvalid_indicesZpadded_indicesZattnZprior_latentsrù   r   r   r   rC   r"   r"   r#   r†   ÿ  sŽ    %ÿ
ú
û
&
ûzVitsModel.forward)NNNNNNN)r   r   r   r   rq   rT  r   r   r   r&  rŠ   r'  r   r   r!   r   r   r†   r‹   r"   r"   r‚   r#   rK  Ü  s(          ø÷rK  )Fr-   r.   r.   r.   )?r   rÝ   Údataclassesr   Útypingr   r   r   Únumpyr>   r   Ztorch.utils.checkpointr   Zactivationsr   Zintegrations.deepspeedr	   Zintegrations.fsdpr
   Zmodeling_attn_mask_utilsr   Zmodeling_layersr   Zmodeling_outputsr   r   Zmodeling_utilsr   ry   r   r   Zconfiguration_vitsr   Z
get_loggerr   Úloggerr   r$   ZjitÚscriptr,   rF   rB   rJ  re   rŒ   rš   r²   rÀ   rÆ   rÌ   r×   rà   ræ   rü   r  r(  r.  r3  r=  rC  rK  Ú__all__r"   r"   r"   r#   Ú<module>   s|   
ÿÿ
     ÷
J 	P>X.+d% '*'D2#ÿ !