a
    h(                    @   s
  d dl Z d dlZd dlmZ d dlmZmZmZ d dlZ	d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8 d dl9m:Z: d dl;m<Z<m=Z=m>Z>m?Z?m@Z@ d dlAmBZB d dlCmDZDmEZE d dlFmGZG d(ddZHdd ZIdd ZJe
jKLddd de	jMgdd ZNe
jKLd d gd!fd dgdfg d"dfgd#d$ ZOd%d& ZPd'd( ZQe
jKRd)d*d+ ZSe
jKRd)d,d- ZTe
jKRd)d.d/ ZUd0d1 ZVe
jKLd2g d3e	Wg d4g d5g d6g d7gfg d8g d9fgd:d; ZXe
jKLd2g d<e	Wg d7g d7g d=g d=g d>gfg d?g d@fgdAdB ZYdCdD ZZdEdF Z[dGdH Z\dIdJ Z]e
jKLdKeDe
jKLdLeEdMdN Z^dOdP Z_e
jKLdQg dRdSdT Z`dUdV ZadWdX Zbe
jKLdYe	Wg dZe	Wg dZd[d\fe	Wg d]e	Wg d]d[d^fe	Wg d]e	Wg dZd[d_fe	Wg d]e	Wg d`d[dafe	Wg dZe	Wg d]d[dbfgdcdd Zce
jKLdee	Wg dfe	Wg dgd[dhfgdidj Zddkdl Zedmdn Zfe
jKLdd de	jMge
jKLdod gd gfge
jKLdpe"ee#ddqe,e-gdrds Zge
jKLdod gd gfge
jKLdpe"ee#ddqe,e-gdtdu Zhdvdw Zidxdy Zjdzd{ Zkd|d} Zle
jKLd~ddgdd Zmdd Zne
jKLdg ddd Zodd Zpdd Zqdd Zre
jKjLdg dfddgdfgddgddd Zse
jKjLddd dgg d"fg dddd Ztdd Zue
jKLdg ddd Zvdd Zwdd Zxdd Zydd Zzdd Z{dd Z|dd Z}dd Z~dd Ze
jKRd)dd Zdd Zdd Zdd Zdd Zdd Zdd ZddĄ Ze
jKLdddgddɄ Ze
jKRd)dd˄ Ze
jKRd)dd̈́ Ze
jKRd)e
jKLdddde	jMe	jMfgdd҄ Ze
jKLddge
jKLdg dԢe
jKLdd de	jMgddք Ze
jKLdg dԢdd؄ Ze
jKLdd de	jMgddڄ Zdd܄ Zddބ Ze
jKLdd de	jMgdd Ze
jKLddd de	jMgdd Ze
jKLddd de	jMgdd Ze
jKLddd de	jMgdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Ze
jKLde	je	je	jgdd Ze
jKLde	je	je	jgdd  Ze
jKLdog dg dfg ddd gd dgdd ggfg d"g dg dg dgfgdd Zdd Zdd	 Zd
d Ze
jKLdg dg dfg dg dfg dg dfgdd Ze
jKLdpe&e"ee#ddqe+e,e-ege
jKLdg ddd Ze
jKLde	Wd dge	Wdd gdfe	Wd dge	Wd dgdfe	Wd dge	Wd d gdfe	Wd d ge	Wd d gdfgdd Ze
jKLde(e"e	jMde(e#de	jMde(e,e	jMde(e-e	jMdgd d! Zd"d# Zd$d% Zd&d' ZdS ()      N)partial)chainpermutationsproduct)linalg)hamming)	bernoulli)datasetssvm)make_multilabel_classification)UndefinedMetricWarning)accuracy_scoreaverage_precision_scorebalanced_accuracy_scorebrier_score_lossclass_likelihood_ratiosclassification_reportcohen_kappa_scoreconfusion_matrixf1_scorefbeta_scorehamming_loss
hinge_lossjaccard_scorelog_lossmake_scorermatthews_corrcoefmultilabel_confusion_matrixprecision_recall_fscore_supportprecision_scorerecall_scorezero_one_loss)_check_targetsd2_log_loss_score)cross_val_score)LabelBinarizerlabel_binarize)DecisionTreeClassifier)MockDataFrame)assert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equalignore_warnings)_nanaverage)CSC_CONTAINERSCSR_CONTAINERS)check_random_stateFc                 C   s  | du rt  } | j}| j}|r:||dk  ||dk   }}|j\}}t|}td}|| || ||  }}t	|d }tj
d}tj|||d| f }tjdddd}	|	|d| |d| ||d }
|r|
ddd	f }
|	||d }||d }|||
fS )
zMake some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      linearT)ZkernelZprobabilityrandom_state   )r	   	load_irisdatatargetshapenparanger1   shuffleintrandomRandomStateZc_Zrandnr
   ZSVCfitZpredict_probaZpredict)datasetbinaryXy	n_samples
n_featuresprngZhalfZclfZy_pred_probay_predy_true rM   e/var/www/html/assistant/venv/lib/python3.9/site-packages/sklearn/metrics/tests/test_classification.pymake_prediction:   s*    


*rO   c               
   C   st  t  } t| dd\}}}ddddddd	d
ddddddddddddddddddd}t||tt| j| jdd}| | ksJ |D ]x}|dkrt	|| t
sJ || || ksJ q||  ||  ksJ || D ]}t|| | || |  qqt	|d d t
s(J t	|d d t
s@J t	|d d tsXJ t	|d d tspJ d S ) NFrC   rD   g7Mo?gUUUUUU?ghQ?   )	precisionrecallf1-scoresupportUUUUUU?gc1Ƹ?g433333?   g)k??$I$I?   gCFQ?gc?gƢ?K   rT   rR   rS   rU   g?gD~WG?g]3p?)setosaZ
versicolorZ	virginica	macro avgaccuracyweighted avgT)labelstarget_namesoutput_dictr_   r]   rR   r^   rU   )r	   r8   rO   r   r<   r=   lenrb   keys
isinstancefloatr*   r?   )irisrL   rK   _expected_reportreportkeymetricrM   rM   rN   ,test_classification_report_dictionary_outputj   s`    "	rn   c                  C   s   t g g dd} dtjtjtjddtjtjtjddd}t| tsHJ |  | ks\J |D ]x}|dkrt| | ts~J | | || ksJ q`| |  ||  ksJ || D ]}t|| | | | |  qq`d S )NT)rL   rK   rc           r   r\   )r_   r^   r`   r_   )r   r<   nanrf   dictre   rg   r*   )rk   rj   rl   rm   rM   rM   rN   2test_classification_report_output_dict_empty_input   s,    rr   zero_divisionwarnr7   c                 C   s   g dg d }}t jddb}t||| dd | dkrjt|dksHJ |D ]}d}|t|jv sLJ qLn|rrJ W d    n1 s0    Y  d S )	Nabc)rv   rw   dTrecord)rs   rc   rt   r7   z7Use `zero_division` parameter to control this behavior.)warningscatch_warningsr   rd   strmessage)rs   rL   rK   r{   itemmsgrM   rM   rN   0test_classification_report_zero_division_warning   s    r   zlabels, show_micro_avgTr   r7   r2   c                 C   s\   ddgddg }}t ||| dd}|r@d|v s2J d|vsXJ nd|v sLJ d|vsXJ dS )a3  Check the behaviour of passing `labels` as a superset or subset of the labels.
    WHen a superset, we expect to show the "accuracy" in the report while it should be
    the micro-averaging if this is a subset.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27927
    r   r7   T)ra   rc   z	micro avgr_   Nr   )ra   Zshow_micro_avgrL   rK   rk   rM   rM   rN   1test_classification_report_labels_subset_superset   s    r   c                  C   s   t g dg dg} t g dg dg}t| |dks>J t| | dksPJ t||dksbJ t|t |dkszJ t| t | dksJ t| t | jdksJ t|t | jdksJ d S )Nr   r7   r7   r7   r   r7   r   r   r7         ?r7   r   )r<   arrayr   logical_notzerosr;   y1y2rM   rM   rN   .test_multilabel_accuracy_score_subset_accuracy   s    r   c               	   C   s:  t dd\} }}t| |d d\}}}}t|ddgd t|ddgd t|d	d
gd t|ddg i ddifD ]}t  td t| |fi |}t|dd t| |fi |}	t|	dd t	| |fi |}
t|
d
d t
t| |fddi|d| |	 d| |	  d W d    qp1 s*0    Y  qpd S )NTrD   averageg\(\?g333333?r2   g)\(?g(\?皙?gRQ?   r   rD   errorbeta      )rO   r   r+   r,   r|   r}   simplefilterr   r    r   r*   r   )rL   rK   ri   rI   rfskwargspsrsfsrM   rM   rN   %test_precision_recall_f1_score_binary   s(    

r   z1ignore::sklearn.exceptions.UndefinedMetricWarningc                   C   s  dt ddgddgksJ dtddgddgks4J dtddgddgksNJ dtddgddgddkslJ dt ddgddgksJ dtddgddgksJ dtddgddgksJ dtddgddgtddksJ tddgddgtddttddgddgddksJ d S )	N      ?r7   r   r   ro   infg     j@)r   r    r   r   rg   pytestapproxrM   rM   rM   rN   +test_precision_recall_f_binary_single_class   s    "r   c               
   C   s  g d} g d}t | tdd}t |tdd}| |f||fg}t|D ]\}\} }t| |g dd d}tg d| t| |g ddd}ttg d| d	D ]<}|d
kr|dkrqtt| |g d|dt| |d |d qqLdD ]}t	t
& t||td|d W d    n1 s&0    Y  t	t
( t||tdd|d W d    q1 sj0    Y  qtg dg dg} tg dg dg}t| |d
ddgd\}}	}
}tt||	|
gtg d d S )N)r7      r   r2   )r7   r7   r   r2   r   classes)r   r7   r2   r   r   ra   r   )ro   r   r   r   ro   macro)microweightedsamplesr   r   )Nr   r   r      r   r   r   r7   r   r   r7   r7   r7   r   r7   r   ra   )      ?r7   竪?)r&   r<   r=   	enumerater    r+   meanr*   r   raises
ValueErrorr   r   )rL   rK   
y_true_bin
y_pred_binr9   iactualr   rI   r   r   ri   rM   rM   rN   $test_precision_recall_f_extra_labels3  s<    6(r   c            	      C   s   g d} g d}t | tdd}t |tdd}| |f||fg}t|D ]\}\} }tt| |ddgd}tt| |d d}tdd	g|d d
 td|dd
 td|dd
 td|dd
 dD ]}||d
||d
ksJ qqLd S )N)r7   r7   r2   r   )r7   r   r   r   r   r   r7   r   ra   r   r   r   r   r   UUUUUU?r   r   )r   r   r   )r&   r<   r=   r   r   r    r+   r*   )	rL   rK   r   r   r9   r   Z	recall_13Z
recall_allr   rM   rM   rN   &test_precision_recall_f_ignored_labelsa  s    r   c               	   C   s   t g dg dg dg dg dg dg} t g dg dg dg d	g d
g dg}d}tjt|d t| |dd W d   n1 s0    Y  dS )z:Test multiclass-multiouptut for `average_precision_score`.)r2   r2   r7   r7   r2   r   r   r7   r2   r7   )r2   r   r7   ffffff?皙?皙?皙?333333?r   r   r   r   r   r   r   )r   r   r   )r   r   r   z.multiclass-multioutput format is not supportedmatchr2   	pos_labelN)r<   r   r   r   r   r   )rL   y_scoreerr_msgrM   rM   rN   -test_average_precision_score_non_binary_classx  s*    

r   zy_true, y_scorer   r   r7   r2   r   r   r   r   )r   r   r   r   r7   r7   r7   r7   r7   r7   r7   )r   r   r   r   r   333333?r   rX   rX   r7   r7   c                 C   s   t | |dksJ dS )a(  
    Duplicate values with precision-recall require a different
    processing than when computing the AUC of a ROC, because the
    precision-recall curve is a decreasing curve
    The following situation corresponds to a perfect
    test statistic, the average_precision_score should be 1.
    r7   Nr   rL   r   rM   rM   rN   -test_average_precision_score_duplicate_values  s    r   )r2   r2   r7   r7   r   )r   r   r   )r   r   r   r   )r   r   r   c                 C   s   t | |dksJ d S )Nr   r   r   rM   rM   rN   (test_average_precision_score_tied_values  s    r   c                  C   sN   d} t jt| d( tg dg dddd W d    n1 s@0    Y  d S )NzNote that pos_label \(set to 2\) is ignored when average != 'binary' \(got 'macro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   r7   r2   r2   r2   r   r   r   )r   warnsUserWarningr   r   rM   rM   rN   (test_precision_recall_f_unused_pos_label  s    r   c                  C   sD   t dd\} }}dd }|| | |dd | D dd |D  d S )NTr   c                 S   s   t | |}t|ddgddgg | \}}}}|| ||  }t|| ||  ||  ||  }|dkrrdn|| }	t| |}
t|
|	dd t|
ddd d S )	N   r         r   r2   decimal=
ףp=?)r   r,   flattenr<   sqrtr   r+   )rL   rK   cmtpfpfntnnumZdenZtrue_mccZmccrM   rM   rN   test  s    
&
z*test_confusion_matrix_binary.<locals>.testc                 S   s   g | ]}t |qS rM   r~   .0rF   rM   rM   rN   
<listcomp>      z0test_confusion_matrix_binary.<locals>.<listcomp>rO   rL   rK   ri   r   rM   rM   rN   test_confusion_matrix_binary  s    
r   c                  C   sD   t dd\} }}dd }|| | |dd | D dd |D  d S )NTr   c                 S   s4   t | |}t|ddgddggddgddggg d S )Nr   r   r   r   r   r,   )rL   rK   r   rM   rM   rN   r     s    
z5test_multilabel_confusion_matrix_binary.<locals>.testc                 S   s   g | ]}t |qS rM   r   r   rM   rM   rN   r      r   z;test_multilabel_confusion_matrix_binary.<locals>.<listcomp>r   r   rM   rM   rN   'test_multilabel_confusion_matrix_binary  s    
r   c                  C   sJ   t dd\} }}d	dd}|| | |dd | D dd |D dd d S )
NFr   c                 S   s   t | |}t|ddgddggddgddggd	d
gddggg |rJg dng d}t | ||d}t|ddgddggd	d
gddggddgddggg |rg dng d}t | ||d}t|ddgddggd	d
gddggddgddggddgddggg d S )N/   r   r      &   r      r      r   r2      )021r   r2   r7   r   )r  r  r  3)r   r2   r7   r   r[   r   r   )rL   rK   string_typer   ra   rM   rM   rN   r     s&    
..z9test_multilabel_confusion_matrix_multiclass.<locals>.testc                 S   s   g | ]}t |qS rM   r   r   rM   rM   rN   r   #  r   z?test_multilabel_confusion_matrix_multiclass.<locals>.<listcomp>T)r  )Fr   r   rM   rM   rN   +test_multilabel_confusion_matrix_multiclass  s    

r  csc_containercsr_containerc                 C   s  t g dg dg dg}t g dg dg dg}||}||}| |}| |}t g d}dd	gddggdd	gddggd	d
gdd	ggg}	|||g}
|||g}|
D ]"}|D ]}t||}t||	 qqt||dd}t|dd	gddggddgd	dggd	dgd
d	ggg t||d
d	gd}t|d	d
gdd	ggdd	gddggg t||d
d	gdd}t|d	d	gddggddgd	d	ggd	dgdd	ggg t|||dd}t|d
d	gd
d
ggddgd	dggd	dgdd	ggg d S )Nr   r   r7   r   r7   r7   r   r   r   r   )r2   r7   r   r7   r   r2   T
samplewiser   )ra   r  )sample_weightr  r   r   )r<   r   r   r,   )r  r	  rL   rK   Z
y_true_csrZ
y_pred_csrZ
y_true_cscZ
y_pred_cscr  Zreal_cmZtruespredsZ
y_true_tmpZ
y_pred_tmpr   rM   rM   rN   +test_multilabel_confusion_matrix_multilabel&  s0    .


4&4r  c                  C   s  t g dg dg dg} t g dg dg dg}tjtdd" t| |d	d
gd W d    n1 sn0    Y  tjtdd0 t| |g dg dg dgd W d    n1 s0    Y  d}tjt|d  t| |dgd W d    n1 s0    Y  d}tjt|d  t| |dgd W d    n1 sB0    Y  tjtdd& tg dg ddd W d    n1 s0    Y  d}tjt|d2 tg dg dgg dg dg W d    n1 s0    Y  d S )Nr   r
  r  r   r   r   inconsistent numbers of samplesr   r7   r2   r  zshould be a 1d arrayr7   r2   r   )r2   r   r   )r   r   r   z%All labels must be in \[0, n labels\)r   r   r   zSamplewise metricsr   r   Tr  z'multiclass-multioutput is not supported)r2   r7   r   r7   r   r2   )r<   r   r   r   r   r   )rL   rK   r   rM   rM   rN   'test_multilabel_confusion_matrix_errorsP  s&    0$.06r  z%normalize, cm_dtype, expected_results))truer   TUU?)predr   r  )allr   geq?)Nr   r2   c                 C   sL   g dd }t ttg d }t||| d}t|| |jj|ksHJ d S )Nr   r   	normalize)listr   r   r   r)   dtypekind)r  Zcm_dtypeZexpected_resultsy_testrK   r   rM   rM   rN   test_confusion_matrix_normalizen  s
    

r   c                  C   s   g d} g d}t | |dd}| tdks4J t * tdt t | |dd}W d    n1 sl0    Y  | tdksJ t * tdt t || dd W d    n1 s0    Y  d S )	N)r   r   r   r   r7   r7   r7   r7   )r   r   r   r   r   r   r   r   r  r         @r   r  r   )r   sumr   r   r|   r}   r   RuntimeWarning)r  rK   Zcm_trueZcm_predrM   rM   rN   ,test_confusion_matrix_normalize_single_class  s    
,
r$  c                  C   sL   g d} g d}t jtdd t||  W d   n1 s>0    Y  dS )z8Test `confusion_matrix` warns when only one label found.r   r   r   r   zA single label was found inr   N)r   r   r   r   )r  rK   rM   rM   rN   "test_confusion_matrix_single_label  s    r&  zparams, warn_msg)r   r   r   r   r   r   rL   rK   z2samples of only one class were seen during testing)r7   r7   r7   r   r   r   z:positive_likelihood_ratio ill-defined and being set to nanz+no samples predicted for the positive classr   r   r   r7   r7   r7   z:negative_likelihood_ratio ill-defined and being set to nanz@no samples of the positive class were present in the testing setc                 C   s@   t jt|d tf i |  W d    n1 s20    Y  d S Nr   )r   r   r   r   )paramswarn_msgrM   rM   rN   test_likelihood_ratios_warnings  s    1r,  zparams, err_msg)r   r7   r   r7   r   r7   r7   r   r   r2   zeclass_likelihood_ratios only supports binary classification problems, got targets of type: multiclassc                 C   s@   t jt|d tf i |  W d    n1 s20    Y  d S r)  )r   r   r   r   )r*  r   rM   rM   rN   test_likelihood_ratios_errors  s    r.  c                  C   s   t dgd dgd  } t dgd dgd  dgd  }t| |\}}t|d t|d	 t| | \}}t|t jd  t|t dd
d t dgd dgd  }t| ||d\}}t|d t|d d S )Nr7   r   r   r   r2   
   r   g?g_B{	%?g-q=)Zrtolr      ro   r   r  gUUUUUU@gqq?)r<   r   r   r)   r,   rp   r   )rL   rK   posnegr  rM   rM   rN   test_likelihood_ratios  s    $


r3  c                  C   s  t dgd dgd  } t dgd dgd  dgd  dgd  }t| |}t|dd	d
 |t|| ksrJ t | dgd } t |dgd }t| |ddgd|ksJ tt| | d t dgd dgd  dgd  } t dgd dgd  dgd  }tt| |ddd
 t dgd dgd  dgd  } t dgd dgd  dgd  }tt| |ddd
 tt| |ddddd
 tt| |ddddd
 d S )Nr   (   r7   <   rZ   r/  2   gʡE?r   r   r2   r   r   r   .   ,   4          g??g+?r5   weightsg_vO?Z	quadraticg#?)r<   r   r   r*   append)r   r   kapparM   rM   rN   test_cohen_kappa  s&    .
$$$$r@  zy_true, y_predrm   r   c                 C   sj   t  ( t d | |||d}W d   n1 s60    Y  t|rZt|sfJ n||ksfJ dS )zmCheck the behaviour of `zero_division` when setting to 0, 1 or np.nan.
    No warnings should be raised.
    r   rs   N)r|   r}   r   r<   isnan)rm   rL   rK   rs   resultrM   rM   rN   !test_zero_division_nan_no_warning  s    

,
rD  c                 C   sH   t t | ||dd}W d   n1 s.0    Y  |dksDJ dS )ztCheck the behaviour of `zero_division` when setting to "warn".
    A `UndefinedMetricWarning` should be raised.
    rt   rA  Nro   )r   r   r   )rm   rL   rK   rC  rM   rM   rN   test_zero_division_nan_warning7  s    ,rE  c                  C   sN   t jd} | jdddd}| jdddd}tt||t ||d d d S )Nr   r2   rZ   sizer   r7   r/  )r<   r@   rA   randintr*   r   Zcorrcoef)rJ   rL   rK   rM   rM   rN   -test_matthews_corrcoef_against_numpy_corrcoefJ  s    rJ  c            	         s   t jd} | jdddd}| jdddd}| d}t|||d t t fddtD }t fddtD }t  fd	dtD }|t 	||  }t
|||d}t||d
 d S )Nr   r2   rZ   rF  r  c              	      sX   g | ]P}t D ]B}t D ]4} ||f  ||f   ||f  ||f   qqqS rM   range)r   kmlCNrM   rN   r   `  s   z9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>c                    s@   g | ]8 d d  f   t  fddtD  qS )Nc                    s.   g | ]&}t D ]}|kr ||f qqS rM   rK  r   r   grQ  rR  rM  rM   rN   r   j  r   Dtest_matthews_corrcoef_against_jurman.<locals>.<listcomp>.<listcomp>r"  r<   rL  r   rP  rM  rN   r   h  s   c                    s@   g | ]8  d d f   t  fddtD  qS )Nc                    s.   g | ]&}t D ]}|kr ||f qqS rM   rK  rS  rU  rM   rN   r   q  r   rV  rW  rX  rP  rY  rN   r   o  s   r/  )r<   r@   rA   rI  Zrandr   rd   r"  rL  r   r   r*   )	rJ   rL   rK   r  Zcov_ytypZcov_ytytZcov_ypypZ
mcc_jurmanZmcc_oursrM   rP  rN   %test_matthews_corrcoef_against_jurmanT  s0    
rZ  c                  C   s,  t jd} dd | jddddD }tt||d dd |D }tt||d	 t|d
dgd}t |d
d}tt||d	 ttg dg dd tt|d
gt| d g d}g d}tt||d dgd dgd  }t	
t$ tt|||dd W d    n1 s0    Y  d S )Nr   c                 S   s   g | ]}|d krdndqS )r   rv   rw   rM   r   r   rM   rM   rN   r   }  r   z*test_matthews_corrcoef.<locals>.<listcomp>r2   rZ   rF  r   c                 S   s   g | ]}|d krdnd qS )rv   rw   rM   r[  rM   rM   rN   r     r   r   rv   rw   r   r%  ro   )r7   r   r7   r7   r   r7   r7   r7   r   r7   r7   r7   r7   r7   r7   r7   r   r7   r7   r7   )r7   r7   r7   r   r   r7   r7   r7   r7   r   r7   r7   r7   r   r7   r7   r7   r   r7   r7   r7   r/  r  )r<   r@   rA   rI  r*   r   r&   whererd   r   r   AssertionError)rJ   rL   Z
y_true_invZy_true_inv2y_1y_2maskrM   rM   rN   test_matthews_corrcoef{  s     ra  c            	         sL  t jd} td d} fdd| jd|ddD }tt||d g d	}g d
}tt||d g d	}g d}tt||dt d  g d}g d}tt||d g d}g d}tt||d g d}g d}tt||d g d}g d}g d}tt|||dd g d}g d}g d}tt|||dd d S )Nr   rv   r   c                    s   g | ]}t  | qS rM   )chrr[  Zord_arM   rN   r     r   z5test_matthews_corrcoef_multiclass.<locals>.<listcomp>rZ   rF  r   )r   r   r7   r7   r2   r2   )r2   r2   r   r   r7   r7   g      )r7   r7   r   r   r   r   ii  r   )r   r   r   ro   	r   r7   r2   r   r7   r2   r   r7   r2   )	r7   r7   r7   r2   r2   r2   r   r   r   )r   r   r7   r7   r2   r-  r7   r7   r7   r7   r   r  r   r   r7   r7   r   r   )r<   r@   rA   ordrI  r*   r   r   )	rJ   	n_classesrL   Z
y_pred_badZ
y_pred_minrK   r^  r_  r  rM   rc  rN   !test_matthews_corrcoef_multiclass  s@    ri  n_pointsd   i'  c                    s   t jd dd } fdd}t ddg| }tt||d t g d| }tt||d || \}}tt||d tt||||| d S )	Ni3c                 S   sx   t | |}|d }|d }|d }t| }|| | }|| | }|| ||  }	|| d|  d|  }
|	t|
 S )Nr7   r7   )r7   r   rH  r7   )r   rd   r<   r   )rL   rK   Zconf_matrixZtrue_posZ	false_posZ	false_negrj  Zpos_rateZactivityZmcc_numeratorZmcc_denominatorrM   rM   rN   mcc_safe  s    
z1test_matthews_corrcoef_overflow.<locals>.mcc_safec                    s8     | }|d  | d   }|dk}|dk}||fS )Nr   r   )Zrandom_sample)rj  Zx_trueZx_predrL   rK   rJ   rM   rN   	random_ys  s
    
z2test_matthews_corrcoef_overflow.<locals>.random_ysro   r   )ro   r   r!  )r<   r@   rA   repeatr*   r   )rj  rm  ro  ZarrrL   rK   rM   rn  rN   test_matthews_corrcoef_overflow  s    rq  c            
      C   s  t dd\} }}t| |d d\}}}}t|g dd t|g dd t|g dd t|g d t| |d	d
d}t|dd t| |d
d}t|dd t| |d
d}	t|	dd t| |dd}t|dd t| |dd}t|dd t| |dd}	t|	dd t| |dd}t|dd t| |dd}t|dd t| |dd}	t|	dd tt	 t| |dd W d    n1 s0    Y  tt	 t| |dd W d    n1 s0    Y  tt	 t| |dd W d    n1 s0    Y  tt	  t
| |ddd W d    n1 s00    Y  t| |g dd d\}}}}t|g dd t|g dd t|g dd t|g d d S )NFr   r   )(\?Q?gzG?r2   )HzG?g
ףp=
?rX   )Q?333333?r   )rQ   rW   rZ   r7   r   r   g(\?r   r   gRQ?r   gGz?r   r   r   r   r  r   )rr  g=
ףp=?rs  )rt  rX   r   )ru  r   rv  )rQ   rZ   rW   )rO   r   r+   r,   r   r    r   r   r   r   r   )
rL   rK   ri   rI   r   r   r   r   r   r   rM   rM   rN   )test_precision_recall_f1_score_multiclass  sN    ...0rx  r   )r   r   r   r   Nc                 C   sv   t g dg}t g dg}t||g dg | d\}}}}t|d t|d t|d | d u rrt|g d d S )Nrf  r   r   r7   r7   )r   r   r7   r2   )ra   warn_forr   r   r   r7   r7   r   )r<   r   r   r,   )r   rL   rK   rI   r   r   r   rM   rM   rN   ;test_precision_refcall_f1_score_multilabel_unordered_labels2  s    


r|  c            
      C   s   t g d} t g d}t| |d d\}}}}t| |dd\}}}}|t |ksZJ |t |kslJ |t |ks~J t| |dd\}}}}t | }	|t j||	dksJ |t j||	dksJ |t j||	dksJ d S )N)r   r7   r   r   r7   r7   r   r7   r   r   r7   r   r7   r   r7   )r7   r7   r   r7   r   r7   r7   r7   r7   r   r7   r   r7   r   r7   r   r   r   r<  )r<   r   r   r   Zbincountr   )
rL   rK   r   r   r   ri   rI   r   r   rU   rM   rM   rN   .test_precision_recall_f1_score_binary_averagedA  s    
r}  c               
   C   s   t jdd} zrt g d}t g d}tt||dddd tt||dddd tt||dddd W t jf i |  nt jf i |  0 d S )	Nraise)r  )r   r7   r2   r   r7   r2   )r2   r   r7   r7   r2   r   r   r   ro   r2   )r<   Zseterrr   r*   r   r    r   )Zold_error_settingsrL   rK   rM   rM   rN   test_zero_precision_recallR  s    r  c                  C   s   t dd\} }}t| |ddgd}t|ddgddgg t| |d	dgd}t|d
d	gddgg t| d }t| |d	|gd}t|d
dgddgg d S )NFr   r   r7   r   r   r   r   r2   r   rQ   )rO   r   r,   r<   max)rL   rK   ri   r   Zextra_labelrM   rM   rN   .test_confusion_matrix_multiclass_subset_labelsc  s    r  zlabels, err_msgz,'labels' should contains at least one label.r   r   z.At least one label specified must be in y_truez
empty listzunknown labels)Zidsc                 C   sP   t dd\}}}tjt|d t||| d W d    n1 sB0    Y  d S )NFr   r   r   )rO   r   r   r   r   )ra   r   rL   rK   ri   rM   rM   rN   test_confusion_matrix_errorv  s    	r  ra   )NonerD   
multiclassc                 C   s>   | rt | nd}tj||ftd}tg g | d}t|| d S )Nr   r  r   )rd   r<   r   r?   r   r,   )ra   Zexpected_n_classesexpectedr   rM   rM   rN   *test_confusion_matrix_on_zero_length_input  s    r  c                  C   s<  g d} t t| }t| | }|jt jks0J t jt jt jfD ],}t| | |j	|ddd}|jt jks@J q@t j
t jd tfD ],}t| | |j	|ddd}|jt jks~J q~t jt| dt jd}t| | |d}|d dksJ |d d	ksJ t jt| d
t jd}t| | |d}|d d
ks&J |d dks8J d S )Nr   F)copyr  l    r  r   r   rl  l    l    )r<   Zonesrd   r   r  int64Zbool_Zint32Zuint64Zastypefloat32float64objectfullZuint32)rF   weightr   r  rM   rM   rN   test_confusion_matrix_dtype  s$    
r  r  )ZInt64ZFloat64booleanc                 C   sZ   t d}tg d}|j|| d}|jg ddd}t||}t||}t|| dS )zkChecks that confusion_matrix works with pandas nullable dtypes.

    Non-regression test for gh-25635.
    pandas)	r7   r   r   r7   r   r7   r7   r   r7   r  )	r   r   r7   r7   r   r7   r7   r7   r7   r  N)r   Zimportorskipr<   r   Seriesr   r,   )r  pdZ	y_ndarrayrL   Zy_predictedoutputZexpected_outputrM   rM   rN   %test_confusion_matrix_pandas_nullable  s    


r  c                  C   sL   t  } t| dd\}}}d}t||tt| j| jd}||ksHJ d S )NFrP   a|                precision    recall  f1-score   support

      setosa       0.83      0.79      0.81        24
  versicolor       0.33      0.10      0.15        31
   virginica       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
ra   rb   r	   r8   rO   r   r<   r=   rd   rb   rh   rL   rK   ri   rj   rk   rM   rM   rN   %test_classification_report_multiclass  s    r  c                  C   s0   g dg d } }d}t | |}||ks,J d S )N)	r   r   r   r7   r7   r7   r2   r2   r2   rd  a|                precision    recall  f1-score   support

           0       0.33      0.33      0.33         3
           1       0.33      0.33      0.33         3
           2       0.33      0.33      0.33         3

    accuracy                           0.33         9
   macro avg       0.33      0.33      0.33         9
weighted avg       0.33      0.33      0.33         9
r   )rL   rK   rj   rk   rM   rM   rN   .test_classification_report_multiclass_balanced  s    
r  c                  C   s8   t  } t| dd\}}}d}t||}||ks4J d S )NFrP   a|                precision    recall  f1-score   support

           0       0.83      0.79      0.81        24
           1       0.33      0.10      0.15        31
           2       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
)r	   r8   rO   r   r  rM   rM   rN   :test_classification_report_multiclass_with_label_detection  s
    
r  c                  C   sN   t  } t| dd\}}}d}t||tt| j| jdd}||ksJJ d S )NFrP   a|                precision    recall  f1-score   support

      setosa    0.82609   0.79167   0.80851        24
  versicolor    0.33333   0.09677   0.15000        31
   virginica    0.41860   0.90000   0.57143        20

    accuracy                        0.53333        75
   macro avg    0.52601   0.59615   0.50998        75
weighted avg    0.51375   0.53333   0.47310        75
r   )ra   rb   digitsr  r  rM   rM   rN   1test_classification_report_multiclass_with_digits  s    r  c                  C   st   t dd\} }}tg d|  } tg d| }d}t| |}||ksNJ d}t| |g dd}||kspJ d S )NFr   )bluegreenreda|                precision    recall  f1-score   support

        blue       0.83      0.79      0.81        24
       green       0.33      0.10      0.15        31
         red       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
a|                precision    recall  f1-score   support

           a       0.83      0.79      0.81        24
           b       0.33      0.10      0.15        31
           c       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
ru   rb   rO   r<   r   r   )rL   rK   ri   rj   rk   rM   rM   rN   7test_classification_report_multiclass_with_string_label  s    
r  c                  C   sL   t dd\} }}tg d}||  } || }d}t| |}||ksHJ d S )NFr   )u   blue¢u   green¢u   red¢u                precision    recall  f1-score   support

       blue¢       0.83      0.79      0.81        24
      green¢       0.33      0.10      0.15        31
        red¢       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r  rL   rK   ri   ra   rj   rk   rM   rM   rN   8test_classification_report_multiclass_with_unicode_label9  s    
r  c                  C   sL   t dd\} }}tg d}||  } || }d}t| |}||ksHJ d S )NFr   )r  Zgreengreengreengreengreenr  a                             precision    recall  f1-score   support

                     blue       0.83      0.79      0.81        24
greengreengreengreengreen       0.33      0.10      0.15        31
                      red       0.42      0.90      0.57        20

                 accuracy                           0.53        75
                macro avg       0.53      0.60      0.51        75
             weighted avg       0.51      0.53      0.47        75
r  r  rM   rM   rN   <test_classification_report_multiclass_with_long_string_labelO  s    
r  c                  C   sb   g d} g d}g d}d}t jt|d$ t| |ddg|d W d    n1 sT0    Y  d S )	Nr   r   r2   r   r   r   r2   r2   r   r   zclass 0zclass 1zclass 2z6labels size, 2, does not match size of target_names, 3r   r   r2   r  )r   r   r   r   )rL   rK   rb   r   rM   rM   rN   =test_classification_report_labels_target_names_unequal_lengthf  s    r  c                  C   s\   g d} g d}g d}d}t jt|d t| ||d W d    n1 sN0    Y  d S )Nr  r  r  zaNumber of classes, 2, does not match size of target_names, 3. Try specifying the labels parameterr   r  )r   r   r   r   )rL   rK   rb   r   rM   rM   rN   @test_classification_report_no_labels_target_names_unequal_lengthp  s    r  c                  C   sN   d} d}t d|| dd\}}t d|| dd\}}d}t||}||ksJJ d S )Nr   r6  r7   r   )rH   rG   rh  r6   a                precision    recall  f1-score   support

           0       0.50      0.67      0.57        24
           1       0.51      0.74      0.61        27
           2       0.29      0.08      0.12        26
           3       0.52      0.56      0.54        27

   micro avg       0.50      0.51      0.50       104
   macro avg       0.45      0.51      0.46       104
weighted avg       0.45      0.51      0.46       104
 samples avg       0.46      0.42      0.40       104
)r   r   )rh  rG   ri   rL   rK   rj   rk   rM   rM   rN   %test_multilabel_classification_report~  s    


r  c                  C   s   t g dg dg} t g dg dg}t| |dks>J t| | dksPJ t||dksbJ t|t |dkszJ t| t | dksJ t| t | jdksJ t|t | jdksJ d S )Nr   r   r   r   r   r7   )r<   r   r!   r   r   r;   r   rM   rM   rN   $test_multilabel_zero_one_loss_subset  s    r  c                  C   sN  t g dg dg} t g dg dg}t ddg}t| |dksLJ t| | dks^J t||dkspJ t|d| dksJ t| d|  dksJ t| t | jdksJ t|t | jd	ksJ t| ||d
dksJ t| d| |d
dksJ t| t | |d
dks J t| d |d t| d |d ksJJ d S )Nr   r   r   r7   r   UUUUUU?r   r   r   r  gUUUUUU?gUUUUUU?)r<   r   r   r   r;   
zeros_like
sp_hamming)r   r   wrM   rM   rN   test_multilabel_hamming_loss  s    r  c                  C   s  t g d} t g d}d}tjt|d  t| |ddd W d    n1 sT0    Y  t g dg dg} t g d	g d
g}d}tjt|d  t| |ddd W d    n1 s0    Y  t g d} t g d}d}tjt|d t| |dd W d    n1 s 0    Y  d}tjt|d t| |dd W d    n1 sb0    Y  d}tjt|d  t| |ddd W d    n1 s0    Y  d S )N)r   r7   r   r7   r7   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   rD   r2   r   r   r   r   r   r   Target is multilabel-indicator but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted', 'samples'\].r   )r   r7   r7   r   r2   re  Target is multiclass but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted'\].r   zJSamplewise metrics are not available outside of multilabel classification.r   zNote that pos_label \(set to 3\) is ignored when average != 'binary' \(got 'micro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   )r<   r   r   r   r   r   r   r   )rL   rK   r   Zmsg1msg2msg3r   rM   rM   rN   test_jaccard_score_validation  s0    ....r  c                 C   sb  t g dg dg}t g dg dg}t||dddksBJ t||dddksXJ t||dddksnJ t|t |dddksJ t|t |dddksJ t|t |jdddksJ t|t |jdddksJ t g dg d	g}t g d
g dg}tt||ddd tt||ddd tt||ddd tt||dddgdd tt||dddgdd tt||d dt g d t g dg dg}t g d
g dg}tt||ddd tt||ddd d}tj	t
|d" t||dgdd W d    n1 s,0    Y  d}tj	t
|d" t||dgdd W d    n1 sr0    Y  d}tjt|d@ tt ddggt ddggdddksJ W d    n1 s0    Y  d}tjt|dL tt ddgddggt ddgddggdddks0J W d    n1 sF0    Y  t| r^J d S )Nr   r   r   r   r   r   r7   r   r   r   r   r   r   r   g?r2   r   r   )r   r   r   r   r   g      ?z	Got 4 > 2r   r   r   z
Got -1 < 0r   zXJaccard is ill-defined and being set to 0.0 in labels with no true or predicted samples.zXJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels.)r<   r   r   r   r   r;   r*   r,   r   r   r   r   r   r  )recwarnr   r   rL   rK   r  r  r   rM   rM   rN   test_multilabel_jaccard_score  sh    22$((	r  c                 C   s^  g d}g d}g d}t  }|| ||}||}tt||}tt||}ddgddgddgdgdgdgd g}	ddgdd	gd	dgdgdgd	gd g}
d
D ]2}t|	|
D ]"\}}t|||d|||d qqtddgddgddgg}tddgddgddgg}t	 ( t||dddks,J W d    n1 sB0    Y  t
| rZJ d S )N)antr  catr  r  r  birdr  )r  r  r  r  r  r  r  r  )r  r  r  r  r  r  r   r7   r2   )r   r   r   Nr   r   r   )r%   rB   Z	transformr   r   zipr*   r<   r   r-   r  )r  rL   rK   ra   Zlbr   r   Zmulti_jaccard_scoreZbin_jaccard_scoreZmulti_labels_listZbin_labels_listr   Zm_labelZb_labelrM   rM   rN   test_multiclass_jaccard_score4  s:    


	$

8r  c                 C   s   t dgdgdddksJ d}tjt|d. t ddgddgdddksLJ W d    n1 s`0    Y  t dgdgdddd	ksJ tg d
}tg d}tt ||ddd tt ||dddd t| rJ d S )Nr7   r   rD   r   ro   zOJaccard is ill-defined and being set to 0.0 due to no true or predicted samplesr   r   r   )r7   r   r7   r7   r   )r7   r   r7   r7   r7   r   r  r   )r   r   r   r   r<   r   r*   r  )r  r   rL   rK   rM   rM   rN   !test_average_binary_jaccard_scoreY  s    <r  c                  C   s   t g dg dg} t g dg dg}d}tjt|d2 t| |ddd}|tdksbJ W d    n1 sv0    Y  d S )	Nr   r   r   r   zJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels. Use `zero_division` parameter to control this behavior.r   r   rt   r   rs   ro   )r<   r   r   r   r   r   r   )rL   rK   r   scorerM   rM   rN   (test_jaccard_score_zero_division_warningp  s    r  zzero_division, expected_scorer  )r7   r   c                 C   s   t g dg dg}t g dg dg}t , tdt t||d| d}W d    n1 sf0    Y  |t|ksJ d S )Nr   r  r   r   r  )	r<   r   r|   r}   r   r   r   r   r   )rs   expected_scorerL   rK   r  rM   rM   rN   *test_jaccard_score_zero_division_set_value  s    
$r  c                  C   s8  t g dg dg dg} t g dg dg dg}t| |d d\}}}}t|g dd t|g dd t|g d	d t|g d
d t| |dd d}|}t|g dd t| |dd\}}}}t|d t|d t|d |d u sJ tt| |dddt | t| |dd\}}}}t|d t|d t|d |d u sPJ tt| |dddd| | d| |   t| |dd\}}}}t|d t|d t|d |d u sJ tt| |dddt j||d t| |dd\}}}}t|d t|d t|d |d u sJ tt| |dddd d S )Nr7   r   r   r   r   r7   r   r   ry  )r7   r   r7   r   r   )ro   r   r   ro   r2   )ro   r   r   ro   )ro   r   r7   ro   )r7   r7   r7   r7   r   r   )r   rr  r7   r   r   g      ?r   g?r   r   r   r   r<  r   r<   r   r   r+   r   r*   r   r   rL   rK   rI   r   r   r   f2rU   rM   rM   rN   +test_precision_recall_f1_score_multilabel_1  sT    











r  c                  C   s:  t g dg dg dg} t g dg dg dg}t| |d d\}}}}t|g dd t|g d	d t|g d
d t|g dd t| |dd d}|}t|g dd t| |dd\}}}}t|d t|d t|d |d u sJ tt| |dddd| | d| |   t| |dd\}}}}t|d t|d t|d |d u s^J tt| |dddt | t| |dd\}}}}t|d t|d t|d |d u sJ tt| |dddt j||d t| |dd\}}}}t|d t|d t|d |d u sJ tt| |ddddd d S )Nr  r  r{  r   r   r   r7   rf  r   )ro   r   ro   ro   r2   )ro   r   ro   ro   )ro   gQ?ro   ro   r7   r2   r7   r   r  )r   皙?r   r   r         ?r   r   r   g      ?r  r   r   rV   r<  r   g&S?r  r  rM   rM   rN   +test_precision_recall_f1_score_multilabel_2  sX    











r  z%zero_division, zero_division_expected)rt   r   rl  c                 C   s  t g dg dg dg}t g dg dg dg}t||d | d\}}}}t||dddgd	 t|dd
d|gd	 d}t||dd|gd	 t|g dd	 t||d	d | d}	|}
t|	|dd|gd	 t||d| d\}}}}t |rdn|}dt |  }t|d	| |  t|d| |  d}t|| |d u sDJ tt||d	d| dt|	d d t||d| d\}}}}t|d t|d
 t|d |d u sJ tt||d	d| dd| | d| |   t||d| d\}}}}t||dkrdnd t|d
 d}t|d|  |d u s.J tt||d	d| dt|	|
d t||dd\}}}}t|d t|d t|d |d u sJ d }tt||d	d| d|d	 d S )!Nr  r  r{  r%  r  r  r   ro   r2   r   r   r   r7   r  r   r   rs   r  r   r         ?g?r<  r   rY   r   r   r   r   g@r   r   rV   gZd;O?)r<   r   r   r+   r   rB  r*   r.   )rs   Zzero_division_expectedrL   rK   rI   r   r   r   Z
expected_fr  rU   Zvalue_to_sumZvalues_to_averageZexpected_resultrM   rM   rN   7test_precision_recall_f1_score_with_an_empty_prediction  s    












r  r   )r   r   r   r   c                 C   s   t d}t |}t F td t|||| |d\}}}}t||| ||d}	W d    n1 sh0    Y  |d u s~J t |r||||	fD ]}
t |
sJ qd S t	|}t
|| t
|| t
|| t
|	t	| d S )NrZ   r   r   r   r   rs   r  )r<   r   r  r|   r}   r   r   r   rB  rg   r*   )r   r   rs   rL   rK   rI   r   r   r   fbetarm   rM   rM   rN   "test_precision_recall_f1_no_labelss  s8    



$



r  c           	      C   s   t d}t |}t}tt( |||| dd\}}}}W d    n1 sP0    Y  t|d t|d t|d |d u sJ tt  t||| dd}W d    n1 s0    Y  t|d d S )Nr  r   rw  r   )	r<   r   r  r   r   r   r   r*   r   )	r   rL   rK   funcrI   r   r   r   r  rM   rM   rN   1test_precision_recall_f1_no_labels_check_warnings  s    

6


.r  c                 C   s   t d}t |}t F td t||d d| d\}}}}t||dd | d}W d    n1 sh0    Y  t | } t	|| | | gd t	|| | | gd t	|| | | gd t	|g dd t	|| | | gd d S )Nr  r   r   r  r  r2   r  )
r<   r   r  r|   r}   r   r   r   r  r+   )rs   rL   rK   rI   r   r   r   r  rM   rM   rN   /test_precision_recall_f1_no_labels_average_none  s(    





$
r  c                  C   s   t d} t | }tt( t| |d dd\}}}}W d    n1 sL0    Y  t|g dd t|g dd t|g dd t|g dd tt  t| |dd d}W d    n1 s0    Y  t|g dd d S )Nr  r7   rw  r  r2   r  )	r<   r   r  r   r   r   r   r+   r   )rL   rK   rI   r   r   r   r  rM   rM   rN   4test_precision_recall_f1_no_labels_average_none_warn  s    


,.r  c               	   C   sv  t t } }dD ]}d}tj||d& | g dg d|d W d    n1 sP0    Y  d}tj||d& | g dg d|d W d    q1 s0    Y  qd}tj||dB | td	d
gd	d
ggtd	d
gd
d
ggdd W d    n1 s 0    Y  d}tj||dB | td	d
gd
d
ggtd	d
gd	d
ggdd W d    n1 sf0    Y  d}tj||dB | td	d	gd	d	ggtd
d
gd
d
ggdd W d    n1 s0    Y  d}tj||dB | td
d
gd
d
ggtd	d	gd	d	ggdd W d    n1 s20    Y  d}tj||d& | d	d	gddgdd W d    n1 s|0    Y  d}tj||d& | ddgd	d	gdd W d    n1 s0    Y  tjdd}td t d
d
gd
d
gdd d}t	|
 j|ksJ d}t	|
 j|ks6J d}t	|
 j|ksRJ W d    n1 sh0    Y  d S )NNr   r   zPrecision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.r   r   r7   r7   r2   r   zRecall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.zPrecision is ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.r7   r   r   zRecall is ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.r   zRecall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.r   rD   Trz   alwaysF-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.)r   r   r   r   r<   r   r|   r}   r   r~   popr   )r   r  r   r   r{   rM   rM   rN   test_prf_warnings  s\    
46RRRR66
r  c                 C   s  t  T t d dD ]4}tg dg d|| d tg dg d|| d qttddgddggtddgddggd| d ttddgddggtddgddggd| d ttddgddggtddgddggd	| d ttddgddggtddgddggd	| d tddgd
d
gd| d td
d
gddgd| d W d    n1 sf0    Y  t jddD}t d tddgddgd| d t|dksJ W d    n1 s0    Y  d S )Nr   r  r   r  r  r7   r   r   r   r   rD   Trz   r  )r|   r}   r   r   r<   r   rd   )rs   r   r{   rM   rM   rN   )test_prf_no_warnings_if_zero_division_setW  sZ    
&
r  c                 C   s:  t  N t d ttddgddggtddgddggd| d W d    n1 s\0    Y  t jdd}t d ttddgddggtddgddggd| d | d	krt| jd
ksJ nt	|dksJ tddgddg | d	krt| jd
ksJ W d    n1 s,0    Y  d S )Nr   r7   r   r   r  Trz   r  rt   r  )
r|   r}   r   r    r<   r   r~   r  r   rd   rs   r{   rM   rM   rN   test_recall_warnings  s0    

$

r  c                 C   s6  t jdd}t d ttddgddggtddgddggd| d | dkrlt| jd	ks|J nt	|dks|J tddgddg | dkrt| jd	ksJ W d    n1 s0    Y  t  N t d
 ttddgddggtddgddggd| d W d    n1 s(0    Y  d S )NTrz   r  r7   r   r   r  rt   r  r   )
r|   r}   r   r   r<   r   r~   r  r   rd   r  rM   rM   rN   test_precision_warnings  s0    
$

r  c              	   C   sD  t jdd }t d tttddfD ]}|tddgddggtddgddggd| d	 t|dksrJ |tddgddggtddgddggd| d	 t|dksJ |tddgddggtddgddggd| d	 | d
krt	|
 jdksJ q*t|dks*J q*W d    n1 s60    Y  d S )NTrz   r  r2   r   r7   r   r   r  rt   r  )r|   r}   r   r   r   r   r<   r   rd   r~   r  r   )rs   r{   r  rM   rM   rN   test_fscore_warnings  s6    


r  c            
   
   C   s   g d} g d}d}t g dg dg dg}t g dg dg dg}d}| ||f|||ffD ]\\}}}tttttd	d
fD ]<}	tjt	|d |	|| W d    q1 s0    Y  qqdd S )N)r7   r2   r   r   )r7   r2   r   r7   r  r   r   r   r
  r  r2   r   r   )
r<   r   r   r    r   r   r   r   r   r   )
Z	y_true_mcZ	y_pred_mcZmsg_mcZ
y_true_indZ
y_pred_indZmsg_indrL   rK   r   rm   rM   rM   rN   'test_prf_average_binary_data_non_binary	  s$    
r  c               *   C   s  d} d}d}d}d}d}| t g dg dg d	gf| t d
dgdd
gddggf|g df|g df|g df|t dgdgdggf|t d
gdgdggf|t dgdgdggf|t d
dgddgddggf|t ddgddgddggfg
}| | f| ||f|||f||| fd || fd ||f|||fd ||fd ||fd | |fd ||fd ||fd ||fd ||fd | |fd ||fd ||fd ||fd | |fd ||fd ||fd i}t|ddD ]\\}}	\}
}z|||
f }W n  ty   ||
|f }Y n0 |d u rtt t|	| W d    n1 s,0    Y  ||
krd||
}tjt|d t|	| W d    n1 s|0    Y  nT|||| fvrd|}tjt|d t|	| W d    n1 s0    Y  nt|	|\}}}||ksJ |	dr*|jdksJ |jdksJJ n t
|t |	 t
|t | tt" t|	d d | W d    n1 s~0    Y  qddg}	d d!g}d"}tjt|d t|	| W d    n1 s0    Y  d S )#Nzmultilabel-indicatorr  rD   Z
continuouszmulticlass-multioutputzcontinuous-multioutputr   r   r   r   r7   )r2   r   r7   )ro   r  r   r2   r   ro   r  r   r   r!  g?g      @)rp  z@Classification metrics can't handle a mix of {0} and {1} targetsr   z{0} is not supportedZ
multilabelZcsrr   )r7   r2   )r   r2   r   )r2   )r   r2   zYou appear to be using a legacy multi-label data representation. Sequence of sequences are no longer supported; use a binary array or sparse matrix instead - the MultiLabelBinarizer transformer can convert to this format.)r<   r   r   KeyErrorr   r   r   r"   format
startswithr,   Zsqueeze)ZINDZMCZBINZCNTZMMCZMCNZEXAMPLESZEXPECTEDZtype1r   Ztype2r   r  r   Zmerged_typeZy1outZy2outr   rM   rM   rN   test__check_targets+	  s    



*
,
,6r  c                  C   s*   ddg} ddg}t | |d dks&J d S )Nr   r7   r   r  )r"   r'  rM   rM   rN   Atest__check_targets_multiclass_with_both_y_true_and_y_pred_binary	  s    r  c                  C   s`   t g d} t g d}t| |dks.J t g d} t g d}t| |dks\J d S )N)r   r7   r7   r   )g      !r   r  g333333ӿr   )r   r2   r2   r   )r<   r   r   rL   pred_decisionrM   rM   rN   test_hinge_loss_binary	  s    r  c               
   C   s  t g dg dg dg dg dg dg} t g d}t d| d d  | d d  d| d d  | d d	  d| d	 d	  | d	 d
  d| d
 d  | d
 d	  d| d d
  | d d	  d| d d	  | d d
  g}t j|dd |d t |}t|| |ksJ d S )N
ףp=
?(\ſ(\gGz)HzGgGz׿Q޿r  333333r  RQؿr  )r  r  r  r  gzGgHzGgHzGѿgQ?)r   r7   r2   r7   r   r2   r7   r   r2   r   r   r   outr<   r   clipr   r   )r  rL   dummy_lossesdummy_hinge_lossrM   rM   rN   test_hinge_loss_multiclass	  s,    


r  c                  C   sp   t g d} t g dg dg dg dg}d}tjt|d t| | W d    n1 sb0    Y  d S )N)r   r7   r2   r2   )gRQ?g rh?g(\gffffffr  r  zDPlease include all labels in y_true or pass labels as third argumentr   )r<   r   r   r   r   r   )rL   r  error_messagerM   rM   rN   :test_hinge_loss_multiclass_missing_labels_with_labels_none	  s    	r  c               
   C   s   t g d} t g d}d}tjtt|d t| |d W d    n1 sV0    Y  t ddgddgddgddgddgddgddgg}g d	}d
}tjtt|d t| ||d W d    n1 s0    Y  d S )N)r2   r7   r   r7   r   r7   r7   )r   r7   r2   r7   r   r2   r7   zThe shape of pred_decision cannot be 1d arraywith a multiclass target. pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7,)r   r  r   r7   r2   r   zThe shape of pred_decision is not consistent with the number of classes. With a multiclass target, pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7, 2))rL   r  ra   )r<   r   r   r   r   reescaper   )rL   r  r  ra   rM   rM   rN   <test_hinge_loss_multiclass_no_consistent_pred_decision_shape	  s    *4r  c               	   C   s  t g dg dg dg dg dg} t g d}t g d}t d| d d  | d d  d| d d  | d d  d| d d  | d d	  d| d	 d  | d	 d  d| d
 d  | d
 d	  g}t j|dd |d t |}t|| |d|ksJ d S )Nr  皙r  r  r  r  )r   r7   r2   r7   r2   )r   r7   r2   r   r7   r   r2   r   r   r   r   r  r  rL   ra   r  r  rM   rM   rN   .test_hinge_loss_multiclass_with_missing_labels	  s*    		
r  c               	   C   s  t g dg dg dg dg dg} t g d}t g d}t d| d d  | d d  d| d d	  | d d  d| d	 d	  | d	 d  d| d
 d  | d
 d	  d| d d	  | d d  g}t j|dd |d t |}tt|| |d| d S )N)r  r  r  )g333333ÿr  r  )r  r  r  )r  g(\gzGڿ)r   r2   r2   r   r2   r   r7   r   r2   r   r   r   r   )r<   r   r  r   r*   r   r  rM   rM   rN   @test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_true 
  s.    		
r  c               
   C   s
  g d} g dg dg dg dg dg dg}t d|d d  |d d  d|d d  |d d  d|d d  |d d	  d|d	 d  |d	 d  d|d
 d	  |d
 d  d|d d  |d d	  g}t j|dd |d t |}t| ||ksJ d S )N)r  r  r  r  whiter  r  r  r  r  r7   r   r2   r   r   r   r   r  )rL   r  r  r  rM   rM   rN   +test_hinge_loss_multiclass_invariance_lists 
  s(    

r  c            	   	   C   s  g d} t ddgddgddgddgddgd	d
gg}t| |}t tt | dk|d d df  }t|| g d} g dg dg dg}t| |dd}t|d | d9 } |d9 }t| |dd}t|d g d} ddgddgddgg}tt	 t| | W d    n1 s0    Y  g d} ddgddgddgddgg}t| |}t|d ddg} ddgddgg}t ddgddgg}d}tjt	|d  t| | W d    n1 s0    Y  ddgddgddgg}d!}tjt	|d  t| | W d    n1 s0    Y  t t 
|d d df  }t| |ddgd"}t|| g d#} g d$g dg d%g}t| |g d&d"}t|t 
d  d S )'Nnor  r  yesr  r  r   r   rX   {Gz?Gz?r   r  gMbP?g+?r  r7   r  r   r   r   )r   r   r   )r   r   r   Tr  g躕ʀ?r2   Fg.L`@r   r   r   r   hamspamr  r  CT?r   r   zly_true contains only one label \(2\). Please provide the true labels explicitly through the labels argument.r   zDFound input variables with inconsistent numbers of samples: \[3, 2\]r   r   )r   r   r   r   r   r   r  )r<   r   r   r   r   Zlogpmfr)   r   r   r   log)	rL   rK   lossZ	loss_truer   Z	error_strZtrue_log_lossZcalculated_log_lossZy_score2rM   rM   rN   test_log_loss<
  sR    &
*


*

**
r   c                 C   s@   t jddg| d}t jddg| d}t||}t |s<J dS )zCheck the behaviour internal eps that changes depending on the input dtype.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/24315
    r   r7   r  N)r<   r   r   isfinite)r  rL   rK   r  rM   rM   rN   test_log_loss_eps}
  s    
r"  c                 C   sp   t g d}t jddgddgddgddgg| d}tjtd	d
 t|| W d   n1 sb0    Y  dS )zGCheck that log_loss raises a warning when y_pred values don't sum to 1.r{  r   r   r   r   r   r   r  z$The y_pred values do not sum to one.r   N)r<   r   r   r   r   r   )r  rL   rK   rM   rM   rN   'test_log_loss_not_probabilities_warning
  s    &r#  r
  r   r   c                 C   s   t | |tdksJ dS )z6Check that log_loss returns 0 for perfect predictions.r   N)r   r   r   r'  rM   rM   rN   !test_log_loss_perfect_predictions
  s    r$  c            
      C   s   t g d} t ddgddgddgddgg}ttfg}z"ddlm}m} |||f W n tyn   Y n0 |D ].\}}|| || }}t||}	t	|	d qtd S )	Nr  r   r   r   r   r   )	DataFramer  r  )
r<   r   r(   r  r%  r  r>  ImportErrorr   r)   )
Zy_trZy_prtypesr%  r  ZTrueInputTypeZPredInputTyperL   rK   r  rM   rM   rN   test_log_loss_pandas_input
  s    "

r(  c                  C   s  t g d} t g d}t| | d t|  }tt| | d tt| || ttd|  || ttd|  d || tt	" t| |dd   W d    n1 s0    Y  tt	 t| |d  W d    n1 s0    Y  tt	 t| |d  W d    n1 s&0    Y  t g d} t g d}d	}tjt	|d
 t| | W d    n1 s0    Y  ttdgdgd ttdgdgd ttdgdgd ttdgdgddd ttdgdgddd d S )Nr   r7   r7   r   r7   r7   r   r   rX   r   r   gffffff?r2   ro   r   r7   )r   r7   r2   r   )r   r   r   r   zMOnly binary classification is supported. The type of the target is multiclassr   r   r   g{Gz?r   r  Zfoobarr   )
r<   r   r   Znormrd   r*   r   r   r   r   )rL   rK   Z
true_scorer  rM   rM   rN   test_brier_score_loss
  s0    0,.*r,  c                  C   sH   d} t jt| d" tg dg d W d    n1 s:0    Y  d S )Nz%y_pred contains classes not in y_truer   r  r   )r   r   r   r   r   rM   rM   rN   #test_balanced_accuracy_score_unseen
  s    r-  zy_true,y_pred)rv   rw   rv   rw   )rv   rv   rv   rw   )rv   rw   rx   rw   c                 C   s   t | |dt| d}t  t| |}W d    n1 s<0    Y  |t|ksXJ t| |dd}t| t| | d }||| d|  ksJ d S )Nr   r   T)adjustedr   r7   )r    r<   uniquer-   r   r   r   Z	full_like)rL   rK   Zmacro_recallZbalancedr.  ZchancerM   rM   rN   test_balanced_accuracy_score
  s    	(r0  r   r   ))FTrH  )ro   r   )zeroonec                 C   sn   t jd}d|d  }}|j||dd}| tu r@|j|d}n| }| |||d}t t |rjJ dS )	zCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    *   r/  r   T)rG  replacerF  r   N)	r<   r@   rA   choicer   uniformr  anyrB  )rm   r   rJ   rG   r   rL   rK   rC  rM   rM   rN   *test_classification_metric_pos_label_types
  s    r8  zy_true, y_pred, expected_scorero   r   c                 C   s    t | |ddt|ksJ dS )zCheck the behaviour of `zero_division` for f1-score.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26965
    r   rA  N)r   r   r   )rL   rK   r  rM   rM   rN   2test_f1_for_small_binary_inputs_with_zero_division  s    r9  scoringrA  r2   )r   rs   c                 C   s<   t jdd\}}tddd||}t|||| ddd dS )	aZ  Check that we validate `np.nan` properly for classification metrics.

    With `n_jobs=2` in cross-validation, the `np.nan` used for the singleton will be
    different in the sub-process and we should not use the `is` operator but
    `math.isnan`.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27563
    r   )r6   r   )	max_depthr6   r2   r~  )r:  Zn_jobsZerror_scoreN)r	   Zmake_classificationr'   rB   r$   )r:  rE   rF   
classifierrM   rM   rN   9test_classification_metric_division_by_zero_nan_validaton&  s    r=  c                  C   s   t g d} t g d}d}tjt|d t| |d W d   n1 sP0    Y  d}tjt|d t| ||d W d   n1 s0    Y  dS )	z)Check the message for future deprecation.r)  r*  z$y_prob was deprecated in version 1.5r   )y_probNz/`y_prob` and `y_proba` cannot be both specified)r>  Zy_proba)r<   r   r   r   FutureWarningr   r   r   )rL   rK   r+  	error_msgrM   rM   rN   )test_brier_score_loss_deprecation_warning?  s    $rA  c               	   C   s  g d} g d}t ddgddgddgddgdd	gd
dgg}t ddgddgddgddgddgddgg}t| |d}t| |dd}t| |dd}d||  }|t|ksJ t g d}|d d  |  |d d df< |dd   |  |d d df< t| ||d}t| ||dd}t| ||dd}d||  }|t|ksVJ t ddgddgddgddgddgddgg}t| |}d|  k rdk sn J t||}	|	t|ksJ t ddgddgddgddgddgddgg}t| |}|dk sJ t||}	|	t|ks.J g d} t ddgddgddgddgddgddgg}t| |}|dks|J t||}	|	dksJ g d} g d}t ddgddgddgddgg}t| |}|dksJ t||}	|	dksJ g d}t| ||d}
|
dksJ g d} g d}t g d g d g d!g d"g}t| |}d|  k rndk stn J t| ||d}d|  k rdk sn J t g d#g d$g d"g d%g}t| |}|dk sJ t| ||d}|dk sJ d S )&Nr(  r  r   rX   r   r   r   gffffff?g?r  r  r'  F)rL   rK   r  r7   )r2   r7   r   r   r   r7   r   r   )rL   rK   r  )rL   rK   r  r  r   r   r   r   r  )r   r7   r7   r7   )r  r  r  r  )r2   r2   r2   r2   r  )highrB  lowZneutral)gffffff?r   r   r   )r   r   r   r   )r   r   r   )r   r   r   r  r  )r<   r   r#   r   r   r   r"  )rL   Zy_true_stringrK   Zy_pred_nullZd2_scoreZlog_likelihoodZlog_likelihood_nullZd2_score_truer  Zd2_score_stringZd2_score_with_sample_weightrM   rM   rN   test_d2_log_loss_scoreU  s    

$$








"



rD  c                  C   s*  g d} ddgddgddgg}d}t jt|d t| | W d	   n1 sP0    Y  g d
} ddgddgddgg}g d}d}t jt|d t| ||d W d	   n1 s0    Y  g d} g dg dg}d}t jt|d t| | W d	   n1 s0    Y  dg} ddgg}d}t jt|d t| | W d	   n1 sZ0    Y  g d} ddgddgddgg}d}t jt|d t| | W d	   n1 s0    Y  g d} dg}ddgddgddgg}d}t jt|d t| ||d W d	   n1 s0    Y  d	S )zPTest that d2_log_loss_score raises the appropriate errors on
    invalid inputs.r   r   r   r   r   r   z#contain different number of classesr   Nru   z(number of classes in labels is differentr   )r   r   r   )r   r   r   r  r7   zscore is not well-definedr   r   zy_true contains only one labelz.The labels array needs to contain at least two)r   r   r   r#   r   r   )rL   rK   errra   rM   rM   rN   test_d2_log_loss_score_raises  s@    (,*
**rF  )NF)r	  r|   	functoolsr   	itertoolsr   r   r   numpyr<   r   Zscipyr   Zscipy.spatial.distancer   r  Zscipy.statsr   Zsklearnr	   r
   Zsklearn.datasetsr   Zsklearn.exceptionsr   Zsklearn.metricsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   Zsklearn.metrics._classificationr"   r#   Zsklearn.model_selectionr$   Zsklearn.preprocessingr%   r&   Zsklearn.treer'   Zsklearn.utils._mockingr(   Zsklearn.utils._testingr)   r*   r+   r,   r-   Zsklearn.utils.extmathr.   Zsklearn.utils.fixesr/   r0   Zsklearn.utils.validationr1   rO   rn   rr   markZparametrizerp   r   r   r   r   filterwarningsr   r   r   r   r   r   r   r   r   r   r  r  r  r   r$  r&  r,  r.  r3  r@  rD  rE  rJ  rZ  ra  ri  rq  rx  r|  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  Zfloat16r"  r#  r$  r(  r,  r-  r0  r8  r9  r=  rA  rD  rF  rM   rM   rM   rN   <module>   sT  \0A
 
"



-



#(	
	



-


	
		
'#8
!:




#


(M%


B

@
^%

#h
=
'
'
&b
 A
	"#			 