
    !g-                         d Z ddlZddlmZmZ ddlZddlm	Z	m
Z
 ddlmZ ddlmZ ddlmZ ddlmZ  G d	 d
ee	      Z G d dee
      Z G d dee      Zd Zd Zd Zd Zd Zd Zd Zd Zd Zy)z>
Created on Wed May 23 12:53:27 2018

Author: Josef Perktold

    N)assert_allcloseassert_equal)PoissonLogit)GLMfamily)PenalizedMixin)VariableScreeningc                       e Zd Zy)PoissonPenalizedN__name__
__module____qualname__     b/var/www/dash_apps/app1/venv/lib/python3.12/site-packages/statsmodels/base/tests/test_screening.pyr   r          r   r   c                       e Zd Zy)LogitPenalizedNr   r   r   r   r   r      r   r   r   c                       e Zd Zy)GLMPenalizedNr   r   r   r   r   r      r   r   r   c                     t         j                  j                  d       d\  } }d}t         j                  j                  | |      dt         j                  j                  | d      dz
  z  z   dz  dz
  }|dz  }||j	                  d	      z
  |j                  d	      z  }d|d d d	f<   t        j                  |      }g d
}dt        j                  d|dz         z  ||<   t        j                  |      }|j                  |      }t        j                  |      }t         j                  j                  |      }||||fS )N d              ?         ?   333333?r   r   r   ,      )nprandomseedrandmeanstdzerosarangesqrtdotexppoisson	nobsk_vars	k_nonzeroxbetaidx_nonzero_truelinpredmuys	            r   _get_poisson_datar?      s   IINN6LD&I	f	%	ryy~~dA&,	-
.12	356	7AHA	
QVVAY!%%("AAadG88FD."))Ay1}"==D	774=DeeDkG	B
		"Aa!4''r   c                     t         j                  j                  d       t               \  } }}}t	        |       }|D cg c]  }d|z  	 }}d|d<   t        j                  ||   |dg      }t        j                  |d d |f   |      }t        | |      j                         }	|	j                  |d<   t        | t        j                  |      |d	z  
      }
t        |
      }|d d dd f   }|j                  |d      }t        t        j                  |j                         |       |j                   D cg c]  }d|z  	 }}d|d<   |j"                  j%                  |       |j&                  j%                          t        |j"                  j(                  d   d       t        j*                  |j"                  j                  |d      }|j-                  |d      }t/        |d   |d   d       y c c}w c c}w )Nr   var%4dconstr   trueindexcolumnsrF   oracler   
pen_weightr!   
   maxiterxname	convergedTfinalrE   nameouterhowh㈵>atol)r)   r*   r+   r?   lenpd	DataFramer   fitparamsr   onesr   screen_exogr   sortidx_nonzeroresults_finalsummaryresults_penmle_retvalsSeriesjoinr   r>   r9   r;   r:   r6   iixnames_true
parametersxframe_true
res_oraclemod_initialscreenerexog_candidates
res_screenxnamespss                   r   test_poisson_screeningru   4   s   IINN6#4#6 Aq
Dq6D+;<R8b=<K<KNd#34K'-h0J ,,q$4!45{KKK(,,.J%,,Jx"1bggdmqIK -H12hO%%or%BJ//02BC&0&<&<=hm=F=F1I $$6$2""$))55kBDI	:++22&w	OB1JJx(*W*=DI9 =" >s   G88G=c                     t         j                  j                  d       d\  t         j                  j                  dz
        dt         j                  j                  d      dz
  z  z   dz  dz
  } | dz  } | | j	                  d      z
  | j                  d      z  } t        j                  t        j                        | f      } dt        j                  ddz         z  }t        j                  |      }| j                  |      }t        j                  |      }t         j                  j                  |      }| d d dd f   j                  d      d d d f   | fd	}t        j                        }d|d dz   t        j                  t        j                        |f      }d
D ]  }t        ||d d d |f   dz        }	t        |	      }
d|
_        |
j#                   |             }g d}t%        |j&                  |       t        j(                  ddgddgddgddggt         j*                        }t%        |j,                  |        y )Nr   )r   r   r!   r    r"   r#   r$   r   c               3   p  K   d} d}t        |      D ]  }dz  t        j                  j                  |       z   dt        j                  j                  d      dz
  z  z   dz  dz
  }|dz  }|dz
  k  rd d |dz   f   |d d d	f<   ||j	                  d
      z
  |j                  d
      z  }|  y w)Nr      g?r    r!   r"   r#   r$   rK   r   )ranger)   r*   r,   r-   r.   )r7   	n_batchesir9   commonr8   r6   	x_nonzeros       r   exog_iteratorz+test_screen_iterated.<locals>.exog_iteratoro   s     	y! 	Af!==biinnT1-34589:<=>AHA9q= $QAX.!R%QVVAY!%%(*AG	s   B3B6r!   r#   r   rI      )var0_10var1_10var2_10var3_10rK      )dtype)r)   r*   r+   r,   r-   r.   column_stackr_   r0   r1   r2   r3   r4   sumr   r   	k_max_addscreen_exog_iteratorr   exog_final_namesarrayint64idx_nonzero_batches)r9   r:   r<   r=   r>   r~   dummy	exog_keepkro   rp   rQ   namesidx_fullr|   r8   r6   r}   s                 @@@@r   test_screen_iteratedr   Z   s   IINN6OD)	i!m	,	biinnT1%+	,
-01	245	6AHA	
QVVAY!%%("A
*+A		!Y]++D774=DeeDkG	B
		"Aq!"uX\\!_QW%FI GGDMEE*419 67I :&q)ArrE*:tczR$[1--mo><U++U388q"X "X "X "X' /1hh8 	U..9:r   c                     t               \  } }}}t        |       }|D cg c]  }d|z  	 }}d|d<   t        j                  ||   |dg      }t        j                  |d d |f   |      }t	        | |t        j                               j                         }	|	j                  |d<   t	        | t        j                  |      t        j                               }
t        |
      }|d d d	d f   }|j                  |d
      }t        t        j                  |j                        |       |j                  D cg c]  }d|z  	 }}d|d<   |j                   j#                  |       |j$                  j#                          t        |j                   j&                  d   d       t        j(                  |j                   j                  |d      }|j+                  |d      }t-        |d   |d   d       y c c}w c c}w )NrA   rB   r   rC   rD   rG   r   rH   r!   rK   rL   rN   rP   TrQ   rR   rT   rU   rW   rX   )r?   rZ   r[   r\   r   r	   r   r]   r^   r)   r_   r   r`   r   ra   rb   rc   rd   re   rf   rg   rh   r   ri   s                   r   test_glmpoisson_screeningr      s   #4#6 Aq
Dq6D+;<R8b=<K<KNd#34KRXQYZJ,,q$4!45{KKaV^^5EFJJLJ%,,Jxq"''$-8HIK -H12hO%%or%BJ//02BC&0&<&<=hm=F=F1I $$6$2""$))55kBDI	:++22&w	OB1JJx(*W*=DI7 =  >s   G<<Hc                     t         j                  j                  d       d\  } }d}t         j                  j                  | |      dt         j                  j                  | d      dz
  z  z   dz  dz
  }|dz  }||j	                  d	      z
  |j                  d	      z  }d|d d d	f<   t        j                  |      }g d
}dt        j                  d|dz         dz  z  ||<   t        j                  |      }|j                  |      }ddt        j                  |       z   z  }t         j                  j                  t        |            |k  j                  t              }||||fS )Nr   )r&   r   r   {Gz?r!   r"   r#   r$   r   r%   r    )r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   rZ   astypeintr5   s	            r   _get_logit_datar      s;   IINN6LD&I	f	%	a(3.	/
034	578	9AHA	
QVVAY!%%("AAadG88FD."))Ay1}"=s"BBD	774=DeeDkG	
a"&&'""	#B	B	 2	%--c2Aa!4''r   c                     t               \  } }}}t        |       }t        |dz  d      }|D cg c]  }d|z  	 }}d|d<   t        j                  ||   |dg      }t        j                  |d d |f   |	      }	t        | |	      j                         }
|
j                  |d
<   t        | t        j                  |      |dz        }t        |fi |}d|_        |d d dd f   }|j                  |d      }t        j                  |j                  j                        dkD  }t!        t        j"                  |j$                  |         |       t        j&                  g d      }t!        t        j"                  |j$                        |       |j$                  D cg c]  }d|z  	 }}d|d<   |j                  j)                  |       |j*                  j)                          t!        |j                  j,                  d   d       t        j.                  |j                  j                  |d      }||d<   t1        |d
   |d   d       y c c}w c c}w )Ngffffff?MbP?)rJ   threshold_trimrA   rB   r   rC   rD   rG   rH   r"   rI   r   r!   rL   皙?)r   J   r      r&   r'   r(   rN   rP   TrQ   rR   {Gzt?rX   )r   rZ   dictr[   r\   r   r]   r^   r   r)   r_   r   r   r`   absrc   r   ra   rb   r   rd   re   rf   rg   r   )r>   r9   r;   r:   r6   screener_kwdsrj   rk   rl   rm   rn   ro   rp   rq   rr   maskidx_rrs   rt   s                      r   test_logit_screeningr      s    #2#4 Aq
Dq6DD3JtDM+;<R8b=<K<KNd#34K'-h0J ,,q$4!45{KKq+&**,J%,,Jx BGGDMdSjIK >>HH12hO%%or%BJ 66***112S8D//568HIHH56E//0%8&0&<&<=hm=F=F1I $$6$2""$))55kBDI	:++22&w	OB JwJx(*W*=EJG =, >s   I Ic                      t               \  } }}}t        |       }t        |dz  dd      }|D cg c]  }d|z  	 }}d|d<   t        j                  ||   |dg	      }t        j                  |d d |f   |
      }	t        | |	t        j                               j                         }
|
j                  |d<   t        | t        j                  |      t        j                               }t        |fi |}d|_        |d d dd f   }|j                  |d      }|j                   |j                    |j                  D cg c]  }d|z  	 }}d|d<   |j                   j#                  |       |j$                  j#                          t'        |j                   j(                  d   d       t        j*                  |j                   j                  |d      }||d<   t-        |d   |d   d       y c c}w c c}w )N      ?r   model.score_factorrJ   r   ranking_attrrA   rB   r   rC   rD   rG   r   rH   rK   r!   r   rL   rN   rP   TrQ   rR   r   rX   )r   rZ   r   r[   r\   r   r	   Binomialr]   r^   r)   r_   r   r   r`   rb   rc   rd   re   r   rf   rg   r   )r>   r9   r;   r:   r6   r   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   s                    r   test_glmlogit_screeningr      s   #2#4 Aq
Dq6D D4K&:<M ,<<R8b=<K<KNd#34K'-h0J ,,q$4!45{KKaV__5FGKKMJ%,,Jx q"''$-8IJK >>HH12hO%%or%BJ&0&<&<=hm=F=F1I $$6$2""$))55kBDI	:++22&w	OB JwJx(*W*=EJE =* >s   G6G;c                  t   t         j                  j                  d       d\  } }d}t         j                  j                  | |      dt         j                  j                  | d      dz
  z  z   dz  dz
  }|dz  }||j	                  d	      z
  |j                  d	      z  }d|d d d	f<   t        j                  |      }g d
}dt        j                  d|dz         z  ||<   t        j                  |      }|j                  |      }|dt         j                  j                  t        |            z  z   }||||fS )Nr   r   r   r   r!   r"   r#   r$   r   )r   r!   r&   r'   r(   r    r   )r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   randnrZ   )r6   r7   r8   r9   r:   r;   r<   r>   s           r   _get_gaussian_datar   "  s   IINN6LD&I	f	%	a(3.	/
034	578	9AHA	
QVVAY!%%("AAadG88FD,"))Ay1}"==D	774=DeeDkG#		G555Aa!4''r   c                  \   t               \  } }}}t        |       }| | j                  d      z
  } t        |dz  dd      }|D cg c]  }d|z  	 }}d|d<   t	        j
                  ||   |dg	      }t	        j
                  |d d |f   |
      }	t        | |	t        j                               j                         }
|
j                  |d<   dD ]O  }t        | |d d d |f   t        j                               }t        |fi |}|d d |d f   }|j                  |d      }t        t        j                  |j                         |       |j                   D cg c]  }d|z  	 }}d|d<   |j"                  j%                  |       |j&                  j%                          t        |j"                  j(                  d   d       t	        j*                  |j"                  j                  |d      }|j-                  |d      }t/        |d   |d   d       |d= R y c c}w c c}w )Nr   r   r   r   r   rA   rB   rC   rD   rG   r   rH   r   r   rL   rN   rP   TrQ   rR   rT   rU   gh㈵>rX   )r   rZ   r-   r   r[   r\   r   r	   Gaussianr]   r^   r   r`   r   r)   ra   rb   rc   rd   re   rf   rg   rh   r   )r>   r9   r;   r:   r6   r   rj   rk   rl   rm   rn   k_keepro   rp   rq   rr   rs   rt   s                     r   test_glmgaussian_screeningr   7  s   #5#7 Aq
Dq6D	AFF1IA D4K&:<M ,<<R8b=<K<KNd#34K'-h0J ,,q$4!45{KKaV__5FGKKMJ%,,Jx  "1a7F7
mFOO<MN$[BMBAvwJ-))/2)F
RWWZ3346FG*4*@*@AB(R-AAq	 	  ((v(6&&(Z--99+FMYYz//66f7S__RW_5

8,j.AMw+  =" Bs   H$H))__doc__numpyr)   numpy.testingr   r   pandasr[   #statsmodels.discrete.discrete_modelr   r   +statsmodels.genmod.generalized_linear_modelr   statsmodels.genmod.familiesr	   statsmodels.base._penalizedr
   statsmodels.base._screeningr   r   r   r   r?   ru   r   r   r   r   r   r   r   r   r   r   <module>r      s     7  > ; . 6 9	~w 		^U 		>3 	(*#JL2:j JF(**KZ+K\(*) r   