e1ddfd?ZEy)BzUtilities for data generation.N) dataclass) TYPE_CHECKINGAnyCallableDict GeneratorList NamedTupleOptionalTupleTypeUnion)request)typing)r)sparse)pandas_pyarrow_mapper) DataFramejoblibz ./cachedir)verbose n_samples n_featuresreturnc#Kddl}tjjd}|j dd||zj ||}tj tjtjtjtjtjtjtjtjtjtj tj"tj$tj&tj(tj*tj,tj.tj0tj2g}|D]A}tj4||}||f|j7|j7fC|D]A}tj4||}|j9|}|j9|} || fC|j;dd||z j ||}tj<t>fD]}tj4||}||f!tj<t>fD]A}tj4||}|j9|}|j9|} || fCyw) z*Enumerate all supported dtypes from numpy.rNlowhighsizedtypeg?r ) pandasnprandom RandomStaterandintreshapeint32int64byteshortintcint_longlonguint32uint64ubyteushortuintcuint ulonglongfloat16float32float64halfsingledoublearraytolistrbinomialbool_bool) rrpdrngorigdtypesr"Xdf_origdfs XC:\Crop_Prediction\Backend\crop-ai-system\venv\Lib\site-packages\xgboost/testing/data.py np_dtypesrL's ))   %C ;;13Y-C; D L L: D                    )F,( HHT 'Ag kkmQXXZ''(  HHT ',,t$ \\!_rk  <<3Y%;< < D D: D((D! HHT 'Ag ((D! HHT ',,t$ \\!_rk sJ?Kc #Kddl}|j|j|j|j |j |j |j|jg}tj}|jdd|dgdd|dgdtj}tjd|jfD]-}|D]&}|jdd|dgdd|dgd|}||f(/tj}|j|jg}|jd d |d gd d |d gdtj}tjd|jfD]m}|D]f}|jd d |d gd d |d gd|}||f|d }|d }t!||j"sJt!||j"sJ||fho|j%d }tjd|jfD]4}|jd d |d gd d |d gd|j'}||f6d|jfD]i}dd|dgdd|dgd}|j||tj(n|j+}|j||j+}||fkyw)z/Enumerate all supported pandas extension types.rNr#rf0f1r!?g@g@rQcategoryTF)r% UInt8Dtype UInt16Dtype UInt32Dtype UInt64Dtype Int8Dtype Int16Dtype Int32Dtype Int64Dtyper&nanrr:NA Float32Dtype Float64Dtype isinstanceSeriesastypeCategoricalDtyperB BooleanDtype) rDrGNullrFr"rJser_origserdatas rK pd_dtypesrjbs         F%'FFD <<1dAq!T1o6bjj  Druu% E1dAq!T1o>eB(N   66Doo!2 3F <<S$$S#tS,AB"**  Druu%   ES$,S#tS4IJRWB(NDzHT(Cc299- -h 2 2C-     ;;z "Druu% \\dC(c40E F%%' Bh ruu UD$/tT47PQ||DDLbooFW|X \\$boo&7\ 8Bh sKKc #Kddl}ddl}t}d|jdfD]}|D]}|j ds|j dr&|j |s|dk(r|nt j}|jdd|dgdd|dgd t j }|jdd|dgdd|dgd | }||f|jd fD]o}|jd d |d gd d |d gd |j }|jd d |d gd d |d gd |j|j }||fqyw) z*Pandas DataFrame with pyarrow backed type.rNr9rCr#rrNrOrPr!FT) r%pyarrowrr^ startswithisnar&r]rr:re ArrowDtyperB)rDparGrfr" orig_nullrFrJs rKpd_arrow_dtypesrrs#F.ruua  E *e.>.>v.F$&GGDMdaiRVVI<<1i+Aq)Q3GHjj D 1dAq!T1o>eB(N " ||%t,UD$4M N//# \\%t,UD$4M N-- + Bh sEErEc|jdjdd}|jd}tj|d<t j t d5tj||dddt j t d5tj||dddy#1swYDxYw#1swYyxYw) zValidate there's no inf in X. r$rO)rzInput data contains `inf`matchN) r'r*r&infpytestraises ValueErrorxgboostQuantileDMatrixDMatrix)rErHys rK check_infrs  ##Aq)A  AffAdG z)D E&1%& z)D E1&&s#B=C =C Cc|tjd}|j}|j|jfS)z2Fetch the California housing dataset from sklearn.sklearn.datasets)rz importorskipfetch_california_housingritargetdatasetsris rKget_california_housingrs6""#56H  , , .D 99dkk !!c|tjd}|j}|j|jfS)z&Fetch the digits dataset from sklearn.r)rzr load_digitsrirrs rK get_digitsrs6""#56H    !D 99dkk !!rcPtjd}|jdS)z-Fetch the breast cancer dataset from sklearn.rT) return_X_y)rzrload_breast_cancer)rs rK get_cancerrs)""#56H  & &$ & 77rctjd}tjj d}d}d}|j ||\}}|j d||j}t|jdD]<}t|jdD]}|||fs tj|||f<!>||fS)zGenerate a sparse dataset.rig?) random_stater#r) rzrr&r'r(make_regressionrAshaperanger]) rrEnsparsityrHrflagijs rK get_sparsers""#56H ))   $C AH  # #AC # 8DAq <<8QWW -D 1771: !qwwqz" !AAqDz&&!Q$ !! a4Krc tjdddltjj ddj }dttttftfdtdjffd }|d d d d ddd|d<|ddddd|d<|dddddd|d<|ddd d!d"d#d$d%d|d&<|d'd(d)d!d*d+|d,<|d-d(d.d/d0d"d1d2d3d|d4<|d5d6d7d8d9d:d;|d<<|d=d>d?d@d$dAd|dB<|dCdDd d"dEd|dF<|d@dGdGdHdI|dJ<dKtdLtdtdjffdM }|dNdOd|dP<|dQdRd|dS<|dTdUd|dV<|dWdXd|dY<|dZd[d|d\<|d]d^d|d_<|d`dad|db<|dcddd|de<|dfdgd|dh<|didjd|dk<t|j}j|||}tjfl}|jD]q}t!||j"j$r:|||j&j(j+tj,z }`|||j.z }s|dm|j1z z}|dn|j3z z }||fS)oamGet a synthetic version of the amse housing dataset. The real one can be obtained via: .. code-block:: from sklearn import datasets datasets.fetch_openml(data_id=42165, as_frame=True, return_X_y=True) Number of samples: 1460 Number of features: 20 Number of categorical features: 10 Number of numerical features: 10 r%rNri name_probadensityrc t d|z z}tjd|z dkDxr|dkD}|rd|z }||tj<t |j }t |j }|dxxdtj|z z cc< j| |} j| jtd|}|S) Nr#rSư>r)r pc"t|tSN)rastr)xs rKz5get_ames_housing..synth_cat..:s As!3rr!) intr&absr]listkeysvaluessumchoicerbrdfilter) rrn_nullshas_nanrrrrseriesrrDrEs rK synth_catz#get_ames_housing..synth_cat(si1w;/0&&w'$.>7Q; W}H!)Jrvv JOO%& ""$ % "rvvay  JJt)qJ 1 %%3T:  rgqu]?gqh.?gsmB.synth_nums} JJ3c J :i1w;/0 66#- 4 'GaKzz)'5zIH&&AhKyy"**y--rgmtF @gOfKhttps://s3-us-west-2.amazonaws.com/xgboost-examples/MQ2008.zipz MQ2008.zip)urlfilenamer)pathNzMQ2008/Fold1/train.txtzMQ2008/Fold1/test.txtzMQ2008/Fold1/vali.txtTF)query_id zero_based) rzrosrjoinexistsr urlretrievezipfileZipFile extractallload_svmlight_files)rrsrcrfx_trainy_train qid_trainx_testy_testqid_testx_validy_valid qid_valids rK get_mq2008rs#""#56H JC WW\\% .F 77>>& !f5  %! %  ! $$ GGLL 8 9 GGLL 7 8 GGLL 7 8  %     /!!s D%%D.Fr) vary_sizern_samples_per_batch n_batchesuse_cupyrrcg}g}g}|r ddl} | jj|} ntjj|} t |D]x} |r|| dzzn|} | j | |} | j | }| j dd| }|j| |j||j|z|||fS)zMake batches of dense data.rN r#r)cupyr'r(r&rrandnuniformappend)rrrrrrrHrwrrErr_X_y_ws rK make_batchesr's A A Akk%%l3ii##L1 9 4='!b&0CV YYy* - YYy ! [[QQY[ 7       a7NrcHeZdZUdZej ed<eje jed<eje jed<eje jed<eje jed<eje jed<y) ClickFoldzCA structure containing information about generated user-click data.rHrqidscoreclickposN) __name__ __module__ __qualname____doc__r csr_matrix__annotations__nptNDArrayr&r+r:r,rrKr)r)spM  {{288 RXX  ;;rzz "" ;;rxx  RXX rr)c<eZdZUdZeed<eed<eed<defdZy) RelDataCVzPSimple data struct for holding a train-test split of a learning to rank dataset.traintestmax_relrc |jdk(S)z6Whether the label consists of binary relevance degree.r#)r;)selfs rK is_binaryzRelDataCV.is_binary2s||q  rN) r.r/r0r1RelDatar3rrCr>r6rrKr8r8+s Z N M L!4!rr8ceZdZdZdeddfdZdejejdejejdejejfdZ y) PBMaSimulate click data with position bias model. There are other models available in `ULTRA `_ like the cascading model. References ---------- Unbiased LambdaMART: An Unbiased Pairwise Learning-to-Rank Algorithm etarNctjgd|_tjgd}tj|||_y)N)rg{Gz?Q?gp= ף?rS) g(\?gQ?gQ?g(\?rDg?g)\(?rg{Gz?gQ?)r&r? click_probpower exam_prob)r=rBrGs rK__init__z PBM.__init__As8((#?@HH H )S1rlabelspositionctj|d}tj|j}d||dk<d||t |j k\<|j |}tj|j}|j |j k(sJtj|d}d|||jj k\<|j|}tjjd}|j|jdtj}tj|jtj}d||||zk<|S) zSample clicks for one query based on input relevance degree and position. Parameters ---------- labels : relevance_degree T)copyrrr)r r"r!r#) r&r?rrlenrEr rGr'rr:r+) r=rIrJrErGranksrEprobclickss rKsample_clicks_for_queryzPBM.sample_clicks_for_queryJs&t,XXfll+ vz13vT__--.__V, HHV\\* }} ++-.0et~~***+NN5) ii##D)zzv||AbjjzA(*RXX(N01ti*,,- r) r.r/r0r1rrHr4r5r&r+r,rQr6rrKrArA7s^2E2d2!kk"((+!7:{{2887L! RXX !rrArc tj|}|j}tjdtjtj |dd|ddddzf}tj tj||f}||}tj|tj|jg}|||fS)zzRun length encoding using numpy, modified from: https://gist.github.com/nvictus/66627b580c13068589957d6ab0919e66 rr#NrT) equal_nan) r&asarrayr r_ flatnonzeroisclosediffr"r?)rrstartslengthsrindptrs rKrlencoder\ns 1 A A UU1bnnbjj12#2$&O%OPSTTT UFggbeeFAI&'G vYF YYvrxx1 2F 7F ""rrHrr* sample_ratectjjd}t|jd|z}tj d|jdtj }|j||d|}||}||}||} tj| } || }|| }| | } tjdd} | j||| | j|} | S) zWe use XGBoost to generate the initial score instead of SVMRank for simplicity. Sample rate is set to 0.1 by default so that we can test with small datasets. rrr!Nz rank:ndcghist) objective tree_method)r*) r&r'rrraranger3rargsortr} XGBRankerfitpredict) rHrr*r]rErindexX_trainrr sorted_idxltrscoress rKinit_rank_scorerl}s ))   %CAGGAJ,-I IIa299 5EKK *9 EhGhGE II&Jj!Gj!G*%I   kv FCGGGW)G,[[^F Mrfold scores_foldc4|\}}}|jtjk(sJtj|}tj|j ftj }tj|j ftj}td}|D]f} | |k(} | j| jd} || } tj| ddd} | || <|| } |j| | }||| <h|jd|jdk(sJ|j|jf|jd|jdk(sJ|j|jft||||||S)zSimulate clicks for one fold.r!rS)rBrNr) r"r&r+uniqueemptyr r,rAr*rrcrQr))rmrnX_foldy_foldqid_foldqidsrJrPpbmqqid_mask query_scoresquery_positionrelevance_degrees query_clickss rKsimulate_one_foldr}sf $FFH >>RXX %% 99X Dxxbhh7H XXv{{nBHH 5F #,C (=##HNN1$56"8, L1$B$7+"8,223DnU 'x ( <<?hnnQ/ /O&,,1OO <<?fll1o -K fll/KK VVX{FH MMrcv_datac dtt|j|j\}}}t j dg|Dcgc]}|j dc}z}t j|}t|dk(sJtj|}t j|}t j|}t|||} td|jD cgc]} | || dz || } } ggggggf\t|jdz D]} t|| || || f| | } j!| j"j!| j$j!| j&j!| j(j!| j*j!| j,t|jdz D cgc]} |  } } tdD]} | | | | k(j/rJtdk(r(t1dddddd}d}||fSfdttD\}}||fScc}wcc} wcc} w)z6Simulate click data using position biased model (PBM).rrNr#rNc 3bK|]&}t||||||(ywr)r)).0rX_lstc_lstp_lstq_lsts_lsty_lsts rK z"simulate_clicks..s@  eAha%(E!HeAha Q s,/)rzipr9r:r&r?rcumsumrMrvstack concatenaterlrr r}r"rHrr*r+r,r-allr))r~rHrr*vr[X_fully_fullqid_full scores_fullrrkrmscores_check_1r9r:rrrrrrs @@@@@@rKsimulate_clicksrsrS 56IAq#XXqc3AQWWQZ33 4F YYv F v;%  ]]1 F ^^A F~~c"H"&&(;K>CAv{{>S Tk&Q-&)4 TF T/12r2r2/E,E5%u 6;;? # !A$!c!f!5vayA TVV TVV TXX TZZ  TZZ  TXX).fkkAo(>?1eAh?N? 1X6q!VAY.33556 5zQ%(E!HeAha%(ERSHU $;   3u:& t $;G4U@s J# 1J( J-rPr-c tj|}||}||}||}||}t|\}}}td|jD]}||dz } ||} | | ks J| | ftj || | jdk(s J| | f|| | } | j dk(sJ| j | j| jdz k\s9J| j| j|tj || | ftj| }|| | ||| | || | ||| | || | ||| | || | ||| | ||||f} | S)z,Sort data based on query index and position.r#r)r&rcr\rr rpminmax) rHrr*rPr-rir[_rbegend query_posris rKsort_ltr_samplesrsCJ * A J F j/C j/CC=LFAq 1fkk "0QUmQiSy$3*$yyS&++q0<3*<CL }}!#4Y]]_4}})..1"44 MMO NN IIc#cl # 7  ZZ * s3Z +#c  S/*5s3s3Z +#c 3s|J/C +0. fa D KrDTypeDMatrixTdevicectjj}||jdddj tj j dd}t|dr|jdddf}n |dddf}|}|||| }tjtd 5tjd |d |dddt|ds||jj dd}||k(jsJ|j j"j$dusJ|j j"j&dusJ|j)|j ||jj dd}||j k(jsJ|}|j+||j} |j+|j d|j,|j} | | k(jsJ|j tj.}|j+||j} | | k(jsJ|j dddd}tjtd 5|j+|dddyy#1swYxYw#1swYyxYw)zRun tests for base margin.rrSdr$2rilocN) base_marginz.*base_margin.*rwr_)rarFTr#rv)r&r'rrrcr:r*hasattrrrzr{r|r}r9get_base_marginrTflags c_contiguous f_contiguousset_infoset_base_marginr r;) rrrrErHrrXygotbm_colbm_rowbm_f64s rKrun_base_margin_infor s ))   !C cjjCcj*11"**=EEb!LMAq& FF1a4L adGK !QK 0B z); <E f?DE 1f B&&(00Q78{"''))}}""//588}}""//477 .B&&(00B78{}}$))++  ;'##% ;..q+2B2BCD##%& %%''"((4  ;'##%& %%''ii1a+ ]]:-? @ ,   { + , ,7 EE< , ,s7KK*K'*K3)F)r)Fr1rr  dataclassesrrrrrrrr r r r r rurllibrnumpyr&rzr4 numpy.randomRNGscipyrr} xgboost.datarcompatr DataFrameTrrMemorymemoryrndarrayrLrjrrrcacherrrrrrr2rrCr'r5r+r?r)r8rAr\rr:rlr}rr,rrrr6rrKrs.$ !     ).0J   X & |Q /88 #8uRZZ+,dD898v<9<~77t 3 4 "bjj"**&< ="""E"**bjj01""8E"**bjj0188  E"**bjj01   q% BJJ 67qqh5 5  JJJJ JJJJ JJJJ 55x      4 T"**-tBJJ/? ?@< !!3;;rxx#8#++bhh:OO P   ! !44n # BHH% #% S[[#++0U*V #& ! ! {{288! RXX ! !  [[ !HN !!3;;rxx#8#++bhh:OO PNRZZ(NNF(Y(5HYA-, -,r