+
    Ú.Gj')  ã                   ó‚  € R t ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIt^ RI	t
^ RIHt ]P                  ! ]P                  RR7       ]P                  ! ]4      t^^ ^^ ^^ ^	^ ^
^^^^^^^^^^^^^^^/tRtR R ltR	 R
 ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R lt]R8X  d
   ]! 4        R# R# )u½  
Feature engineering pipeline.

Produces data/processed/final_feature_matrix.csv â€” the single training
table every model in ml_pipeline/training/ reads from. Also produces
data/processed/climate_clean.csv (already created by clean_climate_data.py,
read directly by the live API's FeatureBuilder for historical averages).

This script must be re-run any time the raw source data is refreshed
(new IMD/ICRISAT/Agmarknet export). Models trained on a stale feature
matrix will silently drift from what the live system computes at
inference time â€” keep this pipeline run as part of your retraining
checklist, not a one-time setup step.

Usage:
    python -m ml_pipeline.feature_engineering.build_feature_matrix         --climate data/processed/climate_clean.csv         --crop data/processed/crop_clean.csv         --market data/processed/market_clean.csv         --crop-lookup data/lookup/crops.json         --output data/processed/final_feature_matrix.csv
N)ÚPath)Ústatsz%%(asctime)s %(levelname)s %(message)s)ÚlevelÚformatiÝ  c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )é   ÚclimateÚreturn©ÚpdÚ	DataFrame)r   s   "ÚaC:\Crop_Prediction\backend\crop-ai-system\ml_pipeline\feature_engineering\build_feature_matrix.pyÚ__annotate__r   0   s"   € ÷ (ñ (¤"§,¡,ð (´2·<±<ñ (ó    c                 ó2  € V P                  . RO4      P                  RR7      p W R,          \        8*  ,          P                  R R.4      P	                  RRR7      P                  4       p\
        P                  ! WR R.RR7      p V R,          P                  4       pV R	,          P                  V4      V R	&   V R
,          P                  V R,          P                  4       4      V R
&   V R
,          P                  ^ ^4      V R
&   V R,          V R	,          ,
          V R
,          ,          V R&   V P                  R 4      R,          P                  ^4      V R&   V P                  R 4      R,          P                  R 4      V R&   V R,          P                  V R,          4      V R&   V R,          P                  \        4      V R&   RV P                  9  d   \         P#                  R4       RV R&   RV P                  9  d   \         P#                  R4       RV R&   V # )ÚdistrictÚyearÚmonthT)ÚdropÚrainfall_mm)Úhist_mean_rainfallÚhist_std_rainfallÚleft©ÚonÚhowr   r   Úrainfall_anomalyÚrainfall_lag1c                 óD   € V P                  ^^R7      P                  4       # )é   ©Úmin_periods)ÚrollingÚmean©Úxs   &r   Ú<lambda>Ú&add_climate_features.<locals>.<lambda>H   s   € ˜QŸY™Y q°a˜YÓ8×=Ñ=Ô?r   Úrainfall_ma3Úseason_indexÚtemp_cuL   No temp_c column in climate data â€” using a flat 30.0C default for all rowsg      >@ÚhumidityuM   No humidity column in climate data â€” using a flat 70.0 default for all rowsg     €Q@©r   r   r   )r   r#   )r   Ústd)Úsort_valuesÚreset_indexÚHISTORICAL_CUTOFF_YEARÚgroupbyÚaggr   Úmerger#   Úfillnar-   ÚreplaceÚshiftÚ	transformÚmapÚ
SEASON_MAPÚcolumnsÚloggerÚwarning)r   ÚhistÚfallback_means   &  r   Úadd_climate_featuresr?   0   s÷  € Ø×!Ñ!Ò"?Ó@×LÑLÐRVÐLÓW€Gà˜6•?Ô&<Ñ<Õ=×EÑEØ	�WÐóç	�cØ2Ø0ð 
ó ÷ �kƒmð 	ô �hŠh�w¨*°gÐ)>ÀFÔK€Gà˜MÕ*×/Ñ/Ó1€MØ$+Ð,@Õ$A×$HÑ$HÈÓ$W€GÐ Ñ!Ø#*Ð+>Õ#?×#FÑ#FÀwÈ}ÕG]×GaÑGaÓGcÓ#d€GÐÑ Ø#*Ð+>Õ#?×#GÑ#GÈÈ1Ó#M€GÐÑ ð 
�Õ	 'Ð*>Õ"?Õ	?À7ÐK^ÕC_Õ_ð ÐÑð  'Ÿ™¨zÓ:¸=ÕI×OÑOÐPQÓR€GˆOÑà�‰˜
Ó# MÕ2ß	‰Ñ?Ó	@ð ˆNÑð  ' Õ7×>Ñ>¸wÀ}Õ?UÓV€GˆOÑà% gÕ.×2Ñ2´:Ó>€GˆNÑð
 �w—‘Ô&Ü�‰ÐeÔfØ ˆ�ÑØ˜Ÿ™Ô(Ü�‰ÐfÔgØ"ˆ�
Ñà€Nr   c                óD   € V ^8„  d   QhR\         R\        P                  /# )r   Úlookup_pathr	   )r   r   r   )r   s   "r   r   r   [   s   € ÷ ñ ¤dð ¬r¯|©|ñ r   c                 ó>  € \        V R R7      ;_uu_ 4       p\        P                  ! V4      pRRR4       . pXP                  4        F+  w  rERV/pVP	                  V4       VP                  V4       K-  	  \        P                  ! V4      #   + '       g   i     Lg; i)z	utf-8-sig)ÚencodingNÚcrop)ÚopenÚjsonÚloadÚitemsÚupdateÚappendr   r   )rA   ÚfÚ
crops_dictÚrowsrD   ÚpropsÚrows   &      r   Úload_crop_propertiesrP   [   sz   € ô
 
ˆk K×	0Õ	0°AÜ—Y’Y˜q“\ˆ
÷ 
1à€DØ!×'Ñ'Ö)‰ˆØ�tˆnˆØ�
‰
�5ÔØ�‰�CÖñ *ô �<Š<˜ÓÐ÷ 
1×	0ús   –BÂB	c                óD   € V ^8„  d   QhR\         P                  R\        /# )r   Úseriesr	   )r   ÚSeriesÚfloat)r   s   "r   r   r   j   s   € ÷ ñ œ2Ÿ9™9ð ¬ñ r   c                 óæ   € V P                  4       P                  p\        V4      ^8  d   R# \        P                  ! \        V4      4      p\
        P                  ! W!4      w  p   p\        V4      # )r   ç        )ÚdropnaÚvaluesÚlenÚnpÚaranger   Ú
linregressrT   )rR   rX   r%   ÚslopeÚ_s   &    r   Ú_compute_sloper_   j   sX   € Ø�]‰]‹_×#Ñ#€FÜ
ˆ6ƒ{�Q„ÙÜ
�	Š	”#�f“+Ó€AÜ×(Ò(¨Ó3Ñ€Eˆ1ˆa��AÜ�‹<Ðr   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r   rD   Ú
crop_propsr	   r
   )r   s   "r   r   r   s   s-   € ÷ ñ œBŸL™Lð ´b·l±lð ÄrÇ|Á|ñ r   c                 óÞ  € \         P                  ! WR RR7      p W R,          P                  4       ,          R ,          P                  4       p\	        V4      ^ 8”  d#   \
        P                  R\        V4       R24       V P                  . RO4      p V P                  RR .4      R,          P                  R 4      P                  ^ ^.R	R
7      pW0R&   V R,          P                  R4      V R&   V # )rD   r   r   Úwater_req_mmz@Crops present in yield data but missing from crops.json lookup: zu. Add them to data/lookup/crops.json or these rows will have null agronomic features and get dropped before training.r   Úyield_ton_hac                 óR   € V P                  ^^R7      P                  \        RR7      # )é   r    F©Úraw©r"   Úapplyr_   ©Úss   &r   r&   Ú#add_crop_features.<locals>.<lambda>�   ó!   € ˜Ÿ™ 1°!˜Ó4×:Ñ:¼>ÈuÐ:ÔUr   T©r   r   Úyield_trend_sloperV   )r   rD   r   )r   r3   ÚisnullÚuniquerY   r;   r<   Úlistr.   r1   rj   r/   r4   )rD   ra   Úmissing_propsÚtrend_slopess   &&  r   Úadd_crop_featuresrv   s   sâ   € Ü�8Š8�D¨°VÔ<€Dà˜nÕ-×4Ñ4Ó6Õ7¸Õ?×FÑFÓH€MÜ
ˆ=Ó˜AÔÜ�‰ØNÜ�MÓ"Ð#ð $VðWô	
ð ×ÑÒ8Ó9€Dà�‰�j &Ð)Ó*¨>Õ:ß	‰ÑUÓ	Vß	‰˜A˜q˜6¨ˆÓ	-ð ð
 !-Ð	ÑØ $Ð%8Õ 9× @Ñ @ÀÓ E€DÐ	Ñà€Kr   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r   Úmarketr	   r
   )r   s   "r   r   r   Š   s"   € ÷ 
ñ 
¤§¡ð 
´·±ñ 
r   c                 ó&  € V P                  . R	O4      p V P                  R R.4      R,          P                  ^4      V R&   V P                  R R.4      R,          P                  R 4      P	                  ^ ^.RR7      pVP                  R4      V R&   V # )
r   rD   Úprice_per_quintalÚ
price_lag1c                 óR   € V P                  ^^R7      P                  \        RR7      # )é   r    Frg   ri   rk   s   &r   r&   Ú%add_market_features.<locals>.<lambda>�   rn   r   Tro   rV   Údemand_trend©r   rD   r   r   )r.   r1   r6   rj   r/   r4   )rx   r   s   & r   Úadd_market_featuresr�   Š   s”   € Ø×ÑÒ EÓF€FØ!Ÿ>™>¨:°vÐ*>Ó?Ð@SÕT×ZÑZÐ[\Ó]€Fˆ<Ñð 	�‰˜
 FÐ+Ó,Ð-@ÕAß	‰ÑUÓ	Vß	‰˜A˜q˜6¨ˆÓ	-ð ð
 *×0Ñ0°Ó5€Fˆ>ÑØ€Mr   c                óD   € V ^8„  d   QhR\         P                  R\        /# ©r   rO   r	   ©r   rS   Ústr)r   s   "r   r   r   –   s   € ÷ ñ œ2Ÿ9™9ð ¬ñ r   c                 óè  € V R ,          V R,          u;8*  ;'       d    V R,          8*  Mu pV R,          V R,          ^,          8¬  p\        V R,          4      R8„  pV P                  R^ 4      ^ 8„  pV'       d   V'       d   V'       g   R# V'       d$   V R,          V R,          ^,          R	,          8  d   R
# V'       d   V'       g   V'       d   R# V'       g   V R,          '       g   V'       g   R
# R# )Ú
min_temp_cr*   Ú
max_temp_cr   rc   r   g      ø?rd   Úsuitableçš™™™™™¹?Únot_suitableÚriskyÚdrought_tolerant)ÚabsÚget)rO   Útemp_okÚrain_okÚanomaly_badÚ	has_yields   &    r   Úlabel_feasibilityr”   –   s²   € Ø�,Õ 3 x¥=×EÔE°C¸Õ4EÔE€GØ�-Õ  S¨Õ%8¸2Õ%=Ñ>€GÜ�cÐ,Õ-Ó.°Ñ4€KØ—‘˜¨Ó*¨QÑ.€I÷ —7§;Ùß˜˜MÕ*¨c°.Õ.AÀBÕ.FÈ$Õ-NÔNÙß	Ÿ;¯7Ùß˜SÐ!3×4Ô4¿YÙÙr   c                ó0   € V ^8„  d   QhR\         R\        /# )r   r]   r	   )rT   r…   )r   s   "r   r   r   ª   s   € ÷ ñ ”uð ¤ñ r   c                 óN   € V R 8”  d   R# V R8  d   R# \        V 4      R8”  d   R# R# )rŠ   ÚupwardÚdownwardgš™™™™™©?ÚvolatileÚstablegš™™™™™¹¿)rŽ   )r]   s   &r   Úlabel_trendr›   ª   s(   € Øˆs„{ÙØ	�ŒÙÜ	ˆU‹�dÔ	ÙÙr   c                óD   € V ^8„  d   QhR\         P                  R\        /# rƒ   r„   )r   s   "r   r   r   ´   s   € ÷ ñ œ2Ÿ9™9ð ¬ñ r   c                 ót   € V R ,          R8X  d   V P                  R^ 4      ^ 8”  d   R# V R ,          R8X  d   R# R# )Úfeasibility_labelr‰   r   Úhighr‹   ÚlowÚmedium)r�   )rO   s   &r   Úlabel_suitabilityr¢   ´   s9   € Ø
ÐÕ :Ô-°#·'±'¸.È!Ó2LÈqÔ2PÙØ	Ð Õ	! ^Ô	3ÙÙr   c          
      ó¸   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  /# )r   r   rD   rx   ra   r	   r
   )r   s   "r   r   r   ¼   sI   € ÷ !ñ !”2—<‘<ð !¤r§|¡|ð !¼R¿\¹\ð !ÔWY×WcÑWcð !Ôhj×htÑhtñ !r   c                 óº  € \        V 4      p \        W4      p\        V4      p\        P                  ! W. RORR7      p\
        P                  R\        V4       R24       \        V4      ^ 8X  d   \        R4      h\        P                  ! WB. RORR7      p\
        P                  R\        V4       R	VR
,          P                  4       P                  4        R24       VP                  R4      R
,          P                  R 4      VR
&   VR,          P                  R4      VR&   VR,          P                  R4      VR&   \        V4      p. ROpVP                  VR7      p\
        P                  RV\        V4      ,
           R24       VP                  \         ^R7      VR&   VR,          P                  \"        4      VR&   VP                  \$        ^R7      VR&   V# )r   Úinnerr   zAfter climate+crop merge: z rowsz³Zero rows after merging climate and crop data. Likely cause: district name spelling mismatch, or year/month ranges that don't overlap between the two sources. Inspect both inputs.rD   r   zAfter market merge: z rows (rz   z missing market price)c                 ó@   € V P                  V P                  4       4      # )N)r4   Úmedianr$   s   &r   r&   Úbuild.<locals>.<lambda>Ï   s   € �!—(‘(˜1Ÿ8™8›:Ô&r   Úprice_volatilityrŠ   r   rV   )ÚsubsetzDropped z) rows missing required label-input fields)Úaxisrž   rp   Útrend_labelÚsuitability_labelr,   r€   )r*   r‡   rˆ   r   rc   )r?   rv   r�   r   r3   r;   ÚinforY   Ú
ValueErrorrq   Úsumr1   r7   r4   rW   rj   r”   r›   r¢   )r   rD   rx   ra   ÚmergedÚbefore_dropÚrequired_for_labelss   &&&&   r   Úbuildr´   ¼   s®  € Ü" 7Ó+€GÜ˜TÓ.€DÜ  Ó(€Fä�XŠX�dÒ(EÈ7ÔS€FÜ
‡K�KÐ,¬S°«[¨M¸Ð?Ô@ä
ˆ6ƒ{�aÔÜðJó
ð 	
ô �XŠX�fÒ)NÐTZÔ[€FÜ
‡K�KÐ&¤s¨6£{ m°7¸6ÐBUÕ;V×;]Ñ;]Ó;_×;cÑ;cÓ;eÐ:fÐf|Ð}Ô~à"(§.¡.°Ó"8Ð9LÕ"M×"WÑ"WÙ&ó#€FÐÑð "(Ð(:Õ!;×!BÑ!BÀ3Ó!G€FÐÑØ# NÕ3×:Ñ:¸3Ó?€Fˆ>Ñä�f“+€KÚ_ÐØ�]‰]Ð"5ˆ]Ó6€FÜ
‡K�K�(˜;¬¨V«Õ4Ð5Ð5^Ð_Ô`à"(§,¡,Ô/@Àq ,Ó"I€FÐÑØ"Ð#6Õ7×=Ñ=¼kÓJ€Fˆ=ÑØ"(§,¡,Ô/@Àq ,Ó"I€FÐÑà€Mr   c                ó   € V ^8„  d   QhRR/# )r   r	   N© )r   s   "r   r   r   à   s   € ÷ !dñ !dˆdñ !dr   c                  óê  € \         P                  ! R R7      p V P                  RRR7       V P                  RRR7       V P                  RRR7       V P                  RRR7       V P                  RRR7       V P                  4       p\        P                  R	4       \        P                  ! VP                  4      p\        P                  ! VP                  4      p\        P                  ! VP                  4      p\        \        VP                  4      4      p\        P                  R
4       \        W#WE4      p\        V4      ^d8  d9   \        P!                  R\        V4       R24       \"        P$                  ! ^4       \        VP&                  4      pVP(                  P+                  RRR7       VP-                  VRR7       \        P                  R\        V4       RV 24       \        P                  RVR,          P/                  4        24       \        P                  RVR,          P/                  4        24       \        P                  RVR,          P/                  4        24       R# )z1Build the final feature matrix for model training)Údescriptionz	--climateT)Úrequiredz--cropz--marketz--crop-lookupz--outputzLoading inputs...zBuilding feature matrix...zOnly uŸ    rows in the final feature matrix. This is too small to train reliable models â€” review the merge keys and source data coverage before proceeding to training.)ÚparentsÚexist_okF)ÚindexzSaved z	 rows to z Feasibility label distribution:
rž   zTrend label distribution:
r¬   z Suitability label distribution:
r­   N)ÚargparseÚArgumentParserÚadd_argumentÚ
parse_argsr;   r®   r   Úread_csvr   rD   rx   rP   r   Úcrop_lookupr´   rY   ÚerrorÚsysÚexitÚoutputÚparentÚmkdirÚto_csvÚvalue_counts)ÚparserÚargsÚ
climate_dfÚcrop_dfÚ	market_dfra   Úfinal_dfÚoutput_paths           r   ÚmainrÒ   à   sì  € Ü×$Ò$Ð1dÔe€FØ
×Ñ˜¨dÐÔ3Ø
×Ñ˜¨4ÐÔ0Ø
×Ñ˜
¨TÐÔ2Ø
×Ñ˜°$ÐÔ7Ø
×Ñ˜
¨TÐÔ2Ø×ÑÓ€Dä
‡K�KÐ#Ô$Ü—’˜TŸ\™\Ó*€JÜ�kŠk˜$Ÿ)™)Ó$€GÜ—’˜DŸK™KÓ(€IÜ%¤d¨4×+;Ñ+;Ó&<Ó=€Jä
‡K�KÐ,Ô-Ü�Z¨)Ó@€Hä
ˆ8ƒ}�sÔÜ�‰Ø”C˜“M�?ð #Bð Cô	
ô
 	�Š�Œä�t—{‘{Ó#€KØ×Ñ×Ñ T°DÐÔ9Ø‡O�O�K u€OÔ-ä
‡K�K�&œ˜X›˜ y°°Ð>Ô?Ü
‡K�KÐ3°HÐ=PÕ4Q×4^Ñ4^Ó4`Ð3aÐbÔcÜ
‡K�KÐ-¨h°}Õ.E×.RÑ.RÓ.TÐ-UÐVÔWÜ
‡K�KÐ3°HÐ=PÕ4Q×4^Ñ4^Ó4`Ð3aÐbÖcr   Ú__main__)Ú__doc__r½   rF   ÚloggingrÄ   Úpathlibr   ÚnumpyrZ   Úpandasr   Úscipyr   ÚbasicConfigÚINFOÚ	getLoggerÚ__name__r;   r9   r0   r?   rP   r_   rv   r�   r”   r›   r¢   r´   rÒ   r¶   r   r   Ú<module>rÞ      s×   ðñó. Û Û Û 
Ý ã Û Ý à × Ò ˜'Ÿ,™,Ð/VÕ WØ	×	Ò	˜8Ó	$€ð €qˆ!ˆQ��1�a˜Øˆˆ2ˆq�"�a˜˜AØ€qˆ!ˆQ��1�a˜ð€
ð Ð õ
(õVõõõ.
õõ(õõ!õH!dðH ˆzÔÙ†Fñ r   