Ë
    9YGj`)  ã                   óÔ  — d Z ddlZddlZddlZddlZddlmZ ddlZddl	Z
ddlmZ  ej                  ej                  d¬«        ej                  e«      Zddddddddddddd	œZd
Zde
j(                  de
j(                  fd„Zdede
j(                  fd„Zde
j.                  defd„Zde
j(                  de
j(                  de
j(                  fd„Zde
j(                  de
j(                  fd„Zde
j.                  defd„Zdedefd„Zde
j.                  defd„Zde
j(                  de
j(                  de
j(                  de
j(                  de
j(                  f
d„Z dd„Z!edk(  r e!«        yy) u½  
Feature engineering pipeline.

Produces data/processed/final_feature_matrix.csv â€” the single training
table every model in ml_pipeline/training/ reads from. Also produces
data/processed/climate_clean.csv (already created by clean_climate_data.py,
read directly by the live API's FeatureBuilder for historical averages).

This script must be re-run any time the raw source data is refreshed
(new IMD/ICRISAT/Agmarknet export). Models trained on a stale feature
matrix will silently drift from what the live system computes at
inference time â€” keep this pipeline run as part of your retraining
checklist, not a one-time setup step.

Usage:
    python -m ml_pipeline.feature_engineering.build_feature_matrix         --climate data/processed/climate_clean.csv         --crop data/processed/crop_clean.csv         --market data/processed/market_clean.csv         --crop-lookup data/lookup/crops.json         --output data/processed/final_feature_matrix.csv
é    N)ÚPath)Ústatsz%%(asctime)s %(levelname)s %(message)s)ÚlevelÚformaté   é   )é   é   é   é	   é
   é   é   r   r   é   é   é   iÝ  ÚclimateÚreturnc                 ó   — | j                  g d¢«      j                  d¬«      } | | d   t        k     j                  ddg«      j	                  dd¬	«      j                  «       }t        j                  | |ddgd
¬«      } | d   j                  «       }| d   j                  |«      | d<   | d   j                  | d   j                  «       «      | d<   | d   j                  dd«      | d<   | d   | d   z
  | d   z  | d<   | j                  d«      d   j                  d«      | d<   | j                  d«      d   j                  d„ «      | d<   | d   j                  | d   «      | d<   | d   j                  t        «      | d<   d| j                  vrt         j#                  d«       d| d<   d| j                  vrt         j#                  d«       d| d<   | S )N©ÚdistrictÚyearÚmonthT)Údropr   r   r   )Úrainfall_mmÚmean)r   Ústd)Úhist_mean_rainfallÚhist_std_rainfallÚleft©ÚonÚhowr   r   r   r   r   Úrainfall_anomalyÚrainfall_lag1c                 óD   — | j                  dd¬«      j                  «       S )Nr   r   ©Úmin_periods)Úrollingr   ©Úxs    úaC:\Crop_Prediction\Backend\crop-ai-system\ml_pipeline\feature_engineering\build_feature_matrix.pyú<lambda>z&add_climate_features.<locals>.<lambda>H   s   € ˜QŸY™Y q°a˜YÓ8×=Ñ=Ó?€ ó    Úrainfall_ma3Úseason_indexÚtemp_cuL   No temp_c column in climate data â€” using a flat 30.0C default for all rowsg      >@ÚhumidityuM   No humidity column in climate data â€” using a flat 70.0 default for all rowsg     €Q@)Úsort_valuesÚreset_indexÚHISTORICAL_CUTOFF_YEARÚgroupbyÚaggÚpdÚmerger   Úfillnar   ÚreplaceÚshiftÚ	transformÚmapÚ
SEASON_MAPÚcolumnsÚloggerÚwarning)r   ÚhistÚfallback_means      r,   Úadd_climate_featuresrE   0   sû  € Ø×!Ñ!Ò"?Ó@×LÑLÐRVÐLÓW€Gà�7˜6‘?Ô&<Ñ<Ñ=×EÑEØ	�WÐóç	�cØ2Ø0ð 
ó ÷ �kƒmð 	ô �h‰h�w ¨*°gÐ)>ÀFÔK€Gà˜MÑ*×/Ñ/Ó1€MØ$+Ð,@Ñ$A×$HÑ$HÈÓ$W€GÐ Ñ!Ø#*Ð+>Ñ#?×#FÑ#FÀwÈ}ÑG]×GaÑGaÓGcÓ#d€GÐÑ Ø#*Ð+>Ñ#?×#GÑ#GÈÈ1Ó#M€GÐÑ ð 
�Ñ	 'Ð*>Ñ"?Ñ	?À7ÐK^ÑC_Ñ_ð ÐÑð  'Ÿ™¨zÓ:¸=ÑI×OÑOÐPQÓR€GˆOÑà�‰˜
Ó# MÑ2ß	‰Ñ?Ó	@ð ˆNÑð  ' Ñ7×>Ñ>¸wÀ}Ñ?UÓV€GˆOÑà% gÑ.×2Ñ2´:Ó>€GˆNÑð
 �w—‘Ñ&Ü�‰ÐeÔfØ ˆ�ÑØ˜Ÿ™Ñ(Ü�‰ÐfÔgØ"ˆ�
Ñà€Nr.   Úlookup_pathc                 ó  — t        | d¬«      5 }t        j                  |«      }d d d «       g }j                  «       D ]+  \  }}d|i}|j	                  |«       |j                  |«       Œ- t        j                  |«      S # 1 sw Y   Œ^xY w)Nz	utf-8-sig)ÚencodingÚcrop)ÚopenÚjsonÚloadÚitemsÚupdateÚappendr8   Ú	DataFrame)rF   ÚfÚ
crops_dictÚrowsrI   ÚpropsÚrows          r,   Úload_crop_propertiesrV   [   s…   € ô
 
ˆk KÔ	0ð "°AÜ—Y‘Y˜q“\ˆ
÷"à€DØ!×'Ñ'Ó)ò ‰ˆˆeØ�tˆnˆØ�
‰
�5ÔØ�‰�CÕðô �<‰<˜ÓÐ÷"ð "ús   ŽBÂB
Úseriesc                 óÞ   — | j                  «       j                  }t        |«      dk  ryt        j                  t        |«      «      }t        j                  ||«      \  }}}}}t        |«      S )Nr   ç        )ÚdropnaÚvaluesÚlenÚnpÚaranger   Ú
linregressÚfloat)rW   r[   r+   ÚslopeÚ_s        r,   Ú_compute_sloperc   j   sZ   € Ø�]‰]‹_×#Ñ#€FÜ
ˆ6ƒ{�Q‚ØÜ
�	‰	”#�f“+Ó€AÜ×(Ñ(¨¨FÓ3Ñ€Eˆ1ˆa��AÜ�‹<Ðr.   rI   Ú
crop_propsc                 ó¶  — t        j                  | |dd¬«      } | | d   j                  «          d   j                  «       }t	        |«      dkD  r"t
        j                  dt        |«      › d�«       | j                  g d¢«      } | j                  d	dg«      d
   j                  d„ «      j                  ddgd¬«      }|| d<   | d   j                  d«      | d<   | S )NrI   r    r!   Úwater_req_mmr   z@Crops present in yield data but missing from crops.json lookup: zu. Add them to data/lookup/crops.json or these rows will have null agronomic features and get dropped before training.)r   rI   r   r   Úyield_ton_hac                 óR   — | j                  dd¬«      j                  t        d¬«      S )Nr   r   r'   F©Úraw©r)   Úapplyrc   ©Úss    r,   r-   z#add_crop_features.<locals>.<lambda>�   ó#   € ˜Ÿ™ 1°!˜Ó4×:Ñ:¼>ÈuÐ:ÓU€ r.   r   T©r   r   Úyield_trend_sloperY   )r8   r9   ÚisnullÚuniquer\   rA   rB   Úlistr3   r6   rl   r4   r:   )rI   rd   Úmissing_propsÚtrend_slopess       r,   Úadd_crop_featuresrw   s   sè   € Ü�8‰8�D˜*¨°VÔ<€Dà˜˜nÑ-×4Ñ4Ó6Ñ7¸Ñ?×FÑFÓH€MÜ
ˆ=Ó˜AÒÜ�‰ØNÜ�MÓ"Ð#ð $VðWô	
ð ×ÑÒ8Ó9€Dà�‰�j &Ð)Ó*¨>Ñ:ß	‰ÑUÓ	Vß	‰˜A˜q˜6¨ˆÓ	-ð ð
 !-€DÐ	ÑØ $Ð%8Ñ 9× @Ñ @ÀÓ E€DÐ	Ñà€Kr.   Úmarketc                 ó  — | j                  g d¢«      } | j                  ddg«      d   j                  d«      | d<   | j                  ddg«      d   j                  d„ «      j	                  ddgd	¬
«      }|j                  d«      | d<   | S )N©r   rI   r   r   r   rI   Úprice_per_quintalr   Ú
price_lag1c                 óR   — | j                  dd¬«      j                  t        d¬«      S )Nr	   r   r'   Fri   rk   rm   s    r,   r-   z%add_market_features.<locals>.<lambda>�   ro   r.   r   Trp   rY   Údemand_trend)r3   r6   r<   rl   r4   r:   )rx   r~   s     r,   Úadd_market_featuresr   Š   s”   € Ø×ÑÒ EÓF€FØ!Ÿ>™>¨:°vÐ*>Ó?Ð@SÑT×ZÑZÐ[\Ó]€Fˆ<Ñð 	�‰˜
 FÐ+Ó,Ð-@ÑAß	‰ÑUÓ	Vß	‰˜A˜q˜6¨ˆÓ	-ð ð
 *×0Ñ0°Ó5€Fˆ>ÑØ€Mr.   rU   c                 óþ   — | d   | d   cxk  xr | d   k  nc }| d   | d   dz  k\  }t        | d   «      dkD  }| j                  d	d
«      d
kD  }| d   | d   dz  dz  k  }|r|ry|r|r|s|ry|r|s|ry|s| d   s|syy)NÚ
min_temp_cr1   Ú
max_temp_cr   rf   r   r$   g      ø?rg   r   çš™™™™™¹?Únot_suitableÚsuitableÚriskyÚdrought_tolerant)ÚabsÚget)rU   Útemp_okÚrain_okÚanomaly_badÚ	has_yieldÚextreme_droughts         r,   Úlabel_feasibilityr�   •   s¹   € Ø�,Ñ 3 x¡=ÖE°C¸Ñ4EÔE€GØ�-Ñ  S¨Ñ%8¸2Ñ%=Ñ>€GÜ�cÐ,Ñ-Ó.°Ñ4€KØ—‘˜¨Ó*¨QÑ.€IØ˜-Ñ(¨C°Ñ,?À"Ñ,DÈÑ+LÑL€Oñ ‘oØñ ‘7¡;±9Øñ ‘{¡yØñ ˜3Ð1Ò2¹9Øàr.   ra   c                 ó:   — | dkD  ry| dk  ryt        | «      dkD  ryy)Nrƒ   Úupwardgš™™™™™¹¿Údownwardgš™™™™™©?ÚvolatileÚstable)rˆ   )ra   s    r,   Úlabel_trendr•   ®   s(   € Øˆs‚{ØØ	�ŠØÜ	ˆU‹�dÒ	ØØr.   c                 óR   — | d   dk(  r| j                  dd«      dkD  ry| d   dk(  ryy)	NÚfeasibility_labelr…   r~   r   Úhighr„   ÚlowÚmedium)r‰   )rU   s    r,   Úlabel_suitabilityr›   ¸   s9   € Ø
ÐÑ :Ò-°#·'±'¸.È!Ó2LÈqÒ2PØØ	Ð Ñ	! ^Ò	3ØØr.   c                 óˆ  — t        | «      } t        ||«      }t        |«      }t        j                  || g d¢d¬«      }t
        j                  dt        |«      › d�«       t        |«      dk(  rt        d«      ‚t        j                  ||g d¢d	¬«      }t
        j                  d
t        |«      › d|d   j                  «       j                  «       › d�«       |j                  d«      d   j                  d„ «      |d<   |d   j                  d«      |d<   |d   j                  d«      |d<   t        |«      }g d¢}|j                  |¬«      }t
        j                  d|t        |«      z
  › d�«       |j                  t         d¬«      |d<   |d   j                  t"        «      |d<   |j                  t$        d¬«      |d<   |S )Nr   Úinnerr!   zAfter climate+crop merge: z rowsr   z³Zero rows after merging climate and crop data. Likely cause: district name spelling mismatch, or year/month ranges that don't overlap between the two sources. Inspect both inputs.rz   r    zAfter market merge: z rows (r{   z missing market price)rI   c                 ó@   — | j                  | j                  «       «      S )N)r:   Úmedianr*   s    r,   r-   zbuild.<locals>.<lambda>Ó   s   € �!—(‘(˜1Ÿ8™8›:Ó&€ r.   Úprice_volatilityrƒ   r~   rY   )r1   r�   r‚   r   rf   )ÚsubsetzDropped z) rows missing required label-input fieldsr   )Úaxisr—   rq   Útrend_labelÚsuitability_label)rE   rw   r   r8   r9   rA   Úinfor\   Ú
ValueErrorrr   Úsumr6   r=   r:   rZ   rl   r�   r•   r›   )r   rI   rx   rd   ÚmergedÚbefore_dropÚrequired_for_labelss          r,   Úbuildr«   À   s´  € Ü" 7Ó+€GÜ˜T :Ó.€DÜ  Ó(€Fä�X‰X�d˜GÒ(EÈ7ÔS€FÜ
‡K�KÐ,¬S°«[¨M¸Ð?Ô@ä
ˆ6ƒ{�aÒÜðJó
ð 	
ô �X‰X�f˜fÒ)NÐTZÔ[€FÜ
‡K�KÐ&¤s¨6£{ m°7¸6ÐBUÑ;V×;]Ñ;]Ó;_×;cÑ;cÓ;eÐ:fÐf|Ð}Ô~à"(§.¡.°Ó"8Ð9LÑ"M×"WÑ"WÙ&ó#€FÐÑð "(Ð(:Ñ!;×!BÑ!BÀ3Ó!G€FÐÑØ# NÑ3×:Ñ:¸3Ó?€Fˆ>Ñä�f“+€KÚ_ÐØ�]‰]Ð"5ˆ]Ó6€FÜ
‡K�K�(˜;¬¨V«Ñ4Ð5Ð5^Ð_Ô`à"(§,¡,Ô/@Àq ,Ó"I€FÐÑØ"Ð#6Ñ7×=Ñ=¼kÓJ€Fˆ=ÑØ"(§,¡,Ô/@Àq ,Ó"I€FÐÑà€Mr.   c                  óÆ  — t        j                  d¬«      } | j                  dd¬«       | j                  dd¬«       | j                  dd¬«       | j                  dd¬«       | j                  d	d¬«       | j                  «       }t        j                  d
«       t        j                  |j                  «      }t        j                  |j                  «      }t        j                  |j                  «      }t        t        |j                  «      «      }t        j                  d«       t        ||||«      }t        |«      dk  r7t        j!                  dt        |«      › d�«       t#        j$                  d«       t        |j&                  «      }|j(                  j+                  dd¬«       |j-                  |d¬«       t        j                  dt        |«      › d|› �«       t        j                  d|d   j/                  «       › �«       t        j                  d|d   j/                  «       › �«       t        j                  d|d   j/                  «       › �«       y )Nz1Build the final feature matrix for model training)Údescriptionz	--climateT)Úrequiredz--cropz--marketz--crop-lookupz--outputzLoading inputs...zBuilding feature matrix...éd   zOnly uŸ    rows in the final feature matrix. This is too small to train reliable models â€” review the merge keys and source data coverage before proceeding to training.r   )ÚparentsÚexist_okF)ÚindexzSaved z	 rows to z Feasibility label distribution:
r—   zTrend label distribution:
r£   z Suitability label distribution:
r¤   )ÚargparseÚArgumentParserÚadd_argumentÚ
parse_argsrA   r¥   r8   Úread_csvr   rI   rx   rV   r   Úcrop_lookupr«   r\   ÚerrorÚsysÚexitÚoutputÚparentÚmkdirÚto_csvÚvalue_counts)ÚparserÚargsÚ
climate_dfÚcrop_dfÚ	market_dfrd   Úfinal_dfÚoutput_paths           r,   ÚmainrÈ   ä   sð  € Ü×$Ñ$Ð1dÔe€FØ
×Ñ˜¨dÐÔ3Ø
×Ñ˜¨4ÐÔ0Ø
×Ñ˜
¨TÐÔ2Ø
×Ñ˜°$ÐÔ7Ø
×Ñ˜
¨TÐÔ2Ø×ÑÓ€Dä
‡K�KÐ#Ô$Ü—‘˜TŸ\™\Ó*€JÜ�k‰k˜$Ÿ)™)Ó$€GÜ—‘˜DŸK™KÓ(€IÜ%¤d¨4×+;Ñ+;Ó&<Ó=€Jä
‡K�KÐ,Ô-Ü�Z ¨)°ZÓ@€Hä
ˆ8ƒ}�sÒÜ�‰Ø”C˜“M�?ð #Bð Cô	
ô
 	�‰�Œä�t—{‘{Ó#€KØ×Ñ×Ñ T°DÐÔ9Ø‡O�O�K u€OÔ-ä
‡K�K�&œ˜X›˜ y°°Ð>Ô?Ü
‡K�KÐ3°HÐ=PÑ4Q×4^Ñ4^Ó4`Ð3aÐbÔcÜ
‡K�KÐ-¨h°}Ñ.E×.RÑ.RÓ.TÐ-UÐVÔWÜ
‡K�KÐ3°HÐ=PÑ4Q×4^Ñ4^Ó4`Ð3aÐbÕcr.   Ú__main__)r   N)"Ú__doc__r³   rK   Úloggingrº   Úpathlibr   Únumpyr]   Úpandasr8   Úscipyr   ÚbasicConfigÚINFOÚ	getLoggerÚ__name__rA   r?   r5   rP   rE   rV   ÚSeriesr`   rc   rw   r   Ústrr�   r•   r›   r«   rÈ   © r.   r,   ú<module>r×      s£  ðñó. Û Û Û 
Ý ã Û Ý à €× Ñ ˜'Ÿ,™,Ð/VÕ WØ	ˆ×	Ñ	˜8Ó	$€ð 	ˆQ�1˜Ø	ˆq�a˜AØˆQ�1˜ñ€
ð Ð ð
( "§,¡,ð (°2·<±<ó (ðV dð ¨r¯|©|ó ð˜2Ÿ9™9ð ¨ó ð˜BŸL™Lð °b·l±lð ÀrÇ|Á|ó ð.
 §¡ð 
°·±ó 
ð˜2Ÿ9™9ð ¨ó ð2�uð  ó ð˜2Ÿ9™9ð ¨ó ð!�2—<‘<ð ! r§|¡|ð !¸R¿\¹\ð !ÐWY×WcÑWcð !Ðhj×htÑhtó !óH!dðH ˆzÒÙ…Fð r.   