U
    Ÿ`¬l  ã                	   @   s  d dl Zd dlZd dlmZmZmZ d dlZd dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZ g g g g g fdd„Zg g g g g fd	d
„Zg g g g g fdd„Zg g g g fdd„Zg g g g fdd„Zddejjdfdd„Zd'dd„Zd(dd „Zd)d#d$„Zd%d&„ ZdS )*é    N)ÚDatasetÚnum2dateÚdate2num)Úpyplot)ÚBasemap)ÚPolygon)ÚPatchCollectionc                 C   sª  t | ƒ}|jd d d … }|jd d d … }|jd d d … }	|jd d d … }
|jd d d … }t|ƒdkrÊ|\}}}}||k ||k@ ||k@ ||k @ }|| }|| }|	| }	|
| }
|d d …|f }|jd }t|d d … |jd�}| ¡  tj |¡}t	 
t	 
|¡j¡}t	j|||
d	�}t|ƒdk�rB||d |d
 … }t|ƒdk�r¾|\}}}|||… }t t |j¡¡ d¡||jd  k}|jd d …|f }|	| }	|
| }
|| }|| }t|ƒdk�r:|\}}}|||… }t t |j¡¡ d¡||jd  k}|jd d …|f }|	| }	|
| }
|| }|| }t|ƒdk�rœ|D ]N}t t| ¡ ƒ|k¡}|jd d …|f }|	| }	|
| }
|| }|| }�qL||	|
||fS )NÚlatÚlonÚidÚnameÚprr   Útime©Úunits©ÚindexÚcolumnsé   )r   Ú	variablesÚlenr   r   ÚcloseÚnpÚmaÚmasked_invalidÚpdÚto_datetimeÚdateÚ	DataFrameÚinvertÚisnanÚvaluesÚsumÚshapeÚilocÚlistÚkeys)ÚncfileÚ_bndsÚ	_timebndsÚ_cov1Ú_cov2Ú_excludeÚncdsetÚ_latsÚ_lonsÚ_idsÚ_namesÚdataÚx1Úx2Úy1Úy2Ú_selÚ_timesÚ_datesÚ_prÚ	_obsdatesÚfyÚlyÚfracÚtestÚ_nn© rA   úI/work/projects/capetowndrought/scripts_trendspaper/functions_rainfall3.pyÚreadSAWSonefile
   sb     

&
&rC   c           &   	   C   s2  d}t |ƒ}|jd d d … }|jd d d … }|jd d d … }	|jd d d … }
|jd d d … }t| ƒdkrÎ| \}}}}||k ||k@ ||k@ ||k @ }|| }|| }|	| }	|
| }
|d d …|f }|jd }t|d d … |jd	�}| ¡  tj |¡}t	j
d
ddd�}t	j|||
d�}tddƒ�*}| ¡ dd … }t dd„ |D ƒ¡}W 5 Q R X t |d d …df ¡}|d d …df }|d d …df }d}|D ]¢}|d d …df |k}t	 dd„ || D ƒ¡}|| }d||dk< | d¡}|dk�röd}|dk�rt	j|||gd�}nt	 |t	j|||gd�gd¡}|d }�qš|j ¡ |_t	j
dddd�}t t|ƒ|jd g¡} tj| d d …< | t	j| |t| ¡ ƒd�¡}| |¡ t|ƒdk�rÊ||d |d … }t|ƒdk�rF|\}!}"}#||!|"… }$t t |$j¡¡ d¡|#|$jd  k}|jd d …|f }|	| }	|
| }
|| }|| }t|ƒdk�rÂ|\}!}"}#||!|"… }$t t |$j¡¡ d¡|#|$jd  k}|jd d …|f }|	| }	|
| }
|| }|| }t|ƒdk�r$|D ]N}%t t| ¡ ƒ|%k¡}|jd d …|f }|	| }	|
| }
|| }|| }�qÔ||	|
||fS )Nz'/work/data/saws/south_africa_2015.pr.ncÚlatitudeÚ	longituder   r   r   r   r   r   z
1850-01-01z
2015-12-31ÚD)Úfreqr   z%../data/bigsix/sawsupdate/alldata.csvÚrr   c                 S   s   g | ]}|  ¡  d ¡‘qS )ú,©ÚstripÚsplit©Ú.0ÚxrA   rA   rB   Ú
<listcomp>a   s     zreadSAWS.<locals>.<listcomp>é   é   c                 S   s   g | ]}|  ¡ ‘qS rA   )rK   rM   rA   rA   rB   rP   i   s     ÚnanÚ ÚfloatzVILLIERSDORP-SOS ARSZVILLIERSDORPz
2016-01-01z
2017-12-31) r   r   r   r   r   r   r   r   r   r   Ú
date_ranger   ÚopenÚ	readlinesÚarrayÚuniquer   ÚastypeÚconcatr   Úzerosr#   rS   Úappendr%   r&   Úupdater   r    r!   r"   r$   )&r(   r)   r*   r+   r,   r'   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   ÚinpfZ_unamesZ_udatesZ_uprÚnÚ_nameÚselZseldatesÚtempÚprupdateZappenddatesÚfillerr<   r=   r>   r?   r@   rA   rA   rB   ÚreadSAWSG   s–     





&
&rg   c                 C   sÖ  t | ƒ}|jd d d … }|jd d d … }|jd d d … }	|jd d d … }
|jd d d … }t|ƒdkrÌ|\}}}}||k ||k@ ||k@ ||k @ }|| }|| }|	| }|
| }|d d …|f }nt |	¡}t |
¡}|jd }t|d d … |jd�}| ¡  tj 	|¡}tj
||dk < tj|t t |¡j¡|d	�}t|ƒdk�rb||d |d
 … }t|ƒdk�rÞ|\}}}|||… }t t |j¡¡ d¡||jd  k}|jd d …|f }|| }|| }|| }|| }t|ƒdk�rZ|\}}}|||… }t t |j¡¡ d¡||jd  k}|jd d …|f }|| }|| }|| }|| }t|ƒdk�rÈ|D ]Z}tt t| ¡ ƒ|k¡ƒ}t|ƒ |jd d …|f }|| }|| }|| }|| }�ql|||||fS )NrD   rE   r   r   r   r   r   r   r   r   )r   r   r   r   Úcopyr   r   r   r   r   rS   r   r   r   r   r   r    r!   r"   r#   r$   Úintr%   r&   Úprint)Z_ncfiler(   r)   r*   r+   r,   r-   r.   r/   Ú_ids0Ú_names0r2   r3   r4   r5   r6   rc   r0   r1   r8   r9   r:   r<   r=   r>   r?   r7   r@   rA   rA   rB   ÚreadDWS2£   sh     



&
&rm   c           "   
      sP  d}d}t |ƒ}|jd d d … }|jd d d … }|jd d d … ‰ |jd d d … ‰|jd d d … }t| ƒdkrÔ| \}	}
}}||k ||k@ ||	k@ ||
k @ }|| }|| }ˆ | }ˆ| }|d d …|f }nt ˆ ¡}t ˆ¡}|jd	 }t|d d … |jd
�}| ¡  tj 	|¡}t
j|t
 t
 |¡j¡|d�}t|d ƒ t d¡}d}|D ]¼}tj |¡ d¡}t|dƒ�}| ¡ dd… }W 5 Q R X t dd„ |D ƒ¡}t
 |d d …df ¡}t
j|d d …df  t¡||gd�}|dk�rö| ¡ }nt
j||dddd�}|d }�qX‡ ‡fdd„t| ¡ ƒD ƒ}||_| |¡ t|ƒdk�rd||d |d … }t|ƒdk�rà|\}}}|||… }t t |j ¡¡ !d¡d|j"d  k} |j#d d …| f }||  }||  }||  }||  }t|ƒdk�rB|D ]N}!t t| ¡ ƒ|!k¡} |j#d d …| f }||  }||  }||  }||  }�qò|||||fS )Nú./work/data/dws/pr_dws_day_19000101-20171208.ncz./work/data/dws/pr_dws_day_19000101-20181231.ncrD   rE   r   r   r   r   r   r   r   éÿÿÿÿú../data/bigsix/dwsupdate/*.txtú.txtrH   é   éïÿÿÿc                 S   s   g | ]}|  ¡  ¡ ‘qS rA   rJ   rM   rA   rA   rB   rP     s     zreadDWS_old.<locals>.<listcomp>r   ÚouterT©ÚhowÚ
left_indexÚright_indexc                    s&   g | ]}ˆt  ˆ |k¡d  d   ‘qS ©r   ©r   ÚwhererM   ©rk   rl   rA   rB   rP     s     çÍÌÌÌÌÌì?)$r   r   r   r   rh   r   r   r   r   r   r   r   r   r   rj   ÚglobÚosÚpathÚbasenamerK   rW   rX   rY   r[   rU   Úmerger%   r&   r   r_   r   r    r!   r"   r#   r$   ©"r(   r)   Ú_covr,   r'   r-   r.   r/   r2   r3   r4   r5   r6   rc   r0   r1   r8   r9   r:   Ú	filenamesÚiÚfnameZstnamer`   Údatesrd   re   Zupdatenamesr<   r=   r>   r?   r7   r@   rA   r|   rB   ÚreadDWS_oldá   sx     



$



&r‰   c           "   
      s@  d}t |ƒ}|jd d d … }|jd d d … }|jd d d … ‰ |jd d d … ‰|jd d d … }t| ƒdkrÐ| \}	}
}}||k ||k@ ||	k@ ||
k @ }|| }|| }ˆ | }ˆ| }|d d …|f }nt ˆ ¡}t ˆ¡}|jd }t|d d … |jd	�}| ¡  tj 	|¡}t
j|t
 t
 |¡j¡|d
�}t d¡}d}|D ]¼}tj |¡ d¡}t|dƒ�}| ¡ dd… }W 5 Q R X t dd„ |D ƒ¡}t
 |d d …df ¡}t
j|d d …df  t¡||gd
�}|dk�ræ| ¡ }nt
j||dddd�}|d }�qH‡ ‡fdd„t| ¡ ƒD ƒ}||_| |¡ t|ƒdk�rT||d |d … }t|ƒdk�rÐ|\}}}|||… }t t |j¡¡  d¡d|j!d  k} |j"d d …| f }||  }||  }||  }||  }t|ƒdk�r2|D ]N}!t t| ¡ ƒ|!k¡} |j"d d …| f }||  }||  }||  }||  }�qâ|||||fS )Nrn   rD   rE   r   r   r   r   r   r   r   rp   rq   rH   rr   rs   c                 S   s   g | ]}|  ¡  ¡ ‘qS rA   rJ   rM   rA   rA   rB   rP   I  s     zreadCoCT.<locals>.<listcomp>r   rt   Tru   c                    s&   g | ]}ˆt  ˆ |k¡d  d   ‘qS ry   rz   rM   r|   rA   rB   rP   R  s     r}   )#r   r   r   r   rh   r   r   r   r   r   r   r   r   r   r~   r   r€   r�   rK   rW   rX   rY   r[   rU   r‚   r%   r&   r   r_   r   r    r!   r"   r#   r$   rƒ   rA   r|   rB   ÚreadCoCT)  st     



$



&rŠ   ÚbluerQ   c                 C   s0  t dddd|d |d |d |d d	d
�	}|jt ddd¡ddddgddgddd� |jt ddd¡ddddgddgddd� |jddd� |jdddd� z| dd¡ W n   | dd¡ Y nX g }	t|j	|j
ƒD ]\}
}|	 tt |¡dƒ¡ qØ|  t|	ddddd�¡ |j||||dd|ddd �	 |S )!NÚmercr†   iâÿÿÿé
   é   é   r   r   i­  )	Ú
projectionÚ
resolutionÚlat_0Úlon_0Ú	llcrnrlatÚ	urcrnrlatÚ	llcrnrlonÚ	urcrnrlonÚepsgg      IÀg        ç      à?g      Ð?z0.5)ÚlabelsÚdashesÚ	linewidthÚcoloréöÿÿÿç      Y@ÚwhiteÚgray)Ú
fill_colorr�   z0.8)r�   Ú
lake_colorÚzorderz#/work/data/gis/southafrica/bigsix2dÚdamsz//work/orange/work/data/gis/southafrica/bigsix2dTÚdeepskyblueç      ð?)Ú	facecolorÚ	edgecolorÚ
linewidthsr¤   rR   )r�   Ú
edgecolorsÚlatlonrœ   Úsr¤   Úalpha)r   Údrawparallelsr   ÚarangeÚdrawmeridiansÚdrawmapboundaryÚfillcontinentsÚreadshapefileÚzipZ	dams_infor¥   r^   r   rY   Úadd_collectionr   Úscatter)Úplr.   r/   Úextr�   r©   ÚcmapÚmsizeÚmÚpatchesÚinfor#   rA   rA   rB   Úplotmapo  s    ***r¿   Fc           	         sè   ddl m} ddlm} tj |¡}tj|| dd�‰ |rVt	d‡ fdd„t
d	d
ƒD ƒƒ |d d…  d¡}|d	d …  d¡}t || d ¡t |d ¡ }|r¨t	d|ƒ ||d	dd�\}}|rÐt	d|ƒ t	dƒ ‡ fdd„t
d	d
ƒD ƒS )Nr   )Úacorr_ljungbox)ÚacfT)r   rh   z%autocorrelation for first three lags:c                    s   g | ]}ˆ   |¡‘qS rA   ©Úautocorr©rN   r†   ©ÚpdfrA   rB   rP   £  s     z!autocorr_test.<locals>.<listcomp>r   rQ   ro   rU   rŽ   z;Durbin-Watson statistic (close to 2 if no autocorrelation):F)ÚlagsZ	boxpiercez+Ljung-Box p-value on lag 1 autocorrelation:rT   c                    s   g | ]}ˆ   |¡‘qS rA   rÂ   rÄ   rÅ   rA   rB   rP   ®  s     )Zstatsmodels.stats.diagnosticrÀ   Zstatsmodels.tsa.stattoolsrÁ   r   r   r   r   ÚSeriesrj   Úranger[   Únansum)	Z_xdataZ_ydataÚverboserÀ   rÁ   ÚaÚbÚ_statÚ_pvaluerA   rÅ   rB   Úautocorr_test�  s     
 

rÐ   éè  é_   Úlinregr™   ç333333ã?r�   Ú	bootstrapc           =   	   C   sŽ  ddl m} ddlm} ddlm} ddlm  m} ddlm	} ddl
m  m} ddlm} ddlm}m}m}m} ddlm}m} |	dkrœ|d	krœd
}	t | ¡} tj |¡}t|ƒ}d| d }d| }|d
krô|dt ttt|ƒƒƒ¡ƒ}n|t ttt|ƒƒƒ¡ƒ}|dk�r||
�r:td|	 d t|ƒ d ƒ tj t !| |g¡ddgd�}| "d|¡}|j#|d�}|j$j%}|j$j&} |j'd }!| (¡ d d }"| (¡ d d }#| )|¡}$||$ }%t*| |%dƒ |	d
k�r|t +|$¡ ,dd¡}&g }'g }(| -|¡D ]†})|)d d }*| "d|j.|*dd…f ¡}zH|j#|d�}|'|j$j&g }'|(|j$j%g }(t /|&| )|¡ ,dd¡d¡}&W n   d}+Y nX �qô|dk�rÊ|
�r¨td|	 d t|ƒ d ƒ | 0| ¡},|j||,ddd�}| #¡ }|j$\}} |j'd }!| (¡ d \}"}#| )|,¡}$||$ }%t*| |%ƒ |	d
k�rÊg }'g }(t +|$¡ ,dd¡}&| -|¡D ]Ž})|)d d }*|j||* |,|*dd…f ddd�}zH| #¡ }|'|j$d g }'|(|j$d g }(t /|&| )|,¡ ,dd¡d¡}&W n   d}+Y nX �q:|d	k�rê|
�rötd|	 d t|ƒ d ƒ || |ƒ\}-}!|
�rtd |-d!|!ƒ |j1|| d"d#�\} }.}"}#|.| |   }$||$ }%t*| |%ƒ |	d
k�rêg }'g }(t +|$¡ ,dd¡}&| -|¡D ]l})|)d d }*|j1||* | |* d"d#�}|'|d g }'|(|d g }(t /|&|d |d |    ,dd¡d¡}&�q||d$k�ræ|
�rtd%ƒ t2|j3|| ||dd&�ƒ}/t tt4|/j5| ƒƒ¡ 6d'¡ ,dd¡}$t +|$¡}&| -|¡D ]’})|)d d }*znt2|j3||* | |* ||dd&�ƒ}/t tt4|/j5| ƒƒ¡ 6d'¡ ,dd¡}/tj |/| d(�}0|0 7¡ j8}1t /|&|1d¡}&W n   d}+Y nX �qR|d$k�rDtj9}"tj9}#tj9} tj9}tj9}!t :|&||gd¡}2|2dd…df }3|2dd…df }4�n2|	d
k�rÌt ;|'d)d*g¡\}"}#||'ƒ}5|5dƒ}!|!d+k�r„d|! }!|
�r”t|&j<ƒ t :|&||gd¡ =¡ }2|2dd…df }3|2dd…df }4nªt | ¡d, }6|6t >|6¡ d }7d,t| ƒ |7t?|7ƒ  d+ }/t @t |¡t A|$¡ d ¡t| ƒ d+ }8dd|d-  d  }9| B|9t| ƒd ¡}:|:|8 |/ };|$|; }4|$|; }3| |#|"|!|$|3|4d.œ}<|<S )/aŽ  calculates bootstrap confidence interval and significance level for trend, ignoring autocorrelation or accounting for it
    Parameters
    ----------
    x_var : list
      independent variable
    y_var : list
      dependent variable, same length as x_var
    q : int, optional, only if trendtype==quantreg
      quantile for which regression is to be calculated
    n : int, optional
      number of bootstrap samples
    ci : int, optional
      confidence level. Default is for 95% confidence interval
    frac : int, optional, only if trendtype==lowess
      lowess parameter (fraction of time period length used in local regression)
    it : int, optional, only if trendtype==lowess
      lowess parameter (numbre of iterations)
    autocorr : str, optional
      way of accounting for autocorrelation, possible values: None, "bootstrap"
    trendtype : str, optional
      method of trend derivation, possible values: lowess, linreg, quantreg, TheilSen
    CItype : str, optional
      method of CI derivation, possible values: "analytical" and "bootstrap". 
      if trendtype is "lowess", CItype will be set to None
      if CItype is "analytical": autocorrelation will be set to None
      

    Results
    -------
    returns library with following elements:
    slope - slope of the trend
    CI_high - CI on the slope value
    CI_low - as above
    pvalue - trend's significance level
    trend - trend line, or rather its y values for all x_var
    trendCI_high - confidence interval for each value of y
    trendCI_low - as above

    Remarks
    -------
    the fit function ocassionally crashes on resampled data. The workaround is to use try statement
    r   N)ÚOLS)ÚQuantReg©ÚECDF)ÚmstatsÚmannwhitneyuÚtÚ
kendalltau©ÚStationaryBootstrapÚIIDBootstrapÚ
analyticalZTheilSenrÕ   éd   rŽ   r�   ÚquantregzQuantile regression, CI type: z, autocorrelation adjustment: Ú
ÚXÚY)r   zY ~ X)Úqr   Fro   rÓ   zLinear regression, CI type: TÚdrop)ÚhasconstÚmissingzTheil-Sen slope, CI type: zraw MK tau:zraw MK pvalue:çffffffî?©r®   ÚlowesszLowess
)r>   Úitrê   rU   )r   rR   rÒ   r™   r§   rŸ   )ÚslopeÚCI_highÚCI_lowÚpvalueZtrendÚtrendCI_lowÚtrendCI_high)CZstatsmodels.apiÚapiZ#statsmodels.regression.linear_modelrÖ   Zstatsmodels.tsaÚtsaZstatsmodels.formula.apiZformulaZ*statsmodels.regression.quantile_regressionr×   Zstatsmodels.nonparametric.apiZnonparametricÚ0statsmodels.distributions.empirical_distributionrÙ   Úscipy.statsrÚ   rÛ   rÜ   rÝ   Úarch.bootstraprß   rà   r   rY   r   r   r   r%   rÉ   rj   Ústrr   r   Úcolumn_stackrã   ÚfitÚparamsZ	Interceptrå   ÚpvaluesZconf_intZpredictrÐ   rh   ÚreshaperÕ   Úixr^   Zadd_constantÚtheilslopesÚdictrí   ÚmapÚgetr[   Úinterpolater!   rS   ÚnanpercentileÚ
percentiler#   Ú	transposeÚmeanr"   rÊ   ÚnanmeanÚppf)=Úx_varÚy_varZn_bootÚciZ	trendtyperç   r>   rî   rÃ   ZCItyperË   ÚsmrÖ   rö   Zsmfr×   ZnpsmrÙ   rÚ   rÛ   rÜ   rÝ   rß   rà   Zn_dataZci_lowZci_highÚbsZxydataÚmodelÚresZintcptrï   rò   rñ   rð   Zy_predÚresidsZ	bs_trendsÚ	bs_slopesZ
bs_intcptsr2   ÚindZgoingdownquietlyZx_varOLSÚtauÚ	interceptrd   ÚpredZtemp_interpZconfintró   rô   ÚecdfZxtempZsquanomZseeZci_adjZtcompZcintÚoutputrA   rA   rB   Útrend_CI°  s   ,




 






" 


*
$
 $




*r  rá   r�   c           !      C   sÊ  ddl m}m} ddlm}m} ddlm} d}t| ƒ}	|	| d }
t	 
|
|	g¡}t	j|d d …< t	 
|
|	g¡}t	j|d d …< t||	d ƒD �]4}t|ƒ |d }|	| }|dt	 tt|ƒƒ¡ƒ}t||ƒD ]ô}||d  }|| }| ||… }t	 |¡}|d	k�r*|j||d
d�\}}}}|||ƒ\}}ntg }| |¡D ]8}|d d }|j|| || d
d�}||d g }�q8t	 |¡}||ƒ} | dƒ}|dk�ržd| }|||| |f< |||| |f< qÊqŠ||gS )Nr   )rÚ   rÝ   rÞ   rØ   râ   r   rŽ   r�   rá   rë   rì   r™   )rø   rÚ   rÝ   rù   rß   rà   r÷   rÙ   r   r   r]   rS   rÉ   rj   rY   r%   r°   r  rÕ   Úmedian)!Ú_tsZ_howZ
_minwindowrÚ   rÝ   rß   rà   rÙ   Z_nbootZ_ntsÚ_nlevelsÚ_slopeÚ_pvalZ_windowÚ_firstÚ_lastr  Ú_blockÚ_x0Ú_x1Z_subtsr  Z_trendr  rñ   rð   Z_taurÏ   r  r2   r  r  r  rA   rA   rB   Útrendpyramid¢  sH    



r&  c                 C   s  |\}}}}|| d }t jdd�}t  ddd¡}	|	jt | d ¡dt jj||d�}
|	jt |¡dk ddd	gd
d� |	 	t
td|dƒƒ¡ |	jt
t|d |dƒƒdd� |	 t
td
|d dƒƒ¡ |	 t
t|ddƒƒ¡ |	 d¡ |	 d¡ |	 d¡ t j|
ddd� t  d¡ t  ¡  d S )Nr   )r�   r�   )Úfigsizer�   Únearest)Úinterpolationrº   ÚvminÚvmaxgš™™™™™©?rT   z///r   )Úhatchesr®   rR   éK   )Úrotationrž   gÍÌÌÌÌÌä?zcentered on yearzwindow width [years]g333333Ã?gš™™™™™Ù?)ÚfractionÚshrinkzrainfall trend [mm/decade])ÚpltÚfigureÚsubplotÚimshowr   ÚflipudÚcmZbwr_rÚcontourfÚ
set_xticksr%   rÉ   Úset_xticklabelsÚ
set_yticksÚset_yticklabelsÚ
set_aspectÚ
set_xlabelÚ
set_ylabelÚcolorbarÚtitleÚshow)r  r   Z
plotparamsr<   r=   r*  r+  ÚnyÚfigr¸   ÚaxrA   rA   rB   ÚplotpyramidÎ  s     " 



rE  )F)	rÑ   rÒ   rÓ   r™   rÔ   r�   NrÕ   F)rá   r�   )Únumpyr   Úpandasr   ÚnetCDF4r   r   r   r   r~   Ú
matplotlibr   r1  Úmpl_toolkits.basemapr   Úmatplotlib.patchesr   Úmatplotlib.collectionsr   rC   rg   rm   r‰   rŠ   r6  ZRdYlBu_rr¿   rÐ   r  r&  rE  rA   rA   rA   rB   Ú<module>   s$   =\>HF!
 
 s
,