def loaddata(dset):
    import functions_rainfall as fun
    import pandas as pd
    import numpy as np
    
    reg="ext"
    root=""

    if reg=="east":
        bnds=[18.8,19.3,-34.3,-33.3]
        region="WCWSS dam region"
    elif reg=="west":
        bnds=[18.31,18.8,-34.4,-33.3]
        region="Coastal region"
    elif reg=="all":
        bnds=[18.31,19.3,-34.4,-33.3]
        region="Cape region"
    elif reg=="south":
        bnds=[17.0,20.0,-35,-32.5]
        region="WCWSS dam region"
    else:
        bnds=[17,20.2,-34.4,-30.3]
        region="Western Cape"


    ext=[17,21,-35,-31] #extent of map

    if dset=="SAWS1930":
        perc=0.9
        fy="1933"
        ly="2018"
        timebnds=[fy,ly]
        timecoverage=['1933','2018',perc]
        timecoverage2=[]
        exclude=[]
        exclude=exclude+['BETTY_S_BAY_SPINNEKOPSNES', 'DISAVLEI', 'JONKERSNEK', 'AVALON'] #most of time has values of 0
        exclude=exclude+[u'NUWEBERG', u'MAITLAND', u'ALTYDGEDACHT', u'LANGEBAAN', u'ALGERIA_-_BOS',
                         u'MERTENHOF', u'DE_HOOP'] # based on buishand test
        exclude=exclude+["S_A_ASTRONOMICAL_OBSERVATORY", "BOSKLOOF"] #manual - because of gaps
        rangeall=[500,1800] #east    
        datafile=root+"/work/data/saws/WC_all_accum-filled_2018.nc"
        pr_day,ids,names,lats,lons=fun.readSAWSonefile(datafile, bnds, timebnds, timecoverage,timecoverage2,  exclude) #updated dataset

    
    
    if dset=="SAWS3a":
        perc=0.9
        fy="1933"
        ly="2017"
        timebnds=[fy,ly]
        timecoverage=['1930','2017',perc]
        timecoverage2=[]
        exclude=[]
        exclude=exclude+['BETTY_S_BAY_SPINNEKOPSNES', 'DISAVLEI', 'JONKERSNEK', 'AVALON'] #most of time have values of 0
        exclude=exclude+[u'NUWEBERG', u'MAITLAND', u'ALTYDGEDACHT', u'LANGEBAAN', u'ALGERIA_-_BOS',
                         u'MERTENHOF', u'VONDELINGSFONTEIN', u'DE_HOOP'] # based on buishand test
        exclude=exclude+["S_A_ASTRONOMICAL_OBSERVATORY", "BOSKLOOF"] #manual - because of gaps
        rangeall=[500,1800] #east    
        datafile=root+"/work/data/saws/WC_all_accum-filled_2018.nc"
        pr_day,ids,names,lats,lons=fun.readSAWSonefile(datafile, bnds, timebnds, timecoverage,timecoverage2,  exclude) #updated dataset


    if dset=="SAWS3a1":
        gapfill=True
        perc=0.9
        fy="1933"
        ly="2018"
        timebnds=[fy,ly]
        timecoverage=['1930','2018',perc]
        timecoverage2=[]
        exclude=[]
        exclude=exclude+['BETTY_S_BAY_SPINNEKOPSNES', 'DISAVLEI', 'JONKERSNEK', 'AVALON'] #most of time has values of 0
        exclude=exclude+[u'NUWEBERG', u'ALTYDGEDACHT', u'LANGEBAAN', u'ALGERIA_-_BOS', u'MERTENHOF', 
                         u'DE_HOOP',u'MAITLAND', u'VONDELINGSFONTEIN'] # based on buishand test
        rangeall=[500,1800] #east    
        datafile=root+"/work/data/saws/WC_all_accum-filled_2018.nc"
        pr_day,ids,names,lats,lons=fun.readSAWSonefile(datafile, bnds, timebnds, timecoverage,timecoverage2,  exclude) #updated dataset



    if dset=="SAWS3b":
        perc=0.1
        fy="1981"
        ly="2017"
        timebnds=[fy,ly]
        timecoverage=['1988','2017',perc]
        timecoverage2=[]
        exclude=[]
        exclude=['BETTY_S_BAY_SPINNEKOPSNES', 'DISAVLEI', 'JONKERSNEK', 'DIE_PLAAT', 'AVALON']
        exclude=exclude+['ALGERIA_-_BOS', "PAARL"]
        exclude=exclude+["S_A_ASTRONOMICAL_OBSERVATORY", "WORCESTER", "MAITLAND"]
        rangeall=[0,800] #ext
        rangeall=[500,1800] #east    
        datafile=root+"/work/data/saws/WC_all_accum-filled.nc"
    #       print datafile, bnds, timebnds, timecoverage, exclude
        pr_day,ids,names,lats,lons=fun.readSAWSonefile(datafile, bnds, timebnds, timecoverage,timecoverage2,  exclude) #updated dataset

    if dset=="SAWS3c":
        perc=0.8
        fy="1900"
        ly="2018"
        timebnds=[fy,ly]
        timecoverage=['1900','2017',perc]
        timecoverage2=[]
        exclude=[]
        exclude=['BETTY_S_BAY_SPINNEKOPSNES', 'DIE_PLAAT', 'DISAVLEI', 'JONKERSNEK', u'AVALON']
    #    exclude=exclude+[u'MAITLAND', u'DARLING_-_THE_TOWERS', u'AVALON', u'PIKETBERG-SAPD']
    #    exclude=exclude+[u'S_A_ASTRONOMICAL_OBSERVATORY', u'ALGERIA_-_BOS']
        rangeall=[0,800] #ext
        rangeall=[500,1800] #east    
        datafile=root+"/work/data/saws/WC_all_accum-filled_2018.nc"
    #        print datafile, bnds, timebnds, timecoverage, exclude
        pr_day,ids,names,lats,lons=fun.readSAWSonefile(datafile, bnds, timebnds, timecoverage,timecoverage2,  exclude) #updated dataset

    if dset=="SAWS3c1":
        gapfill=True
        perc=0.9
        fy="1900"
        ly="2018"
        timebnds=[fy,ly]
        timecoverage=['1900','2018',perc]
        timecoverage2=[]
        exclude=[]
        exclude=['BETTY_S_BAY_SPINNEKOPSNES', 'DIE_PLAAT', 'DISAVLEI', 'JONKERSNEK', u'AVALON']
    #    exclude=exclude+[u'WORCESTER', u'PIKETBERG-SAPD'] #buishand   
    #    exclude=exclude+[u'ALGERIA_-_BOS'] #buishand
    #    exclude=exclude+[u'ROBBEN_ISLAND'] #buishand
    #    exclude=exclude+[u'S_A_ASTRONOMICAL_OBSERVATORY']
    #    exclude=exclude+[u'MERTENHOF'] #count, 2015-2017
    #    exclude=exclude+["WORCESTER", "PAARL"] # large gap
        #no station fails Neumann
        rangeall=[0,800] #ext
        rangeall=[500,1800] #east    
        datafile=root+"/work/data/saws/WC_all_accum-filled_2018.nc"
    #        print datafile, bnds, timebnds, timecoverage, exclude
        pr_day,ids,names,lats,lons=fun.readSAWSonefile(datafile, bnds, timebnds, timecoverage,timecoverage2,  exclude) #updated dataset

    if dset=="SAWS3c2":
        gapfill=False
        perc=0.9
        fy="1900"
        ly="2018"
        timebnds=[fy,ly]
        timecoverage=['1900','2018',perc]
        timecoverage2=[]
        exclude=[]
        exclude=['BETTY_S_BAY_SPINNEKOPSNES', 'DIE_PLAAT', 'DISAVLEI', 'JONKERSNEK', u'AVALON']
        exclude=exclude+[u'ALGERIA_-_BOS',u'S_A_ASTRONOMICAL_OBSERVATORY'] #buis and length
    #    exclude=exclude+[u'WORCESTER'] #all three tests    
    #    exclude=exclude+[, u'MAITLAND'] #count, 2015-2017
    #    exclude=exclude+["WORCESTER", "PAARL"] # large gap
        #no station fails Neumann
        rangeall=[0,800] #ext
        rangeall=[500,1800] #east    
        datafile=root+"/work/data/saws/WC_all_accum-filled_2018.nc"
    #        print datafile, bnds, timebnds, timecoverage, exclude
        pr_day,ids,names,lats,lons=fun.readSAWSonefile(datafile, bnds, timebnds, timecoverage,timecoverage2,  exclude) #updated dataset


    if dset=="DWSa":
        perc=0.8    
        fy="1981"
        ly="2017"
        timebnds=[fy,ly]
        timecoverage=['1981','2017',perc]
        timecoverage2=[]
    #    exclude=["Haweqwas_Stateforest_Stettynskloof_Dam", "Haweqwas_Stateforest_Stettynskloof_Dam"]
        datafile=root+"/work/data/dws/pr_dws_day_19000101-20180710.nc"
        pr_day,ids,names,lats,lons=fun.readDWS2(datafile, bnds, timebnds, timecoverage, timecoverage2, exclude)
        rangeall=[300,1500]


    if dset=="MERGEDa1":
        gapfill=True
        perc=0.9
        fy="1981"
        ly="2017"
        exclude=['BETTY_S_BAY_SPINNEKOPSNES', 'DIE_PLAAT', 'DISAVLEI', 'JONKERSNEK', "AVALON"]
        exclude=exclude+[u'STETTYNSKLOOF', u'LANGEBAAN', u'AVALON', u'WOLSELEY', u'ALGERIA_-_BOS', u'KOEKENAAP']
        exclude=exclude+["NUWEBERG", "ALTYDGEDACHT"]
        exclude=exclude+[u'GREYTON', u'ROBBEN_ISLAND', u'TABLE_MOUNTAIN_HOUSE', u'GROOTE_SCHUUR', u'ASHTON', u'MONTAGU', u'MOORREESBURG', u'LANGGEWENS', u'PORTERVILLE', u'REMHOOGTE', u'CERES', u'DE_DOORNS', u'TOUWSRIVIER', u'AGTERKOP', u'LUTZVILLE_HOTEL', u'CALVINIA_BLINKKLIP', u'Roode_Els_Berg_Roode_Els_Berg_Dam', u'Vredendal', u'Kogel_Baai_Steenbrasdam-Lower', u'Tafelberg', u'Vogel_Vallij_Voelvlei_Dam', u'Doorn_Kwaggaskloof_Dam']
        timebnds=[fy,ly]
        timecoverage=['1981','2017',perc]
        timecoverage2=[]
        datafile=root+"/work/data/dws/pr_dws_day_19000101-20180710.nc"
    #    exclude=["Tafelberg"]
        pr_day2,ids2,names2,lats2,lons2=fun.readDWS2(datafile, bnds, timebnds, timecoverage, timecoverage2, exclude)
    #    pr_day1,ids1,names1,lats1,lons1=fun.readSAWS(bnds, timebnds, timecoverage, exclude) #old dataset
        datafile=root+"/work/data/saws/WC_all_accum-filled_2018.nc"
    #    exclude=[]
    #    exclude=['BETTY_S_BAY_SPINNEKOPSNES', 'DISAVLEI', 'JONKERSNEK', 'DIE_PLAAT', 'AVALON']
    #    exclude=exclude+['ALGERIA_-_BOS', ]
    #    exclude=exclude+["LANGEBAAN", "OU_MURE"]
    #    exclude=exclude+['DIEPDRIFT','PORTERVILLE']
    #    exclude=exclude+['KOEKENAAP', "WOLSELEY", "STETTYNSKLOOF", "TABLE_MOUNTAIN_HOUSE"]
        pr_day1,ids1,names1,lats1,lons1=fun.readSAWSonefile(datafile,bnds, timebnds, timecoverage, timecoverage2, exclude) #updated dataset
        rangeall=[300,800]
        pr_day=pd.concat([pr_day1, pr_day2], axis=1)
        lats=np.append(lats1,lats2)
        lons=np.append(lons1,lons2)
        names=np.append(names1,names2)
        ids=np.append(ids1,ids2)

    if dset=="MERGEDa2":
        gapfill=False
        perc=0.9
        fy="1981"
        ly="2017"
        exclude=['BETTY_S_BAY_SPINNEKOPSNES', 'DIE_PLAAT', 'DISAVLEI', 'JONKERSNEK', "AVALON"]
        exclude=exclude+[u'STETTYNSKLOOF', u'LANGEBAAN', u'AVALON', u'WOLSELEY', u'ALGERIA_-_BOS', u'KOEKENAAP']
        exclude=exclude+["NUWEBERG", "ALTYDGEDACHT"]
        exclude=exclude+[u'GREYTON', u'ROBBEN_ISLAND', u'TABLE_MOUNTAIN_HOUSE', u'GROOTE_SCHUUR', u'ASHTON', u'MONTAGU', u'MOORREESBURG', u'LANGGEWENS', u'PORTERVILLE', u'REMHOOGTE', u'CERES', u'DE_DOORNS', u'TOUWSRIVIER', u'AGTERKOP', u'LUTZVILLE_HOTEL', u'CALVINIA_BLINKKLIP', u'Roode_Els_Berg_Roode_Els_Berg_Dam', u'Vredendal', u'Kogel_Baai_Steenbrasdam-Lower', u'Tafelberg', u'Vogel_Vallij_Voelvlei_Dam', u'Doorn_Kwaggaskloof_Dam']
        timebnds=[fy,ly]
        timecoverage=['1981','2017',perc]
        timecoverage2=[]
        datafile=root+"/work/data/dws/pr_dws_day_19000101-20180710.nc"
    #    exclude=["Tafelberg"]
        pr_day2,ids2,names2,lats2,lons2=fun.readDWS2(datafile, bnds, timebnds, timecoverage, timecoverage2, exclude)
    #    pr_day1,ids1,names1,lats1,lons1=fun.readSAWS(bnds, timebnds, timecoverage, exclude) #old dataset
        datafile=root+"/work/data/saws/WC_all_accum-filled_2018.nc"
    #    exclude=[]
    #    exclude=['BETTY_S_BAY_SPINNEKOPSNES', 'DISAVLEI', 'JONKERSNEK', 'DIE_PLAAT', 'AVALON']
    #    exclude=exclude+['ALGERIA_-_BOS', ]
    #    exclude=exclude+["LANGEBAAN", "OU_MURE"]
    #    exclude=exclude+['DIEPDRIFT','PORTERVILLE']
    #    exclude=exclude+['KOEKENAAP', "WOLSELEY", "STETTYNSKLOOF", "TABLE_MOUNTAIN_HOUSE"]
        pr_day1,ids1,names1,lats1,lons1=fun.readSAWSonefile(datafile,bnds, timebnds, timecoverage, timecoverage2, exclude) #updated dataset
        rangeall=[300,800]
        pr_day=pd.concat([pr_day1, pr_day2], axis=1)
        lats=np.append(lats1,lats2)
        lons=np.append(lons1,lons2)
        names=np.append(names1,names2)
        ids=np.append(ids1,ids2)


    #sel=id_saws!="0040005_W" #langebaan?
    sel=ids!="0021825_W" #paarl
    #names[sel]="PAARL1"
    pr_day=pr_day.iloc[:,sel]
    names=names[sel]
    lats=lats[sel]
    lons=lons[sel]
    ids=ids[sel]

    nstat=len(names)
    #print names
    #print pr_day.shape
    suffix=dset+"_"+fy+"-"+ly+"_"+reg

    print "Station selection criteria:"
    print "dataset: "+dset
    print str(perc*100)+"% of daily data available in the period of "+timecoverage[0]+" to "+timecoverage[1]
    print "Lat-Lon range:", bnds
    print "selected: "+str(nstat)+" stations"
    print names
    print

    return pr_day, names, lats, lons, ids, nstat, suffix, fy, ly