import numpy as np
import pandas as pd
from collections import namedtuple, Counter
from sklearn.ensemble import AdaBoostClassifier

import copy
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor

from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import shap
import xgboost
from sklearn.model_selection import train_test_split
import matplotlib.pylab as pl



from fastai.tabular.data import *
from fastai.tabular.learner import *
from fastai.basics import *
from fastai.tabular.core import *
from fastai.tabular.model import *

def seed_everything(seed):
    print ("setting",seed)
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    #torch.set_deterministic(True)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False


SEED = 1

def process_german(cl_type):
    df=pd.read_csv('./datasets/german.csv')
    l=[]
    for v in df['credit'].values:
        if v==2.0:
            l.append(0)
        else:
            l.append(1)
    df['credit']=l

    orig_df=df


    featlst=list(df.columns)
    featlst.remove('credit')


    for feat in featlst:
        highest=max(df[feat])
        lowest=min(df[feat])
        #if len(list(set(df[feat])))>4:
        #    print (feat,len(list(set(df[feat]))))
        l=df[feat]
        if feat=='month':
            processed=[]
            for v in l:
                if v<=10:
                    processed.append(0)
                elif v<15:
                    processed.append(1)
                elif v<25:
                    processed.append(2)
                else:
                    processed.append(3)
            df[feat]=processed
        if feat=='housing':
            #print (feat)
            processed=[]
            for v in l:
                if v==1:
                    processed.append(1)
                else:
                    processed.append(0)
            df[feat]=processed
        '''if feat=='status':
            #print (feat)
            processed=[]
            for v in l:
                if v==1:
                    processed.append(1)
                elif v==0:
                    processed.append(0)
                else:
                    processed.append(2)
            df[feat]=processed
        '''

        if feat=='installment_plans' or feat=='number_of_credits':
            #print (feat)
            processed=[]
            for v in l:
                if v<=1:
                    processed.append(0)
                else:
                    processed.append(1)
            df[feat]=processed
        if feat=='employment' or feat=='credit_history' or feat=='skill_level':
            #print (feat)
            processed=[]
            for v in l:
                if v<=1:
                    processed.append(0)
                else:
                    processed.append(v)
            df[feat]=processed

        if feat=='credit_amount':
            processed=[]
            for v in l:
                if v<=1500:
                    processed.append(0)
                elif v<3000:
                    processed.append(1)
                elif v<5500:
                    processed.append(2)
                else:
                    processed.append(3)
            df[feat]=processed
        if feat=='purpose':
            l=[]
            for v in df['purpose']:
                if v>4:
                    l.append(5)
                elif v<=2:
                    l.append(2)
                else:
                    l.append(v)
            df[feat]=l
        if feat=='other_debtors':
            processed=[]
            for v in l:
                if v<=0:
                    processed.append(0)
                else:
                    processed.append(1)
            df[feat]=processed
        if feat=='savings':
            processed=[]
            for v in l:
                if v<=0:
                    processed.append(0)
                elif v<=3:
                    processed.append(1)
                else:
                    processed.append(2)
            df[feat]=processed




    df.iloc[605]

    y=df['credit']

    X_train, X_test, y_train, y_test = train_test_split(df, y, test_size=0.8, random_state=1)

    feat=list(df.columns)
    feat.remove('credit')
    if cl_type=="randomforest":
        clf = RandomForestClassifier(max_depth=10, random_state=0)
    elif cl_type=="adaboost":
        clf = AdaBoostClassifier(n_estimators=100, random_state=0)
    elif cl_type=="xgboost":
        xgb_train = xgboost.DMatrix(X_train[feat], label=y_train)
        xgb_test = xgboost.DMatrix(X_test[feat], label=y_test)
        params = {
            "eta": 0.002,
            "max_depth": 3,
            "objective": "binary:logistic",
            "subsample": 0.5
        }
        model_train = xgboost.train(params, xgb_train, 10000, evals = [(xgb_train, "test")], verbose_eval=1000)
        pred=model_train.predict(xgb_test, ntree_limit=5000)
        final_pred=[]
        for p in pred:
            if p>0.5:
                final_pred.append(1)
            else:
                final_pred.append(0)
        pred=final_pred
        print(classification_report(y_test, pred))
        return (X_train[feat],y_train,X_test[feat],pred,model_train)
    elif cl_type=="neural":
        class TabularLearner(Learner):
            "`Learner` for tabular data"
            def predict(self, row):
                "Predict on a Pandas Series"
                dl = self.dls.test_dl(row.to_frame().T)
                dl.dataset.conts = dl.dataset.conts.astype(np.float32)
                inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
                b = (*tuplify(inp),*tuplify(dec_preds))
                full_dec = self.dls.decode(b)
                #print("FD",preds)
                return full_dec,dec_preds[0],preds[0]

        #export
        @delegates(Learner.__init__)
        def tabular_learner(dls, layers=None, emb_szs=None, config=None, n_out=None, y_range=None, **kwargs):
            "Get a `Learner` using `dls`, with `metrics`, including a `TabularModel` created using the remaining params."
            if config is None: config = tabular_config()
            if layers is None: layers = [200,100]
            to = dls.train_ds
            emb_szs = get_emb_sz(dls.train_ds, {} if emb_szs is None else emb_szs)
            if n_out is None: n_out = get_c(dls)
            assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
            if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
            model = TabularModel(emb_szs, len(dls.cont_names), n_out, layers, y_range=y_range, **config)
            return TabularLearner(dls, model, **kwargs)


        pd.options.mode.chained_assignment = None  # default='warn'
        #X_test=pd.read_csv('adult_neural_output.csv')
        #pred=X_test['target']
        
        X_train['credit']=list(y_train)
        X_train['credit']=X_train['credit'].apply(str)
        g = X_train.groupby('credit')
        X_train=g.apply(lambda x: x.sample(g.size().min()).reset_index(drop=True))
        X_train=X_train.sample(frac=1)

        procs = [Categorify, FillMissing, Normalize]
        dls = TabularDataLoaders.from_df(X_train,  procs=procs,
                                         y_names="credit", valid_idx=list(range(0,50)), bs=64)

        learn = tabular_learner(dls)
        
        learn=load_learner('./adultmode.pkl')
        feat=list(X_train.columns)
        feat.remove('credit')
        pred=[]
        for iter,row in X_test.iterrows():
            if iter%1000==0:
                print(iter)
            #print (iter,row)
            row, clas, probs = learn.predict(row[feat])
            pred.append(int(clas))
        X_test['credit']=pred
        print(classification_report(y_test, pred))
        
        return(X_train[feat],y_train,X_test[feat],pred,learn)

    #clf = LogisticRegression(random_state=0)
    if not cl_type=="xgboost":
        clf.fit(X_train[feat], y_train)

        pred= (clf.predict(X_test[feat]))
        pred = [int(round(value)) for value in pred]
    #print(pred)
    print(classification_report(y_test, pred))
    pd.options.mode.chained_assignment = None  # default='warn'

    #print (clf.feature_importances_,feat)
    X_test['credit']  = pred
    #df=X_test
    
    return (X_train[feat],y_train,X_test[feat],pred,clf)



def process_adult(cl_type):
    from sklearn.utils import resample
    seed_everything(SEED)
    df=pd.read_csv('./datasets/train.txt',delimiter=' ')

    backdoor={'Age':[],'sex':[],'country':[],'marital':['country','Age'],'edu':['Age','sex','marital','country'],
              'class':['Age','sex','marital','country','edu'],'occupation':['Age','sex','marital','country','edu'],
              'hours':['Age','sex','marital','country','edu']}
    
    y_train=df['target']

    col=list(df.columns)
    col.remove('target')
    col.remove('Unnamed: 15')

    col=list(backdoor.keys())
    df['sex']=1-df['sex']



    featlst=list(df.columns)
    featlst.remove('target')


    for feat in col:
        highest=max(df[feat])
        lowest=min(df[feat])
        if len(list(set(df[feat])))>4:
            print (feat,len(list(set(df[feat]))))
        l=df[feat]
        if feat=='country':
            proc=[]
            for v in l:
                if v==0:
                    proc.append(v)
                elif v<=4:
                    proc.append(2)
                elif v<=10:
                    proc.append(3)
                else:
                    proc.append(4)
            df[feat]=proc
        elif feat=='Age':
            proc=[]
            for v in l:
                if v<=30:
                    proc.append(1)
                elif v<=40:
                    proc.append(2)
                elif v<=50:
                    proc.append(3)
                else:
                    proc.append(4)
            df[feat]=proc
        elif feat=='marital':
            proc=[]
            for v in l:
                if v<2:
                    proc.append(v)
                elif v<=3:
                    proc.append(3)
                else:
                    proc.append(5)
            df[feat]=proc
        elif feat=='edu':
            proc=[]
            for v in l:
                if v<=1:
                    proc.append(v)
                elif v<=4:
                    proc.append(2)
                elif v==5:
                    proc.append(5)
                else:
                    proc.append(6)
            df[feat]=proc
        elif feat=='class':
            proc=[]
            for v in l:
                if v<=2:
                    proc.append(v)
                else:
                    proc.append(6)
            df[feat]=proc
        elif feat=='occupation':
            proc=[]
            for v in l:
                if v<=2:
                    proc.append(1)
                elif v<=5:
                    proc.append(v)
                elif v<=8:
                    proc.append(8)
                else:
                    proc.append(9)
            df[feat]=proc
        elif feat=='hours':
            proc=[]
            for v in l:
                if v<=25:
                    proc.append(1)
                elif v<=41:
                    proc.append(2)
                elif v<=55:
                    proc.append(8)
                else:
                    proc.append(9)
            df[feat]=proc
            



    X_train=df[col]

    X_test=pd.read_csv('./datasets/test.txt',delimiter=' ')
    y_test=X_test['target']
    X_test=X_test[col]
    X_test['sex']=1-X_test['sex']


    for feat in col:
        if len(list(set(X_test[feat])))>4:
            print (feat,len(list(set(X_test[feat]))))
        l=X_test[feat]
        if feat=='country':
            proc=[]
            for v in l:
                if v==0:
                    proc.append(v)
                elif v<=4:
                    proc.append(2)
                elif v<=10:
                    proc.append(3)
                else:
                    proc.append(4)
            X_test[feat]=proc
        elif feat=='Age':
            proc=[]
            for v in l:
                if v<=30:
                    proc.append(1)
                elif v<=40:
                    proc.append(2)
                elif v<=50:
                    proc.append(3)
                else:
                    proc.append(4)
            X_test[feat]=proc
        elif feat=='marital':
            proc=[]
            for v in l:
                if v<2:
                    proc.append(v)
                elif v<=3:
                    proc.append(3)
                else:
                    proc.append(5)
            X_test[feat]=proc
        elif feat=='edu':
            proc=[]
            for v in l:
                if v<=1:
                    proc.append(v)
                elif v<=4:
                    proc.append(2)
                elif v==5:
                    proc.append(5)
                else:
                    proc.append(6)
            X_test[feat]=proc
        elif feat=='class':
            proc=[]
            for v in l:
                if v<=2:
                    proc.append(v)
                else:
                    proc.append(6)
            X_test[feat]=proc
        elif feat=='occupation':
            proc=[]
            for v in l:
                if v<=2:
                    proc.append(1)
                elif v<=5:
                    proc.append(v)
                elif v<=8:
                    proc.append(8)
                else:
                    proc.append(9)
            X_test[feat]=proc
        elif feat=='hours':
            proc=[]
            for v in l:
                if v<=25:
                    proc.append(1)
                elif v<=41:
                    proc.append(2)
                elif v<=55:
                    proc.append(8)
                else:
                    proc.append(9)
            X_test[feat]=proc
      
    if cl_type=="randomforest":
        clf = RandomForestClassifier(max_depth=10, random_state=0)
        
    elif cl_type=="adaboost":
        clf = AdaBoostClassifier(n_estimators=100, random_state=0)
    elif cl_type=="xgboost":
        feat=list(X_train.columns)
        #feat.remove('target')
        xgb_train = xgboost.DMatrix(X_train[feat], label=y_train)
        xgb_test = xgboost.DMatrix(X_test[feat], label=y_test)
        params = {
            "eta": 0.002,
            "max_depth": 3,
            "objective": "binary:logistic",
            "subsample": 0.5
        }
        model_train = xgboost.train(params, xgb_train, 10000, evals = [(xgb_train, "test")], verbose_eval=1000)
        pred=model_train.predict(xgb_test, ntree_limit=5000)
        final_pred=[]
        for p in pred:
            if p>0.5:
                final_pred.append(1)
            else:
                final_pred.append(0)
        pred=final_pred
        print(classification_report(y_test, pred))
        return (X_train[feat],y_train,X_test[feat],pred,model_train)
    

    elif cl_type=="neural":
        
        class TabularLearner(Learner):
            "`Learner` for tabular data"
            def predict(self, row):
                "Predict on a Pandas Series"
                dl = self.dls.test_dl(row.to_frame().T)
                dl.dataset.conts = dl.dataset.conts.astype(np.float32)
                inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
                b = (*tuplify(inp),*tuplify(dec_preds))
                full_dec = self.dls.decode(b)
                #print("FD",preds)
                return full_dec,dec_preds[0],preds[0]

        #export
        @delegates(Learner.__init__)
        def tabular_learner(dls, layers=None, emb_szs=None, config=None, n_out=None, y_range=None, **kwargs):
            "Get a `Learner` using `dls`, with `metrics`, including a `TabularModel` created using the remaining params."
            if config is None: config = tabular_config()
            if layers is None: layers = [200,100]
            to = dls.train_ds
            emb_szs = get_emb_sz(dls.train_ds, {} if emb_szs is None else emb_szs)
            if n_out is None: n_out = get_c(dls)
            assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
            if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
            model = TabularModel(emb_szs, len(dls.cont_names), n_out, layers, y_range=y_range, **config)
            return TabularLearner(dls, model, **kwargs)

        
        learn=load_learner('./adultmodel.pkl')
        
        pd.options.mode.chained_assignment = None  # default='warn'
        
        
        

        '''
        X_test=X_test[:1000]
        y_test=y_test[:1000]
        X_train['target']=list(y_train)
        X_train['target']=X_train['target'].apply(str)
        g = X_train.groupby('target')
        X_train=g.apply(lambda x: x.sample(g.size().min()).reset_index(drop=True))
        X_train=X_train.sample(frac=1,random_state=0)

        procs = [Categorify, FillMissing, Normalize]
        dls = TabularDataLoaders.from_df(X_train,  procs=procs,
                                         y_names="target", valid_idx=list(range(0,500)), bs=64,seed=0)

        print(dls.dataset)


        learn = tabular_learner(dls)
        feat=list(X_train.columns)
        feat.remove('target')
        pred=[]
        for iter,row in X_test.iterrows():
            if iter%1000==0:
                print(iter)
            row, clas, probs = learn.predict(X_test.iloc[iter])
            pred.append(int(clas))
        X_test['target']=pred
        print(classification_report(y_test, pred))
        '''
        X_test=pd.read_csv('adult_neural_output.csv')
        pred=X_test['target']

        print(classification_report(y_test, pred))
        return(X_train[feat],y_train,X_test,pred,learn)

    if not cl_type=="xgboost":
        clf.fit(X_train, y_train)

        pred= (clf.predict(X_test))
        pred = [int(round(value)) for value in pred]

    print(classification_report(y_test, pred))

    print (clf.feature_importances_)

    X_test['target']=pred
    df=X_test
    return (X_train,y_train,X_test,pred,clf)




#export



def process_compas(predtype,cl_type):

    df=pd.read_csv('../compas/compas-scores-two-years.csv')

    df=df[df['days_b_screening_arrest']>=-30]#.value_counts()#[df['score_text']=='N/A']
    df=df[df['days_b_screening_arrest']<=30]


    # In[262]:

    df=df[df['race']!='Hispanic']# or df['race']=='Caucasian']
    df=df[df['race']!='Other']
    df=df[df['race']!='Asian']
    df=df[df['race']!='Native American']



    df=df[[ 'vr_charge_degree', 'r_charge_degree',
           'is_recid', 'c_charge_degree', 'priors_count', 'juv_other_count',
           'juv_misd_count', 'juv_fel_count', 'race', 'age_cat', 'sex','score_text']]
    
        #score_text

    l=list(df['race'])
    cleaned=[]
    for v in l:
        if v=='Caucasian':
            cleaned.append(1)
        else:
            cleaned.append(0)
            
    df['race']=cleaned


    l=list(df['c_charge_degree'])
    cleaned=[]
    for v in l:
        if v=='F':
            cleaned.append(1)
        else:
            cleaned.append(0)
            
    df['c_charge_degree']=cleaned


    l=list(df['score_text'])
    cleaned=[]
    for v in l:
        if v=='Low':
            cleaned.append(0)
        elif v=='Medium':
            cleaned.append(1)
        else:
            cleaned.append(1)
            
    df['score_text']=cleaned


    l=list(df['vr_charge_degree'])
    cleaned=[]
    for v in l:
        if v=='NaN':
            cleaned.append(0)
        elif v=='(F3)':
            cleaned.append(1)
        elif v=='(M1)':
            cleaned.append(2)
        else:
            cleaned.append(2)
    df['vr_charge_degree']=cleaned


    l=list(df['r_charge_degree'])
    cleaned=[]
    for v in l:
        if v=='NaN':
            cleaned.append(0)
        elif v=='(F3)':
            cleaned.append(1)
        elif v=='(M1)':
            cleaned.append(2)
        else:
            cleaned.append(2)
    df['r_charge_degree']=cleaned

    l=list(df['juv_other_count'])
    cleaned=[]
    for v in l:
        if v==0:
            cleaned.append(0)
        elif v==1:
            cleaned.append(1)
        else:
            cleaned.append(1)
            

    l=list(df['juv_fel_count'])
    cleaned=[]
    for v in l:
        if v==0:
            cleaned.append(0)
        elif v==1:
            cleaned.append(1)
        else:
            cleaned.append(1)
            
    df['juv_fel_count']=cleaned

            

    l=list(df['juv_misd_count'])
    cleaned=[]
    for v in l:
        if v==0:
            cleaned.append(0)
        elif v==1:
            cleaned.append(1)
        else:
            cleaned.append(1)
            
    df['juv_misd_count']=cleaned

    df['juv_fel_count']=df['juv_fel_count']+df['juv_misd_count']+df['juv_other_count']
    df['juv_other_count']=[0]*df.shape[0]
    df['juv_misd_count']=[0]*df.shape[0]
    df=df[['vr_charge_degree',  'r_charge_degree',
           'is_recid', 'c_charge_degree', 'priors_count', 'juv_fel_count', 'race', 'age_cat', 'sex','score_text']]
    l=list(df['juv_fel_count'])
    cleaned=[]
    for v in l:
        if v==0:
            cleaned.append(0)
        elif v==1:
            cleaned.append(1)
        else:
            cleaned.append(1)

    df['juv_fel_count']=cleaned




    l=list(df['priors_count'])
    cleaned=[]
    for v in l:
        
        if v==0:
            cleaned.append(0)
        elif v<=3:
            cleaned.append(1)
        else:
            cleaned.append(2)
        
    df['priors_count']=cleaned

    l=list(df['sex'])
    cleaned=[]
    for v in l:
        if v=='Female':
            cleaned.append(0)
        else:
            cleaned.append(1)
        
    df['sex']=cleaned

    l=list(df['age_cat'])
    cleaned=[]
    for v in l:
        if v=='Greater than 45':
            cleaned.append(2)
        elif v=='25-45':
            cleaned.append(1)
        else:
            cleaned.append(0)
    df['age_cat']=cleaned
    if cl_type==-1 or cl_type=='-1':
        return (df,df[predtype],df,df[predtype],[])

    y=df[predtype]#is_recid']
    X_train, X_test, y_train, y_test = train_test_split(df, y, test_size=0.5, random_state=1)



    feat=list(df.columns)
    #if predtype=='score_text':
    feat.remove('score_text')
    feat.remove('is_recid')
    feat.remove('vr_charge_degree')
    feat.remove('r_charge_degree')
    feat.remove('c_charge_degree')
    X_train=X_train[feat]
    X_test=X_test[feat]

    if cl_type=="randomforest":
        clf = RandomForestClassifier(max_depth=10, random_state=0)    
    elif cl_type=="adaboost":
        clf = AdaBoostClassifier(n_estimators=100, random_state=0)
    elif cl_type=="xgboost":
        feat=list(X_train.columns)
        if predtype in feat:
            feat.remove(predtype)
        
        xgb_train = xgboost.DMatrix(X_train[feat], label=y_train)
        xgb_test = xgboost.DMatrix(X_test[feat], label=y_test)
        params = {
            "eta": 0.002,
            "max_depth": 3,
            "objective": "binary:logistic",
            "subsample": 0.5
        }
        model_train = xgboost.train(params, xgb_train, 10000, evals = [(xgb_train, "test")], verbose_eval=1000)
        pred=model_train.predict(xgb_test, ntree_limit=5000)
        final_pred=[]
        for p in pred:
            if p>0.5:
                final_pred.append(1)
            else:
                final_pred.append(0)
        pred=final_pred
        print(classification_report(y_test, pred))
        return (X_train[feat],y_train,X_test[feat],pred,model_train)
    

    elif cl_type=="neural":
        class TabularLearner(Learner):
            "`Learner` for tabular data"
            def predict(self, row):
                "Predict on a Pandas Series"
                dl = self.dls.test_dl(row.to_frame().T)
                dl.dataset.conts = dl.dataset.conts.astype(np.float32)
                inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
                b = (*tuplify(inp),*tuplify(dec_preds))
                full_dec = self.dls.decode(b)
                #print("FD",preds)
                return full_dec,dec_preds[0],preds[0]

        #export
        @delegates(Learner.__init__)
        def tabular_learner(dls, layers=None, emb_szs=None, config=None, n_out=None, y_range=None, **kwargs):
            "Get a `Learner` using `dls`, with `metrics`, including a `TabularModel` created using the remaining params."
            if config is None: config = tabular_config()
            if layers is None: layers = [200,100]
            to = dls.train_ds
            emb_szs = get_emb_sz(dls.train_ds, {} if emb_szs is None else emb_szs)
            if n_out is None: n_out = get_c(dls)
            assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
            if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
            model = TabularModel(emb_szs, len(dls.cont_names), n_out, layers, y_range=y_range, **config)
            return TabularLearner(dls, model, **kwargs)


        pd.options.mode.chained_assignment = None  # default='warn'

        X_train[predtype]=list(y_train)
        X_train[predtype]=X_train[predtype].apply(str)
        g = X_train.groupby(predtype)
        X_train=g.apply(lambda x: x.sample(g.size().min()).reset_index(drop=True))
        X_train=X_train.sample(frac=1)

        procs = [Categorify, FillMissing, Normalize]
        dls = TabularDataLoaders.from_df(X_train,  procs=procs,
                                         y_names=predtype, valid_idx=list(range(0,100)), bs=64)

        learn = tabular_learner(dls)
        feat=list(X_train.columns)
        feat.remove(predtype)
        pred=[]
        for iter,row in X_test.iterrows():
            if iter%1000==0:
                print(iter)
            row, clas, probs = learn.predict(row)
            pred.append(int(clas))
        X_test[predtype]=pred
        print(classification_report(y_test, pred))
        return(X_train[feat],y_train,X_test[feat],pred,learn)

    if not cl_type=="xgboost":
        clf.fit(X_train[feat], y_train)

        pred= (clf.predict(X_test))

        pred = [int(round(value)) for value in pred]
       
    print(classification_report(y_test, pred))
    pd.options.mode.chained_assignment = None  # default='warn'
    print (clf.feature_importances_)

    X_test[predtype]=pred
    df=X_test
    return (X_train,y_train,X_test,pred,clf)

def process(lst):
    final=[]
    for v in lst:
        if v>0:
            final.append(1)
        else:
            final.append(0)
    return final
def process_drug(cl_type,predvariable):
    df=pd.read_csv('../drug_consumption.data')
    cols=['Age','Gender','Edu','Country','Ethnicity','Nscore','Escore','Oscore','Ascore','Cscore','impulsive','SS',predvariable]
    cleaned_df=df[cols]
    df=df[cols]

    pd.options.mode.chained_assignment = None  # default='warn'

    for feat in cols:
            l=df[feat]
            processed=df[feat]
            if feat==predvariable:
                processed=[]
                for v in l:
                    if v=='CL0':
                        processed.append(0)
                    elif v=='CL1':
                        processed.append(1)
                    elif v=='CL2':
                        processed.append(2)
                    elif v=='CL3':
                        processed.append(2)
                    else:
                        processed.append(2)
            if feat=='Oscore'or feat=='Escore' or feat=='Ascore' or feat=='Cscore'or feat=='impulsive'or feat=='SS':
                processed=[]
                for v in l:
                    if v<=-1.5:
                        processed.append(0)
                    elif v<0:
                        processed.append(1)
                    elif v<1:
                        processed.append(2)
                    else:
                        processed.append(3)
            if feat=='Nscore':
                
                processed=[]
                for v in l:
                    if v<=-1.5:
                        processed.append(0)
                    elif v<0:
                        processed.append(1)
                    elif v<1:
                        processed.append(2)
                    else:
                        processed.append(3)
            if feat=='Age':
                processed=[]
                for v in l:
                    if v<=-0.95 :
                        processed.append(0)
                    elif v<0.0:
                        processed.append(1)
                    elif v<=0.5:
                        processed.append(2)
                    else:
                        processed.append(3)
                df[feat]=processed
            if feat=='Gender':
                processed=[]
                for v in l:
                    if v<=0:
                        processed.append(0)
                    else:
                        processed.append(1)
            if feat=='Edu':
                processed=[]
                for v in l:
                    if v<-1:
                        processed.append(0)
                    elif v<0:
                        processed.append(1)
                    elif v<1:
                        processed.append(2)
                    else:
                        processed.append(3)
            if feat=='Country':
                processed=[]
                for v in l:
                    if v<-0.5:
                        processed.append(0)
                    elif v<0:
                        processed.append(1)
                    elif v<0.9:
                        processed.append(2)
                    else:
                        processed.append(3)
            if feat=='Ethnicity':
                processed=[]
                for v in l:
                    if v==-0.31685:
                        processed.append(0)
                    else:
                        processed.append(1)
            df[feat]=processed

    y=df[predvariable]
    feat=list(df.columns)
    feat.remove(predvariable)
    X_train, X_test, y_train, y_test = train_test_split(df, y, test_size=0.3, random_state=1)

    y_train=process(y_train)
    y_test=process(y_test)


    if cl_type=="randomforest":
        clf = RandomForestClassifier(max_depth=10, random_state=0)    
    elif cl_type=="adaboost":
        clf = AdaBoostClassifier(n_estimators=100, random_state=0)
    elif cl_type=="xgboost":
        feat=list(X_train.columns)
        if predvariable in feat:
            feat.remove(predvariable)
        
        xgb_train = xgboost.DMatrix(X_train[feat], label=y_train)
        xgb_test = xgboost.DMatrix(X_test[feat], label=y_test)
        params = {
            "eta": 0.002,
            "max_depth": 3,
            "objective": "binary:logistic",
            "subsample": 0.5
        }
        model_train = xgboost.train(params, xgb_train, 10000, evals = [(xgb_train, "test")], verbose_eval=1000)
        pred=model_train.predict(xgb_test, ntree_limit=5000)
        final_pred=[]
        for p in pred:
            if p>0.5:
                final_pred.append(1)
            else:
                final_pred.append(0)
        pred=final_pred
        print(classification_report(y_test, pred))
        return (X_train[feat],y_train,X_test[feat],pred,model_train)
    

    elif cl_type=="neural":
        class TabularLearner(Learner):
            "`Learner` for tabular data"
            def predict(self, row):
                "Predict on a Pandas Series"
                dl = self.dls.test_dl(row.to_frame().T)
                dl.dataset.conts = dl.dataset.conts.astype(np.float32)
                inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
                b = (*tuplify(inp),*tuplify(dec_preds))
                full_dec = self.dls.decode(b)
                #print("FD",preds)
                return full_dec,dec_preds[0],preds[0]

        #export
        @delegates(Learner.__init__)
        def tabular_learner(dls, layers=None, emb_szs=None, config=None, n_out=None, y_range=None, **kwargs):
            "Get a `Learner` using `dls`, with `metrics`, including a `TabularModel` created using the remaining params."
            if config is None: config = tabular_config()
            if layers is None: layers = [200,100]
            to = dls.train_ds
            emb_szs = get_emb_sz(dls.train_ds, {} if emb_szs is None else emb_szs)
            if n_out is None: n_out = get_c(dls)
            assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
            if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
            model = TabularModel(emb_szs, len(dls.cont_names), n_out, layers, y_range=y_range, **config)
            return TabularLearner(dls, model, **kwargs)


        pd.options.mode.chained_assignment = None  # default='warn'

        X_train[predvariable]=list(y_train)
        X_train[predvariable]=X_train[predvariable].apply(str)
        g = X_train.groupby(predvariable)
        X_train=g.apply(lambda x: x.sample(g.size().min()).reset_index(drop=True))
        X_train=X_train.sample(frac=1)

        procs = [Categorify, FillMissing, Normalize]
        dls = TabularDataLoaders.from_df(X_train,  procs=procs,
                                         y_names=predvariable, valid_idx=list(range(0,100)), bs=64)

        learn = tabular_learner(dls)
        feat=list(X_train.columns)
        feat.remove(predvariable)
        pred=[]
        for iter,row in X_test.iterrows():
            if iter%1000==0:
                print(iter)
            row, clas, probs = learn.predict(row)
            pred.append(int(clas))
        X_test[predvariable]=pred
        print(classification_report(y_test, pred))
        return(X_train[feat],y_train,X_test[feat],pred,learn)

    if not cl_type=="xgboost":
        clf.fit(X_train[feat], y_train)

        pred= (clf.predict(X_test[feat]))
        pred = [int(round(value)) for value in pred]
       
    print(classification_report(y_test, pred))
    pd.options.mode.chained_assignment = None  # default='warn'
    print (clf.feature_importances_)
    #print (pred[13],y_test[13],"here pred")
    X_test[predvariable]=pred
    df=X_test
    return (X_train,y_train,X_test,pred,clf)

    clf = RandomForestClassifier(max_depth=10, random_state=0)
    clf.fit(X_train[feat], y_train)

    pred= (clf.predict(X_test[feat]))
    pred = [int(round(value)) for value in pred]


def process_ad():
    df=pd.read_csv('../adult/train.txt',delimiter=' ')

    backdoor={'Age':[],'sex':[],'country':[],'marital':['country','Age'],'edu':['Age','sex','marital','country'],
              'class':['Age','sex','marital','country','edu'],'occupation':['Age','sex','marital','country','edu'],
              'hours':['Age','sex','marital','country','edu']}

    y_train=df['target']

    col=list(df.columns)
    col.remove('target')
    col.remove('Unnamed: 15')

    col=list(backdoor.keys())
    df['sex']=1-df['sex']



    featlst=list(df.columns)
    featlst.remove('target')


    for feat in col:
        highest=max(df[feat])
        lowest=min(df[feat])
        if len(list(set(df[feat])))>4:
            print (feat,len(list(set(df[feat]))))
        l=df[feat]
        if feat=='country':
            proc=[]
            for v in l:
                if v==0:
                    proc.append(v)
                elif v<=4:
                    proc.append(2)
                elif v<=10:
                    proc.append(3)
                else:
                    proc.append(4)
            df[feat]=proc
        elif feat=='Age':
            proc=[]
            for v in l:
                if v<=30:
                    proc.append(1)
                elif v<=40:
                    proc.append(2)
                elif v<=50:
                    proc.append(3)
                else:
                    proc.append(4)
            df[feat]=proc
        elif feat=='marital':
            proc=[]
            for v in l:
                if v<2:
                    proc.append(v)
                elif v<=3:
                    proc.append(3)
                else:
                    proc.append(5)
            df[feat]=proc
        elif feat=='edu':
            proc=[]
            for v in l:
                if v<=1:
                    proc.append(v)
                elif v<=4:
                    proc.append(2)
                elif v==5:
                    proc.append(5)
                else:
                    proc.append(6)
            df[feat]=proc
        elif feat=='class':
            proc=[]
            for v in l:
                if v<=2:
                    proc.append(v)
                else:
                    proc.append(6)
            df[feat]=proc
        elif feat=='occupation':
            proc=[]
            for v in l:
                if v<=2:
                    proc.append(1)
                elif v<=5:
                    proc.append(v)
                elif v<=8:
                    proc.append(8)
                else:
                    proc.append(9)
            df[feat]=proc
        elif feat=='hours':
            proc=[]
            for v in l:
                if v<=25:
                    proc.append(1)
                elif v<=41:
                    proc.append(2)
                elif v<=55:
                    proc.append(8)
                else:
                    proc.append(9)
            df[feat]=proc




    X_train=df[col]

    X_test=pd.read_csv('../adult/test.txt',delimiter=' ')
    y_test=X_test['target']
    X_test=X_test[col]
    X_test['sex']=1-X_test['sex']


    for feat in col:
        if len(list(set(X_test[feat])))>4:
            print (feat,len(list(set(X_test[feat]))))
        l=X_test[feat]
        if feat=='country':
            proc=[]
            for v in l:
                if v==0:
                    proc.append(v)
                elif v<=4:
                    proc.append(2)
                elif v<=10:
                    proc.append(3)
                else:
                    proc.append(4)
            X_test[feat]=proc
        elif feat=='Age':
            proc=[]
            for v in l:
                if v<=30:
                    proc.append(1)
                elif v<=40:
                    proc.append(2)
                elif v<=50:
                    proc.append(3)
                else:
                    proc.append(4)
            X_test[feat]=proc
        elif feat=='marital':
            proc=[]
            for v in l:
                if v<2:
                    proc.append(v)
                elif v<=3:
                    proc.append(3)
                else:
                    proc.append(5)
            X_test[feat]=proc
        elif feat=='edu':
            proc=[]
            for v in l:
                if v<=1:
                    proc.append(v)
                elif v<=4:
                    proc.append(2)
                elif v==5:
                    proc.append(5)
                else:
                    proc.append(6)
            X_test[feat]=proc
        elif feat=='class':
            proc=[]
            for v in l:
                if v<=2:
                    proc.append(v)
                else:
                    proc.append(6)
            X_test[feat]=proc
        elif feat=='occupation':
            proc=[]
            for v in l:
                if v<=2:
                    proc.append(1)
                elif v<=5:
                    proc.append(v)
                elif v<=8:
                    proc.append(8)
                else:
                    proc.append(9)
            X_test[feat]=proc
        elif feat=='hours':
            proc=[]
            for v in l:
                if v<=25:
                    proc.append(1)
                elif v<=41:
                    proc.append(2)
                elif v<=55:
                    proc.append(8)
                else:
                    proc.append(9)
            X_test[feat]=proc


    # In[2]:


    learn=load_learner('./adultmodel.pkl')

    print(learn)

