import numpy as np
import pandas as pd
from collections import namedtuple, Counter
from sklearn.ensemble import AdaBoostClassifier
import copy
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor

from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
#import shap
import xgboost
from sklearn.model_selection import train_test_split
import matplotlib.pylab as pl


from fastai.tabular.data import *
from fastai.tabular.learner import *
from fastai.basics import *
from fastai.tabular.core import *
from fastai.tabular.model import *
def seed_everything(seed):
    #print ("setting",seed)
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    #torch.set_deterministic(True)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False


SEED = 1

def process_german(cl_type):

    df=pd.read_csv('./datasets/german_binned.csv')
    featlst=list(df.columns)
    featlst.remove('credit')


    y=df['credit']
    X_train, X_test, y_train, y_test = train_test_split(df, y, test_size=0.5, random_state=1)

    feat=list(df.columns)
    feat.remove('credit')
    if cl_type=="randomforest":
        clf = RandomForestClassifier(max_depth=10, random_state=0)
    elif cl_type=="adaboost":
        clf = AdaBoostClassifier(n_estimators=100, random_state=0)
    elif cl_type=="xgboost":
        xgb_train = xgboost.DMatrix(X_train[feat], label=y_train)
        xgb_test = xgboost.DMatrix(X_test[feat], label=y_test)
        params = {
            "eta": 0.002,
            "max_depth": 3,
            "objective": "binary:logistic",
            "subsample": 0.5
        }
        model_train = xgboost.train(params, xgb_train, 10000, evals = [(xgb_train, "test")], verbose_eval=1000)
        pred=model_train.predict(xgb_test, ntree_limit=5000)
        final_pred=[]
        for p in pred:
            if p>0.5:
                final_pred.append(1)
            else:
                final_pred.append(0)
        pred=final_pred
        #print(classification_report(y_test, pred))
        return (X_train[feat],y_train,X_test[feat],pred,model_train)
    elif cl_type=="neural":
        class TabularLearner(Learner):
            "`Learner` for tabular data"
            def predict(self, row):
                "Predict on a Pandas Series"
                dl = self.dls.test_dl(row.to_frame().T)
                dl.dataset.conts = dl.dataset.conts.astype(np.float32)
                inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
                b = (*tuplify(inp),*tuplify(dec_preds))
                full_dec = self.dls.decode(b)
                #print("FD",preds)
                return full_dec,dec_preds[0],preds[0]

        #export
        @delegates(Learner.__init__)
        def tabular_learner(dls, layers=None, emb_szs=None, config=None, n_out=None, y_range=None, **kwargs):
            "Get a `Learner` using `dls`, with `metrics`, including a `TabularModel` created using the remaining params."
            if config is None: config = tabular_config()
            if layers is None: layers = [200,100]
            to = dls.train_ds
            emb_szs = get_emb_sz(dls.train_ds, {} if emb_szs is None else emb_szs)
            if n_out is None: n_out = get_c(dls)
            assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
            if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
            model = TabularModel(emb_szs, len(dls.cont_names), n_out, layers, y_range=y_range, **config)
            return TabularLearner(dls, model, **kwargs)


        pd.options.mode.chained_assignment = None  # default='warn'
        #X_test=pd.read_csv('adult_neural_output.csv')
        #pred=X_test['target']
        
        X_train['credit']=list(y_train)
        X_train['credit']=X_train['credit'].apply(str)
        g = X_train.groupby('credit')
        X_train=g.apply(lambda x: x.sample(g.size().min()).reset_index(drop=True))
        X_train=X_train.sample(frac=1)

        procs = [Categorify, FillMissing, Normalize]
        dls = TabularDataLoaders.from_df(X_train,  procs=procs,
                                         y_names="credit", valid_idx=list(range(0,50)), bs=64)

        learn = tabular_learner(dls)
        
        learn=load_learner('./adultmode.pkl')
        feat=list(X_train.columns)
        feat.remove('credit')
        pred=[]
        for iter,row in X_test.iterrows():
            #if iter%1000==0:
            #    print(iter)
            #print (iter,row)
            row, clas, probs = learn.predict(row[feat])
            pred.append(int(clas))
        X_test['credit']=pred
        #print(classification_report(y_test, pred))
        
        return(X_train[feat],y_train,X_test[feat],pred,learn)

    #clf = LogisticRegression(random_state=0)
    if not cl_type=="xgboost":
        clf.fit(X_train[feat], y_train)

        pred= (clf.predict(X_test[feat]))
        pred = [int(round(value)) for value in pred]
    #print(pred)
    #print(classification_report(y_test, pred))
    pd.options.mode.chained_assignment = None  # default='warn'

    #print (clf.feature_importances_,feat)
    X_test['credit']  = pred
    #df=X_test
    
    return (X_train[feat],y_train,X_test[feat],pred,clf)



def process_adult(cl_type):
    from sklearn.utils import resample
    seed_everything(SEED)
    backdoor={'Age':[],'sex':[],'country':[],'marital':['country','Age'],'edu':['Age','sex','marital','country'],
              'class':['Age','sex','marital','country','edu'],'occupation':['Age','sex','marital','country','edu'],
              'hours':['Age','sex','marital','country','edu']}
    

    df=pd.read_csv('./datasets/adult_train.csv')
    y_train=df['target']

    col=list(df.columns)
    col.remove('target')

    col=list(backdoor.keys())
   
    featlst=list(df.columns)
    featlst.remove('target')

    X_train=df[col]

    X_test=pd.read_csv('./datasets/adult_test.csv')
    y_test=X_test['target']
    X_test=X_test[col]
    

    if cl_type=="randomforest":
        clf = RandomForestClassifier(max_depth=10, random_state=0)
        
    elif cl_type=="adaboost":
        clf = AdaBoostClassifier(n_estimators=100, random_state=0)
    elif cl_type=="xgboost":
        feat=list(X_train.columns)
        #feat.remove('target')
        xgb_train = xgboost.DMatrix(X_train[feat], label=y_train)
        xgb_test = xgboost.DMatrix(X_test[feat], label=y_test)
        params = {
            "eta": 0.002,
            "max_depth": 3,
            "objective": "binary:logistic",
            "subsample": 0.5
        }
        model_train = xgboost.train(params, xgb_train, 10000, evals = [(xgb_train, "test")], verbose_eval=1000)
        pred=model_train.predict(xgb_test, ntree_limit=5000)
        final_pred=[]
        for p in pred:
            if p>0.5:
                final_pred.append(1)
            else:
                final_pred.append(0)
        pred=final_pred
        #print(classification_report(y_test, pred))
        return (X_train[feat],y_train,X_test[feat],pred,model_train)
    

    elif cl_type=="neural":
        
        class TabularLearner(Learner):
            "`Learner` for tabular data"
            def predict(self, row):
                "Predict on a Pandas Series"
                dl = self.dls.test_dl(row.to_frame().T)
                dl.dataset.conts = dl.dataset.conts.astype(np.float32)
                inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
                b = (*tuplify(inp),*tuplify(dec_preds))
                full_dec = self.dls.decode(b)
                #print("FD",preds)
                return full_dec,dec_preds[0],preds[0]

        #export
        @delegates(Learner.__init__)
        def tabular_learner(dls, layers=None, emb_szs=None, config=None, n_out=None, y_range=None, **kwargs):
            "Get a `Learner` using `dls`, with `metrics`, including a `TabularModel` created using the remaining params."
            if config is None: config = tabular_config()
            if layers is None: layers = [200,100]
            to = dls.train_ds
            emb_szs = get_emb_sz(dls.train_ds, {} if emb_szs is None else emb_szs)
            if n_out is None: n_out = get_c(dls)
            assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
            if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
            model = TabularModel(emb_szs, len(dls.cont_names), n_out, layers, y_range=y_range, **config)
            return TabularLearner(dls, model, **kwargs)

        
        learn=load_learner('./adultmodel.pkl')
        
        pd.options.mode.chained_assignment = None  # default='warn'
        
        
        

        '''
        X_test=X_test[:1000]
        y_test=y_test[:1000]
        X_train['target']=list(y_train)
        X_train['target']=X_train['target'].apply(str)
        g = X_train.groupby('target')
        X_train=g.apply(lambda x: x.sample(g.size().min()).reset_index(drop=True))
        X_train=X_train.sample(frac=1,random_state=0)

        procs = [Categorify, FillMissing, Normalize]
        dls = TabularDataLoaders.from_df(X_train,  procs=procs,
                                         y_names="target", valid_idx=list(range(0,500)), bs=64,seed=0)

        print(dls.dataset)


        learn = tabular_learner(dls)
        feat=list(X_train.columns)
        feat.remove('target')
        pred=[]
        for iter,row in X_test.iterrows():
            if iter%1000==0:
                print(iter)
            row, clas, probs = learn.predict(X_test.iloc[iter])
            pred.append(int(clas))
        X_test['target']=pred
        print(classification_report(y_test, pred))
        '''
        X_test=pd.read_csv('adult_neural_output.csv')
        pred=X_test['target']

        #print(classification_report(y_test, pred))
        return(X_train[col],y_train,X_test,pred,learn)

    if not cl_type=="xgboost":
        clf.fit(X_train, y_train)

        pred= (clf.predict(X_test))
        pred = [int(round(value)) for value in pred]

    #print(classification_report(y_test, pred))

    #print (clf.feature_importances_)

    X_test['target']=pred
    df=X_test
    return (X_train,y_train,X_test,pred,clf)




#export



def process_compas(predtype,cl_type):

    df=pd.read_csv('./datasets/compas_binned.csv')
    
    y=df[predtype]#is_recid']
    X_train, X_test, y_train, y_test = train_test_split(df, y, test_size=0.5, random_state=1)
    #print(cl_type)
    feat=list(df.columns)
    #if predtype=='score_text':
    feat.remove('score_text')
    feat.remove('is_recid')
    feat.remove('vr_charge_degree')
    feat.remove('r_charge_degree')
    feat.remove('c_charge_degree')
    X_train=X_train[feat]
    X_test=X_test[feat]

    if cl_type=="randomforest":
        clf = RandomForestClassifier(max_depth=10, random_state=0)    
    elif cl_type=="adaboost":
        clf = AdaBoostClassifier(n_estimators=100, random_state=0)
    elif cl_type=="xgboost":
        feat=list(X_train.columns)
        if predtype in feat:
            feat.remove(predtype)
        
        xgb_train = xgboost.DMatrix(X_train[feat], label=y_train)
        xgb_test = xgboost.DMatrix(X_test[feat], label=y_test)
        params = {
            "eta": 0.002,
            "max_depth": 3,
            "objective": "binary:logistic",
            "subsample": 0.5
        }
        model_train = xgboost.train(params, xgb_train, 10000, evals = [(xgb_train, "test")], verbose_eval=1000)
        pred=model_train.predict(xgb_test, ntree_limit=5000)
        final_pred=[]
        for p in pred:
            if p>0.5:
                final_pred.append(1)
            else:
                final_pred.append(0)
        pred=final_pred
        #print(classification_report(y_test, pred))
        return (X_train[feat],y_train,X_test[feat],pred,model_train)
    

    elif cl_type=="neural":
        class TabularLearner(Learner):
            "`Learner` for tabular data"
            def predict(self, row):
                "Predict on a Pandas Series"
                dl = self.dls.test_dl(row.to_frame().T)
                dl.dataset.conts = dl.dataset.conts.astype(np.float32)
                inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
                b = (*tuplify(inp),*tuplify(dec_preds))
                full_dec = self.dls.decode(b)
                #print("FD",preds)
                return full_dec,dec_preds[0],preds[0]

        #export
        @delegates(Learner.__init__)
        def tabular_learner(dls, layers=None, emb_szs=None, config=None, n_out=None, y_range=None, **kwargs):
            "Get a `Learner` using `dls`, with `metrics`, including a `TabularModel` created using the remaining params."
            if config is None: config = tabular_config()
            if layers is None: layers = [200,100]
            to = dls.train_ds
            emb_szs = get_emb_sz(dls.train_ds, {} if emb_szs is None else emb_szs)
            if n_out is None: n_out = get_c(dls)
            assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
            if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
            model = TabularModel(emb_szs, len(dls.cont_names), n_out, layers, y_range=y_range, **config)
            return TabularLearner(dls, model, **kwargs)


        pd.options.mode.chained_assignment = None  # default='warn'

        X_train[predtype]=list(y_train)
        X_train[predtype]=X_train[predtype].apply(str)
        g = X_train.groupby(predtype)
        X_train=g.apply(lambda x: x.sample(g.size().min()).reset_index(drop=True))
        X_train=X_train.sample(frac=1)

        procs = [Categorify, FillMissing, Normalize]
        dls = TabularDataLoaders.from_df(X_train,  procs=procs,
                                         y_names=predtype, valid_idx=list(range(0,100)), bs=64)

        learn = tabular_learner(dls)
        feat=list(X_train.columns)
        feat.remove(predtype)
        pred=[]
        for iter,row in X_test.iterrows():
            #if iter%1000==0:
            #    print(iter)
            row, clas, probs = learn.predict(row)
            pred.append(int(clas))
        X_test[predtype]=pred
        #print(classification_report(y_test, pred))
        return(X_train[feat],y_train,X_test[feat],pred,learn)
    else:
        return (df,df,df,df,df)


    if not cl_type=="xgboost":
        clf.fit(X_train[feat], y_train)

        pred= (clf.predict(X_test))

        pred = [int(round(value)) for value in pred]
       
    #print(classification_report(y_test, pred))
    pd.options.mode.chained_assignment = None  # default='warn'
    #print (clf.feature_importances_)

    X_test[predtype]=pred
    df=X_test
    return (X_train,y_train,X_test,pred,clf)

def process(lst):
    final=[]
    for v in lst:
        if v>0:
            final.append(1)
        else:
            final.append(0)
    return final
def process_drug(cl_type,predvariable):
    #df=pd.read_csv('../drug_consumption.data')
    #cols=['Age','Gender','Edu','Country','Ethnicity','Nscore','Escore','Oscore','Ascore','Cscore','impulsive','SS',predvariable]


    #pd.options.mode.chained_assignment = None  # default='warn'
    df=pd.read_csv('./datasets/drug_binned'+predvariable+'.csv')
    #df.to_csv('/Users/sainyam/Documents/sainyam_umass_laptop/Documents/Explainable/Explainability/cleancode/datasets/drug_binned'+predvariable+'.csv',index=False)
    #jfkls
    y=df[predvariable]

    feat=list(df.columns)
    feat.remove(predvariable)
    X_train, X_test, y_train, y_test = train_test_split(df, y, test_size=0.3, random_state=1)


    y_train=process(y_train)
    y_test=process(y_test)


    if cl_type=="randomforest":
        clf = RandomForestClassifier(max_depth=10, random_state=0)    
    elif cl_type=="adaboost":
        clf = AdaBoostClassifier(n_estimators=100, random_state=0)
    elif cl_type=="xgboost":
        feat=list(X_train.columns)
        if predvariable in feat:
            feat.remove(predvariable)
        
        xgb_train = xgboost.DMatrix(X_train[feat], label=y_train)
        xgb_test = xgboost.DMatrix(X_test[feat], label=y_test)
        params = {
            "eta": 0.002,
            "max_depth": 3,
            "objective": "binary:logistic",
            "subsample": 0.5
        }
        model_train = xgboost.train(params, xgb_train, 10000, evals = [(xgb_train, "test")], verbose_eval=1000)
        pred=model_train.predict(xgb_test, ntree_limit=5000)
        final_pred=[]
        for p in pred:
            if p>0.5:
                final_pred.append(1)
            else:
                final_pred.append(0)
        pred=final_pred
        #print(classification_report(y_test, pred))
        return (X_train[feat],y_train,X_test[feat],pred,model_train)
    

    elif cl_type=="neural":
        class TabularLearner(Learner):
            "`Learner` for tabular data"
            def predict(self, row):
                "Predict on a Pandas Series"
                dl = self.dls.test_dl(row.to_frame().T)
                dl.dataset.conts = dl.dataset.conts.astype(np.float32)
                inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
                b = (*tuplify(inp),*tuplify(dec_preds))
                full_dec = self.dls.decode(b)
                #print("FD",preds)
                return full_dec,dec_preds[0],preds[0]

        #export
        @delegates(Learner.__init__)
        def tabular_learner(dls, layers=None, emb_szs=None, config=None, n_out=None, y_range=None, **kwargs):
            "Get a `Learner` using `dls`, with `metrics`, including a `TabularModel` created using the remaining params."
            if config is None: config = tabular_config()
            if layers is None: layers = [200,100]
            to = dls.train_ds
            emb_szs = get_emb_sz(dls.train_ds, {} if emb_szs is None else emb_szs)
            if n_out is None: n_out = get_c(dls)
            assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
            if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
            model = TabularModel(emb_szs, len(dls.cont_names), n_out, layers, y_range=y_range, **config)
            return TabularLearner(dls, model, **kwargs)


        pd.options.mode.chained_assignment = None  # default='warn'

        X_train[predvariable]=list(y_train)
        X_train[predvariable]=X_train[predvariable].apply(str)
        g = X_train.groupby(predvariable)
        X_train=g.apply(lambda x: x.sample(g.size().min()).reset_index(drop=True))
        X_train=X_train.sample(frac=1)

        procs = [Categorify, FillMissing, Normalize]
        dls = TabularDataLoaders.from_df(X_train,  procs=procs,
                                         y_names=predvariable, valid_idx=list(range(0,100)), bs=64)

        learn = tabular_learner(dls)
        feat=list(X_train.columns)
        feat.remove(predvariable)
        pred=[]
        for iter,row in X_test.iterrows():
            #if iter%1000==0:
            #    print(iter)
            row, clas, probs = learn.predict(row)
            pred.append(int(clas))
        X_test[predvariable]=pred
        #print(classification_report(y_test, pred))
        return(X_train[feat],y_train,X_test[feat],pred,learn)

    if not cl_type=="xgboost":
        clf.fit(X_train[feat], y_train)

        pred= (clf.predict(X_test[feat]))
        pred = [int(round(value)) for value in pred]
       
    #print(classification_report(y_test, pred))
    pd.options.mode.chained_assignment = None  # default='warn'
    #print (clf.feature_importances_)
    #print (pred[13],y_test[13],"here pred")
    X_test[predvariable]=pred
    df=X_test
    return (X_train,y_train,X_test,pred,clf)

    clf = RandomForestClassifier(max_depth=10, random_state=0)
    clf.fit(X_train[feat], y_train)

    pred= (clf.predict(X_test[feat]))
    pred = [int(round(value)) for value in pred]


