import datasethelper,lewishelper
import copy
import pandas as pd
import sys
import random,os,torch
import numpy as np

import numpy as np
import pandas as pd
from collections import namedtuple, Counter
from sklearn.ensemble import AdaBoostClassifier

import copy
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor

from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

import xgboost
from sklearn.model_selection import train_test_split
import matplotlib.pylab as pl



from fastai.tabular.data import *
from fastai.tabular.learner import *
from fastai.basics import *
from fastai.tabular.core import *
from fastai.tabular.model import *



def seed_everything(seed):
    #print ("setting",seed)
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    #torch.set_deterministic(True)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

seed_everything(1)
cl_type=sys.argv[1]
#df=datasethelper.process_adult()
#print ("calling process")


(X_train,y_train,df,pred,clf)=datasethelper.process_adult(cl_type)

df['target']=pred

#df=df.sample(n=1000)
#backdoor={'Age':[],'sex':[],'country':[],'marital':['country','Age','sex'],'edu':['Age','sex','marital','country'],
#              'class':['Age','sex','marital','country','edu'],'occupation':['Age','sex','marital','country','edu'],
#              'hours':['Age','sex','marital','country','edu']}
    
backdoor={'Age':[],'sex':[],'country':[],'marital':['country','Age','sex'],'edu':['Age','sex','country'],
              'class':['Age','sex','country'],'occupation':['Age','sex','country'],
              'hours':['Age','sex','country']}
#backdoor={'Age':[],'sex':[],'country':[],'marital':[],'edu':[],
#              'class':[],'occupation':[],
#              'hours':[]}
featlst=list(df.columns)
if 'target' in featlst:
    featlst.remove('target')

def get_majority(dic1,s1,s2,feat):
    score=[0,0,0]
    if feat in dic1.keys():
        score=dic1[feat]
        
    i=0
    while i<3:
        score[i]=max(score[i],s1[i])
        score[i]=max(score[i],s2[i])
        i+=1
    dic1[feat]=score
    return dic1

def get_majority(dic1,s1,s2,feat):
    score=[0,0,0]
    if feat in dic1.keys():
        score=dic1[feat]

    i=0
    while i<3:
        score[i]=max(score[i],s1[i])
        score[i]=max(score[i],s2[i])
        i+=1
    dic1[feat]=score
    return dic1




score_dic={}

featlst=['Age','sex','country','marital','edu','class','occupation','hours']
for feat in featlst:
    fout=open('outputlatestfinalsome1','a')
    #print (feat)
    uniqval=list(set(list(df[feat])))
    val={}
    for v in uniqval:
        sample=df[df[feat]==v]
        try: 
            val[v]=sample['target'].value_counts()[1]*1.0/(len(list(sample['target'].values)))#.value_counts()[0]+sample['target'].value_counts()[1])
        except:
            val[v]=0
    #print (val)
    import operator
    sorted_val = sorted(val.items(), key=operator.itemgetter(1))


    before= (sorted_val[0][0])
    after=(sorted_val[-1][0])


    score1=lewishelper.get_scores_regression(df,[feat],[after],[before],[],[],backdoor[feat],'target')
    score_dic=get_majority(score_dic,score1,(0,0,0),feat)
    fout.write(feat+"  "+str(score_dic[feat][0])+" "+str(score_dic[feat][1])+" "+str(score_dic[feat][2])+"\n")
    #print(feat+"  "+str(score_dic[feat][0])+" "+str(score_dic[feat][1])+" "+str(score_dic[feat][2])+"\n")
    fout.close()



scoreocc=lewishelper.get_scores_regression(df,['occupation'],[3],[8],[],[],backdoor['occupation'],'target')

score_dic=get_majority(score_dic,scoreocc,(0,0,0),'occupation')
'''for feat in featlst:
    fout=open('outputlatestfinalsome1','a')
    print (feat)
    uniqval=list(set(list(df[feat])))
    i=0
    while i<len(uniqval):
        print(i)
        j=i+1
        while j<len(uniqval):
            score1=lewishelper.get_scores_regression(df,[feat],[uniqval[i]],[uniqval[j]],[],[],backdoor[feat],'target')
            score2=lewishelper.get_scores_regression(df,[feat],[uniqval[j]],[uniqval[i]],[],[],backdoor[feat],'target')
            score_dic=get_majority(score_dic,score1,score2,feat)
            print (uniqval[i],uniqval[j],score1,score2)
            j+=1
        i+=1
    fout.write(feat+"  "+str(score_dic[feat][0])+" "+str(score_dic[feat][1])+" "+str(score_dic[feat][2])+"\n")
    print(feat+"  "+str(score_dic[feat][0])+" "+str(score_dic[feat][1])+" "+str(score_dic[feat][2])+"\n")
    fout.close()
'''
#print (score_dic)


# In[17]:

n_score={}
s_score={}
sn_score={}
n=[]
sn=[]
s=[]
snname=[]
for feat in ['Age','sex','class','marital','country','occupation','edu','hours']:

    if feat in score_dic.keys():
        n_score[feat]=score_dic[feat][0]
        s_score[feat]=score_dic[feat][1]
        sn_score[feat]=score_dic[feat][2]
        snname.append(feat)
        n.append(n_score[feat])
        sn.append(sn_score[feat])
        s.append(s_score[feat])
    #print (feat,n_score[feat],s_score[feat],sn_score[feat])


snname=['Age','sex','class','marital','country','occupation','edu','hours']

import pandas as pd
from functools import reduce

sn = pd.DataFrame(list(zip(sn, snname)), columns=['sn', 'name'])
n = pd.DataFrame(list(zip(n, snname)), columns=['n', 'name'])
s = pd.DataFrame(list(zip(s, snname)), columns=['s', 'name'])


dfs = [sn, n, s]
allScores = reduce(lambda left,right: pd.merge(left,right,on='name'), dfs) #join

allScores = allScores.sort_values(by='sn', ascending=False)

featnames = allScores['name'] 

import scipy.stats as ss
nrank=ss.rankdata(allScores['n'])
srank=ss.rankdata(allScores['s'])
snrank=ss.rankdata(allScores['sn'])

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pylab as plot

fsize=20


params = {'legend.fontsize': fsize/1.3,
          'legend.handlelength': 2}
plot.rcParams.update(params)

font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rcParams['font.family'] = "sans-serif"
matplotlib.rc('font', **font)

labels = featnames
x = np.arange(len(labels))  # the label locations
width = 0.25  # the width of the bars

# plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x - width, allScores['n'], width, label='Nec', color='cornflowerblue', edgecolor='black', hatch="//")
rects2 = ax.barh(x, allScores['s'], width, label='Suf', color='gold', edgecolor='black', hatch="\\\\")
rects3 = ax.barh(x + width, allScores['sn'], width, label='NeSuf', color='forestgreen', hatch="|")

plt.gca().invert_yaxis()
# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Scores',labelpad=fsize/2)
ax.set_yticks(x)
ax.set_yticklabels(labels)
ax.legend()
# plt.legend(bbox_to_anchor=(.2, 1.2, 1., .102), loc=3,
#        ncol=3, borderaxespad=0., fontsize=fsize)
ax.xaxis.set_ticks_position('top')
ax.xaxis.set_label_position('top')
plt.xticks(np.arange(0, 1.5, .5))
figure = plt.gcf() # get current figure
figure.set_size_inches(5.8,8)

def autolabel(rects,rank):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        val=len(labels)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height - 0.02, rect.get_y() + 0.32),
                    xytext=(15, 0),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.5)
        i+=1


autolabel(rects1,nrank)
autolabel(rects2,srank)
autolabel(rects3,snrank)
ax.margins(0.1, 0.05)
matplotlib.rcParams['hatch.linewidth'] = 0.3

fig.tight_layout()
#plt.savefig('newplots/adult_globalExplanations'+cl_type+'.pdf')

featnames =  ['country','sex','class','marital','Age','occupation','edu','hours']
model=load_learner('./adultmodel.pkl')
df=pd.read_csv('adult_neural_output.csv')
pred=df['target']


shapley=[0.01107587, 0.00642941, 0.00577296, 0.01347946, 0.00379818, 0.00684064,
 0.01700787, 0.02015363] 

shapley_name=['hours', 'edu', 'marital', 'Age', 'sex', 'country', 'class', 'occupation']


sn=sn['sn']
#print (sn)
#print (snname)
#jklk


print (sn)
print(snname)

from mlxtend.evaluate import feature_importance_permutation
import seaborn as scs
    
normalized_shapley=[]
normalized_feat=[]
normalized_sn=[]
i=0
for feat in ['Age','sex','class','marital','country','occupation','edu','hours']:

    normalized_sn.append(sn[i]*1.0/max(sn))
    i+=1
    
i=0
for feat in featnames:
    normalized_shapley.append(shapley[shapley_name.index(feat)]*1.0/max(shapley))
    i+=1
import scipy.stats as ss
shapley_rank=ss.rankdata(normalized_shapley)
feat_rank=ss.rankdata(normalized_feat)
sn_rank=ss.rankdata(normalized_sn)


import matplotlib
import matplotlib.pyplot as plt
import numpy as np

import pylab as plot
fsize=20
params = {'legend.fontsize': fsize/1.1,
          'legend.handlelength': 2}
plot.rcParams.update(params)

font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rcParams['font.family'] = "sans-serif"
matplotlib.rc('font', **font)

labels = ['country','sex','class','marital','Age','occupation','edu','hours']
x = np.arange(len(labels))  # the label locations
width = 0.25  # the width of the bars
plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x, normalized_shapley, width, label='SHAP', color='lightcoral', edgecolor='black', hatch="//")
#rects2 = ax.barh(x, normalized_feat, width, label='Feat', color='gainsboro', edgecolor='black', hatch="\\\\")
rects3 = ax.barh(x + 0.25, normalized_sn, width, label='Lewis', color='forestgreen', hatch='|')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Scores', fontsize=fsize, labelpad=fsize/2)
ax.set_yticks(x)
ax.set_yticklabels(labels, fontsize=fsize)
ax.xaxis.set_ticks_position('top')
ax.xaxis.set_label_position('top')
plt.xticks(np.arange(0, 1.5, .5))
ax.legend(loc=(0.51,0))
ax.invert_yaxis()
def autolabel(rects,rank):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        val=len(shapley_rank)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height - 0.02, rect.get_y() + 0.25),
                    xytext=(15, 0),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.5)
        i+=1


autolabel(rects1,shapley_rank)
#autolabel(rects2,feat_rank)
autolabel(rects3,sn_rank)

ax.margins(0.1, 0.05)
matplotlib.rcParams['hatch.linewidth'] = 0.2
figure = plt.gcf() # get current figure
figure.set_size_inches(5.5,7.5)
fig.tight_layout()
plt.savefig('newplots/adult_global_related_'+cl_type+'.pdf')
