import datasethelper,lewishelper
import copy
import pandas as pd
import sys

cl_type=sys.argv[1]
#predvariable='coke'
predvariable='mushrooms'
#df=datasethelper.process_adult()
(X_train,y_train,df,pred,model)=datasethelper.process_drug(cl_type,predvariable)

def process(lst):
    final=[]
    for v in lst:
        if v>0:
            final.append(1)
        else:
            final.append(0)
    return final
df[predvariable]=process(pred)




featlst=['Age','Gender','Edu','Country','Ethnicity','Nscore','Escore','Oscore','Ascore','Cscore','impulsive','SS',predvariable]
#df[predvariable]=pred
backdoor={'Age':[],'Gender':[],'Country':['Age','Gender'],'Ethnicity':[],'marital':['Gender','Ethnicity','Age'],'edu':['Age','sex','marital'],
              'class':['Age','sex','marital'],'occupation':['Age','sex','marital'],
              'hours':['Age','sex','marital']}
#backdoor={'Age':[],'Gender':[],'Country':['Age','Gender'],'Ethnicity':[],'marital':['Gender','Country','Ethnicity','Age'],'edu':['Age','sex','marital','country'],
#              'class':['Age','sex','marital','country','edu'],'occupation':['Age','sex','marital','country','edu'],
#              'hours':['Age','sex','marital','country','edu']}
    
featlst=list(df.columns)
if predvariable in featlst:
    featlst.remove(predvariable)

def get_majority(dic1,s1,s2,feat):
    score=[0,0,0]
    if feat in dic1.keys():
        score=dic1[feat]
        
    i=0
    while i<3:
        score[i]=max(score[i],s1[i])
        score[i]=max(score[i],s2[i])
        i+=1
    dic1[feat]=score
    return dic1

def get_majority(dic1,s1,s2,feat):
    score=[0,0,0]
    if feat in dic1.keys():
        score=dic1[feat]

    i=0
    while i<3:
        score[i]=max(score[i],s1[i])
        score[i]=max(score[i],s2[i])
        i+=1
    dic1[feat]=score
    return dic1
#print (df['Age'].value_counts())



score_dic={}
#featlst=['Edu']
featlst=['Age','Gender','Edu','Country','Ethnicity','Nscore','Escore','Oscore','Ascore','Cscore','impulsive','SS']#'Age','Gender','Country','Ethnicity']
for feat in featlst:
    fout=open('outputlatestfinalsome1','a')
    uniqval=list(set(list(df[feat])))
    if feat in ['Age','Gender','Ethnicity','Country']:
        backdoor[feat]=[]
    elif feat  in ['Edu']:
        backdoor[feat]=['Age','Gender','Ethnicity']
        #backdoor[feat]=['Age','Gender','Country','Ethnicity']
    else:
        backdoor[feat]=['Age','Gender','Country']
        #backdoor[feat]=['Age','Gender','Country','Edu','Ethnicity']

    uniqval=list(set(list(df[feat])))
    val={}
    for v in uniqval:
        sample=df[df[feat]==v]
        val[v]=sample[predvariable].value_counts()[1]*1.0/(sample[predvariable].value_counts()[0]+sample[predvariable].value_counts()[1])
    import operator
    sorted_val = sorted(val.items(), key=operator.itemgetter(1))


    before= (sorted_val[0][0])
    after=(sorted_val[-1][0])


    score1=lewishelper.get_scores_regression(df,[feat],[after],[before],[],[],backdoor[feat],predvariable)
    score_dic=get_majority(score_dic,score1,(0,0,0),feat)
    fout.write(feat+"  "+str(score_dic[feat][0])+" "+str(score_dic[feat][1])+" "+str(score_dic[feat][2])+"\n")
    #print(feat+"  "+str(score_dic[feat][0])+" "+str(score_dic[feat][1])+" "+str(score_dic[feat][2])+"\n")
    fout.close()


'''


    i=0
    while i<len(uniqval):
        print(i)
        j=i+1
        while j<len(uniqval):

            score1=lewishelper.get_scores_regression(df,[feat],[uniqval[i]],[uniqval[j]],[],[],backdoor[feat],predvariable)
            print ("z=",uniqval[i],score1)
            score2=lewishelper.get_scores_regression(df,[feat],[uniqval[j]],[uniqval[i]],[],[],backdoor[feat],predvariable)
            print ("z=",uniqval[j],score2)
            score_dic=get_majority(score_dic,score1,score2,feat)
            print (uniqval[i],uniqval[j],score1,score2)
            j+=1
        i+=1
    fout.write(feat+"  "+str(score_dic[feat][0])+" "+str(score_dic[feat][1])+" "+str(score_dic[feat][2])+"\n")
    print(feat+"  "+str(score_dic[feat][0])+" "+str(score_dic[feat][1])+" "+str(score_dic[feat][2])+"\n")
    fout.close()
'''
#print (score_dic)


# In[17]:

n_score={}
s_score={}
sn_score={}
for feat in score_dic.keys():
    n_score[feat]=score_dic[feat][0]
    s_score[feat]=score_dic[feat][1]
    sn_score[feat]=score_dic[feat][2]
    #print (feat,n_score[feat],s_score[feat],sn_score[feat])


sorted_s = sorted(s_score.items(), key=lambda kv: kv[1],reverse=True)
sorted_n = sorted(n_score.items(), key=lambda kv: kv[1],reverse=True)
sorted_sn = sorted(sn_score.items(), key=lambda kv: kv[1],reverse=True)

slst=[]
nlst=[]
snlst=[]
svallst=[]
nvallst=[]
snvallst=[]

i=0
for (x,y) in sorted_s:
    slst.append(x)
    svallst.append(y)
    nlst.append(sorted_n[i][0])
    nvallst.append(sorted_n[i][1])
    snlst.append(sorted_sn[i][0])
    snvallst.append(sorted_sn[i][1])
    i+=1


#print ("SN")
#print (snlst)
#print (snvallst)
# In[ ]:

#print ("N")
#print (nlst)
#print (nvallst)

#print ("S")
#print (slst)
#print(svallst)


sn=snvallst
n=nvallst
s=svallst

snname=snlst

import pandas as pd
from functools import reduce

sn = pd.DataFrame(list(zip(sn, snname)), columns=['sn', 'name'])
n = pd.DataFrame(list(zip(n, snname)), columns=['n', 'name'])
s = pd.DataFrame(list(zip(s, snname)), columns=['s', 'name'])

dfs = [sn, n, s]
allScores = reduce(lambda left,right: pd.merge(left,right,on='name'), dfs) #join



allScores = allScores.sort_values(by='sn', ascending=True)

#allScores['name'] = ["skill", "telephone", "foreign_worker", "other_debtors", "liable_for", "num_credits", "sex", "employment", "residence_since", "purpose", "install_plans", "credit_amt", "age", "investment", "savings", "housing", "property", "month", "status", "credit_hist"]
featnames = allScores['name']#["skill", "telephone", "foreign_worker", "other_debtors", "liable_for", "num_credits", "sex", "employment", "residence_since", "purpose", "install_plans", "cred_amt", "age", "invest", "savings", "housing", "property", "month", "status", "cred_hist"]

allScores = allScores.tail(9)[::-1]
featnames = featnames[-9:]

import scipy.stats as ss
nrank=ss.rankdata(allScores['n'])
srank=ss.rankdata(allScores['s'])
snrank=ss.rankdata(allScores['sn'])
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pylab as plot

fsize=20


params = {'legend.fontsize': fsize/1.4,
          'legend.handlelength': 2}
plot.rcParams.update(params)

font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rcParams['font.family'] = "sans-serif"
matplotlib.rc('font', **font)

labels = ['country','age','sensation','openness','gender','impulsive','consceintous','edu','extraversion']#allScores['name']
x = np.arange(len(labels))  # the label locations
width = 0.25  # the width of the bars

# plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x - width, allScores['n'], width, label='Nec', color='cornflowerblue', edgecolor='black', hatch="//")
rects2 = ax.barh(x, allScores['s'], width, label='Suf', color='gold', edgecolor='black', hatch="\\\\")
rects3 = ax.barh(x + width, allScores['sn'], width, label='NeSuf', color='forestgreen', hatch="|")

plt.gca().invert_yaxis()
# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Scores',labelpad=fsize/2)
ax.set_yticks(x)
ax.set_yticklabels(labels)
ax.legend()
# plt.legend(bbox_to_anchor=(.2, 1.2, 1., .102), loc=3,
#        ncol=3, borderaxespad=0., fontsize=fsize)
ax.xaxis.set_ticks_position('top')
ax.xaxis.set_label_position('top')
plt.xticks(np.arange(0, 1.5, .5))
figure = plt.gcf() # get current figure
figure.set_size_inches(5.8,8)

def autolabel(rects,rank):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        val=len(labels)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height - 0.02, rect.get_y() + 0.32),
                    xytext=(15, 0),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.4)
        i+=1


autolabel(rects1,nrank)
autolabel(rects2,srank)
autolabel(rects3,snrank)
ax.margins(0.07,0.05)
matplotlib.rcParams['hatch.linewidth'] = 0.3

fig.tight_layout()
plt.savefig('newplots/drug_globalExplanations'+cl_type+'.pdf')


import shap
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(df)[0]

#shap.summary_plot(shap_values, df, plot_type="bar")
#print ("shap values",shap_values)




sc=shap_values[0]
iter=0
for l in shap_values:
        if iter==0:
                sc=shap_values[0]
        else:
                i=0
                while i<len(l):
                        sc[i]+=abs(l[i])
                        i+=1
        iter+=1
lst=[]
for v in sc:
        lst.append(v*1.0/len(shap_values))
#print ("shapley")
#print (lst)

shapley=lst



from sklearn.inspection import permutation_importance
feat=list(df.columns)
feat.remove('mushrooms')
r = permutation_importance(model, df[feat], pred,n_repeats=100,random_state=0)
feat_imp=(r.importances_mean)
#print(df.columns)



import seaborn as scs
shapley=[ 0.01121601, 0.023852650000000003, 0.01996573, 0.02119625, 0.02715466, 0.03742173, 0.04368159, 0.06173148, 0.07709118, 0.08539859, 0.11431267, 0.14801137]

shapley_name=['ethnicity', 'ascore', 'escore', 'edu', 'nscore', 'conscientious', 'impulsive', 'gender', 'openness', 'sensation', 'age', 'country']

featval=[0.16886926, 0.07291519, 0.04273852, 0.25491166, 0.01388693, 0.03800353,
 0.03862191, 0.10646643, 0.03106007, 0.05948763, 0.04911661, 0.09174912]
feat_name=['age', 'gender', 'edu', 'country', 'ethnicity', 'nscore', 'escore',
       'openness', 'ascore', 'conscientious', 'impulsive', 'sensation']

#option='sn'

#if option=='sn':
snname=['country', 'age','sensation', 'openness', 'gender','impulsive','conscientious', 'edu' ,'ascore',  'escore', 'nscore', 'ethnicity']
#    sn=[0.7999999774079127, 0.5084598499610704,0.4123716930399792, 0.3507042984262648, 0.289260310532623,0.2590867854546664,0.13646368884354748,  0.11367208242963402,0.07695628171742809, 0.11357273944443974, 0.028786873906365685, 0.008439428978561758]


#snname=sn['name']
sn=sn['sn']
normalized_shapley=[]
normalized_feat=[]
normalized_sn=[]
i=0
for feat in snname:

    normalized_shapley.append(shapley[shapley_name.index(feat)]*1.0/max(shapley))
    normalized_feat.append(featval[feat_name.index(feat)]*1.0/max(featval))
    normalized_sn.append(sn[i]*1.0/max(sn))
    i+=1

import scipy.stats as ss
shapley_rank=ss.rankdata(normalized_shapley)
feat_rank=ss.rankdata(normalized_feat)
sn_rank=ss.rankdata(normalized_sn)

num_feat=9
normalized_shapley=normalized_shapley[:num_feat]
normalized_feat=normalized_feat[:num_feat]
normalized_sn=normalized_sn[:num_feat]
snname=snname[:num_feat]

featnames = snname
featnames=featnames[:num_feat]


import matplotlib
import matplotlib.pyplot as plt
import numpy as np

import pylab as plot
fsize=20
params = {'legend.fontsize': fsize/1.1,
          'legend.handlelength': 2}
plot.rcParams.update(params)

font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rcParams['font.family'] = "sans-serif"
matplotlib.rc('font', **font)

labels = featnames
x = np.arange(len(labels))  # the label locations
width = 0.25  # the width of the bars
plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x - 0.25, normalized_shapley, width, label='SHAP', color='lightcoral', edgecolor='black', hatch="//")
rects2 = ax.barh(x, normalized_feat, width, label='Feat', color='gainsboro', edgecolor='black', hatch="\\\\")
rects3 = ax.barh(x + 0.25, normalized_sn, width, label='Lewis', color='forestgreen', hatch='|')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Scores', fontsize=fsize, labelpad=fsize/2)
ax.set_yticks(x)
ax.set_yticklabels(labels, fontsize=fsize)
ax.xaxis.set_ticks_position('top')
ax.xaxis.set_label_position('top')
plt.xticks(np.arange(0, 1.5, .5))
ax.legend()
ax.invert_yaxis()
def autolabel(rects,rank):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        val=len(shapley_rank)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height - 0.02, rect.get_y() + 0.25),
                    xytext=(15, 0),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.5)
        i+=1


autolabel(rects1,shapley_rank)
autolabel(rects2,feat_rank)
autolabel(rects3,sn_rank)

ax.margins(0.1,0.05)
matplotlib.rcParams['hatch.linewidth'] = 0.2
figure = plt.gcf() # get current figure
figure.set_size_inches(5.5,7.5)
fig.tight_layout()
plt.savefig('newplots/drugs_global_related_mushrooms.pdf')
