#!/usr/bin/env python
# coding: utf-8

# In[1]:


#This code is run after making it binary classification for fairness reasons (lime does not support addition ofscores)


import datasethelper,lewishelper
import copy
import pandas as pd
import sys

cl_type="randomforest"
#predvariable='coke'
predvariable='mushrooms'

#df=datasethelper.process_adult()
(X_train,y_train,df,pred,model)=datasethelper.process_drug(cl_type,predvariable)

def process(lst):
    final=[]
    for v in lst:
        if v>0:
            final.append(1)
        else:
            final.append(0)
    return final
df[predvariable]=process(pred)




featlst=['Age','Gender','Edu','Country','Ethnicity','Nscore','Escore','Oscore','Ascore','Cscore','impulsive','SS',predvariable]
#df[predvariable]=pred
backdoor={'Age':[],'Gender':[],'Country':['Age','Gender'],'Ethnicity':[],'marital':['Gender','Country','Ethnicity','Age'],'edu':['Age','sex','marital','country'],
              'class':['Age','sex','marital','country','edu'],'occupation':['Age','sex','marital','country','edu'],
              'hours':['Age','sex','marital','country','edu']}
    
featlst=list(df.columns)
if predvariable in featlst:
    featlst.remove(predvariable)


# In[2]:


df


# In[3]:


df.iloc[13]


# In[4]:


###LOcal explanations here:  Sufficiency
row=list(df.columns)
value_map={}
for feat in row:
    values=list(set(df[feat]))
    value_map[feat]=values



descendant={'Age':['Edu','Nscore','Escore','Oscore','Ascore','Cscore','SS','impulsive'],'Ethnicity':['Edu','Nscore','Escore','Oscore','Ascore','Cscore','SS','impulsive'],
            'Country':['Edu','Nscore','Escore','Oscore','Ascore','Cscore','SS','impulsive'],'Gender':['Edu','Nscore','Escore','Oscore','Ascore','Cscore','SS','impulsive'],'country':['marital','edu','class','occupation','hours'],'marital':['edu','class','occupation','hours'],'edu':['class','occupation','hours'],
          'class':[],'occupation':[],'Edu':['Nscore','Escore','Oscore','Ascore','Cscore','SS','impulsive'],
          'Nscore':[],'SS':[],'Oscore':[],'Ascore':[],'Cscore':[],'impulsive':[],'Escore':[]}
#Age	Gender	Edu	Country	Ethnicity	Nscore	Escore	Oscore	Ascore	Cscore	impulsive	SS
#print(row)
curr=df.iloc[13]
#print (value_map)
    
featnames=copy.deepcopy(row)
featnames.remove(predvariable)
orig_row=row
row=copy.deepcopy(orig_row)
def get_decrease(row):
    maxscores=[]
    featlst=['Age','Gender','Country','Ethnicity','Edu','Nscore','SS','Escore','Oscore','Ascore','Cscore','impulsive']
    for feat in ['Age','Gender','Country','Ethnicity','Edu','Nscore','SS','Escore','Oscore','Ascore','Cscore','impulsive']:#featnames:
        #print(featnames,feat,"here")
        row.remove(predvariable)
        rowval=[]
        for f in row:
            rowval.append(curr[f])

        curr_feat=feat
        ind=row.index(curr_feat)
        prevval=rowval[ind]
        rowval.pop(ind)
        row.pop(ind)

        descendantlst=descendant[feat]
        for f in descendantlst:
            ind=row.index(f)
            rowval.pop(ind)
            row.pop(ind)

        values=value_map[feat]
        max_suff=-1
        suff_val=-1
        #print("values",values,feat,row,rowval)
        for val in values:
            #if val<=prevval:
            #    continue

            suff=lewishelper.get_scores_regression(df,[curr_feat],[prevval],[val],row,rowval,[],'mushrooms')[1]

            #suff=lewishelper.get_scores_regression(df,[curr_feat],[val],[prevval],row,rowval,[],'mushrooms')[1]
            #print(curr_feat,val,prevval,row,rowval,suff,"scores ehre")
            max_suff=max(max_suff,suff)
            if max_suff==suff:
                suff_val=val
        #print (feat,max_suff,suff_val,prevval)
        maxscores.append(max_suff)
        row.append(feat)
        rowval.append(prevval)
        row=copy.deepcopy(orig_row)
    return (featlst,maxscores)
def get_increase(row):
    maxscores=[]
    featlst=['Age','Gender','Country','Ethnicity','Edu','Nscore','SS','Escore','Oscore','Ascore','Cscore','impulsive']
    for feat in ['Age','Gender','Country','Ethnicity','Edu','Nscore','SS','Escore','Oscore','Ascore','Cscore','impulsive']:#featnames:
        row.remove(predvariable)
        rowval=[]
        for f in row:
            rowval.append(curr[f])

        curr_feat=feat
        ind=row.index(curr_feat)
        prevval=rowval[ind]
        rowval.pop(ind)
        row.pop(ind)

        descendantlst=descendant[feat]
        for f in descendantlst:
            ind=row.index(f)
            rowval.pop(ind)
            row.pop(ind)

        values=value_map[feat]
        max_suff=-1
        suff_val=-1
        #print("values",values,feat,row,rowval)
        for val in values:
            #if val<=prevval:
            #    continue

            #suff=lewishelper.get_scores_regression(df,[curr_feat],[prevval],[val],row,rowval,[],'mushrooms')[1]

            suff=lewishelper.get_scores_regression(df,[curr_feat],[val],[prevval],row,rowval,[],'mushrooms')[1]
            #print(curr_feat,val,prevval,row,rowval,suff,"scores here")
            max_suff=max(max_suff,suff)
            if max_suff==suff:
                suff_val=val
        #print (feat,max_suff,suff_val,prevval)
        maxscores.append(max_suff)
        row.append(feat)
        rowval.append(prevval)
        row=copy.deepcopy(orig_row)
    return (featlst,maxscores)

row1=copy.deepcopy(row)
row2=copy.deepcopy(row)
(feat_inc,inc_score)=get_increase(row1)
(feat_dec,dec_score)=get_decrease(row2)


# In[5]:


#'Gender':['Edu','Nscore','Escore','Oscore','Ascore','Cscore','SS','impulsive']
#lewishelper.get_scores_regression(df,['Gender'],[0],[1],['Age','Country','Ethnicity'],[1,3,0],[],'mushrooms')[1]


# In[6]:


'''
row.remove('impulsive')
rowval=[]
for f in row:
    rowval.append(curr[f])

curr_feat=feat
ind=row.index(curr_feat)
prevval=rowval[ind]
rowval.pop(ind)
row.pop(ind)
print (rowval)
print (row)

ind=row.index('SS')
rowval.pop(ind)
row.pop(ind)
'''


# In[7]:


#df['impulsive'].value_counts()


# In[8]:


#lewishelper.get_scores_regression(df,['impulsive'],[3],[2],row,rowval,[],'mushrooms')[1]


# In[9]:


#rowval.append(3)
#row.append('impulsive')
#row1=copy.deepcopy(row)


# In[ ]:





# In[ ]:





# In[ ]:





# In[10]:







# In[ ]:





# In[ ]:





# In[13]:


#print (curr)
        


# In[ ]:





# In[14]:


#df['Gender'].value_counts()


# In[15]:


#print (inc_score)


# In[16]:


#print (inc_score)
#['Age','Gender','Country','Ethnicity','Edu','Nscore','SS','Escore','Oscore','Ascore','Cscore','impulsive']:#featnames:
#print (dec_score)


# In[17]:


#df.iloc[13]: Negative


#Decreasing(Positive contrib)
#['Age','Gender','Country','Ethnicity','Edu','Nscore','SS','Escore','Oscore','Ascore','Cscore','impulsive']
#[0.09691410629249135, 0.0, 0.0, 0.13454083322865676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]


#Increasing(Negative contrib:)
#['Age','Gender','Country','Ethnicity','Edu','Nscore','SS','Escore','Oscore','Ascore','Cscore','impulsive']:#featnames:
#[0.0, 0.32133500048954455, 0.8218992295569866, 0.0, 0.35791673217563286, 0.08, 0.04, 0.01, 0.48, 0.11, 0.05, 0.32]



#df.iloc[1]
#Decreasing suff (Positive contrib)
#['Age','Gender','Country','Ethnicity','Edu','Nscore','SS','Escore','Oscore','Ascore','Cscore','impulsive']
#[0.0, 0.0, 0.0, 0.041288650720502355, 0.15692808988397217, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]


#Increasing suff (Negative contrib:)

#['Age','Gender','Country','Ethnicity','Edu','Nscore','SS','Escore','Oscore','Ascore','Cscore','impulsive']
#[0.09691410629249135, 0.33543701369606577, 0.6774044170214932, 0.0, 0.0, 0.0, 0.09, 0.0, 0.16, 0.0, 0.17, 0.04]


# In[18]:


#df.iloc[1]


# In[ ]:





# In[1]:


import lime
import numpy as np
import shap
import datasethelper
predvariable="mushrooms"
(X_train,y_train,df,pred,model)=datasethelper.process_drug("randomforest",predvariable)

feature_names = list(X_train.columns)
feature_names.remove('mushrooms')
#feature_names.remove('mushrooms')
class_names = np.unique(pred).tolist()
# Lime explanations
limeExplainer = lime.lime_tabular.LimeTabularExplainer(X_train[feature_names].to_numpy(), feature_names=feature_names,
                                                       class_names=class_names)


# Shapley scores
#shap.initjs()
#shapExplainer = shap.TreeExplainer(model)
#shap_values = shapExplainer.shap_values(X_train)


# In[ ]:





# In[2]:


df=df[feature_names]
model.predict_proba([df.iloc[13]]).astype(float)
#df.iloc[0]


# In[3]:


predict_fn = lambda x: model.predict_proba(x).astype(float)
feat=feature_names
limeExp = limeExplainer.explain_instance(df.iloc[13][feat], predict_fn, num_features=12)


# In[4]:


#limeExp.show_in_notebook(show_all=False)


# In[5]:


limescore=limeExp.as_list()


# In[6]:


feature_names


# In[7]:


import matplotlib.pyplot as plt
feat=list(df.columns)

# Shapley scores
#shap.initjs()
shapExplainer = shap.TreeExplainer(model)
shap_values = shapExplainer.shap_values(df[feature_names])
#shap.force_plot(shapExplainer.expected_value[1], shap_values[1][0], df.iloc[0],show=True,matplotlib=True).savefig('german_shap.pdf')






# In[20]:


lime=limescore#[('0.00 < Country <= 3.00', -0.1718800800038667), ('0.00 < Gender <= 1.00', -0.11563591475749846), ('Oscore <= 1.00', -0.10677682220692122), ('1.00 < SS <= 2.00', 0.07299083924023884), ('Cscore <= 1.00', 0.05513815569612382), ('Ethnicity <= 0.00', 0.0488014111166854), ('Edu > 2.00', -0.02953387905271882), ('1.00 < impulsive <= 2.00', 0.02304550044449712), ('0.00 < Age <= 1.00', 0.02249686165392694), ('Nscore > 2.00', 0.016367667225121847), ('Ascore <= 1.00', 0.012483641359757248), ('Escore <= 1.00', -0.009316685093694769)]



shapname=['Age', 'Gender', 'Edu', 'Country', 'Ethnicity', 'Nscore', 'Escore', 'Oscore', 'Ascore', 'Cscore', 'impulsive', 'SS']

curr=13
shapval=list(shap_values[1][curr])#[-0.01191537, -0.06024459, -0.02423833, -0.15178094,  0.00727729,
#        0.00063126,  0.00402209, -0.05588275, -0.00295422,  0.01295049,
#        0.01407082,  0.0086113 ]
print (shapval)

# In[21]:


import seaborn as scs


# In[22]:


raven=inc_score
ravendec=dec_score
ravenname=feat_inc


# In[23]:


# our = [max(raven[i], ravendec[i]) for i in range(len(raven))]
raven = [ele * -1 for ele in raven]
shapval = [ele  for ele in shapval]
ravenmax = max(max([abs(ele) for ele in raven]), max([abs(ele) for ele in ravendec]))


# In[24]:


# raven is ordered 
lime_ordered=[]
limenamevals=[]
limevals=[]
for i in range(len(ravenname)):
    for j in range(len(lime)):
        if ravenname[i] in lime[j][0] or ravenname[i] in lime[j][0].lower():
            lime_ordered.append((lime[j][0], ravenname[i], lime[j][1]))
            limenamevals.append(lime[j][0])
            limevals.append(lime[j][1])
            
shap_ordered=[]
shapvals=[]
for i in range(len(ravenname)):
    #print (ravenname[i])
    for j in range(len(shapname)):
        if ravenname[i] in shapname[j] or ravenname[i] in shapname[j].lower() :
            shap_ordered.append((shapval[j], ravenname[i]))
            shapvals.append(shapval[j])
            #print(ravenname[i])



lime_sign=[10,11]
for i in lime_sign:
    limevals[i]=-1*limevals[i]

normalized_shapley=[]
normalized_lime=[]
normalized_raven=[]
normalized_ravendec=[]
for i in range(len(raven)):
    #print(i)
    normalized_shapley.append(shapvals[i]*1.0/max([abs(ele) for ele in shapval]))
    normalized_lime.append(limevals[i]*1.0/max([abs(ele) for ele in limevals]))
    normalized_raven.append(raven[i]*1.0/ravenmax)
    normalized_ravendec.append(ravendec[i]*1.0/ravenmax)
    i+=1


# In[26]:

import scipy.stats as ss
shapley_rank=ss.rankdata([abs(ele) for ele in normalized_shapley], method='min')
lime_rank=ss.rankdata([abs(ele) for ele in normalized_lime], method='min')
raven_rank=ss.rankdata([abs(ele) for ele in normalized_raven], method='min')
ravendec_rank=ss.rankdata([abs(ele) for ele in normalized_ravendec], method='min')


# In[27]:


# select features to show
featlst=['Age', 'Gender', 'Edu', 'Country', 'Ethnicity','Cscore','impulsive']#, 'nscore', 'escore', 'oscore', 'Ascore', 'Cscore', 'impulsive', 'SS']
feats=['age=25-34 yrs', 'sex=female', 'edu=masters','country=UK', 'ethnicity=white','conscientious<0','impulsive>0']#, 'Nscore=10th', 'class=Private', 'occup=Other', 'hours=30']

# featlst = ravenname
# feats = featlst

shaprank_f=[]
limerank_f=[]
ravenrank_f=[]
ravendecrank_f=[]

raven_f=[]
ravendec_f=[]
lime_f=[]
shapley_f=[]

for feat in featlst:
    idx = ravenname.index(feat)
    shapley_f.append(normalized_shapley[idx])
    lime_f.append(normalized_lime[idx])
    raven_f.append(normalized_raven[idx])
    ravendec_f.append(normalized_ravendec[idx])
    shaprank_f.append(shapley_rank[idx])
    limerank_f.append(lime_rank[idx])
    ravenrank_f.append(raven_rank[idx])
    ravendecrank_f.append(ravendec_rank[idx])


# In[28]:


import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pylab as plot

fsize=20
params = {'legend.fontsize': fsize,
          'legend.handlelength': 2,
         'axes.labelsize': fsize,
         'axes.titlesize':fsize,
         'xtick.labelsize':fsize}
plot.rcParams.update(params)
font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rc('font', **font)
matplotlib.rcParams['hatch.linewidth'] = 0.3

labels=feats
# labels=limenames
x = np.arange(len(labels))  # the label locations
width = 0.25/1.3  # the width of the bars
plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x - 2 * width, raven_f, width, label='Lewis:-ve', color='khaki', edgecolor='black', hatch="\\")
rects2 = ax.barh(x - width, ravendec_f, width, label='Lewis:+ve', color='goldenrod', edgecolor='black', hatch="\\")
# rects2 = ax.barh(x, ravendec, width, label='Our-Down',color='royalblue', edgecolor='black')
# rects3 = ax.barh(x + width, lime_0, width, label='Lime: O=0',color='lightcoral', edgecolor='black', hatch="/")
# rects3 = ax.barh(x + 2*width, lime_1, width, label='Lime: O=1',color='y', edgecolor='black', hatch="-")
rects3 = ax.barh(x, lime_f, width, label='LIME',color='snow', edgecolor='black', hatch="/")
rects4 = ax.barh(x + width, shapley_f, width, label='SHAP',color='lightcoral', edgecolor='black', hatch="|")

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Normalized score', fontsize=fsize, labelpad=10)
ax.set_yticks(x)
ax.set_yticklabels(labels, fontsize=fsize*1.1)

ax.set_xticks(range(-1, 2, 1))
ax.set_xticklabels([abs(y) for y in range(-1, 2, 1)])
ax.xaxis.set_ticks_position('bottom')
ax.xaxis.set_label_position('bottom')
# ax.legend(loc=0)
ax.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
       ncol=2, mode="expand", borderaxespad=0.)
# ax.invert_yaxis()
plt.xticks(np.arange(-1, 1.2, 1))
matplotlib.pyplot.hlines(labels,-1,1, color='gainsboro', zorder=0, linestyles='dashed')

def autolabel(rects,rank, col):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        offset = 0.07
        if height < 0:
            offset = offset * -1
        val=len(shapley_rank)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height + offset, rect.get_y() + abs(0.08 * offset)),
                    xytext=(0, -5),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.2, color=col)
        i+=1


autolabel(rects1,ravenrank_f, 'black')
autolabel(rects2,ravendecrank_f, 'black')
autolabel(rects3,limerank_f, 'black')
autolabel(rects4,shaprank_f, 'black')

# handles, labels = ax.get_legend_handles_labels()
# ax.legend(handles[::-1], labels[::-1])
# plt.show()
figure = plt.gcf() # get current figure
figure.set_size_inches(8,12)
fig.tight_layout()
plt.savefig('newplots/drugs_local_negative_related.pdf')


# In[ ]:


