#!/usr/bin/env python
# coding: utf-8

# In[1]:


import datasethelper,lewishelper
import copy
import pandas as pd
import sys

model="randomforest"
col='score_text'
#col='is_recid'
(X_train,y_train,df,pred,clf)=datasethelper.process_compas(col,model)
#df=datasethelper.process_compas('is_recid')
df[col]=pred

featlst=list(df.columns)

featlst.remove(col)


if col=='score_text' and 'is_recid' in featlst:
    featlst.remove('is_recid')
def get_majority(dic1,s1,s2,feat):
    score=[0,0,0]
    if feat in dic1.keys():
        score=dic1[feat]
        
    i=0
    while i<3:
        score[i]=max(score[i],s1[i])
        score[i]=max(score[i],s2[i])
        i+=1
    dic1[feat]=score
    return dic1

def get_majority(dic1,s1,s2,feat):
    score=[0,0,0]
    if feat in dic1.keys():
        score=dic1[feat]

    i=0
    while i<3:
        score[i]=max(score[i],s1[i])
        score[i]=max(score[i],s2[i])
        i+=1
    dic1[feat]=score
    return dic1


score_dic={}
for feat in featlst:
    #print (feat)
    uniqval=list(set(list(df[feat])))

    if feat=='age_cat' or feat=='sex' or feat=='race':
        i=0
        while i<len(uniqval):
            j=i+1
            while j<len(uniqval):
                score1=lewishelper.get_scores_regression(df,[feat],[uniqval[i]],[uniqval[j]],[],[],[],col)
                score2=lewishelper.get_scores_regression(df,[feat],[uniqval[j]],[uniqval[i]],[],[],[],col)
                score_dic=get_majority(score_dic,score1,score2,feat)
                j+=1
            i+=1
    else:
        i=0
        while i<len(uniqval):
            j=i+1
            while j<len(uniqval):
                score1=lewishelper.get_scores_regression(df,[feat],[uniqval[i]],[uniqval[j]],[],[],['age_cat','sex','race'],col)
                score2=lewishelper.get_scores_regression(df,[feat],[uniqval[j]],[uniqval[i]],[],[],['age_cat','sex','race'],col)
                score_dic=get_majority(score_dic,score1,score2,feat)
                j+=1
            i+=1

#print (score_dic)



n_score={}
s_score={}
sn_score={}
for feat in score_dic.keys():
    n_score[feat]=score_dic[feat][0]
    s_score[feat]=score_dic[feat][1]
    sn_score[feat]=score_dic[feat][2]
    #print (feat,n_score[feat],s_score[feat],sn_score[feat])


sorted_s = sorted(s_score.items(), key=lambda kv: kv[1],reverse=True)
sorted_n = sorted(n_score.items(), key=lambda kv: kv[1],reverse=True)
sorted_sn = sorted(sn_score.items(), key=lambda kv: kv[1],reverse=True)

slst=[]
nlst=[]
snlst=[]
svallst=[]
nvallst=[]
snvallst=[]

i=0
for (x,y) in sorted_s:
    slst.append(x)
    svallst.append(y)
    nlst.append(sorted_n[i][0])
    nvallst.append(sorted_n[i][1])
    snlst.append(sorted_sn[i][0])
    snvallst.append(sorted_sn[i][1])
    i+=1

#print ("sn")
#print (snvallst)
#print (snlst)
# In[ ]:

#print("n")
#print (nlst)
#print (nvallst)

#print("S")
#print (slst)
#print(svallst)





sn=snvallst
n=nvallst
s=svallst
snname=snlst
nname=nlst
sname=slst


import pandas as pd
from functools import reduce

sn = pd.DataFrame(list(zip(sn, snname)), columns=['sn', 'name'])
n = pd.DataFrame(list(zip(n, nname)), columns=['n', 'name'])
s = pd.DataFrame(list(zip(s, sname)), columns=['s', 'name'])
dfs = [sn, n, s]
allScores = reduce(lambda left,right: pd.merge(left,right,on='name'), dfs) #join

allScores = allScores.sort_values(by='sn', ascending=False)
import scipy.stats as ss
nrank=ss.rankdata(allScores['n'])
srank=ss.rankdata(allScores['s'])
snrank=ss.rankdata(allScores['sn'])

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pylab as plot


# In[2]:


import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pylab as plot

fsize=20


params = {'legend.fontsize': fsize/1.5,
          'legend.handlelength': 2}
plot.rcParams.update(params)

font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rcParams['font.family'] = "sans-serif"
matplotlib.rc('font', **font)
labels=snname
x = np.arange(len(labels))  # the label locations
width = 0.25  # the width of the bars

# plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x - width, allScores['n'], width, label='Nec', color='cornflowerblue', edgecolor='black', hatch="//")
rects2 = ax.barh(x, allScores['s'], width, label='Suf', color='gold', edgecolor='black', hatch="\\\\")
rects3 = ax.barh(x + width, allScores['sn'], width, label='NeSuf', color='forestgreen', hatch="|")

plt.gca().invert_yaxis()
# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Scores',labelpad=fsize/2,fontsize=fsize)
ax.set_yticks(x)
ax.set_yticklabels(labels)
ax.legend()
# plt.legend(bbox_to_anchor=(.2, 1.2, 1., .102), loc=3,
#        ncol=3, borderaxespad=0., fontsize=fsize)
ax.xaxis.set_ticks_position('top')
ax.xaxis.set_label_position('top')
plt.xticks(np.arange(0, 1.5, .5),fontsize=fsize)
figure = plt.gcf() # get current figure
figure.set_size_inches(5.8,8)

def autolabel(rects,rank):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        val=len(labels)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height - 0.02, rect.get_y() + 0.2),
                    xytext=(15, 0),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.5)
        i+=1


autolabel(rects1,nrank)
autolabel(rects2,srank)
autolabel(rects3,snrank)
ax.margins(0.1,0.05)
matplotlib.rcParams['hatch.linewidth'] = 0.3

fig.tight_layout()

plt.savefig('newplots/compas_globalExplanations_score_text.pdf')


# In[ ]:


feature_names=list(df.columns)
feature_names.remove('score_text')


# In[ ]:


import shap#shap.force_plot(explainer.expected_value[0], shap_values[0], df[feat].iloc[1,:])
explainer = shap.TreeExplainer(clf)
shap_values = explainer.shap_values(df[feature_names],check_additivity=False)

#shap.initjs()
shap.summary_plot(shap_values, df[feature_names], plot_type="bar")


# In[ ]:


from sklearn.inspection import permutation_importance
r = permutation_importance(clf, df[feature_names], df['score_text'],n_repeats=100,random_state=0)


# In[ ]:


r.importances_mean


# In[ ]:


feature_names


# In[ ]:


featval=list(r.importances_mean)
#[0.02847, 0.01573, 0.01006, 0.02148, 0.00398, 0.01776, 0.01885,
#       0.00799, 0.00658, 0.0184 , 0.03966, 0.02424, 0.00226, 0.04138,
#       0.00485, 0.01078, 0.00521, 0.09174, 0.01151, 0.03707]
feat_name=feature_names#['month', 'age', 'install_plans', 'credit_amt', 'number_of_credits', 'employment', 'purpose', 'sex', 'housing', 'residence_since', 'credit_hist', 'property', 'foreign_worker', 'investment', 'liable_for', 'telephone', 'other_debtors', 'status', 'skill_level', 'savings']


# In[ ]:


sn=sn['sn']


# In[ ]:


import seaborn as scs
shapley=[0.00944532, 0.06501204, 0.0824449,  0.08450802, 0.14252463]
shapley_name=['sex', 'juv_fel_count', 'race', 'age_cat', 'priors_count']


 
normalized_shapley=[]
normalized_feat=[]
normalized_sn=[]
i=0
for feat in snname:
    normalized_shapley.append(shapley[shapley_name.index(feat)]*1.0/max(shapley))
    normalized_feat.append(featval[feat_name.index(feat)]*1.0/max(featval))
    normalized_sn.append(sn[i]*1.0/max(sn))
    i+=1
    
import scipy.stats as ss
shapley_rank=ss.rankdata(normalized_shapley)
feat_rank=ss.rankdata(normalized_feat)
sn_rank=ss.rankdata(normalized_sn)

normalized_shapley=normalized_shapley[:10]
normalized_feat=normalized_feat[:10]
normalized_sn=normalized_sn[:10]
snname=snname[:10]


# In[ ]:





# In[ ]:


import matplotlib
import matplotlib.pyplot as plt
import numpy as np

import pylab as plot
fsize=20
params = {'legend.fontsize': fsize/1.3,
          'legend.handlelength': 2}
plot.rcParams.update(params)

font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rcParams['font.family'] = "sans-serif"
matplotlib.rc('font', **font)

labels =feature_names
x = np.arange(len(labels))  # the label locations
width = 0.25  # the width of the bars
plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x - 0.25, normalized_shapley, width, label='SHAP', color='lightcoral', edgecolor='black', hatch="//")
rects2 = ax.barh(x, normalized_feat, width, label='Feat', color='gainsboro', edgecolor='black', hatch="\\\\")
rects3 = ax.barh(x + 0.25, normalized_sn, width, label='Lewis', color='forestgreen', hatch='|')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Scores', fontsize=fsize, labelpad=fsize/2)
ax.set_yticks(x)
ax.set_yticklabels(labels, fontsize=fsize)
ax.xaxis.set_ticks_position('top')
ax.xaxis.set_label_position('top')
plt.xticks(np.arange(0, 1.5, .5))
ax.legend(bbox_to_anchor=(0.5, 0.22))
ax.invert_yaxis()
def autolabel(rects,rank):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        val=len(shapley_rank)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height - 0.04, rect.get_y() + 0.25),
                    xytext=(15, 0),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.5)
        i+=1


autolabel(rects1,shapley_rank)
autolabel(rects2,feat_rank)
autolabel(rects3,sn_rank)

ax.margins(0.1, 0.05)
matplotlib.rcParams['hatch.linewidth'] = 0.2
figure = plt.gcf() # get current figure
figure.set_size_inches(5.5,7.5)
fig.tight_layout()

plt.savefig('newplots/compas_score_text_global_related.pdf')



# In[ ]:





# In[ ]:





# In[ ]:





# In[ ]:




