#!/usr/bin/env python
# coding: utf-8

# In[1]:


import datasethelper,lewishelper
import copy
import pandas as pd
import xgboost


import sys

modeltype="randomforest"
#(X_train,y_train,df,pred,clf)=datasethelper.process_german("adaboost")
#(X_train,y_train,df,pred,clf)=datasethelper.process_german("xgboost")
#(X_train,y_train,df,pred,clf)=datasethelper.process_german("randomforest")
(X_train,y_train,df,pred,clf)=datasethelper.process_german(modeltype)


df['credit']=pred

featlst=list(df.columns)
featlst.remove('credit')

def get_majority(dic1,s1,s2,feat):
    score=[0,0,0]
    if feat in dic1.keys():
        score=dic1[feat]
        
    i=0
    while i<3:
        score[i]=max(score[i],s1[i])
        score[i]=max(score[i],s2[i])
        i+=1
    dic1[feat]=score
    return dic1

score_dic={}
for feat in featlst:
    #print (feat)
    uniqval=list(set(list(df[feat])))
    if feat=='age' or feat=='sex':
        i=0
        while i<len(uniqval):
            j=i+1
            while j<len(uniqval):
                score1=lewishelper.get_scores_regression(df,[feat],[uniqval[i]],[uniqval[j]],[],[],[],'credit')
                score2=lewishelper.get_scores_regression(df,[feat],[uniqval[j]],[uniqval[i]],[],[],[],'credit')
                score_dic=get_majority(score_dic,score1,score2,feat)
                j+=1
            i+=1
    else:
        i=0
        while i<len(uniqval):
            j=i+1
            while j<len(uniqval):
                score1=lewishelper.get_scores_regression(df,[feat],[uniqval[i]],[uniqval[j]],[],[],['age','sex'],'credit')
                score2=lewishelper.get_scores_regression(df,[feat],[uniqval[j]],[uniqval[i]],[],[],['age','sex'],'credit')
                score_dic=get_majority(score_dic,score1,score2,feat)
                j+=1
            i+=1
    
#print (score_dic)

n_score={}
s_score={}
sn_score={}
for feat in score_dic.keys():
    n_score[feat]=score_dic[feat][0]
    s_score[feat]=score_dic[feat][1]
    sn_score[feat]=score_dic[feat][2]
    #print (feat,n_score[feat],s_score[feat],sn_score[feat])

# In[269]:


sorted_s = sorted(s_score.items(), key=lambda kv: kv[1],reverse=True)
sorted_n = sorted(n_score.items(), key=lambda kv: kv[1],reverse=True)
sorted_sn = sorted(sn_score.items(), key=lambda kv: kv[1],reverse=True)

slst=[]
nlst=[]
snlst=[]
svallst=[]
nvallst=[]
snvallst=[]

i=0
for (x,y) in sorted_s:
    slst.append(x)
    svallst.append(y)
    nlst.append(sorted_n[i][0])
    nvallst.append(sorted_n[i][1])
    snlst.append(sorted_sn[i][0])
    snvallst.append(sorted_sn[i][1])
    i+=1

sn=snvallst
n=nvallst
s=svallst
snname=snlst
nname=nlst
sname=slst


#print ("SN")
#print (snlst)
#print (snvallst)
# In[ ]:

#print ("N")
#print (nlst)
#print (nvallst)

#print ("S")
#print (slst)
#print(svallst)



import pandas as pd
from functools import reduce

sn = pd.DataFrame(list(zip(sn, snname)), columns=['sn', 'name'])
n = pd.DataFrame(list(zip(n, nname)), columns=['n', 'name'])
s = pd.DataFrame(list(zip(s, sname)), columns=['s', 'name'])

dfs = [sn, n, s]
allScores = reduce(lambda left,right: pd.merge(left,right,on='name'), dfs) #join

allScores = allScores.sort_values(by='sn', ascending=True)

#allScores['name'] = ["skill", "telephone", "foreign_worker", "other_debtors", "liable_for", "num_credits", "sex", "employment", "residence_since", "purpose", "install_plans", "credit_amt", "age", "investment", "savings", "housing", "property", "month", "status", "credit_hist"]
featnames = allScores['name']#["skill", "telephone", "foreign_worker", "other_debtors", "liable_for", "num_credits", "sex", "employment", "residence_since", "purpose", "install_plans", "cred_amt", "age", "invest", "savings", "housing", "property", "month", "status", "cred_hist"]

allScores = allScores.tail(9)[::-1]
featnames = featnames[-9:]

import scipy.stats as ss
nrank=ss.rankdata(allScores['n'])
srank=ss.rankdata(allScores['s'])
snrank=ss.rankdata(allScores['sn'])
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pylab as plot

fsize=20


params = {'legend.fontsize': fsize/1.4,
          'legend.handlelength': 2}
plot.rcParams.update(params)

font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rcParams['font.family'] = "sans-serif"
matplotlib.rc('font', **font)

labels = allScores['name']
x = np.arange(len(labels))  # the label locations
width = 0.25  # the width of the bars

# plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x - width, allScores['n'], width, label='Nec', color='cornflowerblue', edgecolor='black', hatch="//")
rects2 = ax.barh(x, allScores['s'], width, label='Suf', color='gold', edgecolor='black', hatch="\\\\")
rects3 = ax.barh(x + width, allScores['sn'], width, label='NeSuf', color='forestgreen', hatch="|")

plt.gca().invert_yaxis()
# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Scores',labelpad=fsize/2)
ax.set_yticks(x)
ax.set_yticklabels(labels)
ax.legend()
# plt.legend(bbox_to_anchor=(.2, 1.2, 1., .102), loc=3,
#        ncol=3, borderaxespad=0., fontsize=fsize)
ax.xaxis.set_ticks_position('top')
ax.xaxis.set_label_position('top')
plt.xticks(np.arange(0, 1.5, .5))
figure = plt.gcf() # get current figure
figure.set_size_inches(5.8,8)

def autolabel(rects,rank):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        val=len(labels)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height - 0.02, rect.get_y() + 0.32),
                    xytext=(15, 0),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.4)
        i+=1


autolabel(rects1,nrank)
autolabel(rects2,srank)
autolabel(rects3,snrank)
ax.margins(0.07,0.05)
matplotlib.rcParams['hatch.linewidth'] = 0.3

fig.tight_layout()
plt.savefig('newplots/german_globalExplanations.pdf')


# In[ ]:





# In[2]:


labels[7]='investment'


# In[3]:


allScores


# In[4]:


allScores.iloc[7]['name']='invest'
allScores.iloc[0]['name']='credit_hist'


# In[5]:


fsize=20


params = {'legend.fontsize': fsize/1.4,
          'legend.handlelength': 2}
plot.rcParams.update(params)

font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rcParams['font.family'] = "sans-serif"
matplotlib.rc('font', **font)

labels = list(allScores['name'])#allScores['name']
labels[7]='invest'
labels[0]='crdit_hist'
x = np.arange(len(labels))  # the label locations
width = 0.25  # the width of the bars

# plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x - width, allScores['n'], width, label='Nec', color='cornflowerblue', edgecolor='black', hatch="//")
rects2 = ax.barh(x, allScores['s'], width, label='Suf', color='gold', edgecolor='black', hatch="\\\\")
rects3 = ax.barh(x + width, allScores['sn'], width, label='NeSuf', color='forestgreen', hatch="|")

plt.gca().invert_yaxis()
# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Scores',labelpad=fsize/2)
ax.set_yticks(x)
ax.set_yticklabels(labels)
ax.legend()
# plt.legend(bbox_to_anchor=(.2, 1.2, 1., .102), loc=3,
#        ncol=3, borderaxespad=0., fontsize=fsize)
ax.xaxis.set_ticks_position('top')
ax.xaxis.set_label_position('top')
plt.xticks(np.arange(0, 1.5, .5))
figure = plt.gcf() # get current figure
figure.set_size_inches(5.8,8)

def autolabel(rects,rank):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        val=len(labels)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height - 0.02, rect.get_y() + 0.32),
                    xytext=(15, 0),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.4)
        i+=1


autolabel(rects1,nrank)
autolabel(rects2,srank)
autolabel(rects3,snrank)
ax.margins(0.07,0.05)
matplotlib.rcParams['hatch.linewidth'] = 0.3

fig.tight_layout()
plt.savefig('newplots/german_globalExplanations.pdf')


# In[6]:


import shap
feature_names=list(df.columns)
feature_names.remove('credit')
explainer = shap.TreeExplainer(clf)
shap_values = explainer.shap_values(df[feature_names])[0]
shap.summary_plot(shap_values, df[feature_names], plot_type="bar")
#print (shap_values)

#print("shap")

sc=shap_values[0]
iter=0
for l in shap_values:
        if iter==0:
                sc=shap_values[0]
        else:
                i=0
                while i<len(l):
                        sc[i]+=abs(l[i])
                        i+=1
        iter+=1
lst=[]
for v in sc:
        lst.append(v*1.0/len(shap_values))

#print (lst)

shapley=lst
# In[ ]:
#shapley_name=feat


feat=feature_names


# In[8]:


from sklearn.inspection import permutation_importance
r = permutation_importance(clf, df[feat], df['credit'],n_repeats=100,random_state=0)


# In[9]:


list(r.importances_mean)


# In[10]:


feature_names


# In[11]:


snname


# In[12]:


sn=sn['sn']


# In[13]:


sn


# In[14]:


snname[0]="credit_hist"
snname[-8]="liable_for"
snname[7]="invest"
snname[9]="install_plans"
snname[10]="credit_amt"


# In[15]:


feature_names[3]='credit_amt'
feature_names[10]='credit_hist'
feature_names[-7]='invest'
feature_names[2]='install_plans'
feature_names[-6]='liable_for'


# In[16]:


snname


# In[17]:


feature_names


# In[18]:


sn


# In[19]:


import seaborn as scs
#shapley=[0.00197013,0.00517418,0.00534846,0.00673132,0.00724992,0.00812477
#,0.01107258,0.01205751,0.01231464,0.01587817,0.01771147,0.01809804
#,0.02033344,0.02084454,0.02270457,0.02573899,0.03339503,0.04157993
#,0.04388553,0.10160203]

#shapley_name=['foreign_worker', 'liable_for', 'other_debtors', 'number_of_credits', 'telephone', 'skill_level', 'housing', 'residence_since', 'sex', 'employment', 'credit_amt', 'property', 'install_plans', 'purpose', 'age', 'month', 'savings', 'invest', 'credit_hist', 'status']

featval=list(r.importances_mean)
#[0.02847, 0.01573, 0.01006, 0.02148, 0.00398, 0.01776, 0.01885,
#       0.00799, 0.00658, 0.0184 , 0.03966, 0.02424, 0.00226, 0.04138,
#       0.00485, 0.01078, 0.00521, 0.09174, 0.01151, 0.03707]
feat_name=feature_names#['month', 'age', 'install_plans', 'credit_amt', 'number_of_credits', 'employment', 'purpose', 'sex', 'housing', 'residence_since', 'credit_hist', 'property', 'foreign_worker', 'investment', 'liable_for', 'telephone', 'other_debtors', 'status', 'skill_level', 'savings']
shapley_name=feat_name
option='sn'

    
normalized_shapley=[]
normalized_feat=[]
normalized_sn=[]
i=0
for feat in snname:
    #print (i)
    normalized_shapley.append(shapley[shapley_name.index(feat)]*1.0/max(shapley))
    normalized_feat.append(featval[feat_name.index(feat)]*1.0/max(featval))
    normalized_sn.append(sn[i]*1.0/max(sn))
    i+=1

import scipy.stats as ss
shapley_rank=ss.rankdata(normalized_shapley)
feat_rank=ss.rankdata(normalized_feat)
sn_rank=ss.rankdata(normalized_sn)

num_feat=9
normalized_shapley=normalized_shapley[:num_feat]
normalized_feat=normalized_feat[:num_feat]
normalized_sn=normalized_sn[:num_feat]
snname=snname[:num_feat]

featnames = snname#['cred_hist', 'status', 'month', 'property', 'housing', 'savings', 'invest', 'age', 'cred_amt', 'install_plans', 'purpose', 'residence_since', 'employment', 'sex', 'number_of_credits', 'liable_for', 'other_debtors', 'foreign_worker', 'telephone', 'skill_level']
#featnames=featnames[:num_feat]


# In[20]:


snname


# In[ ]:





# In[21]:


import matplotlib
import matplotlib.pyplot as plt
import numpy as np

import pylab as plot
fsize=20
params = {'legend.fontsize': fsize/1.1,
          'legend.handlelength': 2}
plot.rcParams.update(params)

font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rcParams['font.family'] = "sans-serif"
matplotlib.rc('font', **font)

labels = featnames
x = np.arange(len(labels))  # the label locations
width = 0.25  # the width of the bars
plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x - 0.25, normalized_shapley, width, label='SHAP', color='lightcoral', edgecolor='black', hatch="//")
rects2 = ax.barh(x, normalized_feat, width, label='Feat', color='gainsboro', edgecolor='black', hatch="\\\\")
rects3 = ax.barh(x + 0.25, normalized_sn, width, label='Lewis', color='forestgreen', hatch='|')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Scores', fontsize=fsize, labelpad=fsize/2)
ax.set_yticks(x)
ax.set_yticklabels(labels, fontsize=fsize)
ax.xaxis.set_ticks_position('top')
ax.xaxis.set_label_position('top')
plt.xticks(np.arange(0, 1.5, .5))
ax.legend()
ax.invert_yaxis()
def autolabel(rects,rank):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        val=len(shapley_rank)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height - 0.02, rect.get_y() + 0.25),
                    xytext=(15, 0),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.5)
        i+=1


autolabel(rects1,shapley_rank)
autolabel(rects2,feat_rank)
autolabel(rects3,sn_rank)

ax.margins(0.1,0.05)
matplotlib.rcParams['hatch.linewidth'] = 0.2
figure = plt.gcf() # get current figure
figure.set_size_inches(5.8,7.5)
fig.tight_layout()
plt.savefig('newplots/german_global_related.pdf')


# In[ ]:





# In[ ]:





# In[ ]:




