#!/usr/bin/env python
# coding: utf-8

# # Install packages

# In[1]:


import numpy as np
import copy
from collections import namedtuple, Counter
from sklearn.preprocessing import StandardScaler
from sklearn import svm
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, roc_auc_score, average_precision_score
from sklearn.model_selection import GridSearchCV
import matplotlib.pyplot as plt
import pickle
from numpy import linalg
from sklearn.base import BaseEstimator
from sklearn.metrics.pairwise import rbf_kernel
import multiprocessing
from joblib import Parallel, delayed
import pandas as pd
np.random.seed(101)


# # Generate simulation data

# In[2]:


def random_logit(x):
    z = 1./(1+np.exp(-x))
    #print(z) 
    s = np.random.binomial(n=1, p=z)

    return s
def roundlst(x):
    l=[]
    for v in x:
        if v>0.5:
            l.append(1)
        else:
            l.append(0)
    return l
def roundl(x,th):
    l=[]
    for v in x:
        if v>th:
            l.append(1)
        else:
            l.append(0)
    return l

def roundl4(x,th):
    l=[]
    for v in x:
        if v<=th:
            l.append(0)
        elif v<=2*th:
            l.append(1)
        elif v<=3*th:
            l.append(2)
        else:# v<=4*th:
            l.append(3)
    return l


# In[3]:


#data_src,data_tar,sensible_feature,non_separating_feature = gen_synth_shift_data(gamma_shift_src, gamma_shift_tar, gamma_A, C_src, C_tar, N)


# In[4]:



def get_data(N,seed):
    #N=10000
    np.random.seed(seed)
    S=np.random.binomial(n=1, p=0.5,size=N)
    A=np.random.binomial(n=2, p=0.5,size=N)
    noise1=np.random.binomial(n=2, p=0.005,size=N)
    noise2=np.random.binomial(n=2, p=0.005,size=N)
    noise3=np.random.binomial(n=2, p=0.0002,size=N)
    noise4=np.random.binomial(n=2, p=0.003,size=N)
    St=np.array(roundl((2*S+A+ noise1 )/1.9,1))
    
    sav=np.array(roundl((S+A+noise2)/2,0.5))
            
    hous=np.array(roundl((S+noise3)/1.8,0.5))
    
    #X1 = (2*S+A+np.random.normal(loc=0.0, scale=0.2, size=N))#(np.random.normal(loc=100.0, scale=1.16, size=N)) # N(0,1)
    #X2 =(3*A+2*S+np.random.normal(loc=0.0, scale=0.2, size=N))#random_logit(100*X1)#+np.random.normal(loc=0.0, scale=0.16, size=N)
    #Y = random_logit((A+3*St+2*sav+hous)/50)
    Y = ((St/3+sav/3+hous/6))
    df=pd.DataFrame(S,columns=['S'])
    df['A']=A
    df['S']=S
    df['St']=St
    df['sav']=sav
    df['hous']=hous
    df['Y']=Y
    return df
def causal_effect(dolst,df):

    N=df.shape[0]
    if 'S' in dolst.keys():
        S=np.array([dolst['S']]*df.shape[0])#Construct a list of same size as df
    else:
        S=df['S']
        
        
    if 'A' in dolst.keys():
        A=np.array([dolst['A']]*df.shape[0])#Construct a list of same size as df
    else:
        A=df['A']


    noise1=np.random.binomial(n=2, p=0.005,size=N)
    noise2=np.random.binomial(n=2, p=0.005,size=N)
    noise3=np.random.binomial(n=2, p=0.0002,size=N)
    noise4=np.random.binomial(n=2, p=0.003,size=N)
    if 'St' in dolst.keys():
        St= np.array([dolst['St']]*df.shape[0])
    elif 'A' in dolst.keys() or 'S' in dolst.keys():
        St= np.array(roundl((2*S+A+ noise1 )/1.9,1))
    else:
        St=df['St']
        
    if 'sav' in dolst.keys():
        sav=np.array([dolst['sav']]*df.shape[0])
    elif 'A' in dolst.keys() or 'S' in dolst.keys():
        sav=np.array(roundl((S+A+noise2)/3,0.5))
    else:
        sav=df['sav']
        
    if 'hous' in dolst.keys():
        hous=np.array([dolst['hous']]*df.shape[0])
    elif 'A' in dolst.keys() or 'S' in dolst.keys():
        hous=np.array(roundl((S+noise3)/1.8,0.5))
    else:
        hous=df['hous']
    #print (St)
    #X1 = (2*S+A+np.random.normal(loc=0.0, scale=0.2, size=N))#(np.random.normal(loc=100.0, scale=1.16, size=N)) # N(0,1)
    #X2 =(3*A+2*S+np.random.normal(loc=0.0, scale=0.2, size=N))#random_logit(100*X1)#+np.random.normal(loc=0.0, scale=0.16, size=N)
    #Y = random_logit((A+3*St+2*sav+hous)/50)
    Y = roundlst((St/3+sav/3+hous/6))
    df=pd.DataFrame(np.array(list(S)),columns=['S'])
    

    df['A']=np.array(list(A))
    #print(df)
    df['S']=np.array(list(S))
    df['St']=np.array(list(St))
    df['sav']=np.array(list(sav))
    df['hous']=np.array(list(hous))
    df['Y']=np.array(list(Y))
    
    return df
    #For each variable in dolst, change their descendants
        
        

def get_data_old(N,seed):
    #N=10000
    #np.random.seed(seed)
    S=np.random.binomial(n=1, p=0.5,size=N)
    A=np.random.binomial(n=1, p=0.5,size=N)
    noise1=np.random.binomial(n=2, p=0.15,size=N)
    noise2=np.random.binomial(n=2, p=0.15,size=N)
    noise3=np.random.binomial(n=2, p=0.25,size=N)
    noise4=np.random.binomial(n=2, p=0.23,size=N)
    St= np.array(roundl((2*S+A+ noise1 )/2,1))
    
    sav=np.array(roundl((S+A+noise2)/3,0.5))
            
    hous=np.array(roundl((S+noise3)/2,0.5))
    
    #X1 = (2*S+A+np.random.normal(loc=0.0, scale=0.2, size=N))#(np.random.normal(loc=100.0, scale=1.16, size=N)) # N(0,1)
    #X2 =(3*A+2*S+np.random.normal(loc=0.0, scale=0.2, size=N))#random_logit(100*X1)#+np.random.normal(loc=0.0, scale=0.16, size=N)
    #Y = random_logit((A+3*St+2*sav+hous)/50)
    Y = roundlst((2*St/3+sav/3+hous/3+noise4))
    df=pd.DataFrame(S,columns=['S'])
    df['A']=A
    df['S']=S
    df['St']=St
    df['sav']=sav
    df['hous']=hous
    df['Y']=Y
    return df


# In[5]:


df=get_data(10000,0)
feat=list(df.columns)
feat.remove('Y')
test_df=get_data(10000,1)


# In[6]:


x=df[df['St']==1]
x['Y'].value_counts()


# In[7]:


x


# In[8]:


debug=False
def get_combination(lst,tuplelst):
    i=0
    new_tuplelst=[]
    if len(tuplelst)==0:
        l=lst[0]
        for v in l:
            new_tuplelst.append([v])
        if len(lst)>1:
            return get_combination(lst[1:],new_tuplelst)
        else:
            return new_tuplelst
    

    currlst=lst[0]
    for l in tuplelst:
        
        for v in currlst:
            newl=copy.deepcopy(l)
            newl.append(v)
            new_tuplelst.append(newl)
        
    if len(lst)>1:
        return get_combination(lst[1:],new_tuplelst)
    else:
        return new_tuplelst
      
def get_C_set(df,C):
    lst=[]
    for Cvar in C:
        lst.append(list(set(list(df[Cvar]))))
        
    combination_lst= (get_combination(lst,[]))
    
    return combination_lst

def get_scores_regression(df,z_attr,z,zprime,klst,kvallst,C,target):
    
    sample=df
    #if debug:
    #    print ("Given k number of data points",sample.shape)
      
    if len(C)>0:
        Clst=get_C_set(df,C)
    else:
        Clst=['']
    
    
    snum=0
    nnum=0
    snnum=0

    for cval in Clst:
        #print("cval is ",cval,C)
        if Clst[0]=='':
            Clst=[]
            
        
        #print (C,klst,z_attr)
        conditional_lst=copy.deepcopy(klst)
        conditional_lst.extend(C)
        conditional_lst.extend(z_attr)
        
        conditional_val=copy.deepcopy(kvallst)
        conditional_val.extend(cval)
        
        conditional_valzp=copy.deepcopy(conditional_val)
        conditional_val.extend(z)
        conditional_valzp.extend(zprime)
        
        
        
        #print (conditional_lst,conditional_val)
        pogivenczk=get_prob_o_regression(df,conditional_lst,conditional_val,[target],[1])
        
        pogivenczpk=get_prob_o_regression(df,conditional_lst,conditional_valzp,[target],[1])
        
        #print ( "P[o|c,z,k], P[o'|z']",pogivenczk, pogivenczpk)
        if len(C)>0:
            #if (len(klst)>0):
            pck=get_prob_o_regression(df,klst,kvallst,C,cval)#get_prob_c(sample,cval,C)#get_prob_o_regression(df,conditional_lst,conditional_valzp,C,cval)#Change
            #else:
            #    pck=
            
            conditional=copy.deepcopy(klst)
            conditional_val=copy.deepcopy(kvallst)
            conditional.extend(z_attr)
            conditional_val.extend(z)
            #print (cval,C)
            pcgivenzk=get_prob_o_regression(df,conditional,conditional_val,C,cval)#get_prob_c(sample,cval,C)#get_prob_o_regression(df,conditional_lst,conditional_valzp,C,cval)#Change
            
            conditional_val=copy.deepcopy(kvallst)
            conditional_val.extend(zprime)
            pcgivenzpk=get_prob_o_regression(df,conditional,conditional_val,C,cval)#get_prob_c(sample,cval,C)#get_prob_o_regression(df,conditional_lst,conditional_valzp,C,cval)#Change
        else:
            pck=1
            pcgivenzpk=1
            pcgivenzk=1
        #
        #if debug:
        #    print ("P[o|czk]",pogivenczk)
        #    print ("P[o|cz'k]",pogivenczpk,pck)
        popgivenczpk=1-pogivenczpk
        snum+=pogivenczk*pcgivenzpk ##P[o|do(z),k]
        nnum+=popgivenczpk*pcgivenzk ##P[o|do(z'),k]
        snnum+=(pogivenczk-pogivenczpk)*pck ##P[o|do(z),k]-P[o|do(z'),k]
        #print(nnum,popgivenczpk,pcgivenzk)
        
        
        
        #if debug:
        #    print ("numerators",snum,nnum,snnum)
    
    if len(klst)>0:
        pogivenk=get_prob_o_regression(df,klst,kvallst,[target],[1])
    else:
        pogivenk=df[df[target]==1].shape[0]*1.0/df.shape[0]
    #print (pogivenk)
    #sample_target=get_count(sample[target])
    #pogivenk=sample_target[1]*1.0/(sample_target[1]+sample_target[2])
    
    #sample_target_z=get_count(sample_z[target])
    #pogivenzk=sample_target_z[1]*1.0/(sample_target_z[1]+sample_target_z[2])
    conditional=copy.deepcopy(klst)
    conditional.extend(z_attr)
    conditional_val=copy.deepcopy(kvallst)
    conditional_val.extend(z)
    pogivenzk=get_prob_o_regression(df,conditional,conditional_val,[target],[1])
    
    
    conditional_valzp=copy.deepcopy(kvallst)
    conditional_valzp.extend(zprime)
    #sample_target_zp=get_count(sample_zp[target])
    pogivenzpk=get_prob_o_regression(df,conditional,conditional_valzp,[target],[1])
    #sample_target_zp[2]*1.0/(sample_target_zp[1]+sample_target_zp[2])
    popgivenzpk=1-pogivenzpk
    
    #sample_z=sample[sample[z_attr]==z]#|z,k
    #sample_zp=sample[sample[z_attr]==zprime] #|z',k
    
    #pzgivenk=get_prob_o_regression(df,klst,kvallst,z_attr,z)#
    #sample_z.shape[0]*1.0/(sample_z.shape[0]+sample_zp.shape[0])
    #pzpgivenk=get_prob_o_regression(df,klst,kvallst,z_attr,zprime)#
    #pozgivenk = pogivenzk*pzgivenk
    popgivenzk=1-pogivenzk
    #popzpgivenk=popgivenzpk*pzpgivenk
    #pozpgivenk=pogivenzpk*pzpgivenk
    #popzgivenk=popgivenzk*pzgivenk
    #print (snum,pogivenzpk,popgivenzpk,"these")
    
    sn=(snnum)
    s=((snum-pogivenzpk)*1.0/popgivenzpk)
    
    
    #print ("necessity",popgivenzk,nnum,pogivenzk)
    #necessity 0.16558389274525664 0.5556434074361303 0.8344161072547434
    #P([o'|z'] - P[o'|z])/P[o|z]
    n=((-popgivenzk+nnum)*1.0/pogivenzk)
    #print (nnum,popgivenzk,pogivenzk,"scores")
    ##Lower bound calculation below
    '''
    nlb=(pozgivenk+pozpgivenk-pogivenzpk)*1.0/pozgivenk
    slb=(popzgivenk+popzpgivenk-popgivenzk)*1.0/popzpgivenk

    #newlb=(popgivenzpk-popzpgivenk-popzgivenk - popgivenzpk*(1-pzgivenk-pzpgivenk))*1.0/pozgivenk
    newlb=(-popgivenzk + popgivenzpk)*1.0/pogivenzk
    '''
    
    #sn=max(0,snnum)
    #s= max(0,(snum-pogivenk)*1.0/popzpgivenk)
    #n= max(0,(pogivenk-nnum)*1.0/pozgivenk)
    #print (pogivenzk,pogivenk)
    #Validation:
    #rhs=pozgivenk*n+(1-pogivenzpk)*(1-pzgivenk)*s
    #lhs=1#sn
    #print ("Verification",lhs,rhs)
    
    #print (pzgivenk+pzpgivenk)
    return (n,s,sn)#,nlb,slb,newlb)#,sn_ub,n_ub,s_ub)


# In[9]:


from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_regression

def get_val(row,target,target_val):
    i=0
    while i<len(target):
        #print (row[target[i]],target_val[i])
        if not int(row[target[i]])==int(target_val[i]):
            return 0
        i+=1
    return 1
def get_count(lst):
    count={}
    uniq=list(set(lst))
    for v in uniq:
        count[v]=0
    for v in lst:
        count[v]+=1
    if 1 not in count.keys():
        count[1]=0
    if 2 not in count.keys():
        count[2]=0
    return count
    
def get_prob_o_regression(df,conditional,conditional_values,target,target_val):
    #print (target_val,df.size)
    
    #print (df[conditional[0]].value_counts())
    new_lst=[]
    count=0
    for index,row in df.iterrows():
        new_lst.append(get_val(row,target,target_val))
        if new_lst[-1]==1:
            count+=1
    
    if len(conditional)==0 :
        return count*1.0/df.shape[0]
    #pcgivenk=df[df[target]==1].shape[0]*1.0/df.shape[0]
    if len(list(set(new_lst)))==1:
        if new_lst[0]==1:
            return 1
        else:
            return 0
    if len(conditional)>0:
        X=df[conditional]
        #print (df[df[''==c[i]])
        #print (df[df[conditional[0]]==conditional_values[0]].shape)
        #df=df[df[conditional[0]]==conditional_values[0]]
        #c=(get_count(df['credit']))
        #return c[1]*1.0/(c[0]+c[1])
    else:
        X=df
    #conditional.sort()
    #print (conditional)
    #fsdakl
    X=X[conditional]
    #X.to_csv('t2.csv',index=False)
    #print (len(X),conditional,conditional_values)
    regr = RandomForestRegressor(random_state=0)
    #regr = LogisticRegression(random_state=1)
    regr.fit(X, new_lst)
    #print ("target is ",target,target_val,len(new_lst),new_lst)
    #print (len(conditional),regr.coef_.tolist()[0],list(regr.intercept_),conditional,conditional_values)
    #print (regr.predict_proba([conditional_values]),"ASDFDS")
    
    #print (regr.predict([conditional_values]),"this",conditional,conditional_values)
    #print (conditional, conditional_values,regr.predict([conditional_values])[0],X,new_lst)
    return (regr.predict([conditional_values])[0])
    #return(regr.predict_proba([conditional_values])[0][1])
    
  

    


# In[10]:


def get_scores(curr,z,zprime,df):
    sample=df[df[curr]==zprime]
    sample=sample[sample['Y']==0]
    feat=list(df.columns)
    feat.remove('Y')
    #print(sample)
    
    
    sample= (causal_effect({curr:z},sample))

    #print (sample)
    #print(sample)
    #print (sample)
    #sample[curr]=[z]*(sample.shape[0])
    #print (sample['credit'].value_counts())
    
    
    #pred= (clf.predict(sample[feat]))
    #sample['Y']=(pred)
    #print (pred)
    
    valdic=sample['Y'].value_counts()
    
    
    #print(valdic)
    if 1 not in valdic.keys():
        valdic[1]=0
    if 0 not in valdic.keys():
        valdic[0]=0
        
    suff=valdic[1]*1.0/(valdic[0]+valdic[1])
    
    
    sample=df[df[curr]==z]
    sample=sample[sample['Y']==1]
    #print (sample)
    sample= (causal_effect({curr:zprime},sample))
    #sample[curr]=[zprime]*(sample.shape[0])
    

    #pred= (clf.predict(sample[feat]))
    #print (sample.shape)

    #sample['Y']=(pred)
    
    
    #print ("ncess",sample['Y'].value_counts())
    #print (sample)
    valdic=sample['Y'].value_counts()
    if 1 not in valdic.keys():
        valdic[1]=0
    if 0 not in valdic.keys():
        valdic[0]=0
    necess=1-valdic[1]*1.0/(valdic[0]+valdic[1])
    #print ("necessity",valdic)
    
    
    
    sample= (causal_effect({curr:zprime},df))
    
    
    #pred= (clf.predict(sample[feat]))
    #sample['Y']=(pred)
    
    #pop=sample['Y'].value_counts()[0]*1.0/len(pred)
    
    sample=sample[sample['Y']==0]
    #print (sample.shape,"AAA")
    sample= (causal_effect({curr:z},sample))
    
    sample=sample[sample['Y']==1]
    #print (sample.shape,"AAA2")
    #pred= (clf.predict(sample[feat]))
    #sample['Y']=(pred)
    #po=sample['Y'].value_counts()[1]*1.0/len(pred)
    
    #print(po,pop,sample.shape[0]*1.0/df.shape[0])
    
    return (necess,suff,sample.shape[0]*1.0/df.shape[0])




# In[11]:


'''def get_data(N):
    N=1000
    
    S=np.random.binomial(n=1, p=0.5,size=N)
    X1 = random_logit(18*S)
    X2 = random_logit(10*S+X1)#+np.random.normal(loc=0.0, scale=0.16, size=N)
   
    
    #X1 = random_logit(np.random.normal(loc=0.0, scale=1.16, size=N)) # N(0,1)
    #X2 = random_logit(S+1000*X1)#+np.random.normal(loc=0.0, scale=0.16, size=N)
    Y = random_logit(X1+0.5*X2)
    df=pd.DataFrame(S,columns=['S'])
    df['X1']=X1
    df['X2']=X2
    df['Y']=Y
    return df

df

from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(max_depth=10, random_state=0)

clf.fit(df[['X1','X2']], df['Y'])

df2=get_data(10000)
S=df2['S']
df2=df2[['X1','X2']]
#X_test['credit']=clf.predict(X_test)
df2['Y']=clf.predict(df2)
'''


# In[12]:


from sklearn.metrics import classification_report

np.random.seed(0)

scores=[]
true_scores={}
our_scores={}
for feat in list(df.columns):
    true_scores[feat]=[]
    our_scores[feat]=[]
iter=0
while iter<1:


    df=get_data(10000,0)
    feat=list(df.columns)
    feat.remove('Y')
    test_df=get_data(10000,1)


    clf = RandomForestRegressor(max_depth=10, random_state=0)

    clf.fit(df[feat], df['Y'])

    pred= (clf.predict(test_df[feat]))

    test_df2=copy.deepcopy(test_df)
    test_df2['Y']=roundlst(pred)
    
    
    test_df['Y']=roundlst(test_df['Y'])
    

    #print (iter)
    #print(test_df)
    #print(classification_report(test_df['Y'], pred))
    
    featlst= list(df.columns)
    featlst.remove('Y')
    #featlst=['St']
    for feat in featlst:
        #print(feat)
        if feat=='A':
            l=true_scores[feat]
            l.append(get_scores(feat,2,0,test_df))
            true_scores[feat]=l
            
            l=our_scores[feat]
            l.append(get_scores_regression(test_df2,[feat],[2],[0],[],[],[],'Y'))
            our_scores[feat]=l
            #scores.append((feat,get_scores(feat,1,0,test_df),get_scores_regression(test_df,[feat],[1],[0],[],[],[],'Y')))
            #print (feat,scores)
        
        elif feat=='S':
            l=true_scores[feat]
            l.append(get_scores(feat,1,0,test_df))
            true_scores[feat]=l
            
            l=our_scores[feat]
            l.append(get_scores_regression(test_df2,[feat],[1],[0],[],[],[],'Y'))
            our_scores[feat]=l
            #scores.append((feat,get_scores(feat,1,0,test_df),get_scores_regression(test_df,[feat],[1],[0],[],[],[],'Y')))
            #print (feat,scores)
        else:
            l=true_scores[feat]
            l.append(get_scores(feat,1,0,test_df))
            true_scores[feat]=l
            
            
            l=our_scores[feat]
            l.append(get_scores_regression(test_df2,[feat],[1],[0],[],[],['A','S'],'Y'))
            our_scores[feat]=l
            #scores.append((feat,get_scores(feat,1,0,test_df),get_scores_regression(test_df,[feat],[1],[0],[],[],['A','S'],'Y')))
            #print (feat,scores)
    #print(get_scores('St',1,0,test_df))
    #print(get_scores('sav',1,0,test_df))

    #print (get_scores_regression(test_df,['St'],[1],[0],[],[],['A','S'],'Y'))
    #print (get_scores_regression(test_df,['sav'],[1],[0],[],[],['A','S'],'Y'))
    iter+=1
    scores.append((true_scores,our_scores))
    #break
    
    


# In[13]:


#print(scores)


# In[14]:


import xgboost
import shap

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

df=get_data(10000,0)
feat=list(df.columns)
feat.remove('Y')
test_df=get_data(10000,1)


clf = RandomForestRegressor(max_depth=10, random_state=0)

clf.fit(df[feat], df['Y'])

pred= (clf.predict(test_df[feat]))

# load JS visualization code to notebook
explainer = shap.TreeExplainer(clf)
#shap_values = explainer.shap_values(df)
#shap.summary_plot(shap_values, df, plot_type="bar")

# train XGBoost model

import pandas as pd

feat=list(df.columns)
feat.remove('Y')


shap_values = explainer.shap_values(df[feat])

# visualize the first prediction's explanation (use matplotlib=True to avoid Javascript)
#shap.force_plot(explainer.expected_value, shap_values[0,:], df[feat].iloc[0,:])
shap.summary_plot(shap_values, df[feat], plot_type="bar")


# In[ ]:


#print (shap_values)
sc=shap_values[0]
iter=0
for l in shap_values:
        if iter==0:
                sc=shap_values[0]
        else:
                i=0
                while i<len(l):
                        sc[i]+=abs(l[i])
                        i+=1
        iter+=1
lst=[]
for v in sc:
        lst.append(v*1.0/len(shap_values))


#print (lst)
shapley=lst
# In[ ]:
shapley_name=feat



#print(feat)




tr=[]
sn=[]
feat_lst=['A','S','sav','St','hous']# 'Age',  'Saving','Status','Sex', 'Housing']
for f in feat:#feat_lst:
        sn.append(scores[0][0][f][0][2])
        tr.append(scores[0][1][f][0][2])

snname=feat
feat_name=feat
from sklearn.inspection import permutation_importance
r = permutation_importance(clf, df[feat], pred,n_repeats=100,random_state=0)


# In[18]:


featval=(r.importances_mean)
abslst=[]
for v in featval:
    abslst.append(abs(v))
featval=abslst

#print (featval)

# In[19]:

import pandas as pd
from functools import reduce


import seaborn as scs


#print(sn)
#print (scores)
normalized_shapley=[]
normalized_feat=[]
normalized_sn=[]
normalized_tr=[]
i=0
for feat in ['A','sav','St','S','hous']:

    normalized_shapley.append(shapley[shapley_name.index(feat)]*1.0/max(shapley))
    normalized_feat.append(featval[feat_name.index(feat)]*1.0/max(featval))
    normalized_sn.append(sn[feat_name.index(feat)]*1.0/max(sn))
    normalized_tr.append(tr[feat_name.index(feat)]*1.0/max(tr))
    i+=1
    if normalized_shapley[-1]==0:
        normalized_shapley[-1]=0.02
    if normalized_feat[-1]==0:
       normalized_shapley[-1]=0.02
    if normalized_sn[-1]==0:
       normalized_sn[-1]=0.02
import scipy.stats as ss
shapley_rank=ss.rankdata(normalized_shapley)
feat_rank=ss.rankdata(normalized_feat)
sn_rank=ss.rankdata(normalized_sn)
tr_rank=ss.rankdata(normalized_tr)

num_feat=9
normalized_shapley=normalized_shapley[:num_feat]
normalized_feat=normalized_feat[:num_feat]
normalized_sn=normalized_sn[:num_feat]
normalized_tr=normalized_tr[:num_feat]
snname=snname[:num_feat]

featnames = snname
featnames=featnames[:num_feat]


import matplotlib
import matplotlib.pyplot as plt
import numpy as np

import pylab as plot
fsize=20
params = {'legend.fontsize': fsize/1.1,
          'legend.handlelength': 2}
plot.rcParams.update(params)

font = {'family' : "sans serif", 'size'   : fsize}
matplotlib.rcParams['font.family'] = "sans-serif"
matplotlib.rc('font', **font)

labels = ['Age','Saving','Status','Sex','Housing']#featnames
x = np.arange(len(labels))  # the label locations
width = 0.15  # the width of the bars
plt.figure(figsize=(6, 10)) # in inches!

fig, ax = plt.subplots()
rects1 = ax.barh(x - 0.15, normalized_shapley, width, label='SHAP', color='lightcoral', edgecolor='black', hatch="//")
rects2 = ax.barh(x, normalized_feat, width, label='Feat', color='gainsboro', edgecolor='black', hatch="\\\\")
rects3 = ax.barh(x + 0.15, normalized_sn, width, label='Lewis', color='forestgreen', hatch='|')
rects4 = ax.barh(x + 0.3, normalized_tr, width, label='Ground truth', color='darkviolet', hatch='+')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_xlabel('Scores', fontsize=fsize, labelpad=fsize/2)
ax.set_yticks(x)
ax.set_yticklabels(labels, fontsize=fsize)
ax.xaxis.set_ticks_position('top')
ax.xaxis.set_label_position('top')
plt.xticks(np.arange(0, 1.5, .5))
ax.legend(loc=(0.5,0))
ax.invert_yaxis()
def autolabel(rects,rank):
    """Attach a text label above each bar in *rects*, displaying its height."""
    i=0
    for rect in rects:
        height = rect.get_width()
        val=len(shapley_rank)-int(rank[i])+1
        ax.annotate('{}'.format(val),
                    xy=(height - 0.02, rect.get_y() + 0.15),
                    xytext=(15, 0),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom', fontsize=fsize/1.5)
        i+=1


autolabel(rects1,shapley_rank)
autolabel(rects2,feat_rank)
autolabel(rects3,sn_rank)
autolabel(rects4,tr_rank)

ax.margins(0.1,0.05)
matplotlib.rcParams['hatch.linewidth'] = 0.2
figure = plt.gcf() # get current figure
figure.set_size_inches(7,7.5)
fig.tight_layout()
plt.savefig('newplots/germansyn_SN_correctness.pdf')



