import pandas as pd
import numpy as np
import math
from collections import Counter

#计算卡方值
def calcChi2(df,total_col='total_counts',good_col='good_counts',bad_col='bad_counts'):
    e1=df.iloc[0,0]*df[good_col].sum()/df[total_col].sum() #type1_total*(goodtotal/total)
    e2=df.iloc[0,0]*df[bad_col].sum()/df[total_col].sum()  #type1_total*(badtotal/total)
    e3=df.iloc[1,0]*df[good_col].sum()/df[total_col].sum() #type2_total*(goodtotal/total)
    e4=df.iloc[1,0]*df[bad_col].sum()/df[total_col].sum()  #type2_total*(badtotal/total)
    if(e1!=0)&(e2!=0): #goodtotal和badtotal都不为0
        chi2=(df.iloc[0,1]-e1)**2/e1 + (df.iloc[0,2]-e2)**2/e2 + (df.iloc[1,1]-e3)**2/e3 + (df.iloc[1,2]-e4)**2/e4
    else:
        chi2=0
    return chi2


# 离散型数据初始分箱
def initBin(df,varname,gbflag,sort='bad_percent'):
    total_counts = df[varname].value_counts() #总样本数

    bad_counts, good_counts = pd.Series(), pd.Series()

    for i in total_counts.index:
        good_counts = good_counts.append(
            pd.Series(len(df[(df[varname] == i) & (df[gbflag] == 0)]), index=[i])) #好样本数
        bad_counts = bad_counts.append(
            pd.Series(len(df[(df[varname] == i) & (df[gbflag] == 1)]), index=[i])) #坏样本数

    bad_percent = bad_counts / total_counts #坏样本占比

    init_bin = pd.concat([total_counts, good_counts, bad_counts, bad_percent], axis=1)
    init_bin.columns=['total_counts','good_counts','bad_counts','bad_percent']

    if sort=='bad_percent':
        init_bin.sort_values('bad_percent', inplace=True) #离散数据初始分箱，按照坏样本占比排序

    else:#年龄型数据初始分箱
        init_bin.sort_index(inplace=True)  #按照数值大小排序
        var_list=list(init_bin.index)
        var_list.insert(0, '-∞')
        var_list[-1] = '+∞'
        bin_list=[]
        for i in range(len(var_list) - 1):
            bin_list.append('(' + str(var_list[i]) + ',' + str(var_list[i + 1]) + ']')
        init_bin.insert(0,'bin',bin_list)
        print(init_bin.head(3))
        print(init_bin.tail(3))
        init_bin.set_index('bin',drop=True,inplace=True)

    return init_bin


# 连续型数据初始分箱(等频分箱)
def initNumBin(df,varname,gbflag,InitGroup):
    df=df.sort_values(varname)

    df_cut=pd.qcut(df[varname],InitGroup,duplicates='drop') #等频分箱
    print(pd.concat([df[varname], df_cut, df[gbflag]], axis=1).head(3))
    print(pd.concat([df[varname], df_cut, df[gbflag]], axis=1).tail(3))
    df_cut = pd.concat([df_cut,df[gbflag]], axis=1) #df[index,varname(binGroup),gbflag]
    df_cut[varname]=df_cut[varname].astype(str) #varname(binGroup)转为字符串

    total_counts=pd.Series(df_cut[varname].value_counts(), index=df_cut[varname].drop_duplicates())
    bad_counts, good_counts = pd.Series(), pd.Series()
    for i in total_counts.index:
        good_counts = good_counts.append(
            pd.Series(len(df_cut[(df_cut[varname] == i) & (df_cut[gbflag] == 0)]), index=[i]))  # 好样本数
        bad_counts = bad_counts.append(
            pd.Series(len(df_cut[(df_cut[varname] == i) & (df_cut[gbflag] == 1)]), index=[i]))  # 坏样本数
    init_bin = pd.concat([total_counts, good_counts, bad_counts], axis=1)
    init_bin.columns = ['total_counts', 'good_counts', 'bad_counts']
    return init_bin


#合并分箱
def mergeBin(df,index1,index2):
    df.rename(index={df.index[index1]: str(df.index[index1]) + ',' + str(df.index[index2]),
                     df.index[index2]: str(df.index[index1]) + ',' + str(df.index[index2])}, inplace=True)
    df = df[['total_counts', 'good_counts', 'bad_counts']].groupby(df.index,sort=False).sum()
    # df['bad_percent'] = df.bad_counts / df.total_counts
    # df.sort_values('bad_percent', inplace=True)
    return df


#判断正常样本数据分箱是否符合条件
def matchBin(df,BinMax,BinPcntMin):

    # 检查分组数是否<=BinMax
    while len(df)>BinMax:
        minChi2=None
        for i in range(len(df.index)-1):
            chi2=calcChi2(df[['total_counts','good_counts','bad_counts']].iloc[i:i+2]) #分别计算相邻两组的卡方值
            if minChi2 is None or minChi2>chi2:
                minChi2=chi2
                minIndex=i
        df=mergeBin(df, minIndex, minIndex+1)

    #检查每个分组是否同时包含坏样本和好样本
    while len(df)>1 and len(df[(df['good_counts']==0)|(df['bad_counts']==0)])!=0:
        #若存在只包含好样本/坏样本的分组，且分组数>1
        to_bin=df[(df['good_counts']==0)|(df['bad_counts']==0)].head(1)

        for i in range(len(df.index.tolist())):
            if to_bin.index[0] == df.index.tolist()[i]:
                if i == 0:#第一个分组需要合并
                    df=mergeBin(df, i, i+1)
                    break
                elif i==len(df.index.tolist())-1:#最后一个分组需要合并
                    df = mergeBin(df, i-1, i)
                    break
                else:#中间分组需要合并
                    chi2_f = calcChi2(df[['total_counts', 'good_counts', 'bad_counts']].iloc[i-1:i+1])
                    chi2_b = calcChi2(df[['total_counts', 'good_counts', 'bad_counts']].iloc[i:i+2])
                    if chi2_f<=chi2_b:#与前一个分组的卡方值更小
                        df = mergeBin(df, i - 1, i)
                        break
                    else:#与后一个分组的卡方值更小
                        df = mergeBin(df, i, i + 1)
                        break

    #检查每个分组的样本占比是否>=BinPcntMin
    while len(df)>1 and len(df[df.total_counts/df.total_counts.sum()<BinPcntMin])!=0:
        to_bin=df[df.total_counts/df.total_counts.sum()<BinPcntMin].head(1)

        for i in range(len(df.index.tolist())):
            if to_bin.index[0] == df.index.tolist()[i]:
                if i == 0:#第一个分组需要合并
                    df=mergeBin(df, i, i+1)
                    break
                elif i==len(df.index.tolist())-1:#最后一个分组需要合并
                    df = mergeBin(df, i-1, i)
                    break
                else:#中间分组需要合并
                    chi2_f = calcChi2(df[['total_counts', 'good_counts', 'bad_counts']].iloc[i-1:i+1])
                    chi2_b = calcChi2(df[['total_counts', 'good_counts', 'bad_counts']].iloc[i:i+2])
                    if chi2_f<=chi2_b:#与前一个分组的卡方值更小
                        df = mergeBin(df, i - 1, i)
                        break
                    else:#与后一个分组的卡方值更小
                        df = mergeBin(df, i, i + 1)
                        break

    return df


#离散型数据分箱结果
def CharVarChi2Bin(df,varname,gbflag,BinMax,BinPcntMin,abnor):
    #正常数据样本
    df_nor=df[~df[varname].isin(abnor)]
    merge_bin=initBin(df_nor,varname,gbflag)#正常数据样本初始分箱，按照坏样本占比排序
    merge_bin=matchBin(merge_bin, BinMax, BinPcntMin) #正常数据样本合并分箱，直到满足条件

    #特殊值样本
    df_abnor = df[df[varname].isin(abnor)]
    result_abnor = initBin(df_abnor, varname, gbflag)[['total_counts','good_counts','bad_counts']]  # 特殊值数据样本分箱

    #合并正常样本&异常样本
    result=pd.concat([merge_bin,result_abnor],axis=0)
    result.reset_index(inplace=True)
    result.rename(columns={'index':'BinGroup'},inplace=True)
    result.insert(0,'varname',varname)

    return result


#连续型数据分箱结果
def NumVarChi2Bin(df,varname,gbflag,InitGroup,BinMax,BinPcntMin,abnor=[]):
    # -----------------------------------------正常数据样本-----------------------------------------
    df_nor = df[~df[varname].isin(abnor)]

    if len(Counter(df[varname]))>10:
        merge_bin = initNumBin(df_nor, varname, gbflag,InitGroup)  # 等频分箱，按照变数值大小排序
    else:
        merge_bin=initBin(df_nor, varname, gbflag,sort='num') #年龄型变量，无需等频分箱，按照数值大小排序

    merge_bin = matchBin(merge_bin, BinMax, BinPcntMin)  # 正常数据样本合并分箱，直到满足条件

    bin_list=[]
    for i in merge_bin.index:
        bin_list.append('(' + i.split(',')[0].split('(')[1] + ',' + i.split(',')[-1].split(']')[0] + ']')
    merge_bin.insert(0, 'BinGroup', bin_list)
    merge_bin.reset_index(drop=True, inplace=True)
    # ------------------------------------正常数据样本 分箱结束--------------------------------------

    # 特殊值样本
    df_abnor = df[df[varname].isin(abnor)]
    result_abnor = initBin(df_abnor, varname, gbflag)[['total_counts','good_counts','bad_counts']]  # 特殊值数据样本分箱
    result_abnor.reset_index(inplace=True)
    result_abnor.rename(columns={'index': 'BinGroup'}, inplace=True)

    # 合并正常样本&异常样本
    result=pd.concat([merge_bin,result_abnor],axis=0)
    result.reset_index(drop=True,inplace=True)
    result.insert(0,'varname',varname)

    return result


#计算iv ks woe
def cal_var_iv_ks_woe(df,varname):

    df[['good_percent', 'bad_percent']]=df[['good_counts', 'bad_counts']].\
        apply(lambda x: x / df[['good_counts', 'bad_counts']].sum(), axis=1) #计算占比
    df[['good_cumsum', 'bad_cumsum']] = df[['good_percent', 'bad_percent']].cumsum() #计算累计占比

    df['KS']=abs(df['good_cumsum']-df['bad_cumsum']) #计算ks

    if  len(df[df['good_counts']==0])!=0: #存在good_counts=0的分组
        df['good_counts'].replace(0, 1, inplace=True) #0值改为1
        df[['good_percent', 'bad_percent']] = df[['good_counts', 'bad_counts']]. \
            apply(lambda x: x / df[['good_counts', 'bad_counts']].sum(), axis=1)  # 重新计算占比

    df['WOE'] = (df['bad_percent'] / df['good_percent']).apply(lambda x: math.log(x, 10))  # 计算woe
    df['IV'] = (df['bad_percent'] - df['good_percent']) * df['WOE']  # 计算iv

    df.drop(columns=[ 'good_percent','bad_percent', 'good_cumsum', 'bad_cumsum'],inplace=True)

    result=pd.DataFrame(df[['total_counts', 'good_counts', 'bad_counts','IV']].apply(lambda x:x.sum(),axis=0)).T
    result['KS']=np.max(df['KS'])
    result.insert(0, 'varname', varname)

    return df,result




#----------------测试（离散）----------------
# #离散型数据例子——numpy 随机数据
# import string
# x1=pd.DataFrame(np.random.choice(list(range(20)),size=1000,replace=True))
# x2=pd.DataFrame(np.random.choice(list(string.ascii_lowercase)+list(range(20)),size=1000,replace=True),columns=['letter'])
# y=pd.DataFrame(np.random.choice([0,1],size=1000,replace=True),columns=['label'])
# df=pd.concat([x1,x2,y],axis=1)
# result=CharVarChi2Bin(df,'letter','label',BinMax=9,BinPcntMin=0.05,abnor=['x','y'])
# print(result)

#----------------测试（连续）----------------
# # 连续型数据例子1——sklearn 乳腺癌
# from sklearn.datasets import load_breast_cancer
# cancer = load_breast_cancer()
# print(cancer.data.shape)
# df = pd.concat([pd.DataFrame(cancer.data), pd.DataFrame(cancer.target, columns=['ill'])], axis=1)
# result = NumVarChi2Bin(df, 0, 'ill', InitGroup=70, BinMax=50, BinPcntMin=0.02, abnor=[np.max(df[0]), np.min(df[0])])
# print(result)

#连续型数据例子2——numpy 随机数据
import string
x1=pd.DataFrame(np.random.choice(list(range(20,101)),size=1000,replace=True),columns=['age'])#年龄
x2=pd.DataFrame(np.random.choice(list(range(10000)),size=1000,replace=True),columns=['money'])#金额
y=pd.DataFrame(np.random.choice([0,1],size=1000,replace=True),columns=['label'])
df=pd.concat([x1,x2,y],axis=1)
varname='age'
gbflag='label'
abnor=[np.max(df[varname]),np.min(df[varname])]
result=NumVarChi2Bin(df,varname,gbflag,InitGroup=20,BinMax=9,BinPcntMin=0.1,abnor=abnor)

print(result)

# # 连续型数据例子3——sklearn 随机数据
# from sklearn.datasets.samples_generator import make_classification
# x, y = make_classification(n_samples=50, n_features=2, n_redundant=0,n_clusters_per_class=1, n_classes=2)
# df=pd.concat([pd.DataFrame(x,columns=['varname1','varname2']),pd.DataFrame(y,columns=['gbflag'])],axis=1)
# print(len(df))
# df['varname1'][0:5]=0.1
# df['varname1'][5:10]=0.2
# abnor=[0.1,0.2]
# result=NumVarChi2Bin(df,'varname1','gbflag',InitGroup=20,BinMax=10,BinPcntMin=0.1,abnor=abnor)
# print(result)

# #----------------指标计算----------------
# df=pd.read_excel(r'C:\Users\Administrator\AppData\Roaming\eSpace_Desktop\UserData\37910333\ReceiveFile\chi2.xlsx',sheet_name='Sheet3')
# detail,result=cal_var_iv_ks_woe(df)


