#########################

#Example 1

library(mlbench)
data(PimaIndiansDiabetes2) #used in manual Breiman
Diabetes <- na.omit(PimaIndiansDiabetes2)

set.seed(2016)
ri<- randomForest(diabetes  ~ ., data=Diabetes, ntree=500, importance=TRUE,keep.inbag=T) 
importance(ri)

da=Diabetes[,1:8]
ntree=500
pupf=ipmrf(ri,da,ntree) #IPM case-wise computing with OBB 

#global IPM
pua=apply(pupf,2,mean) 
pua

#IPM by classes
attach(Diabetes)
puac=matrix(0,nrow=2,ncol=dim(da)[2])
puac[1,]=apply(pupf[diabetes=='neg',],2,mean) 
puac[2,]=apply(pupf[diabetes=='pos',],2,mean) 

colnames(puac)=colnames(da)
rownames(puac)=c( 'neg', 'pos')
puac

im=importance(ri)
ii=apply(im,2,rank)

ii


apply(puac,1,rank)

#rank global IPM  
 rank(pua)
 apply(puac,1,rank)



##########################

#clasification diabetes
#Example 2



library(verification)
library(caret)

##############
#LDA
##############

library(MASS)

library(mlbench)
data(PimaIndiansDiabetes2) #used in manual Breiman
Diabetes <- na.omit(PimaIndiansDiabetes2)
attach(Diabetes)

dn=as.numeric(diabetes) #negative:1, positive: 2

preaire=c()
da=Diabetes
ninos=1:length(diabetes)

for (i in 1:length(ninos)){

TR= ninos[-i]

z <- lda(diabetes ~ ., da, subset = TR)

TE=ninos[i]


preaire=rbind(preaire,predict(z, da[TE,])$posterior)

}


clre=max.col(preaire[,1:2])


sum(clre==dn) #307 correct from 392

rlda=rps( dn,preaire) #> rlda$rps 0.1513361

confusionMatrix(clre,dn,positive='2')

 b=brier(dn-1,preaire[,2],bins=F)
 b$bs #[1] 0.1513361 coincide with rps





###############
#LR
###############
#Logistic regression


set.seed(2015)
preaire=c()
for (i in 1:length(ninos)){
TR= ninos[-i]

fitd <- glm(diabetes ~., family=binomial, data=da,subset=TR)
stepmodel <- stepAIC(fitd, trace = FALSE)



TE= ninos[i]


preaire=rbind(preaire,predict(stepmodel, da[TE,],type='response'))

}


preaire2=cbind(1-preaire,preaire)
clre=max.col(preaire2[,1:2])


sum(clre==dn) #304


rpolr=rps( dn,preaire2)
confusionMatrix(clre,dn ,positive='2')

###########################
#Random forest

###########
#RF selecting mtry each time LOU
############

library(party)


preai=c()



attach(da)
mtv=c()

who=1:length(ninos)

for (i in who){

TR=ninos[-i]

aooo=c()

for (j in 1:4){ #sqrt(7)
my_cforest_control <- cforest_control(teststat = "max",
    testtype = "Teststatistic", mincriterion = 0, ntree = 500, mtry = j,
    replace = TRUE)
set.seed(2015)
marboli=cforest(factor(diabetes)~.,data=da,subset=TR,controls=my_cforest_control) 


qqi=predict(marboli,OOB=T,type='prob')
pmi=matrix(unlist(qqi),ncol=2,byrow=TRUE)
cli=max.col(pmi[,1:2])

aooo[j]=sum(cli==dn[TR])

}
mtv[i]=which.max(aooo)


my_cforest_control <- cforest_control(teststat = "max",
    testtype = "Teststatistic", mincriterion = 0, ntree = 500, mtry = which.max(aooo),
    replace = TRUE)
set.seed(2015)
marboli=cforest(factor(diabetes)~.,data=da,subset=TR,controls=my_cforest_control) 


TE=ninos[i]

qq=predict(marboli,da[TE,],OOB=FALSE,type='prob')

preai=rbind(preai,matrix(unlist(qq),ncol=2,byrow=TRUE))
cat(i)
}

cl=max.col(preai[,1:2])


sum(cl==dn)  #309


rrf=rps( dn,preai[,1:2])

confusionMatrix(cl,dn,positive='2')  

#mtv: > mtv
#  [1] 3 3 2 4 4 2 2 3 4 4 3 2 4 4 4 2 4 2 3 4 2 2 4 4 3 2 4 3 2 3 3 4 4 3 4 4 4 3 3 3 4 3 4 4 4 3 3 3 4 4 3 4 3 3 2 2 4 3 2 4 3 4 3 3 4 2 4 4 4 3 2 3 4 3 4
# [76] 4 3 3 3 2 2 2 4 3 2 3 4 3 3 4 2 3 4 4 4 3 3 3 2 3 2 2 2 4 2 3 4 3 3 4 3 4 2 3 3 3 4 3 2 4 4 3 3 3 3 2 4 4 3 2 3 4 2 4 3 2 3 4 3 4 2 3 3 4 2 3 3 2 4 2
#[151] 4 4 2 2 4 3 4 4 3 4 4 4 3 2 4 3 2 2 4 3 3 2 3 2 3 4 4 2 4 4 3 4 4 2 4 4 3 2 2 4 3 1 3 3 4 2 3 3 3 2 3 3 3 3 3 3 4 4 4 3 4 4 2 4 2 2 4 2 4 3 2 4 2 2 4
#[226] 4 4 3 3 2 3 4 3 3 4 2 4 3 2 4 4 3 3 2 3 4 3 4 4 3 4 3 2 4 3 3 3 4 2 2 2 3 4 3 4 3 3 4 2 4 2 4 2 4 2 3 3 3 2 4 3 4 3 4 3 3 3 3 2 3 2 4 4 3 3 3 2 3 4 3
#[301] 4 2 4 4 3 3 3 3 4 3 2 2 2 4 2 2 4 2 3 3 4 4 3 2 3 3 3 2 3 3 4 3 2 2 3 3 3 2 4 3 3 3 2 2 3 3 4 4 2 4 4 4 2 2 4 4 4 3 4 4 4 3 2 4 4 3 2 4 4 4 4 3 3 4 3
#[376] 2 3 2 4 3 2 3 4 4 4 2 4 4 4 4 2 3

 b=brier(dn-1,preai[,2],bins=F)
 b$bs #coincide with rrf

##############
# Ensemble LDA-RF
##############

library(verification)

############
#combining: ensemble
##########
#weighting


w=seq(0,1,0.01)


prc=c()
clw=c()
aw=c()

for (i in 1:length(w)){
prc=w[i]*preai[,1:2]+(1-w[i])*preaire
clw=max.col(prc[,1:2])
aw[i]=sum(clw==dn)
}

p=85

1- rrf$rps^p/(rrf$rps^p+ rlda$rps^p)
#0.9280948

 aw[94]
# 311

i=94
prc=w[i]*preai[,1:2]+(1-w[i])*preaire


rrfe=rps( dn,prc)
clw=max.col(prc[,1:2])
confusionMatrix(clw,dn,positive='2') 



##############
# Ensemble GLM-RF
##############

library(verification)

############
#combining: ensemble
##########
#weighting

w=seq(0,1,0.01)


prc=c()
clw=c()
aw=c()

for (i in 1:length(w)){
prc=w[i]*preai[,1:2]+(1-w[i])*preaire2
clw=max.col(prc[,1:2])
aw[i]=sum(clw==dn)
}

p=85

1- rrf$rps^p/(rrf$rps^p+ rpolr$rps^p)
#0.8970669

 aw[91]
#[1] 311

i=91
prc=w[i]*preai[,1:2]+(1-w[i])*preaire2


rrfe=rps( dn,prc)
clw=max.col(prc[,1:2])
confusionMatrix(clw,dn,positive='2') 


###############
#lda-seclection


library(MASS)
library(klaR)



preaire=c()


for (i in 1:length(ninos)){

TR= ninos[-i]

#gw_obj <- greedy.wilks(as.factor(Size_fit)[TR] ~ ., data = da[TR,], niveau = 0.1)
gw_obj <- greedy.wilks(diabetes ~ ., data = da, subset=TR, niveau = 0.1)

z <- lda(gw_obj$formula, da, subset = TR)

TE= ninos[i]


preaire=rbind(preaire,predict(z, da[TE,])$posterior)

}


clre=max.col(preaire[,1:2])


sum(clre==dn) #309

rlda2=rps( dn,preaire)
confusionMatrix(clre,dn,positive='2') 


##############
# Ensemble LDAVariableselection-RF
##############

library(verification)

############
#combining: ensemble
##########
#weighting

w=seq(0,1,0.01)


prc=c()
clw=c()
aw=c()

for (i in 1:length(w)){
prc=w[i]*preai[,1:2]+(1-w[i])*preaire
clw=max.col(prc[,1:2])
aw[i]=sum(clw==dn)
}

p=85

1- rrf$rps^p/(rrf$rps^p+ rlda2$rps^p)
#[1] 0.5697626
> aw[58]
[1] 310


i=58
prc=w[i]*preai[,1:2]+(1-w[i])*preaire
clw=max.col(prc[,1:2])
rlda2rfe=rps( dn,prc)
confusionMatrix(clw,dn,positive='2') 

##################

###################
#k-nn
####################


library(mlbench)
data(PimaIndiansDiabetes2) #used in manual Breiman
Diabetes <- na.omit(PimaIndiansDiabetes2)
attach(Diabetes)

dn=as.numeric(diabetes) #negative:1, positive: 2
ninos=1:length(diabetes)
library(class)

da=Diabetes[,1:8] #


set.seed(20)
#chosing the number of neighbours

wk=c()


preai=c()
for (i in 1:length(ninos)){

TR= ninos[-i]

ac=c()
for (k in 1:7){
qq=knn.cv(da[TR,], dn[TR], k = k)
ac[k]=sum(dn[TR]==qq)
}

km=which.max(ac)
wk[i]=km

TE=ninos[i]
ki=knn(da[TR,], da[TE,], dn[TR],k=km)




preai=c(preai,ki)


}

sum(dn==round(preai))
#277
wk


confusionMatrix(round(preai),dn,positive='2') 


#######################
#MARS
#######################

#Size_fit classification
library(mda)


preai=c()
for (i in 1:length(ninos)){

TR=ninos[-i]


TE=ninos[i]

marsi=mars(da[TR,],dn[TR])




preai=c(preai,predict(marsi,da[TE,]))


}

 sum(dn==round(preai))
 #303
 
 
confusionMatrix(round(preai),dn,positive='2')   

############
#SVM


#clasification
library(e1071)

preai=c()
for (i in 1:length(ninos)){

TR=ninos[-i]

svmi=svm(da[TR,],as.factor(dn[TR]))




TE= ninos[i]



preai=c(preai,predict(svmi,da[TE,]))


}

sum(dn==(preai))
#

confusionMatrix(round(preai),dn,positive='2')  


################
#FDA
################

library(mda)
preai=c()
for (i in 1:length(ninos)){

TR=ninos[-i]

fdai=fda(as.factor(dn[TR])~as.matrix(da[TR,]),method=mars)


TE=ninos[i]



preai=c(preai,predict(fdai,as.matrix(da[TE,])))


}
sum(dn==(preai))

#


confusionMatrix(round(preai),dn,positive='2')  


#####################
#Projection pursuit
#####################

preai=c()
nm=c()
who=1:length(ninos)
for (i in 1:length(ninos)){

TR= ninos[-i]

whoi=who[-i]
appi=c()

for (num in 1:8){
pprii=c()
for(h in whoi){
TRi= ninos[c(-i,-h)]


TEi= ninos[h]


# fit PPR
fiti <- ppr(da[TRi,],dn[TRi],nterms=num)
pprii=c(pprii,predict(fiti,da[TEi,],nterms=num))
}
appi[num]=sum(round(pprii)==dn[TR])

}

nm[i]=which.max(appi)

TE=ninos[i]

ppri=ppr(da[TR,],dn[TR],nterms=nm[i])

preai=c(preai,predict(ppri,da[TE,]))
cat(i)

}

 sum(dn==round(preai))


confusionMatrix(round(preai),dn,positive='2')  


###################
#Cluster SVM
##################

ce=8


library(SwarmSVM)


preai=c()
for (i in 1:length(ninos)){

TR=ninos[-i]


svmi=clusterSVM(da[TR,],as.factor(dn[TR]),centers = ce, seed = 512,verbose=0)



TE=ninos[i]


if(sum(length(TE))>1){
preai=c(preai,predict(svmi,da[TE,])$predictions)}

if(sum(length(TE))==1){
p1=predict(svmi,as.matrix(rbind(da[TE,],da[TE,])))$predictions
preai=c(preai,p1[1])
}
}


sum(dn==(preai))
 

confusionMatrix(round(preai),dn,positive='2')  
