Séance 3 : ACP et clustering

Master 2 siren 2023

Author

Christophe Benavent

Published

April 25, 2023

Data

Code
library(tidyverse)
df <- read_csv("Plateformes3.csv")%>%
  rename(ID=1,
         Rock=2,
         PopInter=3,
         HipHop=4,
         World=5,
         Salsa=6,
         Classique=7,
         Techno=8,
         HardRock=9,
         Baroque=10,
         Folk=11,
         Reggae=12,
         ChansonFrançaise=13,
         RockAlt=14,
         Dance=15,
         Jazz=16,
         AfroPop=17,
         Opera=18,
         Arabic=19,
         Blues=20,
         JK_pop=21,
         Rap=22,
         SoulFunk=23,
         S_travail=24,
         S_Live=25,
         S_Voiture=26,
         S_Menage=27,
         S_Ami=28,
         S_Bars=29,
         S_Transport=30,
         S_Lire=31,
         S_Douche=32,
         D_Vynil=33,
         D_Streaming=34,
         D_smartphone=35,
         D_concert=36,
         D_laptop=37,
         D_ecouteurs=38,
         D_mix=39,
         D_radiotv=40,
         P_Youtube=42,
         P_Spotify=43,
         P_Deezer=44,
         P_Souncloud=45,
         P_Bandcamp=46, 
         P_Discog=47,
         P_Tidal=48,
         P_Amazon=49,
         P_Mixcloud=50,
         P_AppleM=51,
         P_Qobuz=52,
         P_Pandora=53,
         P_Netease=54,
         Time=68,
         Spending=69,
         Value=70, 
         Age=71,
         Education=72,
         Genre=73
         )


df$Age<-factor(df$Age, ordered = TRUE, 
                                levels = c("moins de 18 ans", "de 18 à 20 ans",
                                           "de 21 à 25 ans", "de 26 à 30 ans",
                                           "de 31 à 40 ans", "de 41 à 60 ans",
                                           "plus de 60 ans"))
#color palette
col<-c("Coral","Pink","Gold","Orange")

Comparaison de moyenne

Code
table(df$Time)

 Entre 2 et 4h  Entre 4 et 8h Entre 8 et 16h    Moins de 2h    Plus de 16h 
            33             43             42             30             35 
Code
df$Time2<-NA
df$Time2[df$Time=="Moins de 2h"]<-1
df$Time2[df$Time=="Entre 2 et 4h"]<-3
df$Time2[df$Time=="Entre 4 et 8h"]<-6
df$Time2[df$Time=="Entre 8 et 16h"]<- 12
df$Time2[df$Time=="Plus de 16h"]<- 24

moyenne_time<-round(mean(df$Time2, na.rm=TRUE), 2)
moyenne_time<-round(sd(df$Time2, na.rm=TRUE), 2)


ggplot(df, aes(x=Time2))+
  geom_histogram()+
  labs(title = "distribution des durée d'écoute",
       subtitle = paste0("Moyenne : ",moyenne_time))

Code
foo <- df %>% group_by(Age)%>%
  summarise(n =n(),
            temps_moyen=mean(Time2, na.rm=TRUE),
            temps_sd=sd(Time2, na.rm=TRUE),
            sd=temps_sd/sqrt(n)) 
foo
# A tibble: 7 x 5
  Age                 n temps_moyen temps_sd    sd
  <ord>           <int>       <dbl>    <dbl> <dbl>
1 moins de 18 ans     5        6        3.67 1.64 
2 de 18 à 20 ans      4        7.5      5.20 2.60 
3 de 21 à 25 ans     97       10.6      8.17 0.829
4 de 26 à 30 ans      8        9.38     9.77 3.45 
5 de 31 à 40 ans     17       10.4      8.67 2.10 
6 de 41 à 60 ans     40        8.05     8.01 1.27 
7 plus de 60 ans     14        5.93     5.92 1.58 
Code
fit<-lm(Time2 ~ Age,data=df)
anova(fit)
Analysis of Variance Table

Response: Time2
           Df  Sum Sq Mean Sq F value Pr(>F)
Age         6   470.7  78.457  1.2286 0.2937
Residuals 176 11238.7  63.856               
Code
ggplot(foo, aes(x=Age, y=temps_moyen, group=1))+
  geom_line(size=2, color="green")+
  geom_errorbar(aes(ymin=temps_moyen-sd, ymax=temps_moyen+sd), 
                width=.2, position=position_dodge(.9)) +
  coord_flip()

Correlations

Code
table(df$Spending)

           0 euros  0 euros, 10 euros           10 euros 10 euros, 20 euros 
                38                  1                 96                  2 
         100 euros           20 euros 20 euros, 50 euros          200 euros 
                 6                 22                  1                  1 
 200 euros et plus           50 euros 
                 5                 13 
Code
df$SpendingMax<-NA
df$SpendingMax[df$Spending=="0 euros"] <-0
df$SpendingMax[df$Spending=="0 euros, 10 euros"] <-10
df$SpendingMax[df$Spending=="10 euros"] <-10
df$SpendingMax[df$Spending=="10 euros, 20 euros"] <-20
df$SpendingMax[df$Spending=="20 euros"] <-20
df$SpendingMax[df$Spending=="50 euros"] <-50
df$SpendingMax[df$Spending=="100 euros"] <-100
df$SpendingMax[df$Spending=="200 euros"] <-200
df$SpendingMax[df$Spending=="200 euros et plus"] <-400

table(df$SpendingMax)

  0  10  20  50 100 200 400 
 38  97  24  13   6   1   5 
Code
moyenne_Spending<-round(mean(df$SpendingMax, na.rm=TRUE), 2)
moyenne_Spending<-round(sd(df$SpendingMax, na.rm=TRUE), 2)

foo<-df %>% 
  select(Time2,SpendingMax) %>%
  drop_na()

r<-cor(foo$Time2,foo$SpendingMax)

res<-cor.test(foo$Time2,foo$SpendingMax, method="pearson")
res

    Pearson's product-moment correlation

data:  foo$Time2 and foo$SpendingMax
t = 4.0193, df = 180, p-value = 8.57e-05
alternative hypothesis: true correlation is not equal to 0
95 percent confidence interval:
 0.1476916 0.4151096
sample estimates:
     cor 
0.286982 
Code
ggplot(foo,aes(Time2, SpendingMax))+geom_point()+
  geom_smooth()+
  geom_smooth(method="lm", color="red")+scale_y_log10()

Correlation entre plus de 3 variables

Code
table(df$ValueMax)
< table of extent 0 >
Code
df$ValueMax[df$Value=="moins de 50 euros"] <-50
df$ValueMax[df$Value=="moins de 50 euros, 100 euros"] <-100
df$ValueMax[df$Value=="100 euros"] <-100
df$ValueMax[df$Value=="200 euros"] <-200
df$ValueMax[df$Value=="500 euros"] <-500
df$ValueMax[df$Value=="1000 euros"] <-1000
df$ValueMax[df$Value=="2000 euros"] <-2000
df$ValueMax[df$Value=="200 euros, 2000 euros"] <-2000

df$ValueMax[df$Value=="2000 euros, 5000 euros et plus"] <-5000
df$ValueMax[df$Value=="5000 euros et plus"] <-5000

ggplot(df, aes(x=ValueMax))+geom_bar()+coord_flip()

Code
foo<-df %>% 
  select(Time2,SpendingMax, ValueMax) %>%
  drop_na()

r<-cor(foo)

r
                Time2 SpendingMax  ValueMax
Time2       1.0000000   0.2856928 0.3839856
SpendingMax 0.2856928   1.0000000 0.3668294
ValueMax    0.3839856   0.3668294 1.0000000
Code
library(ggcorrplot)

ggcorrplot(r, hc.order = TRUE, type = "lower",
   outline.col = "white",
   ggtheme = ggplot2::theme_gray,
   colors = col, 
    lab = TRUE)

Linear model

Code
fit <-lm(SpendingMax~Time2+ValueMax+Age,df)
summary(fit)

Call:
lm(formula = SpendingMax ~ Time2 + ValueMax + Age, data = df)

Residuals:
   Min     1Q Median     3Q    Max 
-93.98 -19.53  -6.22   5.57 385.84 

Coefficients:
              Estimate Std. Error t value Pr(>|t|)    
(Intercept)   1.938671   9.119501   0.213 0.831902    
Time2         1.620299   0.647062   2.504 0.013208 *  
ValueMax      0.012198   0.003499   3.486 0.000622 ***
Age.L        -4.297832  22.578781  -0.190 0.849261    
Age.Q         6.852185  21.071283   0.325 0.745432    
Age.C       -24.786176  20.088918  -1.234 0.218952    
Age^4        -5.331586  22.867327  -0.233 0.815920    
Age^5        -3.476675  17.290562  -0.201 0.840879    
Age^6         4.551914  17.935529   0.254 0.799959    
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Residual standard error: 62.45 on 172 degrees of freedom
  (4 observations deleted due to missingness)
Multiple R-squared:  0.1768,    Adjusted R-squared:  0.1385 
F-statistic: 4.618 on 8 and 172 DF,  p-value: 3.995e-05
Code
library(jtools)
summ(fit)
Observations 181 (4 missing obs. deleted)
Dependent variable SpendingMax
Type OLS linear regression
F(8,172) 4.62
R² 0.18
Adj. R² 0.14
Est. S.E. t val. p
(Intercept) 1.94 9.12 0.21 0.83
Time2 1.62 0.65 2.50 0.01
ValueMax 0.01 0.00 3.49 0.00
Age.L -4.30 22.58 -0.19 0.85
Age.Q 6.85 21.07 0.33 0.75
Age.C -24.79 20.09 -1.23 0.22
Age^4 -5.33 22.87 -0.23 0.82
Age^5 -3.48 17.29 -0.20 0.84
Age^6 4.55 17.94 0.25 0.80
Standard errors: OLS
Code
effect_plot(fit,pred=Age,interval = TRUE)

Code
effect_plot(fit,pred=Time2,interval = TRUE, plot.points = TRUE, 
            jitter = 0.05)

Question à choix multiple

Une belle application de l’approche tidy

1) l’art du pivot

Code
foo<- df %>%
  select(1:23) %>% 
  pivot_longer(-ID,names_to="variable", values_to = "value")
  1. recodage
Code
foo$value[foo$value=="J'aime beaucoup"]<-"J'adore"
foo$value[foo$value=="J'aime pas mal"]<-"Je peux aimer"
foo$value[foo$value=="J'aime un peu"]<-"Je peux aimer"
foo$value[foo$value=="J'aime pas trop"]<-"Je n'aime pas"

#variable définition
foo$value <- factor(foo$value, levels = c("Je n'aime pas",
                                          "Je ne connais pas",
                                          "Je peux aimer",
                                          "J'adore"))
  1. le graphique
Code
#just counting
foo<-foo%>%
  group_by(variable, value)%>%
  summarise(n=n())


#plot
ggplot(foo, aes(x = reorder(variable, value), y=n, group=value))+
  geom_bar(stat="identity", aes(fill=value))+
  coord_flip()+
  labs(x=NULL, y =NULL)+
  scale_fill_manual(values=col)

Code
#construire le bon tableau

foo<- df %>%
  select(1:23) %>% 
  pivot_longer(-ID,names_to="variable", values_to = "value")
table(foo$value)

                J'adore         J'aime beaucoup          J'aime pas mal 
                    895                       7                       2 
J'aime pas mal, J'adore         J'aime pas trop           J'aime un peu 
                      1                       3                       6 
          Je n'aime pas       Je ne connais pas           Je peux aimer 
                    849                     510                    1797 
Code
foo$value[foo$value=="J'aime beaucoup"]<-"J'adore"
foo$value[foo$value=="J'aime pas mal"]<-"Je peux aimer"
foo$value[foo$value=="J'aime un peu"]<-"Je peux aimer"
foo$value[foo$value=="J'aime pas trop"]<-"Je n'aime pas"

foo$value <- factor(foo$value, levels = c("Je n'aime pas",
                                          "Je ne connais pas",
                                          "Je peux aimer",
                                          "J'adore"))


foo$value2[foo$value=="Je n'aime pas"]<-1
foo$value2[foo$value=="Je ne connais pas"]<-2
foo$value2[foo$value=="Je peux aimer"]<-3
foo$value2[foo$value=="J'adore"]<-4

foo<-foo %>%
  select(-value)%>%
  pivot_wider(id_cols=ID,names_from = "variable", values_from = "value2")

corr<-foo[,2:23]%>%
  drop_na()%>%
  cor()
ggcorrplot(corr, hc.order = TRUE, type = "lower",
   outline.col = "white",
   ggtheme = ggplot2::theme_gray,
   colors = c("#6D9EC1", "white", "#E46726"))

ACP

Code
#PCA

library(FactoMineR)
library(factoextra)
fit<-PCA(foo[2:19],ncp = 4, graph =FALSE)
fviz_eig(fit, addlabels = TRUE, ylim = c(0, 30))

Code
fviz_pca_var(fit,  axes = c(1, 2), col.var = "cos2",
             gradient.cols = c("#00AFBB", "#E7B800", "#FC4E07"),
             repel = TRUE # Évite le chevauchement de texte
             )

Code
fviz_pca_var(fit,  axes = c(2, 3), col.var = "cos2",
             gradient.cols = c("#00AFBB", "#E7B800", "#FC4E07"),
             repel = TRUE # Évite le chevauchement de texte
             )

Clustering

Code
# Dissimilarity matrix
d <- dist(t(foo[2:19]), method = "manhattan")

# Hierarchical clustering using Ward Linkage
hc1 <- hclust(d, method = "ward" )
sub_grp <- cutree(hc1, k = 7)

# Plot the obtained dendrogram
plot(hc1, cex = 0.6, hang = -1)
rect.hclust(hc1, k = 7, border = 2:5)

Code
fviz_cluster(list(data = d, cluster = sub_grp))

Segmentation

Code
#number of groups
k=5

# Dissimilarity matrix
d <- dist(foo[2:23], method = "manhattan")

# Hierarchical clustering using Ward Linkage
hc1 <- hclust(d, method = "ward" )
sub_grp <- cutree(hc1, k = k)

# Plot the obtained dendrogram
plot(hc1, cex = 0.6, hang = -1)
rect.hclust(hc1, k = k, border = 2:5)

Code
fviz_cluster(list(data = d, cluster = sub_grp,  ellipse.type = "norm"))

Code
foo1<-cbind(foo,sub_grp)%>%
  pivot_longer(-c(ID,sub_grp),names_to="Variable", values_to="Value") %>%
  group_by(sub_grp, Variable, Value)%>%
  summarise(n=n())

foo2<-cbind(foo,sub_grp)%>%
  pivot_longer(-c(ID,sub_grp),names_to="Variable", values_to="Value") %>%
  group_by(sub_grp, Variable)%>%
  summarise(m=n())

foo3<- foo1 %>%
  left_join(foo2) %>%
  mutate(f=n / m)


ggplot(foo3, aes(x = Variable, y=f, group=Value))+
  geom_bar(stat="identity", aes(fill=as.factor(Value)))+
  facet_wrap(vars(sub_grp), ncol=5)+
  coord_flip()+ 
  scale_fill_manual(values = col)+
  labs(x=NULL, y=NULL, title=" Segmentation")