Parallel Processing in R

parallel
Author
Published

Wednesday, June 5, 2024

Modified

Thursday, June 20, 2024

library(parallel)
library(lme4)
Loading required package: Matrix
### Check the number of cores
detectCores()
[1] 8

mclapply

### mclapply works on unix system, it will call lapply in windows
f <- function(i) {
  lmer(Petal.Width ~ . - Species + (1 | Species), data = iris)
}

system.time(save1 <- lapply(1:100, f))
   user  system elapsed 
  0.681   0.002   0.691 
system.time(save2 <- mclapply(1:100, f))
   user  system elapsed 
  0.033   0.029   0.562 

parlapply

### Works on windows, but slower than mclapply
numCores <- detectCores()

### Starting a cluster
cl <- makeCluster(numCores)
parSapply(cl, Orange, mean, na.rm = TRUE)
         Tree           age circumference 
           NA      922.1429      115.8571 
### Close the cluster, best practise
stopCluster(cl)
### lapply
system.time({save1 <- lapply(1:100, f)})
   user  system elapsed 
  0.718   0.011   0.781 
### mclapply
system.time({save2 <- mclapply(1:100, f)})
   user  system elapsed 
  0.415   0.072   0.549 
###
system.time(
    {
        cl <- makeCluster(detectCores())
        clusterEvalQ(cl, library(lme4))
        save3 <- parLapply(cl, 1:100, f)
        stopCluster(cl)
    }
)
   user  system elapsed 
  0.125   0.019   1.622 

Reference