mirai_map much slower than its synchronous sequential and parallel::parLapply 'alternatives'

[olivier7121]

I'm still struggling to use mirai_map to run a piece of code (that takes quite a long time to complete) in an asynchronous parallel way within a Shiny app.

In this context I was looking for benchmarks in the synchronous world and was quite surprised to see that in this specific example mirai_map completed on my computer (with 8 logical cores) in approx. 01:41 while the sequential and parallel::parLapply synchronous 'alternatives' completed in 01:01 and 00:22, respectively.

Why is there such a big difference between mirai_map and parallel::parLapply?

Below are the three pieces of code:

1. Sequential

library(shiny)
library(bslib)
library(openxlsx)

TIMER_NOT_STARTED <- 0L
TIMER_RUNNING <- 1L
TIMER_FINISHED <- 2L

ui <- fluidPage(
	
	titlePanel("1 - Sequential"),
	
	sidebarLayout(
		sidebarPanel(
			input_task_button(id = "run", "Run")
			#actionButton('run', 'Run')
		),
		
		mainPanel(
			textOutput("Timer"),
			#tableOutput("result")
		)
	)
)

server <- function(input, output) {
	Timer.Status <- reactiveVal(0)
	Timer.Start <- reactiveVal(0)
	Timer.End <- reactiveVal(0)
	
	x <- 1:1000
	
	GenerateExcelFiles <- function(Row)
	{
		Number <- paste0(paste0(rep("0", NCharTotalNRows - nchar(Row)), collapse = ""), Row)
		OutputFile_Individual <- file.path(OutputFolder, paste0("1 - Sequential_", Number, ".xlsx"))
		Workbook_Individual <- createWorkbook()
		saveWorkbook(wb = Workbook_Individual, file = OutputFile_Individual, overwrite = TRUE)
	}
	
	# Handle button click
	observeEvent(input$run,{
		Timer.Status(TIMER_RUNNING)
		Timer.Start(as.integer(Sys.time()))
		
		NCharTotalNRows <<- nchar(length(x))
		OutputFolder <<- file.path(".", "Output")
		if(!dir.exists(OutputFolder))	dir.create(OutputFolder)
		lapply(X = x, FUN = GenerateExcelFiles)
		Timer.End(as.integer(Sys.time()))
		Timer.Status(TIMER_FINISHED)
	})
	output$Timer <- renderText({
		Status <- Timer.Status()
		if(Status == TIMER_RUNNING)
		{
			invalidateLater(1000L)
			Timer.End(as.integer(Sys.time()))
			paste("Time elapsed:", format(as.POSIXct(Timer.End() - Timer.Start() - 3600L), "%H:%M:%S"))
		} else if(Status == TIMER_FINISHED)
		{
			paste("Time elapsed:", format(as.POSIXct(Timer.End() - Timer.Start() - 3600L), "%H:%M:%S"))
		}
	})
}

# Run the application
shinyApp(ui = ui, server = server)

2. parallel::parLapply

library(shiny)
library(bslib)
library(parallel)
library(openxlsx)

TIMER_NOT_STARTED <- 0L
TIMER_RUNNING <- 1L
TIMER_FINISHED <- 2L

ui <- fluidPage(
	
	titlePanel("2 - Parallel"),
	
	sidebarLayout(
		sidebarPanel(
			input_task_button(id = "run", "Run")
			#actionButton('run', 'Run')
		),
		
		mainPanel(
			textOutput("Timer"),
			#tableOutput("result")
		)
	)
)

server <- function(input, output) {
	Timer.Status <- reactiveVal(0)
	Timer.Start <- reactiveVal(0)
	Timer.End <- reactiveVal(0)
	
	x <- 1:1000
	
	GenerateExcelFiles <- function(Row)
	{
		Number <- paste0(paste0(rep("0", NCharTotalNRows - nchar(Row)), collapse = ""), Row)
		OutputFile_Individual <- file.path(OutputFolder, paste0("2 - Parallel_", Number, ".xlsx"))
		Workbook_Individual <- createWorkbook()
		saveWorkbook(wb = Workbook_Individual, file = OutputFile_Individual, overwrite = TRUE)
	}
	
	# Handle button click
	observeEvent(input$run,{
		Timer.Status(TIMER_RUNNING)
		Timer.Start(as.integer(Sys.time()))
		
		NCharTotalNRows <<- nchar(length(x))
		OutputFolder <<- file.path(".", "Output")
		if(!dir.exists(OutputFolder))	dir.create(OutputFolder)
		
		clusterMassiveExport <- makePSOCKcluster(names = 8)
		clusterEvalQ(clusterMassiveExport, {
			lapply(c("openxlsx"), library, character.only = TRUE)
		})
		clusterExport(clusterMassiveExport, c("NCharTotalNRows", "OutputFolder"))
		
		parLapply(cl = clusterMassiveExport, X = x, fun = GenerateExcelFiles)
		Timer.End(as.integer(Sys.time()))
		Timer.Status(TIMER_FINISHED)
		stopCluster(clusterMassiveExport)
	})
	output$Timer <- renderText({
		Status <- Timer.Status()
		if(Status == TIMER_RUNNING)
		{
			invalidateLater(1000L)
			Timer.End(as.integer(Sys.time()))
			paste("Time elapsed:", format(as.POSIXct(Timer.End() - Timer.Start() - 3600L), "%H:%M:%S"))
		} else if(Status == TIMER_FINISHED)
		{
			paste("Time elapsed:", format(as.POSIXct(Timer.End() - Timer.Start() - 3600L), "%H:%M:%S"))
		}
	})
}

# Run the application
shinyApp(ui = ui, server = server)

3. mirai_map

library(shiny)
library(bslib)
library(mirai)
library(promises)
library(openxlsx)

TIMER_NOT_STARTED <- 0L
TIMER_RUNNING <- 1L
TIMER_FINISHED <- 2L

ui <- fluidPage(
	
	titlePanel("3 - Mirai_map"),
	
	sidebarLayout(
		sidebarPanel(
			input_task_button(id = "run", "Run")
			#actionButton('run', 'Run')
		),
		
		mainPanel(
			textOutput("Timer"),
			#tableOutput("result")
		)
	)
)

server <- function(input, output) {
	Timer.Status <- reactiveVal(0)
	Timer.Start <- reactiveVal(0)
	Timer.End <- reactiveVal(0)
	
	x <- 1:1000
	
	GenerateExcelFiles <- function(Row)
	{
		Number <- paste0(paste0(rep("0", NCharTotalNRows - nchar(Row)), collapse = ""), Row)
		OutputFile_Individual <- file.path(OutputFolder, paste0("3 - mirai_map_", Number, ".xlsx"))
		Workbook_Individual <- createWorkbook()
		saveWorkbook(wb = Workbook_Individual, file = OutputFile_Individual, overwrite = TRUE)
	}
	
	# Handle button click
	observeEvent(input$run,{
		Timer.Status(TIMER_RUNNING)
		Timer.Start(as.integer(Sys.time()))
		
		NCharTotalNRows <<- nchar(length(x))
		OutputFolder <<- file.path(".", "Output")
		if(!dir.exists(OutputFolder))	dir.create(OutputFolder)
		
		daemons(7)
		
		Variables2Share <- list("NCharTotalNRows" = NCharTotalNRows, "OutputFolder" = OutputFolder)
		everywhere({
			lapply(c("openxlsx"), library, character.only = TRUE)
			},
			as.environment(Variables2Share)
		)
		
		mirai_map(
			.x = x,
			.f = GenerateExcelFiles
		) %...>% {
			Timer.End(as.integer(Sys.time()))
			Timer.Status(TIMER_FINISHED)
			daemons(0)
		}
		NULL
	})
	
	output$Timer <- renderText({
		Status <- Timer.Status()
		if(Status == TIMER_RUNNING)
		{
			invalidateLater(1000L)
			Timer.End(as.integer(Sys.time()))
			paste("Time elapsed:", format(as.POSIXct(Timer.End() - Timer.Start() - 3600L), "%H:%M:%S"))
		} else if(Status == TIMER_FINISHED)
		{
			paste("Time elapsed:", format(as.POSIXct(Timer.End() - Timer.Start() - 3600L), "%H:%M:%S"))
		}
	})
}

# Run the application
shinyApp(ui = ui, server = server)

[shikokuchuo]

I'm afraid it's difficult for me to comment given all the packages involved. In particular I can't really speak to 'openxlsx'. If you want me to look at mirai performance, then ideally it would be something just involving mirai.

I'm also happy to look at things in a Shiny context, but then it becomes more difficult to narrow down.

[shikokuchuo]

This may be a case of #300 where you are accidentally shipping large objects to the parallel workers. Please take care when you are passing environments or functions.

[shikokuchuo]

I have no way to reproduce this, so am closing for now. If you have a more minimal example, and still run into this issue, please feel free to open another discussion/issue. Thanks.