Using ggplot2 grobs to display multiple plots. Is this causing a memory leak?

The following code seems to always increase memory use. Is there a memory leak?

This is an extract of using multiple tabs to display multiple plots. It uses ggplotGlob to create multiple groups of plots. When object_size (output) is used, the value seems to always be increasing, event when 10 tabs 3 groups of 10 plots each and then tab 1 is changed to 1 group of 1 plot. In the full application this increasing memory use eventually leads to a segment fault.


library(shiny)
library(pryr)
library(ggplot2)
library(grid)
library(gridExtra)

totalTabs <<- 1
lastMemorySize <<- 0

# Define UI for application that draws a histogram
ui <- fluidPage(
   
   # Application title
   titlePanel("test"),
   
   # Sidebar with a slider input for number of bins 
   sidebarLayout(
      sidebarPanel(
        sliderInput("tabNumber",
                    "Tab Number to use:",
                    min = 1,
                    max = totalTabs,
                    value = 1),
        sliderInput("ngroups",
                    "Number of groups:",
                    min = 1,
                    max = 3,
                    value = 1),        
         sliderInput("nplots",
                     "Number of plots in each group:",
                     min = 1,
                     max = 10,
                     value = 30),
        actionButton(inputId = "addTab", label = "Update Tab" ),
        textOutput("memoryValue")
      ),
      
      # Show a plot of the generated distribution
      mainPanel(
         uiOutput("distPlot")
      )
   )
)

# Define server logic required to draw a histogram
server <- function(input, output, session) {
  rv <- reactiveValues(
    plotList = list()
  )
  
  output$memoryValue <- renderText ({
    input$tabNumber
    input$ngroups
    input$nplots
    input$addTab
    currentSize <- object_size(output)
    diff <- currentSize - lastMemorySize
    lastMemorySize <- currentSize
if(diff < 0) browser()
    str <- paste("Difference in output memory:", diff )  
    
  })
  
  clearPlots <- function () {
    if (length(rv$plotList) == 0) return ()
    if (length(rv$plotList) < input$tabNumber) return ()
    if (is.null(rv$plotList[[input$tabNumber]]))  return()
    if (is.na(rv$plotList[[input$tabNumber]]))  return()

    for (g in 1:rv$plotList[[input$tabNumber]][["groups"]]) {
      plotname <- rv$plotList[[input$tabNumber]][["name"]][[g]]
      output[[plotname]] <- NULL
    }
    rv$plotList[[input$tabNumber]] <- list()
  }

  observeEvent(input$addTab, {
    addNewTab()
  })
  
  addNewTab <- function() {

   clearPlots()

    if (input$tabNumber == totalTabs) {
      totalTabs <<- totalTabs + 1
      updateSliderInput(session, inputId = "tabNumber", label = "Tab Number to use:",
                         value = input$tabNumber, min = 1, max = totalTabs, step = 1)
    }
    
    p <- list()
    df <- list()
    pgrob <- list()
    plt <- list()
    rv$plotList[[input$tabNumber]] <- list()

    for (g in 1:input$ngroups) {
      p[[g]] <- list()
      pgrob[[g]] <- list()
      for (i in 1:input$nplots) {
        df[[i]] <- as.data.frame(matrix(rexp(20, rate=.1), ncol=2))
        colnames(df[[i]]) <- c("x", "y")
        p[[g]][[i]] <- qplot(x,y,data = df[[i]])
        pgrob[[g]][[i]] <- ggplotGrob(p[[g]][[i]])
      }
      plotname <- paste0("plot-", input$tabNumber, "-", g)
      rv$plotList[[input$tabNumber]][["groups"]] <- input$ngroups
      rv$plotList[[input$tabNumber]][["name"]][[g]] <- plotname
      ncols <- 3
      if (ncols < 3) ncols <- input$nplots

      output[[plotname]] <- renderPlot  ( {

        if (input$nplots == 1)
          p[[g]][[i]]
        else
          do.call("grid.arrange", c(pgrob[[g]], top = paste("Group", g, "with", input$nplots, "Images"), ncol = ncols))
      })
    }
  }

  output$distPlot <- renderUI({
    plt <- list()
    if (length(rv$plotList) == 0) return ()
    if (length(rv$plotList) < input$tabNumber) return ()

    for(g in 1:rv$plotList[[input$tabNumber]][["groups"]]) {
      plotname <- rv$plotList[[input$tabNumber]][["name"]][[g]] 
      plt[[g]] <- plotOutput(plotname)
    }
    if (length(plt) == 0)
      return (NULL)
    else
      return(plt)
  })
}

# Run the application 
shinyApp(ui = ui, server = server)


Digging through the code, I do not believe it is a memory leak.

Could you explain to me the end goal of what you are trying to achieve with this app? (I'll help me possibly provide a cleaner solution.)

I have modified the code (below) to place each "tab" into a new environment, but still the amount of memory increases.
The output for plotting the same number of plots in the same tab are shown after the code. This includes in clearPlots:

  1. a list of the variables within the tab environment,
  2. these variables are then removed from the tab environment with
    rm(list = ls(envir = rv$plotList[[input$tabNumber]] ), envir = rv$plotList[[input$tabNumber]] ),
  3. garbage collection is forced(?) with gc(verbose = TRUE)
  4. The (verbose = TRUE) displays memory being used, and
  5. then the variables within the environment is listed again (which is empty).

The memory reported by gs(verbose = TRUE) is increasing for both the amount recovered and the "cons cells", "vectors used, and the ". The amount memory reported by the primary object_size(output) also continues to increase. Eventually this crashes the applications.

Does anyone know why this is happening and how to fix it?

After some more testing,

library(shiny)
library(pryr)
library(ggplot2)
library(grid)
library(gridExtra)

totalTabs <<- 1
lastMemorySize <<- 0

# Define UI for application that draws a histogram
ui <- fluidPage(
   
   # Application title
   titlePanel("test"),
   
   # Sidebar with a slider input for number of bins 
   sidebarLayout(
      sidebarPanel(
        sliderInput("tabNumber",
                    "Tab Number to use:",
                    min = 1,
                    max = totalTabs,
                    value = 1),
        sliderInput("ngroups",
                    "Number of groups:",
                    min = 1,
                    max = 3,
                    value = 1),        
         sliderInput("nplots",
                     "Number of plots in each group:",
                     min = 1,
                     max = 10,
                     value = 30),
        actionButton(inputId = "addTab", label = "Update Tab" ),
        textOutput("memoryValue")
      ),
      
      # Show a plot of the generated distribution
      mainPanel(
         uiOutput("distPlot")
      )
   )
)

# Define server logic required to draw a histogram
server <- function(input, output, session) {
  
    rv <- reactiveValues(
      plotList = list()
    )
 
  
  output$memoryValue <- renderText ({
    input$tabNumber
    input$ngroups
    input$nplots
    input$addTab
    currentSize <- object_size(output)
    diff <- currentSize - lastMemorySize
    lastMemorySize <- currentSize
if(diff < 0) browser()
    str <- paste("output object size:", currentSize)  
    
  })
  
  clearPlots <- function () {

     if (length(rv$plotList) > 0){
      if (length(rv$plotList) >= input$tabNumber) {
        cat(paste(ls(envir = rv$plotList[[input$tabNumber]]),"\n"), file = stderr())
        rm(list = ls(envir = rv$plotList[[input$tabNumber]] ), envir = rv$plotList[[input$tabNumber]] )
        gc(verbose = TRUE)
        cat(paste(ls(envir = rv$plotList[[input$tabNumber]]),"\n"), file = stderr())
      } else
        rv$plotList[[input$tabNumber]] <- new.env()
    }
    else
      rv$plotList <- list(new.env())
    cat(paste("\ncreate envirnment", input$tabNumber, ":", object_size(rv$plotList[[input$tabNumber]]), object.size(rv$plotList[[input$tabNumber]]), "\n"), file = stderr())
  }

  observeEvent(input$addTab, {
    addNewTab()
  })
  
  addNewTab <- function() {

    
    clearPlots()
    
    if (input$tabNumber == totalTabs) {
          totalTabs <<- totalTabs + 1
          updateSliderInput(session, inputId = "tabNumber", label = "Tab Number to use:",
                             value = input$tabNumber, min = 1, max = totalTabs, step = 1)
    }
    

    assign("tabNumber", input$tabNumber, envir = rv$plotList[[input$tabNumber]])  
    assign("ngroups", input$ngroups, envir = rv$plotList[[input$tabNumber]])  
    assign("nplots", input$nplots, envir = rv$plotList[[input$tabNumber]])  
    
    local({
        p <- list()
        df <- list()
        pgrob <- list()
        plt <- list()
    
        for (g in 1:ngroups) {
          p[[g]] <- list()
          pgrob[[g]] <- list()
          for (i in 1:input$nplots) {
            df[[i]] <- as.data.frame(matrix(rexp(20, rate=.1), ncol=2))
            colnames(df[[i]]) <- c("x", "y")
            local({
              p[[g]][[i]] <- qplot(x,y,data = df[[i]])
              pgrob[[g]][[i]] <- ggplotGrob(p[[g]][[i]])
            },  envir = rv$plotList[[input$tabNumber]])
          }
          plotname <- paste0("plot-", tabNumber, "-", g)
       
          ncols <- 3
          if (ncols < 3) ncols <- nplots
         
          output[[plotname]] <- renderPlot  ( {
            if (input$nplots == 1)
              p[[g]][[i]]
            else
              do.call("grid.arrange", c(pgrob[[g]], top = paste("Group", g, "with", input$nplots, "Images"), ncol = ncols))
          })
       }
    }, envir =  rv$plotList[[input$tabNumber]])       
    cat(paste("fill envirnment", input$tabNumber, ":", object_size(rv$plotList[[input$tabNumber]]), object.size(rv$plotList[[input$tabNumber]]), "\n"), file = stderr())
  }

  output$distPlot <- renderUI({
    plt <- list()
    if (length(rv$plotList) == 0) return ()
    if (length(rv$plotList) < input$tabNumber) return ()
   
    assign("tabNumber", input$tabNumber, envir = rv$plotList[[input$tabNumber]])
    assign("ngroups", input$ngroups, envir = rv$plotList[[input$tabNumber]])  
   
    local({
      for(g in 1:ngroups) {
        plotname <-  paste0("plot-", tabNumber, "-", g)
        plt[[g]] <- plotOutput(plotname)
      }
      if (length(plt) == 0)
        return (NULL)
      else
        return(plt)
   }, envir = rv$plotList[[input$tabNumber]])
  })
}

# Run the application 
shinyApp(ui = ui, server = server)

My main concern is that the object_size(output) grows with every set of plots displays and never seems to be garbage collected.

I have noticed that if the plots that are use to create the grobs are deleted before the grobs are displayed, the plots are not displayed. I modified the code (below) to remove each component of the output, one layer at a time before the next set of plots are created. This zeros out the memory for all variables, but the output object. If I keep clicking the "Update Tab" button for tab 1, the output object just keeps growing in size. I have gotten to above 1 GB of memory and as the size of the output object grows, the application seems to slows down. Unless the garbage collection routine waits for the application to use and exorbitant amount of memory before collecting unused memory or I don't understand how the output object works, this seems like a memory leak.


library(shiny)
library(pryr)
library(ggplot2)
library(grid)
library(gridExtra)

totalTabs <<- 1
lastMemorySize <<- 0

# Define UI for application that draws a histogram
ui <- fluidPage(
   
   # Application title
   titlePanel("test"),
   
   # Sidebar with a slider input for number of bins 
   sidebarLayout(
      sidebarPanel(
        sliderInput("tabNumber",
                    "Tab Number to use:",
                    min = 1,
                    max = totalTabs,
                    value = 1),
        sliderInput("ngroups",
                    "Number of groups:",
                    min = 1,
                    max = 3,
                    value = 1),        
         sliderInput("nplots",
                     "Number of plots in each group:",
                     min = 1,
                     max = 50,
                     value = 20),
        actionButton(inputId = "addTab", label = "Update Tab" ),
        textOutput("memoryValue")
      ),
      
      # Show a plot of the generated distribution
      mainPanel(
         uiOutput("distPlot")
      )
   )
)

# Define server logic required to draw a histogram
server <- function(input, output, session) {
  
    rv <- reactiveValues(
      plotList = list()
    )
 
  
  output$memoryValue <- renderText ({
    input$tabNumber
    input$ngroups
    input$nplots
    input$addTab
    currentSize <- object_size(output)
    diff <- currentSize - lastMemorySize
    lastMemorySize <- currentSize
if(diff < 0) browser()
    str <- paste("output object size (MB):", format(currentSize/ 1024 / 1024))  
    
  })
  
  # clearPlots <- function () {
  # 
  #    if (length(rv$plotList) > 0){
  #     if (length(rv$plotList) >= input$tabNumber) {
  #       local({
  #         for (g in 1:local(ngroups, envir = rv$plotList[[input$tabNumber]]) ) {
  #           plotname <- paste0("plot-", tabNumber, "-", g)
  #   
  #           output[[plotname]] <- NULL
  #         }
  #         output <- NULL
  #         rm(list = ls())
  #       }, envir =rv$plotList[[input$tabNumber]])
  #     } else
  #       rv$plotList[[input$tabNumber]] <- new.env()
  #   }
  #   else
  #     rv$plotList <- list(new.env())
  # }

  observeEvent(input$addTab, {
    addNewTab()
  })
  
  addNewTab <- function() {

    if (input$tabNumber == totalTabs) {
          totalTabs <<- totalTabs + 1
          updateSliderInput(session, inputId = "tabNumber", label = "Tab Number to use:",
                             value = input$tabNumber, min = 1, max = totalTabs, step = 1)
    }
    
    if(length(rv$plotList) == 0) { rv$plotList <- list(new.env()) }  else {
      if (length(rv$plotList) < input$tabNumber) { rv$plotList[[input$tabNumber]] <- new.env() } else {
       local ( {
# # cat("start of local\n", file = stderr())  
# # gc(verbose = TRUE, full = TRUE)

          # lnames <- c()
          for (g in ngroups:1) {
            output[[paste0("plot-", tabNumber, "-", g)]] <- NULL
          }
        },  envir = rv$plotList[[input$tabNumber]])
      }
#browser()
   }

    assign("tabNumber", input$tabNumber, envir = rv$plotList[[input$tabNumber]])  
    assign("ngroups", input$ngroups, envir = rv$plotList[[input$tabNumber]])  
    assign("nplots", input$nplots, envir = rv$plotList[[input$tabNumber]])  
 

    local({
        p <- list()
        df <- list()
        pgrob <- list()
        plt <- list()

        
        for (g in 1:ngroups) {
          p[[g]] <- list()
          pgrob[[g]] <- list()
          for (i in 1:input$nplots) {
            df[[i]] <- as.data.frame(matrix(rexp(20, rate=.1), ncol=2))
            colnames(df[[i]]) <- c("x", "y")
            p[[g]][[i]] <- qplot(x,y,data = df[[i]])
            pgrob[[g]][[i]] <- ggplotGrob(p[[g]][[i]])
          }
          ncols = 3
          if (input$nplots < 3) ncols <- input$inplots
          output[[paste0("plot-", tabNumber, "-", g)]] <- renderPlot  ( {
            myg <<- myg + 1
            if(myg > input$ngroups) myg <<- 1
            do.call("grid.arrange", c(pgrob[[myg]], top = paste("Group", myg, "with", input$nplots, "Images"), ncol = ncols))
          } )
        }
    }, envir =  rv$plotList[[input$tabNumber]]) 
  }

  output$distPlot <- renderUI({

    if (length(rv$plotList) == 0) return ()
    if (length(rv$plotList) < input$tabNumber) return ()

    assign("tabNumber", input$tabNumber, envir = rv$plotList[[input$tabNumber]])
    assign("ngroups", input$ngroups, envir = rv$plotList[[input$tabNumber]])  
 
    local({
      for(g in 1:ngroups) {
        plotname <-  paste0("plot-", tabNumber, "-", g)
        myg <<- 0
        plt[[g]] <- plotOutput(plotname)
#        cat(paste("distPlot plt", g, object_size(plt[[g]]), "\n"), file = stderr())
      }
#      cat(paste("distPlot plt", object_size(plt), "\n"), file = stderr())
      if (length(plt) == 0)
        return (NULL)
      else
        return(plt)
   }, envir = rv$plotList[[input$tabNumber]])
  })

  observeEvent(input$ngroups, {}) 
  observeEvent(input$nplots, {}) 
  observeEvent(input$tabNumber, {}) 
}

# Run the application 
shinyApp(ui = ui, server = server)

You also asked me what I wanted to do.

I have built an application to display plots that forecast of the progression of several infectious and vector born diseases. There are multiple plots for each forecast. Multiple tabs can be created, each with its own disease, location, date range, and other parameters. This all works, but when it was run in a Debian Docker container, it started to crash with a segment fault after some number of plots were displayed. After a little bit of debugging, I found what looks like a memory leak and assumed that it was creating the segment fault.

I have just been testing my full application some more. The object_size(output) value just keeps climbing. This is not a phantom value either because top also reports that the % memory being used by R just keeps climbing also. This is the definition of a memory leak.

How free up memory in output?

I think I have found the problem. In the example below, if you only change the the number of plots, the memory behaves correctly. If you only change the number of groups, the amount of memory climbs. If you again only change the number of plots, the amount of memory again behaves correctly and collects all the memory built up when changing the number of groups.

If the code is modified to always pass the same number of renderPlot functions (some of them may be blank but must be renderPlot) to the uiOutput, then this memory leak does happen.

I am not sure what is going on, but I have a work around.
NOTE: The "Display Current Memory Values" button must be clicked to see the current memory usage.

rm(list = ls())
library(shiny)
library(pryr)
library(ggplot2)
library(grid)
library(gridExtra)

totalTabs <<- 1
lastMemorySize <<- 0


# Define UI for application that draws a histogram
ui <- fluidPage(
  
  # Application title
  titlePanel("test"),
  
  # Sidebar with a slider input for number of bins 
  sidebarLayout(
    sidebarPanel(
      sliderInput("ngroups",
                  "Number of groups:",
                  min = 1,
                  max = 3,
                  value = 2),        
      sliderInput("nplots",
                  "Number of plots in each group:",
                  min = 1,
                  max = 50,
                  value = 2), 
      actionButton(inputId = "updateMemory", label = "Display Current Memory Values" ),
      verbatimTextOutput("memoryValue")
    ),
    
    # Show a plot of the generated distribution
    mainPanel(
      uiOutput("distPlot")
    )
  )
)


# Define server logic required to draw a histogram
server <- function(input, output, session) {
  
  env <<- new.env()
  myg <<- 0
  rv <- reactiveValues(
    displayMemory = 0
  )
  
 
  
  output$distPlot <- renderUI({
    myg <<- 0
    createPlots()

  } )
  
  createPlots <- function () {
    local ( {  
      
      plt <- list()
      df <- list()
      pgrob <- list()
      
      myg <<- 0      
      for (g in 1:input$ngroups) {
        p <- list()
        pgrob[[g]] <- list()
        
        for (i in 1:input$nplots) {
          df[[i]] <- as.data.frame(matrix(rexp(20, rate=.1), ncol=2))
          colnames(df[[i]]) <- c("x", "y")
          p[[i]] <- qplot(x,y,data = df[[i]])
          pgrob[[g]][[i]] <- ggplotGrob(p[[i]])
        }
        ncols = 3
        if (input$nplots < 3) ncols <- input$inplots
        
        plotname <-   paste0("plot-", g)    
        output[[plotname]] <- renderPlot  ( {
          myg <<- myg + 1
          grid <- do.call("grid.arrange", c(pgrob[[myg]], top = paste("Group", myg, "with", input$nplots, "Images"), ncol = ncols))
          grid
        } )
        plt[[g]] <- plotOutput(plotname)
        
      }
      
      return (plt)
    }, envir = env )
    
  }

  output$memoryValue <- renderText ({
    input$updateMemory
    rv$displayMemory
    currentSize <- object_size(output)
    diff <- currentSize - lastMemorySize
    lastMemorySize <- currentSize
    if(diff < 0) browser()
    str <- HTML(paste("output object size (MB):", format(currentSize/ 1024 / 1024, format = "f", digits = 2), 
                 "\ntotal memory:",formatC(mem_used()/1024/1024, format = "f", digits = 2)  ))
    
  })  

  observeEvent(input$nplots, { rv$displayMemory <- rv$displayMemory + 1})
  observeEvent(input$ngroups, { rv$displayMemory <- rv$displayMemory + 1})
}

# Run the application 
shinyApp(ui = ui, server = server)

This topic was automatically closed 7 days after the last reply. New replies are no longer allowed.

If you have a query related to it or one of the replies, start a new topic and refer back with a link.