In this vignette we compare computational requirements (time and memory) of common operations using data.table and tidyverse functions.

Setup

library(data.table)
hostname <- system("hostname",intern=TRUE)
(max.threads <- as.integer(Sys.getenv("SLURM_JOB_CPUS_PER_NODE", "1")))
#> [1] 4
threads.vec <- unique(as.integer(c(1, ceiling(max.threads/2), max.threads)))
seconds.limit <- 1
cache.list <- list()
cache <- function(symbol, code){
  cache.symb <- substitute(symbol)
  suffix <- if(grepl("devel",R.version.string)){
    "devel"
  }else{
    gsub("R version | .*", "", R.version.string)
  }
  cache.dir <- paste0("~/R/atime-cache-",suffix)
  cache.rds <- file.path(cache.dir, paste0(cache.symb, ".RDS"))
  if(file.exists(cache.rds)){
    value <- readRDS(cache.rds)
  }else{
    to.eval <- substitute(code)
    value <- eval(to.eval)
    value$hostname <- hostname
    if(dir.exists(cache.dir))saveRDS(value, cache.rds)
  }
  cache.list[[paste(cache.symb)]] <<- value
  assign(paste(cache.symb), value, parent.frame())
}
aplot <- function(atime.list, my.title, xmax, max.seconds, xlab, color.vec=NULL){
  best.list <- atime::references_best(atime.list)
  blank.dt <- data.table(x=best.list$meas$N[1], y=max.seconds, unit="seconds")
  if(require(ggplot2)){
    hline.df <- with(atime.list, data.frame(seconds.limit, unit="seconds"))
    gg <- ggplot()+
      ggtitle(tit <<- paste(my.title,"on",hostname))+
      theme_bw()+
      geom_blank(aes(
        x, y),
        data=blank.dt)+
      facet_grid(unit ~ ., scales="free")+
      geom_hline(aes(
        yintercept=seconds.limit),
        color="grey",
        data=hline.df)+
      geom_line(aes(
        N, empirical, color=expr.name),
        data=best.list$meas)+
      geom_ribbon(aes(
        N, ymin=q25, ymax=q75, fill=expr.name),
        data=best.list$meas[unit=="seconds"],
        alpha=0.5)+
      scale_x_log10(xlab)+
      scale_y_log10("median line, quartiles band")
    if(!is.null(color.vec)){
      gg <- gg+
        scale_color_manual(values=color.vec)+
        scale_fill_manual(values=color.vec)
    }
    if(require(directlabels)){
      gg+
        directlabels::geom_dl(aes(
          N, empirical, color=expr.name, label=expr.name),
          method="right.polygons",
          data=best.list$meas)+
        theme(legend.position="none")+
        coord_cartesian(xlim=c(NA,xmax))
    }else{
      gg
    }
  }
}

Writing CSV

First we define some code which will be used in all of the writing benchmarks,

atime_write <- function(make.input.fun, fmt){
  grid.args <- list(
    list(THREADS=threads.vec),
    "data.table::fwrite"=quote({
      data.table::setDTthreads(THREADS)
      data.table::fwrite(input, name.list[["fwrite"]], showProgress = FALSE)
    }))
  small.input <- make.input.fun(2,2)
  if(requireNamespace("arrow") && is.data.frame(small.input)){
    grid.args[["write_csv_arrow"]] <- quote({
      arrow::set_cpu_count(THREADS)
      arrow::write_csv_arrow(input, name.list[["write_csv_arrow"]])
    })
  }
  if(requireNamespace("readr") && is.data.frame(small.input)){
    ##readr can't handle matrix input.
    grid.args[["readr::write_csv"]] <- quote({
      readr::write_csv(
        input, name.list[["write_csv"]], progress = FALSE, num_threads = THREADS)
    })
  }
  expr.list <- do.call(atime::atime_grid, grid.args)
  atime::atime(
    N=as.integer(10^seq(0, 6, by=0.5)),
    setup={
      input <- make.input.fun(N)
      name.list <- list()
      for(fun in c("fwrite", "write_csv", "write_csv_arrow", "write.csv")){
        name.list[[fun]] <- file.path(
          tempdir(), sprintf(fmt, fun, N))
      }
    },
    seconds.limit = seconds.limit,
    expr.list=expr.list,
    "utils::write.csv"=utils::write.csv(input, name.list[["write.csv"]]))
}
one.thread <- function(DT)DT[grepl("utils|scan|THREADS=1$", expr.name)]
facetPlot <- function(atime.list, fun.name.vec=c("N^2","N"), N.min=1e2){
  best.list <- atime::references_best(atime.list)
  meas.dt <- one.thread(best.list$meas)
  ref.dt <- one.thread(best.list$ref)[
    fun.name %in% fun.name.vec & N >= N.min]
  if(require(ggplot2)){
    hline.df <- with(write.real.vary.rows, data.frame(
      seconds.limit, unit="seconds"))
    gg <- ggplot()+
      ggtitle(paste0(tit,", asymptotic complexity"))+
      theme_bw()+
      facet_grid(unit ~ expr.name, scales="free")+
      geom_hline(aes(
        yintercept=seconds.limit),
        color="grey",
        data=hline.df)+
      geom_line(aes(
        N, reference, group=paste(expr.name, fun.name)),
        linewidth=2,
        data=ref.dt)+
      geom_line(aes(
        N, empirical, color=expr.name),
        linewidth=1,
        data=meas.dt)+
      geom_ribbon(aes(
        N, ymin=q25, ymax=q75, fill=expr.name),
        data=meas.dt[unit=="seconds"],
        alpha=0.5)+
      scale_x_log10("N = Number of columns")+
      scale_y_log10("median line, quartiles band")+
      scale_color_manual(values=write.colors)+
      scale_fill_manual(values=write.colors)
    if(require(directlabels)){
      gg+
        directlabels::geom_dl(aes(
          N, reference,
          label.group=paste(expr.name, fun.name),
          label=fun.name),
          method="left.polygons",
          data=ref.dt)+
        theme(legend.position="none")
    }else{
      gg
    }
  }
}
if(FALSE){
  RColorBrewer::display.brewer.all()
  dput(RColorBrewer::brewer.pal(Inf, "Set2"))
  dput(RColorBrewer::brewer.pal(Inf, "RdGy"))
}
NAME <- function(prefix, ...){
  structure(
    c(...)[1:length(threads.vec)],
    names=sprintf("%sTHREADS=%d", prefix, threads.vec))
}
write.colors <- c(
  NAME("readr::write_csv ", "#9970AB","#762A83", "#40004B"), #purple
  "#5AAE61", "#1B7837", "#00441B",#green
  NAME("data.table::fwrite ", "#D6604D", "#B2182B", "#67001F"),#reds
  "#878787", "#4D4D4D", "#1A1A1A",#greys
  NAME("write_csv_arrow ", "#BF812D", "#8C510A", "#543005"),#browns
  "#35978F", "#01665E", "#003C30",#teal polars
  "utils::write.csv"="deepskyblue")
(write.colors <- write.colors[names(write.colors)!=""])
#>   readr::write_csv THREADS=1   readr::write_csv THREADS=2 
#>                    "#9970AB"                    "#762A83" 
#>   readr::write_csv THREADS=4 data.table::fwrite THREADS=1 
#>                    "#40004B"                    "#D6604D" 
#> data.table::fwrite THREADS=2 data.table::fwrite THREADS=4 
#>                    "#B2182B"                    "#67001F" 
#>    write_csv_arrow THREADS=1    write_csv_arrow THREADS=2 
#>                    "#BF812D"                    "#8C510A" 
#>    write_csv_arrow THREADS=4             utils::write.csv 
#>                    "#543005"                "deepskyblue"

Writing CSV with real numbers

The code below is for real numbers with a constant number of columns, and a variable number of rows.

random_real <- function(N.rows, N.cols){
  set.seed(1)
  matrix(rnorm(N.rows*N.cols), N.rows, N.cols)
}
cache(write.real.vary.rows, atime_write(
  function(N.rows, N.cols=10)random_real(N.rows, N.cols),
  "10_real_cols_%s_%d.csv"))
#> Loading required namespace: arrow
#> Loading required namespace: readr
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
aplot(write.real.vary.rows, "Write CSV with 10 random normal real columns", 1e9, 1e1, "Number of rows", write.colors)
#> Loading required package: ggplot2
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

The plot above shows that all methods are the same, except utils::write.csv memory is increasing with data size, and others are contant.

facetPlot(write.real.vary.rows)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

The plot above shows that the memory usage of utils::write.csv is linear.

The code below writes real numbers with a constant number of rows, and a variable number of columns.

cache(write.real.vary.cols, atime_write(
  function(N.cols, N.rows=10)random_real(N.rows, N.cols),
  "10_real_rows_%s_%d.csv"))
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
aplot(write.real.vary.cols, "Write CSV with 10 random normal real rows", 1e9, 1e1, "Number of columns", write.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

The plot above shows that data.table::fread uses asymptotically less time and memory than the others.

Write CSV from character matrix

The code below is for a character data matrix with a constant number of columns, and a variable number of rows.

chr_mat <- function(N.rows, N.cols){
  data.vec <- paste0("'quoted", c(" ", "_"), "data'")
  matrix(data.vec, N.rows, N.cols)
}
cache(write.chrmat.vary.rows, atime_write(
  function(N.rows,N.cols=10)chr_mat(N.rows, N.cols),
  "10_chrmat_cols_%s_%d.csv"))
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
aplot(write.chrmat.vary.rows, "Write CSV from matrix with 10 character columns", 1e9, 1e1, "Number of rows", write.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO

facetPlot(write.chrmat.vary.rows)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO

cache(write.chrmat.vary.cols, atime_write(
  function(N.cols, N.rows=10)chr_mat(N.rows, N.cols),
  "10_chrmat_rows_%s_%d.csv"))
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> x being coerced from class: matrix to data.table
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
aplot(write.chrmat.vary.cols, "Write CSV from matrix with 10 character rows", 1e9, 1e1, "Number of columns", write.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO

facetPlot(write.chrmat.vary.cols)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO

Write CSV from character data.table

The code below is for a character data.table with a constant number of columns, and a variable number of rows.

chr_dt <- function(N.rows, N.cols){
  data.table(chr_mat(N.rows, N.cols))
}
cache(write.chr.vary.rows, atime_write(
  function(N.rows,N.cols=10)chr_dt(N.rows, N.cols),
  "10_chr_cols_%s_%d.csv"))
aplot(write.chr.vary.rows, "Write CSV from data.table with 10 character columns", 1e9, 1e1, "Number of rows", write.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

The figure above is useful for comparing different functions, and shows that all have the same asymptotic time complexity class. However, we observe a difference in memory usage: linear for write.csv and constant for others. Below, we draw reference lines, so we can see the complexity class.

facetPlot(write.chr.vary.rows)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

The figure above shows that all functions are linear time, and write.csv is linear memory. The code below is for a character data.frame with a constant number of rows, and a variable number of columns.

cache(write.chr.vary.cols, atime_write(
  function(N.cols, N.rows=10)chr_dt(N.rows, N.cols),
  "10_chr_rows_%s_%d.csv"))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.

#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.

#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
aplot(write.chr.vary.cols, "Write CSV from data.table with 10 character rows", 1e9, 1e1, "Number of columns", write.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

The figure above shows that data.table::fwrite clearly has a smaller slope (linear complexity in number of columns) than the other methods (quadratic complexity), as shown in the plot below, which includes best reference lines above and below each empirical measurement asymptote.

facetPlot(write.chr.vary.cols)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

The comparisons above show significant advantages for data.table for writing CSV data with a large number of columns: asymptotically less time and memory (linear rather than quadratic in number of columns).

Write CSV from data.table with factor columns

The code below is for factor data with a constant number of columns, and a variable number of rows.

fac_dt <- function(N.rows, N.cols){
  data.vec <- factor(paste0("'quoted", c(" ", "_"), "data'"))
  as.data.table(lapply(1:N.cols, function(col.i)rep(data.vec,l=N.rows)))
}
cache(write.fac.vary.rows, atime_write(
  function(N.rows,N.cols=10)fac_dt(N.rows, N.cols),
  "10_fac_cols_%s_%d.csv"))
aplot(write.fac.vary.rows, "Write CSV from data.table with 10 factor columns", 1e9, 1e1, "Number of rows", write.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO. Below, we draw reference lines, so we can see the complexity class.

facetPlot(write.fac.vary.rows)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO The code below is for factor data with a constant number of rows, and a variable number of columns.

cache(write.fac.vary.cols, atime_write(
  function(N.cols, N.rows=10)fac_dt(N.rows, N.cols),
  "10_fac_rows_%s_%d.csv"))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.

#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
aplot(write.fac.vary.cols, "Write CSV with 10 factor rows", 1e9, 1e1, "Number of columns", write.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO. In the plot below, we include best reference lines above and below each empirical measurement asymptote.

facetPlot(write.fac.vary.cols)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO

Write CSV from data.table with POSIXct columns

The code below is for factor data with a constant number of columns, and a variable number of rows.

POSIXct_dt <- function(N.rows, N.cols){
  as.data.table(lapply(1:N.cols, function(col.i)rep(Sys.time(),l=N.rows)))
}
cache(write.POSIXct.vary.rows, atime_write(
  function(N.rows,N.cols=10)POSIXct_dt(N.rows, N.cols),
  "10_fac_cols_%s_%d.csv"))
aplot(write.POSIXct.vary.rows, "Write CSV from data.table with 10 POSIXct columns", 1e9, 1e1, "Number of rows", write.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO. Below, we draw reference lines, so we can see the complexity class.

facetPlot(write.POSIXct.vary.rows)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO The code below is for factor data with a constant number of rows, and a variable number of columns.

cache(write.POSIXct.vary.cols, atime_write(
  function(N.cols, N.rows=10)POSIXct_dt(N.rows, N.cols),
  "10_POSIXct_rows_%s_%d.csv"))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
aplot(write.POSIXct.vary.cols, "Write CSV with 10 POSIXct rows", 1e9, 1e1, "Number of columns", write.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO. In the plot below, we include best reference lines above and below each empirical measurement asymptote.

facetPlot(write.POSIXct.vary.cols)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

TODO

Reading CSV

First we define a function which we will use for all of the read benchmarks,

read.expr.list <- c(
  if(requireNamespace("readr"))atime::atime_grid(
    list(LAZY=c(TRUE, FALSE), THREADS=threads.vec),
    "readr::read_csv"={
      readr::read_csv(
        f.csv, num_threads = THREADS, lazy = LAZY,
        show_col_types=FALSE, progress=FALSE)
    }),
  atime::atime_grid(
    list(THREADS=threads.vec),
    "data.table::fread"={
      data.table::setDTthreads(THREADS)
      data.table::fread(f.csv, showProgress=FALSE)
    }),
  if(FALSE && requireNamespace("polars"))atime::atime_grid(
    ##TODO wait until we know how to set max number of threads.
    "polars::pl$read_csv"={
    },
    "polars::pl$read_csv_lazy"={
    }),
  if(requireNamespace("arrow"))atime::atime_grid(
    list(THREADS=threads.vec),
    "read_csv_arrow"={
      arrow::set_cpu_count(THREADS)#https://github.com/apache/arrow/issues/30205#issuecomment-1378060874
      arrow::read_csv_arrow(f.csv)
    }),
  atime::atime_grid(
    "utils::read.csv"={
      utils::read.csv(f.csv)
    }))
atime_read <- function(glob, compute=FALSE, colClasses, N.col){  
  fmt <- sub("[*]", "%d", glob)
  csv.dt <- nc::capture_first_vec(
    Sys.glob(file.path(tempdir(), glob)),
    N="[0-9]+", as.integer,
    ".csv")[order(N)]
  read.more.list <- c(
    read.expr.list,
    if(!missing(colClasses))list(
      "read.csv(colClasses)"=substitute(
        utils::read.csv(f.csv, colClasses = CLASS),
        list(CLASS=colClasses)),
      "list2DF(scan)"=substitute({
        what <- `names<-`(
          rep(list(FUN()), NCOL),
          paste0("V",seq_len(NCOL))
        )
        list2DF(scan(f.csv, what=what, sep=",", skip=1, multi.line=FALSE))
      }, list(
        FUN=as.symbol(colClasses),
        NCOL=if(missing(N.col))quote(N) else N.col)
      )))
  expr.list <- if(compute){
    read.compute.expr.list <- list()
    for(expr.name in names(read.more.list)){
      lang.list <- as.list(read.more.list[[expr.name]])
      LAST <- length(lang.list)
      lang.list[[LAST]] <- as.call(c(
        quote(`<-`),
        quote(DF),
        lang.list[[LAST]]))
      read.compute.expr.list[[expr.name]] <- as.call(c(
        lang.list,
        quote(apply(DF, 1, paste, collapse=","))))
    }
    read.compute.expr.list
  }else{
    read.more.list
  }
  atime::atime(
    N=csv.dt$N,
    setup={
      f.csv <- file.path(tempdir(), sprintf(fmt, N))
    },
    seconds.limit = seconds.limit,
    expr.list=expr.list)
}
PRGn <- c(
  NAME("readr::read_csv LAZY=FALSE,", "#9970AB","#762A83", "#40004B"), #purple
  "#5AAE61", "#1B7837", "#00441B",#green
  NAME("data.table::fread ", "#D6604D", "#B2182B", "#67001F"),#reds
  NAME("readr::read_csv LAZY=TRUE,", "#878787", "#4D4D4D", "#1A1A1A"),#greys
  NAME("read_csv_arrow ", "#BF812D", "#8C510A", "#543005"),#browns
  "#35978F", "#01665E", "#003C30",#teal polars
  "utils::read.csv"="#00FFFF",#"deepskyblue",
  "read.csv(colClasses)"="#00CCCC",
  "list2DF(scan)"="#009999")
(read.colors <- PRGn[names(PRGn)!=""&!is.na(names(PRGn))])
#> readr::read_csv LAZY=FALSE,THREADS=1 readr::read_csv LAZY=FALSE,THREADS=2 
#>                            "#9970AB"                            "#762A83" 
#> readr::read_csv LAZY=FALSE,THREADS=4          data.table::fread THREADS=1 
#>                            "#40004B"                            "#D6604D" 
#>          data.table::fread THREADS=2          data.table::fread THREADS=4 
#>                            "#B2182B"                            "#67001F" 
#>  readr::read_csv LAZY=TRUE,THREADS=1  readr::read_csv LAZY=TRUE,THREADS=2 
#>                            "#878787"                            "#4D4D4D" 
#>  readr::read_csv LAZY=TRUE,THREADS=4             read_csv_arrow THREADS=1 
#>                            "#1A1A1A"                            "#BF812D" 
#>             read_csv_arrow THREADS=2             read_csv_arrow THREADS=4 
#>                            "#8C510A"                            "#543005" 
#>                      utils::read.csv                 read.csv(colClasses) 
#>                            "#00FFFF"                            "#00CCCC" 
#>                        list2DF(scan) 
#>                            "#009999"

Below we read real numbers with a constant number of columns, and a variable number of rows.

cache(read.real.vary.rows, atime_read("10_real_cols_fwrite_*.csv", compute=FALSE, colClasses="numeric",N.col=10))
aplot(read.real.vary.rows, "Read CSV with 10 real columns", 1e9, 1e1, "Number of rows", read.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'

It can be seen in the plot above that the green results, read_csv with LAZY=TRUE are fastest, which is normal because lazy reading does not actually read the data values into memory. A more fair comparison is below, which computes a text string for every row after reading the CSV,

cache(compute.real.vary.rows, atime_read("10_real_cols_fwrite_*.csv", compute=TRUE))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
aplot(compute.real.vary.rows, "Read CSV with 10 real columns, then collapse each row", 1e9, 1e1, "Number of rows", read.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'

Below we read real numbers with a constant number of rows, and a variable number of columns.

cache(read.real.vary.cols, atime_read("10_real_rows_fwrite_*.csv", compute=FALSE, colClasses="numeric"))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.

#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.

#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
aplot(read.real.vary.cols, "Read CSV with 10 real rows", 1e8, 1e1, "Number of columns", read.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

The plot above shows that all functions have the same asymptotic memory usage, but read.csv has a larger asymptotic time complexity class than the others. The plot below shows that the time complexity class of read.csv is in fact quadratic, whereas the others are linear.

facetPlot(read.real.vary.cols,c("N^2","N log N", "N"))
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

Below we read character data with a constant number of columns, and a variable number of rows.

cache(read.chr.vary.rows, atime_read("10_chr_cols_fwrite_*.csv", compute=FALSE, colClasses="character",N.col=10))
aplot(read.chr.vary.rows, "Read CSV with 10 character columns", 1e9, 1e1, "Number of rows", read.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'

As with the previous result for real data, the green results above, read_csv with LAZY=TRUE are fastest, which is normal because lazy reading does not actually read the data values into memory. A more fair comparison is below, which computes a text string for every row after reading the CSV,

cache(compute.chr.vary.rows, atime_read("10_chr_cols_fwrite_*.csv", compute=TRUE))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.

#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
aplot(compute.chr.vary.rows, "Read CSV with 10 character columns, then collapse each row", 1e9, 1e1, "Number of rows", read.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'

Below we read character data with a constant number of rows, and a variable number of columns.

cache(read.chr.vary.cols, atime_read("10_chr_rows_fwrite_*.csv", compute=FALSE, colClasses="character"))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
aplot(read.chr.vary.cols, "Read CSV with 10 character rows", 1e8, 1e1, "Number of columns", read.colors)
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

From the comparisons above, it can be seen that for a small number of columns, and a large number of rows, all the methods are about the same (constant factor differences, using more than one thread also results in small constant factor speedups). However for a small number of rows and a large number of columns, data.table::fread is clearly the most efficient:

• data.table::fread uses constant factors less time and memory than readr::read_csv.
• utils::read.csv uses asymptotically more time (super-linear in the number of columns).

facetPlot(read.chr.vary.cols,c("N^2","N log N", "N"))
#> Loading required package: directlabels
#> Warning in library(package, lib.loc = lib.loc, character.only = TRUE,
#> logical.return = TRUE, : there is no package called 'directlabels'
#> Warning: Transformation introduced infinite values in continuous y-axis

Total time

Sys.time() - time.begin
#> Time difference of 1.338563 hours
seconds.vec <- sapply(cache.list, function(L){
  do.call(sum, L$meas$time)
})
sum(seconds.vec)+time.begin-time.begin
#> Time difference of 1.118916 hours

session info

sessionInfo()
#> R version 4.3.0 (2023-04-21)
#> Platform: x86_64-pc-linux-gnu (64-bit)
#> Running under: Red Hat Enterprise Linux 8.7 (Ootpa)
#> 
#> Matrix products: default
#> BLAS:   /projects/genomic-ml/R/R-4.3.0/lib/libRblas.so 
#> LAPACK: /projects/genomic-ml/R/R-4.3.0/lib/libRlapack.so;  LAPACK version 3.11.0
#> 
#> locale:
#> [1] C
#> 
#> time zone: America/Phoenix
#> tzcode source: system (glibc)
#> 
#> attached base packages:
#> [1] stats     graphics  grDevices utils     datasets  methods   base     
#> 
#> other attached packages:
#> [1] ggplot2_3.4.2     data.table_1.14.8
#> 
#> loaded via a namespace (and not attached):
#>  [1] sass_0.4.5       utf8_1.2.3       generics_0.1.3   lattice_0.21-8  
#>  [5] hms_1.1.3        digest_0.6.31    magrittr_2.0.3   evaluate_0.20   
#>  [9] grid_4.3.0       fastmap_1.1.1    jsonlite_1.8.4   bench_1.1.3     
#> [13] purrr_1.0.1      fansi_1.0.4      scales_1.2.1     jquerylib_0.1.4 
#> [17] cli_3.6.1        rlang_1.1.0      crayon_1.5.2     atime_2023.4.27 
#> [21] bit64_4.0.5      munsell_0.5.0    withr_2.5.0      cachem_1.0.7    
#> [25] nc_2020.8.6      tools_4.3.0      tzdb_0.3.0       dplyr_1.1.2     
#> [29] colorspace_2.1-1 profmem_0.6.0    assertthat_0.2.1 vctrs_0.6.2     
#> [33] R6_2.5.1         lifecycle_1.0.3  bit_4.0.5        vroom_1.6.1     
#> [37] arrow_11.0.0.3   pkgconfig_2.0.3  pillar_1.9.0     bslib_0.4.2     
#> [41] gtable_0.3.3     glue_1.6.2       xfun_0.39        tibble_3.2.1    
#> [45] tidyselect_1.2.0 highr_0.10       knitr_1.42       farver_2.1.1    
#> [49] htmltools_0.5.5  rmarkdown_2.21   readr_2.1.4      compiler_4.3.0