I do not have Excel with which to test this but it works when I open the xlsx file in OpenOffice. Cells B1 and B3 have a light red background. The code is largely from the openxlsx package documentation.
library(openxlsx)
wb <- createWorkbook()
addWorksheet(wb, "MySheet")
negStyle <- createStyle(fontColour = "#9C0006", bgFill = "#FFC7CE")
writeData(wb, "MySheet", c("ABCDE", "ABCDEFGHIJK", "AB"), startCol=2)
conditionalFormatting(wb, "MySheet", cols=2, rows=1:3, rule="LEN(B1)<10", style = negStyle)
saveWorkbook(wb, "ExcelOut.xlsx", overwrite = TRUE)