Thank you! That (almost) does it 
I'm hoping this is just as simple; how do I separate the groups when adding more than one group in filter? Note that I've added a few more groups in case_when.
library(tidyverse)
library(ggforce)
#>
#> Attaching package: 'ggforce'
#> The following objects are masked from 'package:ggplot2':
#>
#> stat_ellipse, StatEllipse
Df <- tibble::tribble(
~x, ~y,
4, 2,
1, 1,
3, 2,
1, 0,
1, 0,
3, 0,
5, 2,
5, 4,
3, 2,
3, 0,
5, 2,
4, 3,
1, 0,
4, 1,
1, 0,
1, 0,
4, 1,
3, 0,
5, 3,
1, 0,
1, 0,
1, 0,
1, 0,
4, 0,
1, 0,
1, 0,
5, 5,
1, 0,
5, 0,
3, 3,
1, 2,
4, 2,
1, 0,
5, 5,
4, 4,
5, 4,
3, 0,
1, 0,
1, 0,
4, 4,
1, 0,
1, 0,
5, 4,
1, 0,
1, 1,
5, 3,
5, 4,
5, 5,
1, 4,
5, 3,
1, 0,
1, 0,
5, 1,
1, 0,
1, 0,
4, 4,
1, 0,
1, 0,
1, 0,
5, 3,
5, 2,
1, 0,
1, 0,
1, 1,
5, 3,
1, 0,
5, 4,
1, 2,
3, 1,
1, 0,
1, 0,
1, 0,
5, 3,
1, 0,
1, 0,
5, 5,
1, 1,
1, 0,
5, 3,
1, 0,
5, 2,
1, 0,
4, 1,
5, 3,
3, 0,
3, 0,
1, 0,
5, 2,
1, 3,
5, 2,
5, 5,
5, 3,
3, 2,
1, 0,
4, 2,
5, 3,
5, 4,
3, 0,
1, 0,
5, 0
) %>%
mutate_if(is.numeric, ordered) %>%
mutate(cm = case_when(
x >= 3 & y >= 2 ~ "TP",
x == 1 & y <= 1 ~ "TN",
x >= 3 & y <= 1 ~ "FP",
x == 1 & y >= 2 ~ "FN"
))
ggplot(Df, aes(x, y)) +
geom_count(aes(color = y, size = stat(prop), group = y)) +
scale_size_area(max_size = 25) +
geom_mark_rect(aes(fill = cm, group = "fill", label = cm, filter = cm == "TP"))

ggplot(Df, aes(x, y)) +
geom_count(aes(color = y, size = stat(prop), group = y)) +
scale_size_area(max_size = 25) +
geom_mark_rect(aes(fill = cm, group = "fill", label = cm, filter = cm == c("TP", "FN")))

Created on 2019-03-04 by the reprex package (v0.2.1)