I sometimes put them along the bottom or top of the plot area. In addition, you might consider using the y-values as the markers, which would be less cluttered than putting them next to the point markers. Also, I noticed that you used stat_summary to generate the text values, but you can use geom_text directly if you slice down to one value per year.
Below are a couple of variations using toy_data.
toy_data %>%
ggplot(aes(x = year, y = delay , color = stats)) +
geom_line(size=0.3, alpha=0.5) +
geom_text(aes(label=round(delay)), show.legend=FALSE, size=2.8) +
labs(
subtitle = "Yearly Stat",
color = "Statistics",
y = "Delay",
x = "Year"
) +
scale_x_continuous(breaks = x$breaks, labels=str_wrap(x$labs,5)) +
theme_bw() +
theme(text = element_text(size = 9)) +
guides(colour=guide_legend(override.aes=list(size=2, alpha=1)))

toy_data %>%
ggplot(aes(x = year, y = delay , color = stats)) +
geom_hline(yintercept=0, colour="grey20", size=0.3) +
geom_line(size=0.3, alpha=0.5) +
geom_text(aes(label=round(delay)), show.legend=FALSE, size=2.8) +
geom_text(data= . %>% group_by(year, stats) %>% slice(1),
aes(label=number_obs, y=-1.5), colour="grey40", size=2.3) +
labs(
subtitle = "Yearly Stat",
color = "Statistics",
y = "Delay",
x = "Year"
) +
scale_x_continuous(breaks = x$breaks) +
scale_y_continuous(limits=c(-5,NA), expand=expansion(c(0,0.05))) +
theme_bw() +
theme(text = element_text(size = 9)) +
guides(colour=guide_legend(override.aes=list(size=2, alpha=1)))
