I have only used it with dates, but I imagine that you could put it in to a function where you strip out the date then use it to calculate the interval.
Maybe something like this:
library(lubridate)
library(glue)
df <- tibble(datetime = c(ymd_hms("2020-02-10 09:00:00"), ymd_hms("2020-02-11 13:00:00")))
# A tibble: 2 x 1
datetime
<dttm>
1 2020-02-10 09:00:00
2 2020-02-11 13:00:00
df %>%
mutate(date = date(datetime)) %>%
mutate(int2 = interval(ymd_hms(glue("{date} 08:00:00")), ymd_hms(glue("{date} 12:00:00")))) %>%
mutate(within_int = if_else(datetime %within% int2, "yes", "no"))
# A tibble: 2 x 4
datetime date int2 within_int
<dttm> <date> <Interval> <chr>
1 2020-02-10 09:00:00 2020-02-10 2020-02-10 08:00:00 UTC--2020-02-10 12:00:00 UTC yes
2 2020-02-11 13:00:00 2020-02-11 2020-02-11 08:00:00 UTC--2020-02-11 12:00:00 UTC no