It's kind of ugly, but it works by having separate geom_functions for each subset of data by type, and passing named parameters a and b as arguments to the function f.
library("ggplot2")
library("dplyr")
dat <- data.frame(type = rep(LETTERS[1:2], each = 4),
a = rep(c(-0.32, -0.19), each = 4),
b = -0.02,
d = rep(seq(0,6,2), length = 8),
points = c(1.08, 0.39, 0.25, 0.06, 1.07, 0.55, 0.43, 0.17),
end = c(1.14, 0.45, 0.32, 0.07, 1.12, 0.58, 0.48, 0.20),
start = c(1.03, 0.32, 0.17, 0.04, 1.01, 0.52, 0.39, 0.15))
# Define function
f <- function(x, a, b) exp(a * x + b * x^2)
# Subset data for each geom_function
ggplot(dat, aes(x = d, y = points, color = type, group = type)) +
geom_point() +
geom_function(data = dat %>% filter(type == "A"), fun = f,
args = list(a = -0.32, b = -0.02)) +
geom_function(data = dat %>% filter(type == "B"), fun = f,
args = list(a = -0.19, b = -0.02)) +
geom_segment(aes(y = start, xend = d, yend = end)) +
scale_y_log10() +
scale_color_brewer(palette = 6, type = "qual", direction = -1) +
theme_bw()

Created on 2021-01-08 by the reprex package (v0.3.0)