This is basically what i'm trying to achieve but in a cleaner way, making proper use of legends instead of annotate.
plot_deaths<-ggplot()+
geom_vline(xintercept = 0,linetype="dashed")+theme_minimal()+theme(text = element_text(size = 10),legend.text=element_text(size=10),legend.title=element_text(size=12,face="bold"),axis.title = element_text(size=10))+
labs(x="Number of days since first intervention",y="Number of daily COVID-19 cases",color="Country")+
geom_smooth(data=DatasetForDeaths,aes(x=DaysDifference,y=logvalueDeaths,group=ADM0NAME,color=ADM0NAME),se=FALSE)+
geom_point(data=DatasetEaseDates_Deaths,aes(x=DaysDifference,y=SmoothedValueDeaths,group=ADM0NAME,color=ADM0NAME),shape=21,fill="white",size=5,stroke=1,show.legend=FALSE)+
geom_point(data=DatasetEaseDates_Deaths,aes(x=DaysDifference,y=SmoothedValueDeaths,group=ADM0NAME,color=ADM0NAME),shape=4,stroke=1,show.legend=FALSE)+
scale_color_manual(values=MyPalette)+
scale_x_continuous(breaks=seq(-50,100,10))+
scale_y_continuous(labels=function(x) round(10^x))+
theme(legend.position = c(0.15, 0.85),
legend.background = element_rect(fill='white',color='white'),
panel.border = element_rect(colour = "black", fill=NA, size=1))+
geom_label(x=0,y=100,label="Test", size=4, color="#69b3a2")+
guides(color=guide_legend(ncol=2))+
annotate(geom="label",x=-35, y=2, label="X Ease of restrictions",fill="white",label.size = NA)+
ggsave(filename=paste0(CurrentFolder,"/DeathsAgainstFirstMeasure.jpg"),width = 11, height = 8)