I would do it like this.
library(dplyr, warn.conflicts = FALSE)
Df <- data.frame(order=c(0,0,1,0,0,0,0),order_prev=c(NA,0,0,1,0,0,0))
Df <- Df %>% mutate(trialtype=case_when(
order==1 & order_prev == 1 ~ 0,
order == 0 & order_prev == 0 ~ 0,
order == 1 & order_prev == 0 ~ 1,
order == 0 & order_prev == 1 ~ 1,
))
Df
#> order order_prev trialtype
#> 1 0 NA NA
#> 2 0 0 0
#> 3 1 0 1
#> 4 0 1 1
#> 5 0 0 0
#> 6 0 0 0
#> 7 0 0 0
Created on 2020-12-23 by the reprex package (v0.3.0)