Why is ifelse returning NA as false?

For some reason, my ifelse statement is returning NAs as if they are false and not as NAs. Any idea why might be happening?

The column in question has numbers from 1 to 10.

library(dplyr)

data <- read.csv('210901_CLEANN_Risks_Research.csv')

data <- data %>% mutate_if(is.character, as.factor)

data[data==""]<-NaN
data[data=="Refused to answer"]<-NaN

table(data$safety)
unique(data$safety)

a <- c(1,2,3,4,5)

data$safety <- ifelse(data$safety %in% a , "Yes", "No")

unique(data$safety)
table(data$safety)

The output is the following:

data <- read.csv('210901_CLEANN_Risks_Research.csv')

data <- data %>% mutate_if(is.character, as.factor)

data[data==""]<-NA
data[data=="Refused to answer"]<-NA 
table(data$safety)

>    1    2    3    4    5    6    7    8    9   10 
> 2936 1112  836  548  479  261  165   91   51   12 
unique(data$safety)
> [1]  1  2  3  7  5  6  4  8  9 10 NA
  
a <- c(1,2,3,4,5)
data$safety <- ifelse(data$safety %in% a , "Yes", "No")
 
unique(data$safety)
> [1] "Yes" "No" 
table(data$safety)
>  No  Yes 
> 583 5911 

Any clue why this might be happening?