If else in ggplot + switching values between character and numeric for geom_vline

I need to integrate if else block in ggplot, where depending on value for variable_to_switch it will:

• if value is numeric -> plot line
• value is text -> do NOT plot line

I provide my code in the most complete version to take in account possible conflicts with provided solution.

#  Creating Data ----

obs_number <- 500
mean <- 2
median <- 3
dens_mode <- 2
variable_to_switch <- "Text"
dlt_standard_mean <- 2
cut_off <- 12
max_dlt <- 15

metrics_test <- data.frame(xintercept = c(obs_number,cut_off,max_dlt, 0, mean, median, dens_mode,variable_to_switch, dlt_standard_mean ),
                      label = c("Label 1",
                                "Label 2",
                                "Label 3",
                                "", 
                                "Label 4",
                                "Label 5",
                                "Label 6",
                                "Label 7",
                                "Label 8"))


mean_by_batch_route <- c(rep(1,81), rep(2,252), rep(3,170), rep(4,41))
Label <- c(rep('Label',544))

data_to_plot_test <- data.frame(cbind(Label,mean_by_batch_route))
data_to_plot_test$mean_by_batch_route <- as.numeric(data_to_plot_test$mean_by_batch_route)

# Plotting ---

x_data <- 4

histo_density_plot <- ggplot(data_to_plot_test, aes(x = mean_by_batch_route)) +
  
  geom_histogram(aes(y = ..density..), binwidth = 1, colour= "black", fill = "white") +
  geom_density(adjust = 1.75, aes(y=..density..), fill="blue", alpha = .25)

if(variable_to_switch == "Text"){
  
  histo_density_plot <- histo_density_plot +
    
    geom_vline(aes(xintercept = xintercept, color = label),
               data = metrics_test[c(6,7,9), ], linetype = "dashed", size = 0.5, alpha = 1,
               key_glyph = draw_key_text) +
    
    
    
    scale_color_manual(
      name= "Values",
      limits = metrics_test$label,
      values = c("navyblue","darkcyan", "cyan4", "white", "black", "chartreuse4", "purple","darkgoldenrod3", "red"),
      guide = guide_legend(override.aes = list(label = metrics_test$xintercept, 
                                               size = 3)))
  
} else {
  
  histo_density_plot <- histo_density_plot +
    geom_vline(aes(xintercept = xintercept, color = label),
               data = metrics_test[6:nrow(metrics_test), ], linetype = "dashed", size = 0.5, alpha = 1,
               key_glyph = draw_key_text) +
    
    
    
    scale_color_manual(
      name= "Values",
      limits = metrics_test$label,
      values = c("navyblue","darkcyan", "cyan4", "white", "black", "chartreuse4", "purple","darkgoldenrod3", "red"),
      guide = guide_legend(override.aes = list(label = metrics_test$xintercept, 
                                               size = 3)))
  
  
}

histo_density_plot <- histo_density_plot +
  
  theme(plot.title = element_text(hjust = 0.5, size = 8), 
        axis.title.x = element_text(colour = "darkblue", size=8),
        axis.text.x = element_text(face="plain", color="black", 
                                   size=8, angle=0),
        axis.title.y = element_text(colour = "darkblue", size=8),
        axis.text.y = element_text(face="plain", color="black", 
                                   size=8, angle=0),
        legend.position = c(.80, .70),
        legend.title=element_text(size=8),
        legend.text = element_text(size = 8)
  )+
  
  
  labs(title = relevant_title, y = "Distribution fors DLT values, frequency", x = "DLT for the route: average DLT per batch, days")+
  
  scale_x_continuous(breaks = seq(0, x_data,
                                  ifelse(x_data <= 20, 1,
                                         ifelse((x_data > 20 & x_data < 50), 5, 10))))

histo_density_plot

If variable_to_switch == "Text" I will have this error: Error: Discrete value supplied to continuous scale.

For variable_to_switch equal to any numeric value plot works.

I want to switch between character and numeric value for variable_to_switch.
Desired output for case with variable_to_switch == "Text" :