This setTimeout function isn't making sense to me

This is a simple match card game I am trying to build. In the first if statement, I am checking to see if the urls in the array do not match. If they don't, I reshow the back of the card image. However, in the else section, I want to remove both cards from the board but only that last clicked card is removing, not both the first and last card. I want to remove the two that were compared if they have the same url. Thanks

const card = document.querySelector(".main-board");
var arr = [];

card.addEventListener("click", function(e){
    e.target.src = randomOrder(images);
    arr.push(e.target.src);




//This if statement is what's giving me trouble
    if(arr[0] !== arr[1]) {
        setTimeout(function(){
            e.target.src = "https://i.pinimg.com/originals/c1/59/b4/c159b4738dae9c9d8d6417228024de8d.jpg";
        }, 2000);
    } else {
        setTimeout(function(){
            e.target.style.display = "none";
        }, 2000);
    }
    if(arr.length > 1) {
        arr = [];
    }
});

function randomOrder(images) {
    return images[Math.floor(Math.random()*images.length)];
}

const images = [
    "https://images.unsplash.com/photo-1541233349642-6e425fe6190e?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&w=1000&q=80",
    "https://images.wallpaperscraft.com/image/butterflies_mushrooms_forest_130001_1280x1280.jpg",
    "https://image.shutterstock.com/image-photo/bright-spring-view-cameo-island-260nw-1048185397.jpg",
    "https://i2.wp.com/buzzonearth.com/wp-content/uploads/2018/12/pexels-photo-814499.jpeg?resize=500%2C333&ssl=1",
    "https://images.unsplash.com/photo-1542725577-5240e614e436?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=1000&q=80"
]