If/else statement not executing correctly with event listener inside; Javascript

The goal here is to have a score keeper. Each time you click a button pertaining to a certain player they will have their score increase by one. Once one of the players reach the designated winning score you can not add anymore to the score. My question is why does this code not work, the logic makes sense in my head and I want to understand why it will not give me the alert and why you can keep adding to each score. I have figured out a way to make it work but still believe this should be able to execute correctly. Thank you for helping me out!

var p1Button = document.querySelector("#p1");
var p2Button = document.getElementById("p2");
var p1Display = document.getElementById("p1Display");
var p2Display = document.getElementById("p2Display");
var p1Score = 0;
var p2Score = 0;
var winningScore = 5;


if(p1Score !== winningScore && p2Score !== winningScore){
p1Button.addEventListener("click", function(){
  p1Score++;
  p1Display.textContent = p1Score;
  return p1Score;
});
p2Button.addEventListener("click", function(){
  p2Score++;
  p2Display.textContent = p2Score;
  return p2Score;
});
}
else {
  alert("game over");
}