JavaScript weird issue with else if statement

I'm making a game of Guess the Number, and I want to test if a variable guess is greater than a variable difficulty. difficulty has been taken from my HTML page, and it is not comparing correctly with guess.

//Initialize variables for player guess, guess counter and previous guesses
var guess = 0;
var guessCount = 0;
var previousGuesses = [];

function startGame() {
        
        //Calculate difficulty
        var difficulty = document.getElementById("difficulty").value;
        
        //Calculate secret number
        var secretNumber = Math.floor((Math.random() * difficulty) + 1);
        
        //Repeats while player has not guessed the secret number
        while (guess != secretNumber) {
                
                //Checks for Cancel button pressed
                guess = prompt("Enter your guess: ");
                if (guess == null) {
                        return;
                } 
                
                //Checks for empty string/no input
                else if (guess == "") {
                        alert("Please enter a number");
                }
                
                //Checks if previously guessed
                else if (previousGuesses.includes(guess)) {
                        alert("You have guessed this number before. Please try a different number.");
                }
                
                else if (guess < 1) {
                        alert("Please enter a number between 1-" + difficulty);
                }
                
                //Checks if guess is higher than secretNumber
                else if (guess > secretNumber) {
                        alert("Your guess is too high");
                        //Increments guess counter
                        guessCount++;
                        //Adds the previous guess to previousGuesses
                        previousGuesses.push(guess);
                } 
                
                //Checks if guess is lower than secretNumber
                else if (guess < secretNumber) {
                        alert("Your guess is too low");
                        //Increments guess counter
                        guessCount++;
                        //Adds the previous guess to previousGuesses
                        previousGuesses.push(guess);
                }
                
                //Checks for correct guess
                else if (guess == secretNumber) {
                        //Increments guess counter
                        guessCount++;
                        
                        //Checks for correct grammar - guesses or guess
                        if (guessCount > 1) {
                                alert("Congratulations, you guessed the correct number in " + guessCount + " guesses!");
                        }
                        else {
                                alert("Congratulations, you guessed the correct number in " + guessCount + " guess!");
                        }
                }
                
        }
        //Resets variables to play again
        guess = 0;
        guessCount = 0;
        previousGuesses = [];
}

<h1>Guess the Number</h1>

<button onclick="startGame()">Start the Game</button>

<h2>Difficulty</h2>

<select id="difficulty">
  <option value="10">Beginner</option>
  <option value="50">Intermediate</option>
  <option value="100">Hard</option>
</select>

This code works, but I want something to occur. When your guess is greater than difficulty, I want to print "Please enter a number between 1-" + difficulty. However, when I change this code:

else if (guess < 1) {
    alert("Please enter a number between 1-" + difficulty);
}

into this:

else if (guess < 1 || guess > difficulty) {...}

then what happens is that EVERY guess except 1, difficulty and anything more than difficulty is alerted by Please enter a number.

How should I fix this?