No matter what the last if statement goes into effect when it shouldn't

I'm making a program that does math for a person and then asks if it is correct (this is based off of a class project we did that I wanted to add on to). I have 3 if statements that check if the user put yes or no and if they didn't asks them to answer again. The problem I am having is no matter what you put it will ask you to enter yes or no.

using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;

namespace
{
class Program
{
    static void Main(string[] args)
    {
        Console.WriteLine("Please enter a number");

        int num1 = Convert.ToInt32(Console.ReadLine());

        Console.WriteLine("You have entered {0}", num1);
        Console.WriteLine("Please enter another number");

        int num2 = Convert.ToInt32(Console.ReadLine());

        Console.WriteLine("You have entered {0}", num2);
        Console.WriteLine("Now we will add the numbers");

        int sum = num1 + num2;

        Console.WriteLine("{0} + {1} = {2}", num1, num2, sum);
        Console.WriteLine("Was this correct?");

        string answer = Console.ReadLine();

        if (answer == "yes")
        {
            Console.WriteLine("Thank you for using this service!");
        }                

        if (answer == "no")
        {
            Console.WriteLine("We are sorry for the mistake!");
        }                

        if (answer != "yes" || answer != "no")
        {
            Console.WriteLine("please enter yes or no");
            Console.ReadLine();
        }
    }
}

}