Changing a value in array in react state

The idea is that when a user clicks on a building(Marker), it shows information via an overlay modal and adds points to a variable called totalXP. I want to check whether the user has clicked on a marker already using my if statement. I thought I had it, but it never fires the else block, which means the clicked value is probably never changing to true or, changing back to false. I am unsure how to change the clicked value to true permanently? I think it has something to do with mutable objects and is a react thing??? console logging before the

//MY ARRAY AND STATE
  const [visibleBuilding, setVisibleBuilding] = useState(0);
  const buildings = [
    {
      name: "Building 1",
      description:
        " Information1 ",
      xp: 25,
      clicked: false,
    },
    {
      name: "Building 2",
      description:
        "Information 1",
      xp: 30,
      clicked: false,
    },
  ];

//MY FUNCTION TO ADD POINTS AND CHANGE CLICKED TO TRUE

  const [totalXP, setTotalXP] = useState(0);

  const handleXP = function (points) {
    if (buildings[visibleBuilding].clicked === false) {
      // console.log(buildings[visibleBuilding].clicked);<---shows false??
      let totalPoints = totalXP;
      totalPoints += points;
      setTotalXP(totalPoints);
      buildings[visibleBuilding].clicked = true;<----Need this to work?
      // console.log(buildings[visibleBuilding].clicked);<---shows true??

    } else {
      alert("You already got your xp homie");
    }
  };

//THE RENDERING STUFF
return(
     <OverlayBox
          building={buildings[visibleBuilding]}
          onPress={() => toggleOverlay()}
          key={visibleBuilding}
          title={buildings[visibleBuilding].name}
          info={buildings[visibleBuilding].description}
          xp={buildings[visibleBuilding].xp}
        />
     <Text>Total XP: {totalXP} </Text>
     <Marker
          onPress={() => {
            toggleOverlay();
            setVisibleBuilding(0);
            handleXP(buildings[visibleBuilding].xp);
          }}
          coordinate=
          pinColor="maroon"
          title={"Rutledge Building"}
          description={"25 XP"}
        />
        

)