raycast navmesh click to move

Im trying to get indiviual soldiers to move around the map by first clicking on them (detected through raycast which seems to be working) and then clicking somewhere on the navmesh for them to move to. The seem to move for a single frame currently and only if i click on them which is not what i was aiming for. Can anyone help?
{
private NavMeshAgent agents;
private NavMeshAgent selectedSoldier;
private bool unitSelected = false;
private Vector3 targetPosition;
private Vector3 lookAtTarget;
private Quaternion playerRot;
private NavMeshHit navHitPosition;
private float agentSpeed = 300f;
private bool somethingSelected = false;

// Start is called before the first frame update
void Start()
{
    agents = GetComponent<NavMeshAgent[]>();
}

// Update is called once per frame
void Update()
{
    if (Input.GetMouseButtonDown(0))
    {
        SelectUnit();
        SetTargetPosition(selectedSoldier);
    }
}

private void SelectUnit()
{
    Ray ray = Camera.main.ScreenPointToRay(Input.mousePosition);

    RaycastHit hitInfo;

    if (Physics.Raycast(ray, out hitInfo, 2000.0f))
    {
        if (hitInfo.transform != null)
        {
            Debug.Log(hitInfo.transform.gameObject);
            if (selectedSoldier = hitInfo.transform.root.GetComponent<NavMeshAgent>())
            {
                SetTargetPosition(selectedSoldier);
            }
        }
    }
}
private void SetTargetPosition(NavMeshAgent selectedSoldier)
{
    if (Input.GetMouseButtonDown(0))
    {
        Ray ray = Camera.main.ScreenPointToRay(Input.mousePosition);
        RaycastHit hit;

        if (Physics.Raycast(ray, out hit, 1000))
        {
            targetPosition = hit.point;
            lookAtTarget = new Vector3(targetPosition.x - transform.position.x, transform.position.y,
            targetPosition.z - transform.position.z);
            playerRot = Quaternion.LookRotation(lookAtTarget);
            selectedSoldier.SetDestination(hit.point);
            selectedSoldier.speed = agentSpeed * Time.deltaTime;
        }
    }
}

}

So at the moment, i believe whats happening is, i click on the agent via the box collider, and the raycast goes through to hit the floor and moves the agent just marginally. What i need is the delay so that i can click twice. Once to select and once to move to a new position