RTS Drag Selection trouble

Right now I have the correct UI box being drawn when I drag but the actual selection part is not working properly and units that are clearly in the area aren’t selected and sometimes units completely outside it is being selected.


        if (Input.GetMouseButton(0))
        {
            UpdateSelectionBox(Input.mousePosition);
        }

        if (Input.GetMouseButtonUp(0))
        {
            selectionBox.gameObject.SetActive(false);
            RaycastHit[] hitColliders = Physics.BoxCastAll(cam.ScreenToWorldPoint(new Vector3(selectionBox.position.x, selectionBox.position.y, 1)), new Vector3((selectionBox.rect.width / 100) / 2, (selectionBox.rect.height / 100) / 2, 5), cam.transform.forward, cam.transform.rotation, 100, unitLayer);
            foreach (RaycastHit hit in hitColliders)
            {
                if (!unitsSelected.Contains(hit.transform))
                {
                    unitsSelected.Add(hit.transform);
                    unitAgentsSelected.Add(hit.transform.GetComponent<NavMeshAgent>());
                }
            }
        }
    }

    void UpdateSelectionBox(Vector2 curMousePos)
    {
        if (!selectionBox.gameObject.activeInHierarchy) selectionBox.gameObject.SetActive(true);

        float width = curMousePos.x - boxStartPos.x;
        float height = curMousePos.y - boxStartPos.y;

        selectionBox.sizeDelta = new Vector2(Mathf.Abs(width), Mathf.Abs(height));
        selectionBox.anchoredPosition = boxStartPos + new Vector2(width / 2, height / 2);
    }