Wanting to change Action based controller prefab/model with a script during game

So far I have this and it is changing the fields in the editor but not reflecting that in the game:

public class HandSwitcher : MonoBehaviour
{
    private bool isPressed;
    private ActionBasedController controller;

    [SerializeField] private GameObject sword;
    [SerializeField] private GameObject gun;

    private void Start()
    {
        controller = GetComponent<ActionBasedController>();

        controller.selectAction.action.performed += Action_performed;
        controller.selectAction.action.canceled += Action_canceled;
    }

    private void Update()
    {
        if (isPressed)
        {
            controller.modelPrefab = gun.transform;
            controller.model = gun.transform;
        }
        else
        {
            controller.modelPrefab = sword.transform;
            controller.model = sword.transform;
        }
    }

    private void Action_canceled (InputAction.CallbackContext obj)
    {
        isPressed = false;
        Debug.Log("Grip released");
    }

    private void Action_performed(InputAction.CallbackContext obj)
    {
        isPressed = true;
        Debug.Log("Grip pressed");
    }
}

Did you ever figure this out?