Smooth crouching with Mathf.Lerp

I am trying to get my character to crouch smoothly, which works when decreasing height, but when increasing the height back to the original, it’s much more ‘stuttery’. However, when I do this while moving, the transition is smooth ‘both ways’. How can I make it so that it is always smooth, not just while moving?

if (Input.GetKey(KeyCode.LeftControl))
        {
            characterController.height = Mathf.Lerp(characterController.height, 0.5f, 10f * Time.deltaTime);
            canRun = false;
            canJump = false;
            walkingSpeed = 2f;
        }

        else
        {
            characterController.height = Mathf.Lerp(characterController.height, 2.0f, 10f * Time.deltaTime);
            canRun = true;
            canJump = true;
            walkingSpeed = 4f;
        }

^ This is the crouching code so far, I don’t see why it doesn’t work (or rather, only works while moving)

If the rest of the code is necessary, here it is

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

[RequireComponent(typeof(CharacterController))]

public class FirstPersonController : MonoBehaviour
{
    public float walkingSpeed = 4f;
    public float runningSpeed = 5.5f;
    public float jumpSpeed = 4.0f;
    public float gravity = 15.0f;

    public Camera playerCamera;
    public CharacterController Player;

    public float lookSpeed = 2.0f;
    public float lookXLimit = 90.0f;

    public static bool canRun = true;
    public static bool running = false;
    
    CharacterController characterController;
    Vector3 moveDirection = Vector3.zero;
    float rotationX = 0;

    [HideInInspector]
    public bool canMove = true;
    public bool canJump = true;

    void Start()
    {
        characterController = GetComponent<CharacterController>();

        // Lock cursor
        Cursor.lockState = CursorLockMode.Locked;
        Cursor.visible = false;
    }

    void Update()
    {
        Vector3 forward = transform.TransformDirection(Vector3.forward);
        Vector3 right = transform.TransformDirection(Vector3.right);

        bool isRunning = Input.GetKey(KeyCode.LeftShift) && canRun == true;
        float curSpeedX = canMove ? (isRunning ? runningSpeed : walkingSpeed) * Input.GetAxis("Vertical") : 0;
        float curSpeedY = canMove ? (isRunning ? runningSpeed : walkingSpeed) * Input.GetAxis("Horizontal") : 0;
        float movementDirectionY = moveDirection.y;

        if (isRunning)
        {
            moveDirection = Vector3.ClampMagnitude((forward * curSpeedX) + (right * curSpeedY), 5.5f);
            running = true;
            if (Input.GetKey(KeyCode.W) || Input.GetKey(KeyCode.A) || Input.GetKey(KeyCode.S) || Input.GetKey(KeyCode.D))
            {
                Camera.main.fieldOfView = Mathf.Lerp(Camera.main.fieldOfView, 65f, 10f * Time.deltaTime);
            }
        }

        else
        {
            moveDirection = Vector3.ClampMagnitude((forward * curSpeedX) + (right * curSpeedY), 4.0f);
            running = false;
            Camera.main.fieldOfView = Mathf.Lerp(Camera.main.fieldOfView, 60f, 10f * Time.deltaTime);
        }

        if (Input.GetButton("Jump") && canMove && characterController.isGrounded && canJump)
        {
            moveDirection.y = jumpSpeed;
        }
        else
        {
            moveDirection.y = movementDirectionY;
        }

        if (!characterController.isGrounded)
        {
            moveDirection.y -= gravity * Time.deltaTime;
        }

        characterController.Move(moveDirection * Time.deltaTime);

        if (canMove)
        {
            rotationX += -Input.GetAxis("Mouse Y") * lookSpeed;
            rotationX = Mathf.Clamp(rotationX, -lookXLimit, lookXLimit);
            playerCamera.transform.localRotation = Quaternion.Euler(rotationX, 0, 0);
            transform.rotation *= Quaternion.Euler(0, Input.GetAxis("Mouse X") * lookSpeed, 0);   
        }

        if (Input.GetKey(KeyCode.LeftControl))
        {
            characterController.height = Mathf.Lerp(characterController.height, 0.5f, 10f * Time.deltaTime);
            canRun = false;
            canJump = false;
            walkingSpeed = 2f;
        }

        else
        {
            characterController.height = Mathf.Lerp(characterController.height, 2.0f, 10f * Time.deltaTime);
            canRun = true;
            canJump = true;
            walkingSpeed = 4f;
        }
    }
}