How do I stop the camera from going farther away while I am moving the player and rotating the camera at the same time?

Hello, I made a simple camera controller which follows the player around from an upper perspective, I’m also using Lerp to make the camera less stiff, and I also use the Q and E keys to rotate it around the player object.
I move the player using the NavMeshAgent.destination function.

The problem is that whenever I move the player and rotate the camera at the same time the camera “drifts” and the player moves further away from the camera. This effect is fixed if I remove the Lerp but I want the camera to not be so stiff but also not go further than a fixed distance away from the player, what would be the easiest way to accomplish this?

Here is the CameraController script:

public class CameraController : MonoBehaviour
{
    public float rotationSpeed; //this controls the speed of the rotation 
    public float damping = 1; //this adds "floatiness" to the camera

    private Transform player;
    private float movement;
    private Vector3 offset;

    void Start()
    {
        //Find the Object with the Player tag and set the initial offset
        player = GameObject.FindGameObjectWithTag("Player").transform;
        if (player == null)
        {
            Debug.LogError("Could not find GameObject with 'Player' tag");
        }

        offset = transform.position - player.position;
    }

    void LateUpdate()
    {
        Vector3 desiredPosition = player.position + offset;
        Vector3 position = Vector3.Lerp(transform.position,
            desiredPosition,
            Time.deltaTime * damping);

        transform.position = position;

        //Rotation has Q and E as Keys
        float movement = -Input.GetAxis("Rotation") * rotationSpeed * Time.deltaTime;
        if (!Mathf.Approximately(movement, 0.0f))
        {
            transform.RotateAround(player.position, Vector3.up, movement);
            offset = transform.position - player.position;
        }

        transform.LookAt(player);
    }
}

I solved it using ClampMagnitude(), first I save the original offset in the distance variable, than when I rotate the camera around I clamp the magnitude of the Offset vector using the distance , I also set the Y coordinate equal to the Y coordinate of the distance vector.

EDIT: The only problem now is if I move the player towards the camera stays above the player and does not move back to the original offset…

    public float rotationSpeed; //this controls the speed of the rotation
    public float damping = 1; //this adds "floatiness" to the camera
 
    private Transform player;
    private float movement;
    private Vector3 offset;
    private Vector3 distance;
 
    void Start()
    {
        //Find the Object with the Player tag and set the initial offset
        player = GameObject.FindGameObjectWithTag("Player").transform;
        if (player == null)
        {
            Debug.LogError("Could not find GameObject with 'Player' tag");
        }
 
        offset = transform.position - player.position;
        distance = offset;
    }
 
    void LateUpdate()
    {
        Debug.Log("Distance" + distance);
        Debug.Log("Offset" + offset);
        Vector3 desiredPosition = player.position + offset;
        Vector3 position = Vector3.Slerp(transform.position,
            desiredPosition,
            Time.deltaTime * damping);
 
        transform.position = position;
 
        //Rotation has Q and E as Keys
        float movement = -Input.GetAxis("Rotation") * rotationSpeed * Time.deltaTime;
        if (!Mathf.Approximately(movement, 0.0f))
        {
            transform.RotateAround(player.position, Vector3.up, movement);
            offset = Vector3.ClampMagnitude(transform.position - player.position, distance.magnitude);
            offset.y = distance.y;
        }
 
        transform.LookAt(player);
    }