How To Prevent Cursor-Object Offset When Trying To Drag Object Past Clamped Position

Hi I have a problem within my game wherein moving the cursor past the clamped position of the object and then back causes the clamped object to move backwards too, causing an offset. However what would be optimal is if the object would only start moving with the cursor when it has “caught up” with the object. Below is my script that handles dragging, clamping and flinging objects

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class DragObject : MonoBehaviour
{
    private float mZCoord;
    private Vector3 mOffset;
    private Vector3 lastMousePosition;
    private Vector3 velocity;

    float flingPowerScale = 0.33f;
    private Rigidbody rb;
    private Collider objectCollider;

    // Boundaries (these will be adjusted dynamically)
    private float maxXBoundary = 480f;
    private float minXBoundary = -480f;
    private float minZBoundary = -450f;
    private float maxZBoundary = 510f;
    private float minYBoundary = -5f;

    private void Start()
    {
        rb = GetComponent<Rigidbody>();
        objectCollider = GetComponent<Collider>();

        if (objectCollider == null)
        {
            Debug.LogError("No collider found on the object. Please attach a collider.");
        }

        if (rb == null)
        {
            Debug.LogError("No Rigidbody found on the object. Please attach a Rigidbody.");
        }
    }

    private void OnMouseDown()
    {
        mZCoord = Camera.main.WorldToScreenPoint(gameObject.transform.position).z;

        // Store offset = gameobject mouse pos - mouse world pos
        mOffset = gameObject.transform.position - GetMouseWorldPos();

        // Initialize the last mouse position
        lastMousePosition = GetMouseWorldPos();
    }

    void OnMouseDrag()
    {
        Vector3 currentMousePosition = GetMouseWorldPos();

        // Calculate velocity based on the difference between current and last mouse positions
        velocity = (currentMousePosition - lastMousePosition) / Time.deltaTime;

        // Update position based on mouse drag
        Vector3 newPosition = currentMousePosition + mOffset;

        // Clamp the position within boundaries
        newPosition = ClampPosition(newPosition);

        // Adjust the offset to avoid the tug-of-war effect
        mOffset = newPosition - currentMousePosition;

        // Apply the clamped position
        rb.MovePosition(newPosition);

        // Update last mouse position
        lastMousePosition = currentMousePosition;
    }

    private void OnMouseUp()
    {
        // When the mouse is released, apply the calculated velocity to the Rigidbody
        if (rb != null)
        {
            rb.velocity = velocity * flingPowerScale;
        }
    }

    private Vector3 GetMouseWorldPos()
    {
        // pixel coordinates (x,y)
        Vector3 mousePoint = Input.mousePosition;

        // z coordinate of gameobject on screen
        mousePoint.z = mZCoord;

        return Camera.main.ScreenToWorldPoint(mousePoint);
    }

    private Vector3 ClampPosition(Vector3 position)
    {
        if (objectCollider == null)
            return position;

        // Get the bounds of the object in world space
        Bounds bounds = objectCollider.bounds;

        // Clamp X axis
        if (bounds.min.x < minXBoundary)
        {
            position.x += minXBoundary - bounds.min.x;
        }
        else if (bounds.max.x > maxXBoundary)
        {
            position.x -= bounds.max.x - maxXBoundary;
        }

        // Clamp Z axis
        if (bounds.min.z < minZBoundary)
        {
            position.z += minZBoundary - bounds.min.z;
        }
        else if (bounds.max.z > maxZBoundary)
        {
            position.z -= bounds.max.z - maxZBoundary;
        }

        // Clamp Y axis (assuming only a lower bound, no upper bound)
        if (bounds.min.y < minYBoundary)
        {
            position.y += minYBoundary - bounds.min.y;
        }

        return position;

    }
}