Drag Rigidbody that respects collisions.

Hi there, i have a game where you drag blocks only across one axis, either the Z or the X. The problem im having at the minute is the movement isn’t 1:1, the further away the block gets from the camera the slower it travels. I need it to be a constant speed that will also respect collisions, as moving to fast of course goes through other colliders.

Here is the current code

    public virtual void FixedUpdate()
    {
        
        if (!ReferenceManager.moveController.CanMove) { return; }

        //Debug.DrawRay(_rayPos.transform.position + new Vector3(raycastOffset.x, 0, raycastOffset.y), Vector3.down, Color.red);


        if (Input.GetMouseButton(0) && CanMove)
        {
            ray = Camera.main.ScreenPointToRay(Input.mousePosition);
            if (!dragObj && (Physics.Raycast(ray, out hit) && hit.rigidbody))
            {
                if (!gameObject) return;    //check we're still there
                dragObj = hit.transform;
                Length = hit.distance;
                Debug.Log(Length);
                _offset = hit.point - dragObj.position;

                _rigidbody.constraints = RigidbodyConstraints.FreezeAll;
                _rigidbody.isKinematic = true;
                _rigidbody.mass = TempMass;
              
                if (dragObj.GetComponentInParent<BlockBehaviour>().blockType == BlockType.Horizontal)
                {
                    dragObj.GetComponent<Rigidbody>().constraints = RigidbodyConstraints.FreezePositionY |
                                                                    RigidbodyConstraints.FreezePositionZ |
                                                                    RigidbodyConstraints.FreezeRotation;
                    dragObj.GetComponent<Rigidbody>().mass = CachedMass;
                    dragObj.GetComponent<Rigidbody>().isKinematic = false;
                }
                else
                {
                    dragObj.GetComponent<Rigidbody>().constraints = RigidbodyConstraints.FreezePositionX |
                                                                    RigidbodyConstraints.FreezePositionY |
                                                                    RigidbodyConstraints.FreezeRotation;
                    dragObj.GetComponent<Rigidbody>().mass = CachedMass;
                    dragObj.GetComponent<Rigidbody>().isKinematic = false;
                }
            }
        }
        else
        {
            dragObj = null;
        }
        if (!dragObj) return;
        if (dragObj == null) return;

//        var wantedPos = Camera.main.ScreenToWorldPoint(new Vector3(Input.mousePosition.x, Input.mousePosition.y, 10)); 
//        _rigidbody.MovePosition(new Vector3(wantedPos.x, transform.position.y, transform.position.z)) * Time.deltaTime;
//        transform.position = new Vector3(wantedPos.x, transform.position.y, transform.position.z);

        var vel = (ray.GetPoint(Length) - (dragObj.position + _offset)) * Speed;
        if (vel.magnitude > MaxSpeed)
            vel *= MaxSpeed / vel.magnitude;
        dragObj.gameObject.GetComponentInChildren<Rigidbody>().velocity = vel;
    }

You have the distance moved this frame already computed as _offset, why not use that to compute the velocity?
Velocity= distance / time

   vel =  _offset / Time.fixedDeltaTime;