Pick up objects like in Amnesia - problem with smooth object rotation on bumping onto things

I’m making Amnesia-like mechanics and currently I’m working on picking objects up. The behaviour I’m after is as follows: when I pick a book (for example) up, I want this book to be able to bump onto things that are on the scene, but the book should always try to return to the direction towards camera it had when it was picked up. After some trial and error I managed to code it, but the movement is very jittery.

My code:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

[RequireComponent(typeof(Rigidbody))]
public class Grabbable : Interactable
{
    private Rigidbody rigidbody;

    private bool grabbed = false;

    private Vector3 targetDirection;
    private Quaternion originalRotation;

    private Player player;

    [SerializeField] private float sensitivity = 0.05f;

    void Awake()
    {
        rigidbody = GetComponent<Rigidbody>();
    }

    void FixedUpdate()
    {
        if (grabbed)
        {
            Ray playerAim = player.PlayerCam.ScreenPointToRay(new Vector3(Screen.width / 2.0f, Screen.height / 2.0f));

            Vector3 nextPos = player.PlayerCam.transform.position + playerAim.direction.normalized * player.GrabbedObjectDistance;
            Vector3 currentPos = transform.position;

            rigidbody.velocity = (nextPos - currentPos) * 1000.0f * Time.fixedDeltaTime;

            Vector3 currentDirection = (player.PlayerCam.transform.position - transform.position);
            transform.rotation = Quaternion.Lerp(transform.rotation,
                Quaternion.LookRotation(currentDirection) * Quaternion.Inverse(Quaternion.LookRotation(targetDirection)) * originalRotation,
                sensitivity * Time.fixedDeltaTime);
        }
    }

    public override void OnInteract(Player playerObject)
    {
        grabbed = true;
        player = playerObject;
        rigidbody.useGravity = false;
        rigidbody.detectCollisions = true;
        targetDirection = (player.PlayerCam.transform.position - transform.position);
        originalRotation = transform.rotation;
    }

    public override void OnRelease()
    {
        grabbed = false;
        rigidbody.useGravity = true;
    }
}

As for rigidbody that’s on the object I’m picking up, there are no constraints, it is not set to kinematic, no kind of interpolation. Mass set to 50, both drags to 0, use gravity checked and collision type set to continuous dynamic.

The code above actually works almost as intended, but when an object bumps onto something, the movement is very jittery. Original Frictional Games titles (SOMA and first Amnesia game are the ones I’m comparing my mechanics to) have very smooth movement.

I think that I should rather calculate angular velocity for rotating object, but I’m at a loss on how to calculate it.

GIF of how this mechanic currently looks:
coolwellwornabyssiniangroundhornbill

“In most cases you should not modify the velocity directly, as this can result in unrealistic behaviour - use AddForce instead. Do not set the velocity of an object every physics step, this will lead to unrealistic physics simulation.”

Thanks! I managed to get it working using AddForce() and AddTorque(). I searched Internet for how to calculate torque and I don’t understand all the math yet, but it works much better now. Almost like in Amnesia or SOMA, though the held object can still jitter a bit when you drag it across the table, but in the mentioned games objects also jitter a bit now and then, so I think it’s all right.

For anyone wondering, here’s the code:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

[RequireComponent(typeof(Rigidbody))]
public class Grabbable : Interactable
{
    private Rigidbody rigidbody;

    private bool grabbed = false;

    private Vector3 targetDirection;
    private Quaternion originalRotation;

    private Player player;

    [SerializeField] private float positionForce = 1000.0f;
    [SerializeField] private float angularForce = 50.0f;

    // Start is called before the first frame update
    void Awake()
    {
        rigidbody = GetComponent<Rigidbody>();
    }

    // Update is called once per frame
    void FixedUpdate()
    {
        if (grabbed)
        {
            rigidbody.velocity = Vector3.zero;
            rigidbody.angularVelocity = Vector3.zero;
            Ray playerAim = player.PlayerCam.ScreenPointToRay(new Vector3(Screen.width / 2.0f, Screen.height / 2.0f));

            Vector3 nextPos = player.PlayerCam.transform.position + playerAim.direction.normalized * player.GrabbedObjectDistance;
            Vector3 currentPos = transform.position;

            rigidbody.AddForce((nextPos - currentPos) * positionForce * Time.fixedDeltaTime, ForceMode.VelocityChange);

            Vector3 currentDirection = (player.PlayerCam.transform.position - transform.position).normalized;
            Quaternion target = Quaternion.LookRotation(currentDirection) * Quaternion.Inverse(Quaternion.LookRotation(targetDirection)) * originalRotation;
            rigidbody.AddTorque(ComputeTorque(target) * angularForce * Time.fixedDeltaTime);
        }
    }

    public override void OnInteract(Player playerObject)
    {
        grabbed = true;
        player = playerObject;
        rigidbody.useGravity = false;
        rigidbody.detectCollisions = true;
        targetDirection = (player.PlayerCam.transform.position - transform.position).normalized;
        originalRotation = transform.rotation;
    }

    public override void OnRelease()
    {
        grabbed = false;
        rigidbody.useGravity = true;
        rigidbody.isKinematic = false;
        rigidbody.velocity = Vector3.zero;
        rigidbody.angularVelocity = Vector3.zero;
    }

    Vector3 ComputeTorque(Quaternion desiredRotation)
    {
        Quaternion q = desiredRotation * Quaternion.Inverse(transform.rotation);

        Vector3 x;
        float xMag;
        q.ToAngleAxis(out xMag, out x);
        x.Normalize();
        Vector3 w = x * xMag * Mathf.Deg2Rad / Time.fixedDeltaTime;
        w -= rigidbody.angularVelocity;
        Vector3 wl = transform.InverseTransformDirection(w);
        Vector3 Tl;
        Vector3 wll = wl;
        wll = rigidbody.inertiaTensorRotation * wll;
        wll.Scale(rigidbody.inertiaTensor);
        Tl = Quaternion.Inverse(rigidbody.inertiaTensorRotation) * wll;
        Vector3 T = transform.TransformDirection(Tl);
        return T;
    }
}