Hello
I am creating an app for the Vision Pro that spawns bouncing balls that move around the player. I am using rigidbodys to make the balls movement more natural. I am trying to add in a feature where the user can swat the balls out of the way - the balls need to move in the direction of the users hand after they have been touched. Does anyone have any experience of setting up hand interaction like this in Vision OS?
Hi there! I think the easiest way to do this would just be with physics. Put a rigidbody on all of the balls, and one for each joint of the hand (or maybe a big one for the palm, depending on how detailed you want the interaction to be). It’s important when updating the rigidbody for the hand that you use velocities instead of setting its position. The easiest way to do that is to record the position from last frame and use the difference between the current position and the one from last frame to set the rigidbody velocity. I have a script somewhere that does this for the whole hand if you’re interested. I just need to find it 
yes the script would be great to look at if its available!
this is the logic i’m currently using to apply invisible cubes to the users hands, and apply a rigidbody:
using UnityEngine;
//#if UNITY_INCLUDE_XR_HANDS && (UNITY_VISIONOS || UNITY_EDITOR)
using System.Collections.Generic;
using UnityEngine.XR.Hands;
using UnityEngine.XR.VisionOS;
//#endif
namespace PolySpatial.Samples
{
public class HandVisualizer : MonoBehaviour
{
[SerializeField]
private Vector3 cubeScale = new Vector3(0.1f, 0.1f, 0.1f);
[SerializeField]
private bool isTrigger = false;
[SerializeField]
private bool providesContacts = false;
[SerializeField]
private bool useGravity = false;
[SerializeField]
private bool isKinematic = true;
[SerializeField]
private float isMass = 0;
[SerializeField]
private Vector3 positionOffset = Vector3.zero; // Offset added to the cube's position
//#if UNITY_INCLUDE_XR_HANDS && (UNITY_VISIONOS || UNITY_EDITOR)
private XRHandSubsystem m_Subsystem;
private GameObject m_LeftHandCube;
private GameObject m_RightHandCube;
private static readonly List<XRHandSubsystem> k_SubsystemsReuse = new();
protected void OnEnable()
{
Debug.Log("HandCubeVisualizer OnEnable");
if (m_Subsystem == null)
{
Debug.Log("No hand subsystem found on enable.");
return;
}
EnsureCubesExist();
UpdateCubeVisibility(m_LeftHandCube, m_Subsystem.leftHand.isTracked);
UpdateCubeVisibility(m_RightHandCube, m_Subsystem.rightHand.isTracked);
}
protected void OnDisable()
{
UpdateCubeVisibility(m_LeftHandCube, false);
UpdateCubeVisibility(m_RightHandCube, false);
}
protected void FixedUpdate()
{
if (m_Subsystem == null || !m_Subsystem.running)
{
FindAndAssignHandSubsystem();
EnsureCubesExist();
return;
}
UpdateHandCubes();
}
private void FindAndAssignHandSubsystem()
{
SubsystemManager.GetSubsystems(k_SubsystemsReuse);
foreach (var subsystem in k_SubsystemsReuse)
{
if (subsystem.running)
{
Debug.Log("Hand subsystem found and assigned.");
m_Subsystem = subsystem;
EnsureCubesExist();
return;
}
}
Debug.LogWarning("No running hand subsystem found.");
}
private void EnsureCubesExist()
{
if (m_LeftHandCube == null)
{
Debug.Log("Creating Left Hand Cube");
m_LeftHandCube = CreateHandCube("Left Hand Cube");
}
if (m_RightHandCube == null)
{
Debug.Log("Creating Right Hand Cube");
m_RightHandCube = CreateHandCube("Right Hand Cube");
}
}
private GameObject CreateHandCube(string name)
{
Debug.Log($"Creating cube: {name}");
var cube = GameObject.CreatePrimitive(PrimitiveType.Cube);
cube.transform.localScale = cubeScale;
cube.name = name;
// Assign the tag "Hand" to the cube
cube.tag = "Hand";
// Configure BoxCollider
var collider = cube.GetComponent<BoxCollider>();
if (collider != null)
collider.isTrigger = isTrigger;
collider.providesContacts = providesContacts;
// Add Rigidbody for physics
var rb = cube.GetComponent<Rigidbody>();
if (rb == null)
{
rb = cube.AddComponent<Rigidbody>();
Debug.Log("Rigidbody added to the cube.");
}
// Configure Rigidbody properties
rb.useGravity = useGravity;
rb.isKinematic = isKinematic;
rb.mass = isMass;
// Set collision detection to Continuous Dynamic
rb.collisionDetectionMode = CollisionDetectionMode.ContinuousDynamic;
// Disable the MeshRenderer to make the cube invisible
var renderer = cube.GetComponent<MeshRenderer>();
if (renderer != null)
renderer.enabled = false;
cube.SetActive(false);
Debug.Log($"Cube {name} created successfully with Rigidbody.");
return cube;
}
private void UpdateHandCubes()
{
if (m_LeftHandCube == null || m_RightHandCube == null)
{
Debug.LogWarning("Cubes not initialized. Calling EnsureCubesExist.");
EnsureCubesExist();
}
UpdateCubeTransform(m_Subsystem.leftHand, m_LeftHandCube);
UpdateCubeTransform(m_Subsystem.rightHand, m_RightHandCube);
}
private void UpdateCubeTransform(XRHand hand, GameObject cube)
{
if (hand == null)
{
Debug.LogError("Hand reference is null.");
return;
}
if (cube == null)
{
Debug.LogError("Cube reference is null.");
return;
}
if (hand.isTracked && hand.GetJoint(XRHandJointID.Wrist).TryGetPose(out var pose))
{
// Apply offset to the cube's position
var offsetPosition = pose.position + pose.rotation * positionOffset;
cube.transform.SetPositionAndRotation(offsetPosition, pose.rotation);
cube.SetActive(true);
}
else
{
cube.SetActive(false);
}
}
private static void UpdateCubeVisibility(GameObject cube, bool isTracked)
{
if (cube != null)
cube.SetActive(isTracked);
}
//#endif
}
}
When a collision is registered with the users hand I call this function to add a force to the ball game object:
public void ApplyForce(Collision collision)
{
// Calculate a force based on the collision impact
Vector3 collisionNormal = collision.contacts[0].normal;
Vector3 bounceForce = (collisionNormal + collision.relativeVelocity.normalized);
rb.AddForce(bounceForce, ForceMode.Impulse);
}
These are the values applied to the ball game objects:
Hey there! Sorry for the delay in getting back to you. Happy new year! 
This isn’t exactly what I meant about using velocities. While this may work for your use case, it’s a little more “manual” an approach (if you’ll forgive my pun 
).
What I meant was to replace the following line from your hand-cube script:
cube.transform.SetPositionAndRotation(offsetPosition, pose.rotation);
with something like
rigidbody.velocity = (targetPosition - rigidbody.position) * Time.deltaTime;
assuming rigidbody is referring to the rigidbody of the cube that’s following the wrist pose. What this does is allow the physics solver to “integrate” its solution based on the velocity of the hand, rather than instantaneously setting its position every frame. It gets a little more complicated when you try to introduce a fixed offset like you are doing here, but it’s important that you actually use the delta between the real position and the target, otherwise you’ll be able to “deform” the cube’s position relative to the hand by pushing into stuff.
I finally found my old example script based on HandVisualizer.cs. I’m going to share it in another post since… it’s kind of a lot
Here’s the full script for my Hand Physics sample. I’ll try to get this integrated into the main VR sample scene soon for an upcoming release. In the meantime, you basically just need to set up the following script similar to the existing hand visualizer (put it under the CameraOffset object in your XR Rig).
The demo I was building this for was a “sphere fountain” where I just created another simple script to constantly spawn a “waterfall” of spheres with rigidbodies, and the user could “scoop” up a handful of them or let them “trickle” through their fingers, hence it was important to include colliders for each individual joint. You could greatly cut down on the complexity here (both in terms of code and runtime efficiency) by just deleting all the finger joint stuff. You could start by just deleting the m_JointPhysicsPrefab field and keep pulling on the thread by fixing up compile errors due to missing references. Or just give it an empty prefab and pay for some wasted CPU cycles.
It’s up to you whether or not you want to add rendering visuals to these objects, but note that the default capsule will not adjust its height/radius without applying a scale, and that will mess up the physics collider. I recommend handling visuals completely separately from this script, since the collider locations may also not match up 1:1 with hand position. One of the “benefits” of this approach is that the hand will stop when it encounters a static collider, even though the user’s real hand has passed through it, and you may want your visual affordances to behave differently.
Note that the script calls for two new prefabs:
m_JointPhysicsPrefab is an object with a Z-aligned capsule collider whose radius more or less matches to the user’s hand size. You may want to get fancy and allow users to change this, or you may want to just strip out all the finger joint stuff and simply use the logic for the palm. The length of the capsules is adjusted dynamically by the script. You should leave the center at 0,0,0.m_PalmPhysicsPrefab is an object with a box collider shaped roughly to the size/aspect ratio of the palm. The offset from the wrist to this collider is also “baked into” this object. I found it was possible to just use collider offsets to get this good enough for my use case, but you should also be able to add the collider as a child of the prefab offset to account for more control over rotation/translation as needed.And without further ado, the script:
// Requires hands package and VisionOSHandExtensions which is only compiled for visionOS and Editor
#if INCLUDE_UNITY_XR_HANDS && (UNITY_VISIONOS || UNITY_EDITOR)
using System.Collections.Generic;
using UnityEngine.XR.Hands;
#endif
namespace UnityEngine.XR.VisionOS.Samples.URP
{
public class HandPhysics : MonoBehaviour
{
[SerializeField]
GameObject m_JointPhysicsPrefab;
[SerializeField]
GameObject m_PalmPhysicsPrefab;
#if INCLUDE_UNITY_XR_HANDS && (UNITY_VISIONOS || UNITY_EDITOR)
XRHandSubsystem m_Subsystem;
HandGameObjects m_LeftHandGameObjects;
HandGameObjects m_RightHandGameObjects;
float m_JointRadius;
XRHandSubsystem.UpdateType m_UpdateType = XRHandSubsystem.UpdateType.Fixed;
static readonly List<XRHandSubsystem> k_SubsystemsReuse = new();
public void SetUpdateType(XRHandSubsystem.UpdateType updateType)
{
m_UpdateType = updateType;
}
void Awake()
{
m_JointRadius = m_JointPhysicsPrefab.GetComponent<CapsuleCollider>().radius;
}
protected void OnEnable()
{
if (m_Subsystem == null)
return;
UpdateRenderingVisibility(m_LeftHandGameObjects, m_Subsystem.leftHand.isTracked);
UpdateRenderingVisibility(m_RightHandGameObjects, m_Subsystem.rightHand.isTracked);
}
protected void OnDisable()
{
if (m_Subsystem != null)
UnsubscribeSubsystem();
UpdateRenderingVisibility(m_LeftHandGameObjects, false);
UpdateRenderingVisibility(m_RightHandGameObjects, false);
}
void UnsubscribeSubsystem()
{
m_Subsystem.trackingAcquired -= OnTrackingAcquired;
m_Subsystem.trackingLost -= OnTrackingLost;
m_Subsystem.updatedHands -= OnUpdatedHands;
m_Subsystem = null;
}
protected void OnDestroy()
{
if (m_LeftHandGameObjects != null)
{
m_LeftHandGameObjects.OnDestroy();
m_LeftHandGameObjects = null;
}
if (m_RightHandGameObjects != null)
{
m_RightHandGameObjects.OnDestroy();
m_RightHandGameObjects = null;
}
}
protected void Update()
{
if (m_Subsystem != null)
{
if (m_Subsystem.running)
return;
UnsubscribeSubsystem();
UpdateRenderingVisibility(m_LeftHandGameObjects, false);
UpdateRenderingVisibility(m_RightHandGameObjects, false);
return;
}
SubsystemManager.GetSubsystems(k_SubsystemsReuse);
for (var i = 0; i < k_SubsystemsReuse.Count; ++i)
{
var handSubsystem = k_SubsystemsReuse[i];
if (handSubsystem.running)
{
UnsubscribeHandSubsystem();
m_Subsystem = handSubsystem;
break;
}
}
if (m_Subsystem == null)
return;
if (m_LeftHandGameObjects == null)
{
m_LeftHandGameObjects = new HandGameObjects(
Handedness.Left,
transform,
m_JointPhysicsPrefab,
m_PalmPhysicsPrefab);
}
if (m_RightHandGameObjects == null)
{
m_RightHandGameObjects = new HandGameObjects(
Handedness.Right,
transform,
m_JointPhysicsPrefab,
m_PalmPhysicsPrefab);
}
UpdateRenderingVisibility(m_LeftHandGameObjects, m_Subsystem.leftHand.isTracked);
UpdateRenderingVisibility(m_RightHandGameObjects, m_Subsystem.rightHand.isTracked);
SubscribeHandSubsystem();
}
void SubscribeHandSubsystem()
{
if (m_Subsystem == null)
return;
m_Subsystem.trackingAcquired += OnTrackingAcquired;
m_Subsystem.trackingLost += OnTrackingLost;
m_Subsystem.updatedHands += OnUpdatedHands;
}
void UnsubscribeHandSubsystem()
{
if (m_Subsystem == null)
return;
m_Subsystem.trackingAcquired -= OnTrackingAcquired;
m_Subsystem.trackingLost -= OnTrackingLost;
m_Subsystem.updatedHands -= OnUpdatedHands;
}
static void UpdateRenderingVisibility(HandGameObjects handGameObjects, bool isTracked)
{
if (handGameObjects == null)
return;
handGameObjects.SetHandActive(isTracked);
}
void OnTrackingAcquired(XRHand hand)
{
switch (hand.handedness)
{
case Handedness.Left:
UpdateRenderingVisibility(m_LeftHandGameObjects, true);
break;
case Handedness.Right:
UpdateRenderingVisibility(m_RightHandGameObjects, true);
break;
}
}
void OnTrackingLost(XRHand hand)
{
switch (hand.handedness)
{
case Handedness.Left:
UpdateRenderingVisibility(m_LeftHandGameObjects, false);
break;
case Handedness.Right:
UpdateRenderingVisibility(m_RightHandGameObjects, false);
break;
}
}
void OnUpdatedHands(XRHandSubsystem subsystem, XRHandSubsystem.UpdateSuccessFlags updateSuccessFlags, XRHandSubsystem.UpdateType updateType)
{
// We have no game logic depending on the Transforms, so early out here
// (add game logic before this return here, directly querying from
// subsystem.leftHand and subsystem.rightHand using GetJoint on each hand)
if (updateType != m_UpdateType)
return;
var thisTransform = transform;
m_LeftHandGameObjects.UpdateJoints(
subsystem.leftHand,
thisTransform,
m_JointRadius,
(updateSuccessFlags & XRHandSubsystem.UpdateSuccessFlags.LeftHandJoints) != 0,
m_UpdateType);
m_RightHandGameObjects.UpdateJoints(
subsystem.rightHand,
thisTransform,
m_JointRadius,
(updateSuccessFlags & XRHandSubsystem.UpdateSuccessFlags.RightHandJoints) != 0,
m_UpdateType);
}
class HandGameObjects
{
GameObject m_JointsParent;
readonly Rigidbody[] m_JointRigidBodies = new Rigidbody[XRHandJointID.EndMarker.ToIndex()];
readonly CapsuleCollider[] m_JointCapsules = new CapsuleCollider[XRHandJointID.EndMarker.ToIndex()];
readonly bool[] m_JointHasBeenUpdated = new bool[XRHandJointID.EndMarker.ToIndex()];
public HandGameObjects(
Handedness handedness,
Transform parent,
GameObject jointPhysicsPrefab,
GameObject palmPhysicsPrefab)
{
void AssignJoint(
XRHandJointID jointID,
Transform drawJointsParent)
{
var jointIndex = jointID.ToIndex();
var jointName = jointID.ToString();
if (jointID == XRHandJointID.Wrist)
{
var palmPhysicsObject = Instantiate(palmPhysicsPrefab, drawJointsParent, false);
palmPhysicsObject.name = jointName;
m_JointRigidBodies[jointIndex] = palmPhysicsObject.GetComponent<Rigidbody>();
return;
}
var jointVisualsObject = Instantiate(jointPhysicsPrefab, drawJointsParent, false);
jointVisualsObject.name = $"{jointName}";
var jointRigidBody = jointVisualsObject.GetComponent<Rigidbody>();
var jointCapsule = jointVisualsObject.GetComponent<CapsuleCollider>();
m_JointRigidBodies[jointIndex] = jointRigidBody;
m_JointCapsules[jointIndex] = jointCapsule;
}
m_JointsParent = new GameObject();
var parentTransform = m_JointsParent.transform;
parentTransform.parent = parent;
parentTransform.localPosition = Vector3.zero;
parentTransform.localRotation = Quaternion.identity;
m_JointsParent.name = $"{handedness} Hand physics";
AssignJoint(XRHandJointID.Wrist, parentTransform);
for (var fingerIndex = (int)XRHandFingerID.Thumb;
fingerIndex <= (int)XRHandFingerID.Little;
++fingerIndex)
{
var fingerId = (XRHandFingerID)fingerIndex;
// Ignore fingertips (capsule from distal extends to tip)
var jointIndexBack = fingerId.GetBackJointID().ToIndex() - 1;
// Ignore metacarpals (covered by palm collider)
var jointIndexFront = fingerId.GetFrontJointID().ToIndex() + 1;
for (var jointIndex = jointIndexFront;
jointIndex <= jointIndexBack;
++jointIndex)
{
AssignJoint(XRHandJointIDUtility.FromIndex(jointIndex), parentTransform);
}
}
}
public void OnDestroy()
{
var length = m_JointRigidBodies.Length;
for (var jointIndex = 0; jointIndex < length; ++jointIndex)
{
var visuals = m_JointRigidBodies[jointIndex];
if (visuals == null)
continue;
Destroy(visuals.gameObject);
m_JointRigidBodies[jointIndex] = default;
}
Destroy(m_JointsParent);
m_JointsParent = null;
}
public void SetHandActive(bool isActive)
{
m_JointsParent.SetActive(isActive);
}
public void UpdateJoints(
XRHand hand,
Transform rootTransform,
float jointRadius,
bool areJointsTracked,
XRHandSubsystem.UpdateType updateType)
{
if (!areJointsTracked)
return;
UpdateJoint(hand.GetJoint(XRHandJointID.Wrist), default, rootTransform, jointRadius, updateType);
for (var fingerIndex = (int)XRHandFingerID.Thumb;
fingerIndex <= (int)XRHandFingerID.Little;
++fingerIndex)
{
var fingerId = (XRHandFingerID)fingerIndex;
// Ignore fingertips (capsule from distal extends to tip)
var jointIndexBack = fingerId.GetBackJointID().ToIndex() - 1;
// Ignore metacarpals (covered by palm collider)
var jointIndexFront = fingerId.GetFrontJointID().ToIndex() + 1;
for (var jointIndex = jointIndexFront;
jointIndex <= jointIndexBack;
++jointIndex)
{
var joint = hand.GetJoint(XRHandJointIDUtility.FromIndex(jointIndex));
var nextJoint = hand.GetJoint(XRHandJointIDUtility.FromIndex(jointIndex + 1));
UpdateJoint(joint, nextJoint, rootTransform, jointRadius, updateType);
}
}
}
static readonly Vector3 k_WristOffset = new Vector3(0, 0, 0.05f);
void UpdateJoint(
XRHandJoint joint,
XRHandJoint nextJoint,
Transform rootTransform,
float jointRadius,
XRHandSubsystem.UpdateType updateType)
{
if (joint.id == XRHandJointID.Invalid)
return;
var jointIndex = joint.id.ToIndex();
var rigidBody = m_JointRigidBodies[jointIndex];
if (!joint.TryGetPose(out var pose))
{
rigidBody.gameObject.SetActive(false);
return;
}
Vector3 offset;
var targetRotation = rootTransform.rotation * pose.rotation;
var targetPosition = rootTransform.TransformPoint(pose.position);
if (nextJoint.TryGetPose(out var nextPose))
{
var nextPosition = rootTransform.TransformPoint(nextPose.position);
var jointLength = Vector3.Distance(nextPosition, targetPosition);
offset = Vector3.forward * jointLength * 0.5f;
var capsule = m_JointCapsules[jointIndex];
capsule.height = jointLength - jointRadius;
//capsule.center = Vector3.forward * (jointLength * 0.5f);
}
else
{
// Palm collider has a fixed offset from the wrist
offset = k_WristOffset;
}
// Adjust actual position to be in the center
targetPosition += targetRotation * offset;
if (updateType == XRHandSubsystem.UpdateType.Fixed)
{
var hasBeenUpdated = m_JointHasBeenUpdated[jointIndex];
if (!hasBeenUpdated)
{
m_JointHasBeenUpdated[jointIndex] = true;
rigidBody.position = targetPosition;
rigidBody.rotation = targetRotation;
}
else
{
var fixedDeltaInverse = 1 / Time.fixedDeltaTime;
rigidBody.linearVelocity = (targetPosition - rigidBody.position) * fixedDeltaInverse;
rigidBody.rotation = targetRotation;
}
}
else
{
rigidBody.transform.SetLocalPositionAndRotation(pose.position, pose.rotation);
}
}
}
#endif
}
}
P.S. Your mileage may vary here… I don’t have this in a working state anymore so I’m just pasting in the final “check-in” state of this script I could find. As I said I’ll try to turn this into a real sample in the coming weeks, since I think it’s kind of a fun thing to play around with, if nothing else.
Hope this helps. Good luck!
thanks for the reply,ive added this into my scene, but I cant seem to get the capsule prefabs to show up with my hands. I had to make a couple of amendments to the code because I kept getting build errors:
for off, I had to change the line:
//XRHandSubsystem.UpdateType m_UpdateType = XRHandSubsystem.UpdateType.Fixed
XRHandSubsystem.UpdateType m_UpdateType = XRHandSubsystem.UpdateType.Dynamic;
It wouldn’t recognise a fixed value.
I also had to change this line because the rigidbody value wouldn’t recognise linearvelocity:
//rigidBody.linearVelocity = (targetPosition - rigidBody.position) * fixedDeltaInverse;
rigidBody.velocity = (targetPosition - rigidBody.position) * fixedDeltaInverse;
Could any of these changes be what’s causing issues?
Hm… what version of Unity and com.unity.xr.hands are you using?