UI Toolkit Gestures

Is there any plan to support not just single clicks but also multi-clicks (double, triple), pinch gestures, etc.? Currently, it’s quite challenging to implement both drag and pinch gestures simultaneously.

Here is my implementation of a manipulator that can handle single clicks, double clicks, and hold gestures:

public class ClickManipulator : PointerManipulator
{
    private int clickCount;
    private float lastClickTime;
    private const int holdDuration = 500; // in milliseconds
    private const int doubleClickDuration = 200; // in milliseconds

    private Action<ClickData> _onClick;
    private Action<ClickData> _onHold;
    private Action<ClickData> _onDoubleClick;

    private bool isHolding;
    private long pointerDownTime;

    public ClickManipulator(Action<ClickData> onClick, Action<ClickData> onHold = null, Action<ClickData> onDoubleClick = null)
    {
        _onClick = onClick;
        _onHold = onHold;
        _onDoubleClick = onDoubleClick;
        clickCount = 0;
        lastClickTime = 0;
    }

    protected override void RegisterCallbacksOnTarget()
    {
        if (target == null) return;
        target.RegisterCallback<PointerDownEvent>(OnPointerDown);
        target.RegisterCallback<PointerUpEvent>(OnPointerUp);
    }

    protected override void UnregisterCallbacksFromTarget()
    {
        if (target == null) return;
        target.UnregisterCallback<PointerDownEvent>(OnPointerDown);
        target.UnregisterCallback<PointerUpEvent>(OnPointerUp);
    }

    private void OnPointerDown(PointerDownEvent evt)
    {
        pointerDownTime = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond;
        isHolding = true;
        target.schedule.Execute(() =>
        {
            if (isHolding)
            {
                _onHold?.Invoke(new ClickData(target, evt.localPosition, evt));
            }
        }).StartingIn(holdDuration);
    }

    private void OnPointerUp(PointerUpEvent evt)
    {
        var pos = evt.localPosition;
        isHolding = false;
        long pointerUpTime = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond;
        float clickInterval = pointerUpTime - pointerDownTime;

        if (clickInterval < holdDuration)
        {
            float currentTime = Time.realtimeSinceStartup * 1000;
            if (currentTime - lastClickTime <= doubleClickDuration)
            {
                clickCount++;
            }
            else
            {
                clickCount = 1;
            }

            if (clickCount == 2)
            {
                _onDoubleClick?.Invoke(new ClickData(target, pos, evt));
                clickCount = 0;
            }
            else
            {
                target.schedule.Execute(() =>
                {
                    if (clickCount == 1)
                    {
                        _onClick?.Invoke(new ClickData(target, pos, evt));
                        clickCount = 0;
                    }
                }).StartingIn(doubleClickDuration);
            }

            lastClickTime = currentTime;
        }
    }
}

And here is an implementation that detects both drag and pinch gestures:

public class TouchManipulator : PointerManipulator
{
    private class TouchData
    {
        public int ID;
        public Vector3 StartPosition;
        public Vector3 CurrentPosition;
        public float DistanceFromTouch;
    }
    
    [Flags] 
    public enum TouchMode
    {
        Nothing = 0,
        Drag = 1 << 0,
        Scale = 1 << 1,
        Resize = 1 << 2,
        Everything = Drag | Scale,
    }

    private TouchMode touchMode;
    private VisualElement dragElement, scaleElement;
    private (float min, float max) scaleRange;
    private Action onChanged;
    private float pickingZoneScale;
    private Length boundsOffset;
    public bool IsActive = false;
    
    private bool isDragging, isScaling;
    private int touchCount;
    private List<TouchData> pointers = new List<TouchData>();

    private Vector2 offset;
    private float startDistance, distance;
    private float scale, initialScale;
    private Vector2 initialSize;
    
    public TouchManipulator(
        TouchMode _touchMode, 
        VisualElement _dragElement = null, 
        VisualElement _scaleElement = null,
        (float min, float max) _scaleRange = default,
        Action _onChanged = null,
        float _pickingZoneScale = 1f, 
        Length _boundsOffset = default)
    {
        touchMode = _touchMode;
        dragElement = _dragElement;
        scaleElement = _scaleElement;
        scaleRange = _scaleRange;
        onChanged = _onChanged;
        pickingZoneScale = _pickingZoneScale;
        boundsOffset = _boundsOffset;
        IsActive = true;
    }
    
    protected override void RegisterCallbacksOnTarget()
    {
        dragElement = dragElement ?? target;
        scaleElement = scaleElement ?? target;
        IsActive = true;

        target.RegisterCallback<PointerDownEvent>(OnPointerDown);
        target.RegisterCallback<PointerMoveEvent>(OnPointerMove);
        target.RegisterCallback<PointerUpEvent>(OnPointerUp);
        target.RegisterCallback<PointerOutEvent>(OnPointerOut);
    }
    
    protected override void UnregisterCallbacksFromTarget()
    {
        target.UnregisterCallback<PointerDownEvent>(OnPointerDown);
        target.UnregisterCallback<PointerMoveEvent>(OnPointerMove);
        target.UnregisterCallback<PointerUpEvent>(OnPointerUp);
        target.UnregisterCallback<PointerOutEvent>(OnPointerOut);
    }

    private void OnPointerDown(PointerDownEvent e)
    {
        if (!IsActive) return;
        if (e.pointerId > 0)
        {
            touchCount++;
            pointers.Add(new TouchData { ID = touchCount, StartPosition = e.localPosition, CurrentPosition = e.localPosition});
        }
        if (touchCount == 2)
        {
            startDistance = (pointers[1].StartPosition - pointers[0].StartPosition).magnitude;
            initialScale = scaleElement.resolvedStyle.scale.value.x;
        }
        initialSize = new Vector2(scaleElement.resolvedStyle.width, scaleElement.resolvedStyle.height);
        e.StopImmediatePropagation();
    }
    
    private void OnPointerUp(PointerUpEvent e)
    {
        if (!IsActive) return;
        if (touchCount > 0 && e.pointerId > 0)
        {
            pointers.RemoveAll(p => p.ID == touchCount);
            touchCount--;
        }
        if (touchCount == 1) scaleElement.ReturnToParentBounds(boundsOffset);
        if (touchCount == 0) dragElement.ReturnToParentBounds(boundsOffset);
        e.StopImmediatePropagation();
        onChanged?.Invoke();
    }
    
    private void OnPointerOut(PointerOutEvent e) => OnPointerUp(PointerUpEvent.GetPooled(e));

    private void OnPointerMove(PointerMoveEvent e)
    {
        if (!IsActive || touchCount <= 0) return;

        switch (touchCount)
        {
            case 1:
                if (!touchMode.HasFlag(TouchMode.Drag) || isScaling) return;
                offset = e.localPosition - pointers[0].StartPosition;
                dragElement.style.left = dragElement.layout.x + offset.x;
                dragElement.style.top = dragElement.layout.y + offset.y;
                break;
            case 2:
                isScaling = true;
                if (touchMode.HasFlag(TouchMode.Scale))
                {
                    pointers[0].DistanceFromTouch = (pointers[0].CurrentPosition - e.localPosition).magnitude;
                    pointers[1].DistanceFromTouch = (pointers[1].CurrentPosition - e.localPosition).magnitude;
                    pointers[pointers[0].DistanceFromTouch < pointers[1].DistanceFromTouch ? 0 : 1].CurrentPosition = e.localPosition;

                    distance = (pointers[1].CurrentPosition - pointers[0].CurrentPosition).magnitude;
                    scale = (distance / startDistance) * initialScale;
                    scale = Mathf.Clamp(scale, scaleRange.min, scaleRange.max);
                    scaleElement.style.scale = new Vector2(scale, scale);
                }
                break;
        }
        e.StopImmediatePropagation();
        onChanged?.Invoke();
    }
}

However, this approach is not perfect. Have you encountered a more robust or native solution for handling multiple gestures in UI Toolkit? Any suggestions for improving this implementation would be highly appreciated!

1 Like