SetStereoProjectionMatrix is not working in URP

Unity Engine
, , ,
1

I’m encountering an issue while working with the projection matrix in my Unity VR project. In a project using the Built-in Render Pipeline, I was able to successfully modify the projection matrices for both eyes using the code below. However, the same code doesn’t seem to work in a URP (Universal Render Pipeline) project. I’ve verified that all the functions involved are executing correctly in URP, which adds to my confusion — the new projection matrices simply don’t appear to be applied to the cameras when using URP. What’s happening?

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Experimental.Rendering;
using UnityEngine.Rendering;
public class ProjetctionMatrix : MonoBehaviour
{
    private void Awake()
    {
    }
    private void OnEnable()
    {
        RenderPipelineManager.beginFrameRendering += ScenePreCull;
    }

    private void OnDisable()
    {
        RenderPipelineManager.beginFrameRendering -= ScenePreCull;
        GetComponent<Camera>().ResetStereoProjectionMatrices();
    }

    private void ScenePreCull(ScriptableRenderContext context, Camera[] cam)
    {
        OnPreCull();
    }

    Camera cam;
    //the matrices which is going to be customized
    Matrix4x4 matR, matL;

    float ll, rl, bl, tl;
    float lr, rr, br, tr;

    float n, f;

    public float gain;

    public float fov_ver, fov_hor;
    float v, h;
    public float imageScale;
    // Start is called before the first frame update
    void Start()
    {
        cam = GetComponent<Camera>();
        n = cam.nearClipPlane;
        f = cam.farClipPlane;
        ll = -1.349f; rl = 1.202f; tl = 1.425f; bl = -1.413f;
        lr = -1.205f; rr = 1.349f; tr = 1.424f; br = -1.416f;
        gain = 1f;

        fov_hor = Mathf.Atan(rr) * Mathf.Rad2Deg - Mathf.Atan(ll) * Mathf.Rad2Deg;
        fov_ver = Mathf.Atan(tl) * Mathf.Rad2Deg - Mathf.Atan(bl) * Mathf.Rad2Deg;
        h = fov_hor;
        v = fov_ver;

        matR = cam.GetStereoProjectionMatrix(Camera.StereoscopicEye.Right);
        matL = cam.GetStereoProjectionMatrix(Camera.StereoscopicEye.Left);
    }


    void Update()
    {
        if (Input.GetKeyDown(KeyCode.UpArrow))
            gain += 0.1f;

        if (Input.GetKeyDown(KeyCode.DownArrow))
            gain -= 0.1f;

    }

    private void OnPreCull()
    {
        cam.ResetStereoProjectionMatrices();
        newProjection(gain);
        
        cam.SetStereoProjectionMatrix(Camera.StereoscopicEye.Right, matR);
        cam.SetStereoProjectionMatrix(Camera.StereoscopicEye.Left, matL);
    }

    void newProjection(float gain)
    {
        float lR = Mathf.Tan(Mathf.Atan(lr) * gain);
        float rR = Mathf.Tan(Mathf.Atan(rr) * gain);
        float bR = Mathf.Tan(Mathf.Atan(br) * gain);
        float tR = Mathf.Tan(Mathf.Atan(tr) * gain);

        matR = Matrix4x4.Frustum(lR * n, rR * n, bR * n, tR * n, n, f);

        float lL = Mathf.Tan(Mathf.Atan(ll) * gain);
        float rL = Mathf.Tan(Mathf.Atan(rl) * gain);
        float bL = Mathf.Tan(Mathf.Atan(bl) * gain);
        float tL = Mathf.Tan(Mathf.Atan(tl) * gain);

        matL = Matrix4x4.Frustum(lL * n, rL * n, bL * n, tL * n, n, f);

        fov_hor = Mathf.Atan(rR) * Mathf.Rad2Deg - Mathf.Atan(lL) * Mathf.Rad2Deg;
        fov_ver = Mathf.Atan(tL) * Mathf.Rad2Deg - Mathf.Atan(bL) * Mathf.Rad2Deg;

        imageScale = rr / rR;
        //imageScale = Mathf.Tan(v/2 * Mathf.Deg2Rad) / Mathf.Tan(fov_ver/2 * Mathf.Deg2Rad);
    }
}
2

I am also having great difficulty modifying the projection matrices in an XR setup. I notice that if I detach the USB cable (PC Link), then my custom projection matrices work. It’s only when the USB cable is attached to the headset that the custom projection matrices are ignored. When I connect the cable, that also coincides with myCamera.stereoEnabled changing from false to true.

@JerryPai What happens when you detach your USB cable (before launching the game) and just look at the game in the Unity editor window? Do your custom projection matrices work then?

To me it seems more related to XR than to the render pipeline, but I’m happy to run more specific tests. If anyone has a solution I’d love to know.

FWIW I’m mainly using Unity 6.1 and Unity’s OpenXR package (com.unity.xr.openxr), but I’ve also tried Unity 6.0 and 2022.3. I have very few other packages enabled. This same code worked ages ago when I was using a (long obsolete) version of the Oculus SDK. I am using the Quest 3 and Quest 2 headsets.

using UnityEngine;

namespace Foo
{
    public class MirrorManager : MonoBehaviour
    {
        public enum Mode
        { Normal, Mirrored };

        [SerializeField]
        private Camera? _camera;

        public Mode CurrentMode { get; private set; }

        public void SetMode(Mode mode)
        {
            if (mode == CurrentMode)
                return;

            var scale = mode == Mode.Mirrored ? -1 : 1;
            var vScale = new Vector3(scale, 1, 1);
            var mScale = Matrix4x4.Scale(vScale);

            if (_camera != null)
            {
                _camera.ResetWorldToCameraMatrix();
                _camera.ResetProjectionMatrix();
                _camera.ResetStereoProjectionMatrices();
                var cpm = _camera.projectionMatrix;
                var lpm = _camera.GetStereoProjectionMatrix(Camera.StereoscopicEye.Left);
                var rpm = _camera.GetStereoProjectionMatrix(Camera.StereoscopicEye.Right);
                _camera.SetStereoProjectionMatrix(Camera.StereoscopicEye.Left, lpm * mScale);
                _camera.SetStereoProjectionMatrix(Camera.StereoscopicEye.Right, rpm * mScale);
                _camera.projectionMatrix = cpm * mScale;
            }
            GL.invertCulling = mode == Mode.Mirrored;

            CurrentMode = mode;
        }

        private void Awake()
        {
            // Initialize everything in Normal mode
            CurrentMode = Mode.Normal;
            GL.invertCulling = false;
        }

        private void Update()
        {
            if (Input.GetKeyDown(KeyCode.M))
            {
                SetMode(CurrentMode == Mode.Normal ? Mode.Mirrored : Mode.Normal);
            }
        }

        private void OnApplicationQuit()
        {
            GL.invertCulling = false;
        }
    }
}
3

@JerryPai According to Unity documentation, the OnPreCull callback only affects the Built-in Render Pipeline. See here: Unity - Scripting API: MonoBehaviour.OnPreCull()

If you manage to get your code working with XR I’d love to see your solution.