The mirror cannot work in VisionPro

Hi, I wrote a real-time rendering mirror in Unity 6.0.23f1 with Poly Spatial version 2.1.2. It works well in the Unity editor, but when Build to AVP or viewed in AVP using Play to Device, the mirror object appears completely white. The RenderTexture format is R8G8B8A8_UNORM. Could you help me figure out what might be causing this issue? Here 's my code, Thx!

using System;
using UnityEngine;


//[ExecuteAlways]
public class PlannarReflectionTest1 : MonoBehaviour
{
    public LayerMask reflectMask = -1;

    [SerializeField] private MeshRenderer m_targetPlaneRenderer;
    [SerializeField] private int m_targetTextureWidth = 1024;
    [SerializeField] private string m_reflectTexName = "_MirrorReflectTex";
    [SerializeField] private float m_scale = 1.0f;
    [HideInInspector] private float m_clipPlaneOffset = 0.01f;

    private Camera m_cameraMain;
    private Transform m_cameraMainTransform;
    private Camera m_cameraReflection;
    private Transform m_cameraReflectionTransform;
    public RenderTexture m_targetTexture;
    private int m_reflectTexInt;
    private Vector3 m_planeNormal;
    private Vector3 m_planePosition;
    private Transform m_normalTransform;
    private Vector4 m_reflectionPlane;

    private void Awake()
    {
        // 查找主相机
        m_cameraMain = Camera.main;
        m_cameraMainTransform = m_cameraMain.transform;

        // 创建一个反射相机
        var go = new GameObject("ReflectionCamera");
        m_cameraReflection = go.AddComponent<Camera>();
        m_cameraReflection.aspect = m_cameraMain.aspect;
        m_cameraReflection.fieldOfView = m_cameraMain.fieldOfView;
        m_cameraReflection.enabled = false; // 禁用自动渲染

        m_cameraReflectionTransform = m_cameraReflection.transform;

        // 创建贴图,赋值到反摄像机的 targetTexture
        m_cameraReflection.targetTexture = m_targetTexture;
        m_cameraReflection.cullingMask = reflectMask.value;

        // 创建法线
        m_normalTransform = new GameObject("Normal").transform;
        var planeTransform = m_targetPlaneRenderer.transform;
        m_normalTransform.SetPositionAndRotation(planeTransform.position, planeTransform.rotation);
        m_normalTransform.SetParent(planeTransform);
        m_planePosition = m_normalTransform.position;
        m_planeNormal = m_normalTransform.up;

        m_reflectTexInt = Shader.PropertyToID(m_reflectTexName);
        Shader.SetGlobalTexture(m_reflectTexName, m_targetTexture);
    }

    private void Update()
    {
        RenderReflection();
    }

    private void RenderReflection()
    {
        // 判断相机是否在法线下方,如果在下方就不做渲染
        Vector3 localPos = m_normalTransform.worldToLocalMatrix.MultiplyPoint3x4(m_cameraMainTransform.position);
        if (localPos.y < 0) return;

        // 调整位置
        Vector3 normal = m_normalTransform.up;
        Vector3 pos = m_normalTransform.position;

        float d = -Vector3.Dot(normal, pos) - m_clipPlaneOffset;
        m_reflectionPlane = new Vector4(normal.x, normal.y, normal.z, d);

        var reflection = Matrix4x4.identity;
        reflection *= Matrix4x4.Scale(new Vector3(1, -1, 1));
        CalculateReflectionMatrix(ref reflection, ref m_reflectionPlane);

        // 设置反射相机的视图矩阵
        Matrix4x4 worldToCameraMatrix = m_cameraMain.worldToCameraMatrix * reflection;
        m_cameraReflection.worldToCameraMatrix = worldToCameraMatrix;

        // 计算投影矩阵
        Vector3 offsetPos = pos + normal * m_clipPlaneOffset;
        Vector3 cpos = worldToCameraMatrix.MultiplyPoint3x4(offsetPos);
        Vector3 cnormal = worldToCameraMatrix.MultiplyVector(normal).normalized;
        Vector4 clipPlane = new Vector4(cnormal.x, cnormal.y, cnormal.z, -Vector3.Dot(cpos, cnormal));
        m_cameraReflection.projectionMatrix = m_cameraMain.CalculateObliqueMatrix(clipPlane);

        // 渲染到目标 RenderTexture
        GL.invertCulling = true;
        m_cameraReflection.Render();
        GL.invertCulling = false;

        Shader.SetGlobalTexture(m_reflectTexName, m_targetTexture);
        m_cameraReflection.targetTexture = m_targetTexture;
        Unity.PolySpatial.PolySpatialObjectUtils.MarkDirty(m_targetTexture);
    }

    private static void CalculateReflectionMatrix(ref Matrix4x4 reflectionMat, ref Vector4 plane)
    {
        reflectionMat.m00 = (1F - 2F * plane[0] * plane[0]);
        reflectionMat.m01 = (-2F * plane[0] * plane[1]);
        reflectionMat.m02 = (-2F * plane[0] * plane[2]);
        reflectionMat.m03 = (-2F * plane[3] * plane[0]);

        reflectionMat.m10 = (-2F * plane[1] * plane[0]);
        reflectionMat.m11 = (1F - 2F * plane[1] * plane[1]);
        reflectionMat.m12 = (-2F * plane[1] * plane[2]);
        reflectionMat.m13 = (-2F * plane[3] * plane[1]);

        reflectionMat.m20 = (-2F * plane[2] * plane[0]);
        reflectionMat.m21 = (-2F * plane[2] * plane[1]);
        reflectionMat.m22 = (1F - 2F * plane[2] * plane[2]);
        reflectionMat.m23 = (-2F * plane[3] * plane[2]);

        reflectionMat.m30 = 0F;
        reflectionMat.m31 = 0F;
        reflectionMat.m32 = 0F;
        reflectionMat.m33 = 1F;
    }
} 

Hello! The piece you’re missing is that globals for PolySpatial (visionOS RealityKit mode) need to be set using the PolySpatial-specific methods (which also set the Unity globals):

Unity.PolySpatial.PolySpatialShaderGlobals.SetTexture(m_reflectTexName, m_targetTexture);

Once I made that change (and used a shader graph with a global texture input), your script worked for me:

Great! :smiley: I used the method you told and created a new RenderTexture(used a shader graph with a global texture) ,applying it to the output of the m_cameraReflection. It worked in AVP. However, I found that the left and right eyes see different things, which makes me feel very dizzy and doesn’t align with natural human vision. For example, the right eye can see the cube reflected in the mirror, but the left eye only sees the skybox. I have to turn my head significantly to see the reflected cube in the left eye, but by that time, the right eye only sees the skybox. Would you know how to solve the visual effects for the left and right eyes in AVP?

Basically, you need to render to two RenderTextures (one for each eye) and use the Eye Index node in your shader graph to switch between them. This is rendered somewhat more complicated by the fact that the Apple Vision Pro doesn’t expose the eye parameters (such as the inter-pupillary distance), so you 'll have to estimate them. There are a couple previous threads on this topic here and here that may help.

1 Like

Thank you so much for your reply! I will review the topic you provided and try the methods you told. Thanks again! :face_with_peeking_eye:

1 Like

@kapolka hi, after a few days of test, I created two m_cameraReflection cameras and two RenderTextures(one for each eye), and I used two global textures in ShaderGraph, using screenPosition as their UVs, and finally used a Branch node with the eye index, connecting it to the fragment base color. I added two buttons to adjust the interpupillary distance, with the default value set to 0.06f. I adjust the position of the reflection cameras, but it doesn’t seem to have any effect. I found that the reflection for the right eye is correct, but the left eye is not. What should I do to make both eyes see the same texture? Or is there something I’m missing that would make the stereo vision work properly for both eyes? :thinking:Do you have any good ideas for this? Thx!

I realized I made a mistake. I’m using this code for the m_cameraReflection cameras rendering, so adjusting the position of the reflection cameras doesn’t work.

// 设置反射相机的视图矩阵
        Matrix4x4 worldToCameraMatrix = m_cameraMain.worldToCameraMatrix * reflection;
        m_cameraReflection.worldToCameraMatrix = worldToCameraMatrix;

How should I modify it to make the stereo vision for both eyes work properly?

Sorry for all the extra replies! :stuck_out_tongue_closed_eyes: I did some more testing and found the problem. The RenderTextures for the left and right eyes are reversed. For example, when I close my left eye and slightly turn my head to the right, I can see an image, because the mirror is rendered in real-time,so let’s call it Pic1. However, when I close my right eye, I need to turn my head to the left to see the same image Pic1. Is there a way to flip the RenderTexture for the left eye? Or is there another way to solve this problem?

I guess I’m not quite sure what you mean by “flip” in this case. If you mean flip left-to-right, you can certain do that (multiply the UV by (-1, 1) and add (1, 0)–and you can use the Eye Index to lerp between the flipped and unflipped coords). It sounds like the issue might be in the Camera matrix, though, based on the way you described it (as if it needs a reflection about the local YZ plane). If you share the code and shader graph, I can take a look.

Sure, I can definitely share my code and shader graph with you, because I really hope to support planar mirror reflections in AVP. I tried adding two buttons to adjust the interpupillary distance (IPD). At a certain moment, I can see the focus correctly, but as soon as I move my headset, the situation becomes less than ideal, and I experience dizziness caused by visual inconsistency between the left and right eyes. Below are my code and shader graph screenshots,

using System;
using TMPro;
using UnityEngine;


//[ExecuteAlways]
public class PlannarReflectionTest2 : MonoBehaviour
{
    public static PlannarReflectionTest2 instance;
    public LayerMask reflectMask = -1;

    [SerializeField] private MeshRenderer m_targetPlaneRenderer;
    [SerializeField] private int m_targetTextureWidth = 1024;
    [SerializeField] private string m_leftReflectTexName = "_MirrorReflectTex_leftEye";
    [SerializeField] private string m_rightReflectTexName = "_MirrorReflectTex_rightEye";
    [SerializeField] private float m_scale = 1.0f;
    [HideInInspector] private float m_clipPlaneOffset = 0.01f;

    private Camera m_cameraMain;
    private Transform m_cameraMainTransform;

    //改为渲染两个renderTexture
    private Camera m_cameraReflectionLeft;
    private Camera m_cameraReflectionRight;
    private Transform m_cameraReflectionTransformLeft;
    private Transform m_cameraReflectionTransformRight;
    public RenderTexture m_leftTargetTexture;
    public RenderTexture m_rightTargetTexture;
    private int m_leftReflectTexInt;
    private int m_rightReflectTexInt;
    public Vector3 leftCameraLocalPos;
    public Vector3 rightCameraLocalPos;

    private Vector3 m_planeNormal;
    private Vector3 m_planePosition;
    private Transform m_normalTransform;
    private Vector4 m_reflectionPlane;

    public float interpupillaryDistance = 0.06f;

    public TMP_Text tMP_Text;

    private void Awake()
    {
        instance = this;
        interpupillaryDistance = 0.06f;
        // 查找主相机
        m_cameraMain = Camera.main;
        m_cameraMainTransform = m_cameraMain.transform;

        
        // var go = new GameObject("ReflectionCamera");
        // m_cameraReflection = go.AddComponent<Camera>();
        // m_cameraReflection.aspect = m_cameraMain.aspect;
        // m_cameraReflection.fieldOfView = m_cameraMain.fieldOfView;
        // m_cameraReflection.enabled = false; // 禁用自动渲染

        // m_cameraReflectionTransform = m_cameraReflection.transform;

        // 创建两个贴图,赋值到反摄像机的 targetTexture
        int newWidth = (int)(m_targetTextureWidth * m_scale);
        int newHeight = (int)(m_targetTextureWidth * m_scale);
        //int newHeight = (int)(newWidth * ((float)Screen.height / Screen.width));
        m_leftTargetTexture = new RenderTexture(newWidth, newHeight, 24, RenderTextureFormat.ARGB32);
        m_rightTargetTexture = new RenderTexture(newWidth, newHeight, 24, RenderTextureFormat.ARGB32);
        m_leftTargetTexture.antiAliasing = 4;
        m_rightTargetTexture.antiAliasing = 4;

        // 创建两个反射相机
        m_cameraReflectionLeft = createReflectionCamera("ReflectCamera_left", m_leftTargetTexture);
        m_cameraReflectionRight = createReflectionCamera("ReflectCamera_right",m_rightTargetTexture);
        
        m_cameraReflectionTransformLeft = m_cameraReflectionLeft.transform;
        m_cameraReflectionTransformRight = m_cameraReflectionRight.transform;
        leftCameraLocalPos = m_cameraReflectionTransformLeft.localPosition;
        rightCameraLocalPos = m_cameraReflectionTransformRight.localPosition;

        // m_cameraReflection.targetTexture = m_targetTexture;
        // m_cameraReflection.cullingMask = reflectMask.value;

        // 创建法线
        m_normalTransform = new GameObject("Normal").transform;
        var planeTransform = m_targetPlaneRenderer.transform;
        m_normalTransform.SetPositionAndRotation(planeTransform.position, planeTransform.rotation);
        m_normalTransform.SetParent(planeTransform);
        m_planePosition = m_normalTransform.position;
        m_planeNormal = m_normalTransform.up;

        // m_reflectTexInt = Shader.PropertyToID(m_reflectTexName);
        // Unity.PolySpatial.PolySpatialShaderGlobals.SetTexture(m_reflectTexName, m_targetTexture);
        //Shader.SetGlobalTexture(m_reflectTexName, m_targetTexture);
        UpdateEyeCameraPos(m_cameraReflectionLeft, m_cameraReflectionRight);

        //设置全局纹理
        m_leftReflectTexInt = Shader.PropertyToID(m_leftReflectTexName);
        Unity.PolySpatial.PolySpatialShaderGlobals.SetTexture(m_leftReflectTexName, m_leftTargetTexture);
         m_rightReflectTexInt = Shader.PropertyToID(m_rightReflectTexName);
        Unity.PolySpatial.PolySpatialShaderGlobals.SetTexture(m_rightReflectTexName, m_rightTargetTexture);
    }

    private Camera createReflectionCamera(string name, RenderTexture renderTexture)
    {
        var go = new GameObject(name);
        var camera = go.AddComponent<Camera>();
        camera.aspect = m_cameraMain.aspect;
        camera.fieldOfView = m_cameraMain.fieldOfView;
        camera.enabled = false; // 禁用自动渲染
        camera.targetTexture = renderTexture;
        camera.cullingMask = reflectMask.value;
        
        return camera;
    }

    public void updatecamera()
    {
        UpdateEyeCameraPos(m_cameraReflectionLeft, m_cameraReflectionRight);
    }

    private void Update()
    {
        tMP_Text.text = "interpupillaryDistance:  " + interpupillaryDistance;
        
        RenderReflection(m_cameraReflectionLeft,m_leftTargetTexture);
        RenderReflection(m_cameraReflectionRight,m_rightTargetTexture);
    }

    private void RenderReflection(Camera reflectionCamera, RenderTexture targetTexture)
    {
        // 判断相机是否在法线下方,如果在下方就不做渲染
        Vector3 localPos = m_normalTransform.worldToLocalMatrix.MultiplyPoint3x4(m_cameraMainTransform.position);
        if (localPos.y < 0) return;

        // 调整位置
        Vector3 normal = m_normalTransform.up;
        Vector3 pos = m_normalTransform.position;

        float d = -Vector3.Dot(normal, pos) - m_clipPlaneOffset;
        m_reflectionPlane = new Vector4(normal.x, normal.y, normal.z, d);

        var reflection = Matrix4x4.identity;
        //reflection *= Matrix4x4.Scale(new Vector3(1, -1, 1));
        CalculateReflectionMatrix(ref reflection, ref m_reflectionPlane);

        // 设置反射相机的视图矩阵
        Matrix4x4 worldToCameraMatrix = m_cameraMain.worldToCameraMatrix * reflection;
        if(reflectionCamera == m_cameraReflectionLeft)
        {
            worldToCameraMatrix = adjustViewMatrixForEye(worldToCameraMatrix, -interpupillaryDistance/2f);
        }
        else if(reflectionCamera == m_cameraReflectionRight)
        {
            worldToCameraMatrix = adjustViewMatrixForEye(worldToCameraMatrix, interpupillaryDistance/2);
        }

        reflectionCamera.worldToCameraMatrix = worldToCameraMatrix;

        // 计算投影矩阵
        Vector3 offsetPos = pos + normal * m_clipPlaneOffset;
        Vector3 cpos = worldToCameraMatrix.MultiplyPoint3x4(offsetPos);
        Vector3 cnormal = worldToCameraMatrix.MultiplyVector(normal).normalized;
        Vector4 clipPlane = new Vector4(cnormal.x, cnormal.y, cnormal.z, -Vector3.Dot(cpos, cnormal));
        reflectionCamera.projectionMatrix = m_cameraMain.CalculateObliqueMatrix(clipPlane);

        // 渲染到目标 RenderTexture
        GL.invertCulling = true;
        //Debug.Log("xuanran camera");
        reflectionCamera.Render();
        GL.invertCulling = false;

        //Shader.SetGlobalTexture(m_reflectTexName, m_targetTexture);
        var m_reflectTexName = targetTexture == m_leftTargetTexture ? m_leftReflectTexName: m_rightReflectTexName;
        Unity.PolySpatial.PolySpatialShaderGlobals.SetTexture(m_reflectTexName, targetTexture);
        reflectionCamera.targetTexture = targetTexture;
        Unity.PolySpatial.PolySpatialObjectUtils.MarkDirty(targetTexture);
    }

    private Matrix4x4 adjustViewMatrixForEye(Matrix4x4 worldToCameraMatrix, float eyeOffset)
    {
        Matrix4x4 translationMatrix = Matrix4x4.Translate(new Vector3(eyeOffset, 0, 0));

        return worldToCameraMatrix * translationMatrix;
    }

    // private void UpdateEyeCameraPos(Camera lefteyecamera, Camera righteyecamera)
    // {
    //     Vector3 offset = new Vector3(interpupillaryDistance / 2f, 0f,0f);
    //     lefteyecamera.transform.localPosition = leftCameraLocalPos - offset;
    //     righteyecamera.transform.localPosition = rightCameraLocalPos + offset; 
    // }
     private void UpdateEyeCameraPos(Camera lefteyecamera, Camera righteyecamera)
    {
        Vector3 offset = new Vector3(interpupillaryDistance / 2f, 0f,0f);
        lefteyecamera.transform.localPosition = leftCameraLocalPos - m_cameraMainTransform.right * offset.x;
        righteyecamera.transform.localPosition = rightCameraLocalPos + m_cameraMainTransform.right * offset.x; 
    }
    private static void CalculateReflectionMatrix(ref Matrix4x4 reflectionMat, ref Vector4 plane)
    {
        reflectionMat.m00 = (1F - 2F * plane[0] * plane[0]);
        reflectionMat.m01 = (-2F * plane[0] * plane[1]);
        reflectionMat.m02 = (-2F * plane[0] * plane[2]);
        reflectionMat.m03 = (-2F * plane[3] * plane[0]);

        reflectionMat.m10 = (-2F * plane[1] * plane[0]);
        reflectionMat.m11 = (1F - 2F * plane[1] * plane[1]);
        reflectionMat.m12 = (-2F * plane[1] * plane[2]);
        reflectionMat.m13 = (-2F * plane[3] * plane[1]);

        reflectionMat.m20 = (-2F * plane[2] * plane[0]);
        reflectionMat.m21 = (-2F * plane[2] * plane[1]);
        reflectionMat.m22 = (1F - 2F * plane[2] * plane[2]);
        reflectionMat.m23 = (-2F * plane[3] * plane[2]);

        reflectionMat.m30 = 0F;
        reflectionMat.m31 = 0F;
        reflectionMat.m32 = 0F;
        reflectionMat.m33 = 1F;
    }
}

and also, the code for adjusting the IPD with two buttons looks like this:

PlannarReflectionTest2.instance.interpupillaryDistance += 0.01f;
PlannarReflectionTest2.instance.updatecamera();

Could you please help me figure out where the problem is? Thx!

Finally, I achieved focus by change the offset.x of two RenderTextures, but I noticed that when I rotate my head around the Z-axis with my head as the coordinate center, there is a slight vertical blur effect. Do you have any good ideas about this? :face_with_peeking_eye:

No, nothing comes immediately to mind.

Thank you so much for your enthusiastic help. It’s great to have Unity staff like you. Thanks again! :smiley: :heart:

1 Like