Outputting Motion Vectors data to image

I can see the Motion Vector data, as color changes and lines using the Rederer Debug window. How would I go about outputting that image (with not lines/vectors) colors to an texture/image ? In other words an optical flow.

I know that it has to do with the camera depth texture and I might need to write a shader. But l can't get any specific information, is that the right solution? I tried to get this shader to work but with no luck:
https://github.com/immersive-limit/Unity-ComputerVisionSim/blob/master/Unity/Assets/ImageSynthesis/Shaders/OpticalFlow.shader

Am headed in the right direction with this?

First of all, motion vectors and optical flow are not the same. Motion vectors describe where certain points move on the screen and can only be generated from 3D data. Optical flow describes where points seem to move based on a 2D analysis of the picture (at least that's how I understand it).
See here:
https://dsp.stackexchange.com/a/34942

If you are fine with the format that Unity's motion vectors are stored (I think they are u/v space vectors), you should be able to just read the motion vector render texture with AsyncGPUReadback and save it with ImageConversion.EncodeToExr(). You need to use EXR because PNG doesn't support floating point values. If you need to do this in real time it might get tricky, though.

I think you can get the motion vector texture by calling Shader.GetGlobalTexture("_CameraMotionVectorsTexture") in the Camera.onPostRender callback, not 100% sure, though.

1 Like

Excellent feedback. Thank you @c0d3_m0nk3y .

1 Like

So I was finally able to get this shader working.

Shader"Unlit/Test"
{
    Properties
    {
        _MainTex ("Texture", 2D) = "white" {}
        [Toggle] _drawArrows("DrawArrows ?", float) = 0

        _MinX ("MinX", Float) = 0
        _MaxX ("MaxX", Float) = 0

        _MinY ("MinX", Float) = 0
        _MaxY ("MaxX", Float) = 0
    }
    SubShader
    {
        Tags { "RenderType"="Opaque" }
        LOD 100

        Pass
        {
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            // make fog work
            #pragma multi_compile_fog

            #include "UnityCG.cginc"

            float _MinX;
            float _MaxX;
            float _MinY;
            float _MaxY;

            struct appdata
            {
                float4 vertex : POSITION;
                float2 uv : TEXCOORD0;
            };

            struct v2f
            {
                float2 uv : TEXCOORD0;
                UNITY_FOG_COORDS(1)
                float4 vertex : SV_POSITION;
            };

            sampler2D _MainTex;
            float4 _MainTex_ST;

            float _drawArrows;
            v2f vert (appdata v)
            {
                v2f o;
                o.vertex = UnityObjectToClipPos(v.vertex);
                o.uv = TRANSFORM_TEX(v.uv, _MainTex);
                UNITY_TRANSFER_FOG(o,o.vertex);
                return o;
            }

            // Debug Motion Vector from https://github.com/Unity-Technologies/Graphics/blob/master/com.unity.render-pipelines.high-definition/Runtime/Debug/DebugFullScreen.shader#L221

            // Motion vector debug utilities
            float DistanceToLine(float2 p, float2 p1, float2 p2)
            {
                float2 center = (p1 + p2) * 0.5;
                float len = length(p2 - p1);
                float2 dir = (p2 - p1) / len;
                float2 rel_p = p - center;
                return dot(rel_p, float2(dir.y, -dir.x));
            }

            float DistanceToSegment(float2 p, float2 p1, float2 p2)
            {
                float2 center = (p1 + p2) * 0.5;
                float len = length(p2 - p1);
                float2 dir = (p2 - p1) / len;
                float2 rel_p = p - center;
                float dist1 = abs(dot(rel_p, float2(dir.y, -dir.x)));
                float dist2 = abs(dot(rel_p, dir)) - 0.5 * len;
                return max(dist1, dist2);
            }

            float2 SampleMotionVectors(float2 coords)
            {
                float4 col = tex2D(_MainTex, coords);

                // In case material is set as no motion
                if (col.x > 1)
                    return 0;
                else
                    return col.xy;
            }

            float DrawArrow(float2 texcoord, float body, float head, float height, float linewidth, float antialias)
            {
                float w = linewidth / 2.0 + antialias;
                float2 start = -float2(body / 2.0, 0.0);
                float2 end = float2(body / 2.0, 0.0);

                // Head: 3 lines
                float d1 = DistanceToLine(texcoord, end, end - head * float2(1.0, -height));
                float d2 = DistanceToLine(texcoord, end - head * float2(1.0, height), end);
                float d3 = texcoord.x - end.x + head;

                // Body: 1 segment
                float d4 = DistanceToSegment(texcoord, start, end - float2(linewidth, 0.0));

                float d = min(max(max(d1, d2), -d3), d4);
                return d;
            }

            #define PI 3.14159265359
            float4 frag (v2f i) : SV_Target
            {
                float motionVectMin = -1;
                float motionVectMax = 1;

                float rgMin = 0;
                float rgMax = 1;

                float motionVectorRange =  motionVectMax - motionVectMin;
                float rgRange =  rgMax - rgMin;

                float2 mv = SampleMotionVectors(i.uv);

                _MaxX = 1.3f; //    max(_MaxX, mv.x);
                _MaxY = max(_MaxY, mv.y);

                _MinX = min(_MinX, mv.x);
                _MinY = min(_MinY, mv.y);

                // Background color intensity - keep this low unless you want to make your eyes bleed
                const float kMinIntensity = 0.03f;
                const float kMaxIntensity = 0.50f;

                // Map motion vector direction to color wheel (hue between 0 and 360deg)
                float phi = atan2(mv.x, mv.y);
                float hue = (phi / PI + 1.0) * 0.5;
                float r = abs(hue * 6.0 - 3.0) - 1.0;
                float g = 2.0 - abs(hue * 6.0 - 2.0);
                float b = 2.0 - abs(hue * 6.0 - 4.0);

                float maxSpeed = 60.0f / 0.15f; // Admit that 15% of a move the viewport by second at 60 fps is really fast
                float absoluteLength = saturate(length(mv.xy) * maxSpeed);
                float3 color = float3(r, g, b) * lerp(kMinIntensity, kMaxIntensity, absoluteLength);
                color = saturate(color);

                if (!_drawArrows)
                    return float4(color, 1);

                // Grid subdivisions - should be dynamic
                const float kGrid = 64.0;

                float arrowSize = 500;
                float4 screenSize = float4(arrowSize, arrowSize, 1.0 / arrowSize, 1.0 / arrowSize);

                // Arrow grid (aspect ratio is kept)
                float aspect = screenSize.y * screenSize.z;
                float rows = floor(kGrid * aspect);
                float cols = kGrid;
                float2 size = screenSize.xy / float2(cols, rows);
                float body = min(size.x, size.y) / sqrt(2.0);
                float2 positionSS = i.uv;
                positionSS *= screenSize.xy;
                float2 center = (floor(positionSS / size) + 0.5) * size;
                positionSS -= center;

                // Sample the center of the cell to get the current arrow vector
                float2 mv_arrow = 0.0f;
#if DONT_USE_NINE_TAP_FILTER
                mv_arrow = SampleMotionVectors(center * screenSize.zw);
#else
                for (int i = -1; i <= 1; ++i) for (int j = -1; j <= 1; ++j)
                    mv_arrow += SampleMotionVectors((center + float2(i, j)) * screenSize.zw);
                mv_arrow /= 9.0f;
#endif
                mv_arrow.y *= -1;

                // Skip empty motion
                float d = 0.0;
                if (any(mv_arrow))
                {
                    // Rotate the arrow according to the direction
                    mv_arrow = normalize(mv_arrow);
                    float2x2 rot = float2x2(mv_arrow.x, -mv_arrow.y, mv_arrow.y, mv_arrow.x);
                    positionSS = mul(rot, positionSS);

                    d = DrawArrow(positionSS, body, 0.25 * body, 0.5, 2.0, 1.0);
                    d = 1.0 - saturate(d);
                }

                // Explicitly handling the case where mv == float2(0, 0) as atan2(mv.x, mv.y) above would be atan2(0,0) which
                // is undefined and in practice can be incosistent between compilers (e.g. NaN on FXC and ~pi/2 on DXC)
                if(!any(mv))
                    color = float3(0, 0, 0);

                return float4(color + d.xxx, 1);
            }
            ENDCG
        }
    }
}

But I am having trouble retrieving the min max values. I will post a question for it in the shaders thread.

Ok I am back to your suggestions here is that I came up with but all it outputs is blue :( Do you have any thoughts for me?

public class MotionVectorManager : MonoBehaviour
{
    int frameNumber = 0;

    public void Start()
    {
        RenderPipelineManager.endCameraRendering += OnEndCameraRendering;
    }

    void OnEndCameraRendering(ScriptableRenderContext context, Camera camera)
    {
        Texture vectorMotionTex = Shader.GetGlobalTexture("_CameraMotionVectorsTexture");
        SaveTextureToFile(vectorMotionTex, "C:/tmp/Image" + frameNumber++ + ".exr");
    }

    static public void SaveTextureToFile(Texture source, string filePath, bool asynchronous = true, System.Action<bool> done = null)
    {
        if (!(source is Texture2D || source is RenderTexture))
        {
            done?.Invoke(false);
            return;
        }

        // resize the original image:
        var resizeRT = RenderTexture.GetTemporary(source.width, source.height, 0, RenderTextureFormat.RGHalf);
        Graphics.Blit(source, resizeRT);

        // create a native array to receive data from the GPU:
        var narray = new NativeArray<byte>(source.width * source.height * 4, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);

        // request the texture data back from the GPU:
        var request = AsyncGPUReadback.RequestIntoNativeArray(ref narray, resizeRT, 0, (AsyncGPUReadbackRequest request) =>
        {
            // if the readback was successful, encode and write the results to disk
            if (!request.hasError)
            {
                NativeArray<byte> encoded;
                encoded = ImageConversion.EncodeNativeArrayToEXR(narray, resizeRT.graphicsFormat, (uint)source.width, (uint)source.height);
                System.IO.File.WriteAllBytes(filePath, encoded.ToArray());
                encoded.Dispose();
            }

            narray.Dispose();

            // notify the user that the operation is done, and its outcome.
            done?.Invoke(!request.hasError);
        });

        if (!asynchronous)
            request.WaitForCompletion();
    }
}

I know the motion vectors are being generated because the above Shader worked. But I need to get the min max information from the data and I can't figure out how to pass that information from the Shader to the CPU. So I tried the above.

Never mind the above actually works ! :) I just must have the format incorrect. The background is all blue and the motion information is there but very very faint :)
Well at least I have something! Thank you @c0d3_m0nk3y I owe you.

1 Like