Can I override some PlayerLoopSystems?

I want to change some PlayerLoopSystem conditionally. I am using unity to render picture from camera and I want it to be very low latency.

As far as I know, In Unity’s PlayerLoop, it presents (calling Graphics.PresentAndSync) rendered-frame from last PlayerLoop, which brings about 1-frame latency.

For better understandings, these are some important systems in a PlayerLoop:

  • EarlyUpdate.PresentAndSync => present frame rendered from last PlayerLoop.

  • FixedUpdate => before update and handles physics

  • Update => most common function to handle every frame’s change

  • PostLateUpdate.FinishFrameRendering => main rendering process such as culling/drawing/GUI rendering.

  • TimeUpdate.WaitForLastPresentationAndUpdateTime => wait until current interval of fps (eg. 16.67ms for 60fps)

When I received new picture from Android camera, I want unity to render and present AFAP.

And I use newCameraAvaliableAfterDrawing to denote there is new frame after current drawing:

private static bool newCameraAvaliableAfterDrawing = false;
private static presented = false;
private static onFrameAvailable() { // called from Android/jni
    newCameraAvaliableAfterDrawing = true;
}

So I want to solve this problem by overriding the default PlayerLoopSystems in these ways:

  1. insert a system after PostLateUpdate.FinishFrameRendering:
mySystemAfterFinishFrameRendering() {
    if (newCameraAvaliableAfterDrawing) {
        call update_Texture_to_unity();
        PostLateUpdate.FinishFrameRendering.Invoke();
        call Graphics.PresentAndSync or call jni.eglSwapBuffers();
        presented = true;
    }
}
  1. optionally overrides EarlyUpdate.PresentAndSync:
myPresent() {
    if (presented) {
        // skip
    } else {
        call original EarlyUpdate.PresentAndSync
    }
}
  1. skip original TimeUpdate.WaitForLastPresentationAndUpdateTime:
static void CustomWaitForLastPresentationAndUpdateTimeFunction()
{
    if (newCameraAvaliableAfterDrawing) {
        // skip;
    } else {
        // origWaitForFpsFunctionDelegate?.Invoke();
        Thread.Sleep(10);
    }
}

I have tried these methods:

  • method 1, partially worked, I can present the frame using jni.eglSwapBuffers();
  • method 2, when I call original EarlyUpdate.PresentAndSync it crashes.
  • method 3, when I call origWaitForFpsFunctionDelegate tie crashes, when I only sleep, the frame not shows.

Here’s my sample code:

[RuntimeInitializeOnLoadMethod]
static void Initialize()
{
    try
    {
        var playerLoop = PlayerLoop.GetCurrentPlayerLoop();

        var earlyUpdateSystemIndex = Array.FindIndex(playerLoop.subSystemList, s => s.type == typeof(EarlyUpdate));
        var earlyUpdateSystem = playerLoop.subSystemList[earlyUpdateSystemIndex];

        var presentBeforeUpdateSystemIndex = Array.FindIndex(earlyUpdateSystem.subSystemList, s => s.type == typeof(EarlyUpdate.PresentBeforeUpdate));
        var presentBeforeUpdateSystem = earlyUpdateSystem.subSystemList[presentBeforeUpdateSystemIndex];

        if (override_earlyupdate) {
            originalPresentBeforeUpdateFunctionPtr = presentBeforeUpdateSystem.updateFunction;
            if (originalPresentBeforeUpdateFunctionPtr != IntPtr.Zero)
            {
                originalPresentBeforeUpdateFunction = Marshal.GetDelegateForFunctionPointer<PlayerLoopSystem.UpdateFunction>(originalPresentBeforeUpdateFunctionPtr);
            }

            // customUpdateFunctionDelegate = new PlayerLoopSystem.UpdateFunction(CustomPresentBeforeUpdate);
            // presentBeforeUpdateSystem.updateFunction = Marshal.GetFunctionPointerForDelegate(customUpdateFunctionDelegate);

            var customPresentSystem = new PlayerLoopSystem
            {
                type = typeof(CustomPresent),
                updateDelegate = CustomPresentBeforeUpdate
            };  

            earlyUpdateSystem.subSystemList[presentBeforeUpdateSystemIndex] = customPresentSystem;//presentBeforeUpdateSystem;
            playerLoop.subSystemList[earlyUpdateSystemIndex] = earlyUpdateSystem;
        }

        var postLateUpdateIndex = Array.FindIndex(playerLoop.subSystemList, s => s.type == typeof(PostLateUpdate));
        if (override_timeupdate) {
            var timeUpdateSystemIndex = Array.FindIndex(playerLoop.subSystemList, s => s.type == typeof(TimeUpdate));
            var timeUpdateSystem = playerLoop.subSystemList[timeUpdateSystemIndex];

            var waitForLastPresentSystemIndex = Array.FindIndex(timeUpdateSystem.subSystemList, s => s.type == typeof(TimeUpdate.WaitForLastPresentationAndUpdateTime));
            var waitForLastPresentSystem = timeUpdateSystem.subSystemList[waitForLastPresentSystemIndex];

            origWaitForFpsFunctionPtr = waitForLastPresentSystem.updateFunction;
            if (origWaitForFpsFunctionPtr != IntPtr.Zero)
            {
                origWaitForFpsFunction = Marshal.GetDelegateForFunctionPointer<PlayerLoopSystem.UpdateFunction>(origWaitForFpsFunctionPtr);
            }
            origWaitForFpsFunctionDelegate = waitForLastPresentSystem.updateDelegate;

            var customWaitForLastPresentationAndUpdateTimeSystem = new PlayerLoopSystem
            {
                type = typeof(CustomWaitForLastPresentationAndUpdateTime),
                updateDelegate = new PlayerLoopSystem.UpdateFunction(CustomWaitForLastPresentationAndUpdateTimeFunction)
            };
            timeUpdateSystem.subSystemList[waitForLastPresentSystemIndex] = customWaitForLastPresentationAndUpdateTimeSystem;
            playerLoop.subSystemList[timeUpdateSystemIndex] = timeUpdateSystem;

            Debug.Log($"timeUpdateSystem: {timeUpdateSystem}, waitForLastPresentSystem:{waitForLastPresentSystem}, origWaitForFpsFunction:{origWaitForFpsFunction}");
        }

        if (override_lateupdate) {
            var postLateUpdate = playerLoop.subSystemList[postLateUpdateIndex];

            var customUpdateSystem = new PlayerLoopSystem
            {
                type = typeof(CustomUpdate),
                updateDelegate = CustomUpdateMethod_static
            };

            var presentAfterDrawIndex = Array.FindIndex(postLateUpdate.subSystemList, s => s.type == typeof(PostLateUpdate.PresentAfterDraw));

            var newSubSystems = new PlayerLoopSystem[postLateUpdate.subSystemList.Length + 1];
            Array.Copy(postLateUpdate.subSystemList, 0, newSubSystems, 0, presentAfterDrawIndex + 1);
            newSubSystems[presentAfterDrawIndex + 1] = customUpdateSystem;
            Array.Copy(postLateUpdate.subSystemList, presentAfterDrawIndex + 1, newSubSystems, presentAfterDrawIndex + 2, postLateUpdate.subSystemList.Length - presentAfterDrawIndex - 1);
            
            postLateUpdate.subSystemList = newSubSystems;
            playerLoop.subSystemList[postLateUpdateIndex] = postLateUpdate;

            Debug.Log($"EarlyUpdate system: {earlyUpdateSystem.type}, PresentBeforeUpdate system replaced: {presentBeforeUpdateSystem.type}");
            Debug.Log($"PostLateUpdate system: {postLateUpdate.type}, CustomUpdate system inserted after PresentAfterDraw");
        }

        PlayerLoop.SetPlayerLoop(playerLoop);
    }
    catch (Exception ex)
    {
        Debug.LogError($"An error occurred while initializing the custom player loop: {ex.Message}\n{ex.StackTrace}");
    }
}