Display Video Input With Minimal CPU Usage

Questions & Answers
1

I am using Microsoft Hololens 2 and I want to be able to take live video input (via capture card) and display it. Unfortunately I cannot use a RawImage UI object as it won’t display on the Hololens. Moreover, I want the displayed video to intractable so that I can move it around.

I was able to achieve this, but at the cost of CPU usage.

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.SceneManagement;
using System.Linq;

public class VideoInputManager : MonoBehaviour
{
    private const float default_scale = 0.25f;
    private const float default_z_scale = 0.0001f;

    private int device_index = 0;

    public GameObject VideoPanel;

    private WebCamTexture webcamTexture;
    private Texture2D texture;

    private List<WebCamDevice> devices = new List<WebCamDevice>();

    void Start()
    {
        // Get all available devices
        devices.AddRange(WebCamTexture.devices);

        // Check if any devices are available
        if (devices.Count == 0)
        {
            Debug.LogError("No camera devices found.");
            return;
        }

        SelectDevice(device_index);

        // Start playing the camera feed
        webcamTexture.Play();

        // Start the coroutine to update the texture
        StartCoroutine(UpdateTexture());
    }

    void Update()
    {
        if (Input.GetKey(KeyCode.RightArrow))
        {
            SetToNextDeviceIndex();
            SelectDevice(device_index);
        }

        if (Input.GetKey(KeyCode.LeftArrow))
        {
            SetToPreviousDeviceIndex();
            SelectDevice(device_index);
        }
    }

    void OnDisable()
    {
        // Stop the camera feed when the script is disabled
        if (webcamTexture != null)
        {
            webcamTexture.Stop();
        }
    }

    private IEnumerator UpdateTexture()
    {
        while (true)
        {
            // Wait for a new frame
            yield return new WaitUntil(() => webcamTexture.didUpdateThisFrame);

            // Check if the texture needs to be resized
            if (webcamTexture.width != texture.width || webcamTexture.height != texture.height)
            {
                texture.Reinitialize(webcamTexture.width, webcamTexture.height, TextureFormat.RGB24, false);

                // Properly Scale Image
                ScaleImage();
            }

            // Update the texture with the current camera frame
            texture.SetPixels32(webcamTexture.GetPixels32());
            texture.Apply();
        }
    }

    public void SetToNextDeviceIndex() 
    {
        int length = devices.Count;
        device_index = (device_index+1) % length;
        Debug.Log("[NEXT] Setting Index to " + (device_index+1) + "/" + length);
    }

    public void SetToPreviousDeviceIndex()
    {
        int length = devices.Count;
        device_index = (device_index-1) % length;
        Debug.Log("[PREVIOUS] Setting Index to " + (device_index + 1) + "/" + length);
    }

    private void SelectDevice(int index)
    {
        // Stop the current camera feed
        if (webcamTexture != null && webcamTexture.isPlaying)
        {
            webcamTexture.Stop();
        }

        // Select the next device
        WebCamDevice device = devices[index];

        // Create a new WebCamTexture using the selected device
        webcamTexture = new WebCamTexture(device.name);

        // Create a new Texture2D object
        texture = new Texture2D(webcamTexture.width, webcamTexture.height, TextureFormat.RGB24, false);

        // Assign the texture to the cube
        VideoPanel.GetComponent<Renderer>().material.mainTexture = texture;

        // Properly Scale Image
        ScaleImage();

        // Start playing the camera feed
        webcamTexture.Play();
    }


    private void ScaleImage() {
        // Get the aspect ratio of the video input
        float videoAspectRatio = (float)webcamTexture.width / (float)webcamTexture.height;

        // Calculate the scale factor based on the aspect ratio and desired scale
        float scaleFactor = Mathf.Min(default_scale / webcamTexture.width, default_scale / webcamTexture.height);
        float scaledWidth = webcamTexture.width * scaleFactor;
        float scaledHeight = webcamTexture.height * scaleFactor;

        // Scale the cube proportionally & flip vertically & horizontally
        VideoPanel.transform.localScale = new Vector3(-scaledWidth, scaledHeight, -default_z_scale);
    }
}

I was able to determine (using the profiler) that the following lines of code use 26% of the overall CPU.

    private IEnumerator UpdateTexture()
    {
        while (true)
        {
            // Wait for a new frame
            yield return new WaitUntil(() => webcamTexture.didUpdateThisFrame);

            // Check if the texture needs to be resized
            if (webcamTexture.width != texture.width || webcamTexture.height != texture.height)
            {
                texture.Reinitialize(webcamTexture.width, webcamTexture.height, TextureFormat.RGB24, false);

                // Properly Scale Image
                ScaleImage();
            }

            // Update the texture with the current camera frame
            texture.SetPixels32(webcamTexture.GetPixels32());
            texture.Apply();
        }
    }

How can I optimize/what can I do to minimize CPU usage?