Streaming in Unity - Poor performance issue.

Hi there !

Three days ago I started to try to make my own streaming / video chat in Unity thinking it will be quick and easy ( ha-ha-ha ). Actually I have kind of “succeed”. My server run the device webcam, transform the texture in arrays of bytes and send them to the client for reconstruction. The client receive it and - woohoo - I have my image !

{
}

My problem : the time taken for the entire process for one image. It take about 6-7 seconds to send it… What a stream ! erk.

{
}

Maybe I made a big mistake in my script but I’m so “in it” since 3 days I can’t figure it out.
If someone could provide some help to figure why the performance are so bad - or maybe my logic is bad.

{
}

Here is the core part of the Script on the “Screen”, actually a plane with a texture

public override void OnStartServer()
{

    //Webcam initialisation,  so only on server.
    webcamTexture = new WebCamTexture();
    serverRenderer = GetComponent<Renderer>();
    serverRenderer.material.mainTexture = webcamTexture;
    webcamTexture.Play();

    currentTexture = new Texture2D(webcamTexture.width, webcamTexture.height);

    //mean the server can send an image
    serverReadyToStream = true;

    h = currentTexture.height;
    w = currentTexture.width ;

    //the transmitter is responsible of send bytes arrays in right order
    transmitter = GetComponent<Transmitter>();

    //Some event to catch  when datas are sent
    transmitter.OnDataComepletelySent += StreamCompletelySent;
    transmitter.OnDataFragmentSent += StreamFragmentSent;

}

public void Start()
{
    this.transform.position = new Vector3(-10, 2, 25);
    this.transform.rotation = Quaternion.Euler(90, 0, 180);

    //return if server, i will set up client stuff 
    if (isServer)
    {
        return;
    }

    h = 480;
    w = 640;

    transmitter = GetComponent<Transmitter>();

    //event to catch when datas are received
    transmitter.OnDataCompletelyReceived += StreamCompletelyReceived;
    transmitter.OnDataFragmentReceived += StreamFragmentReceived;

     //client plane to put the texture on
    clientScreen = GameObject.FindObjectOfType<myScreen>();
    clientRenderer = clientScreen.GetComponent<Renderer>();

}

public void Update()
{
   
    if (isServer)
    {

        if (serverReadyToStream)
        {            

             //update texture
              currentTexture.SetPixels(webcamTexture.GetPixels());
            //change texture to a byte array 
            colorInByte = currentTexture.EncodeToPNG();

            serverReadyToStream = false;
            currentTransmission++;
            Debug.Log("New data transmission started with id... " + currentTransmission);
            StartCoroutine(transmitter.SendBytesToClientsRoutine(currentTransmission, colorInByte));
        }
    }

}

[Server]
private void StreamCompletelySent(int transmissionId, byte[] data)
{
    Debug.Log(" Stream COMPLETELY Sended " + transmissionId);
    serverReadyToStream = true;
}

[Server]
private void StreamFragmentSent(int transmissionId, byte[] data)
{
  //  Debug.Log(" Stream Fragment Sended " + transmissionId);
}

[Client]
private void StreamCompletelyReceived ( int transmissionId, byte[] data )
{
    Debug.Log(" Stream COMPLETELY Received " + transmissionId);

     //server don't need the stream
    if (isServer)
    {
        return;
    }

    //image reconstruction for the client
    receivedTexture = new Texture2D(w, h);
    receivedTexture.LoadImage(data);
    clientRenderer.material.mainTexture = receivedTexture;
    clientScreen.clientReadyToStream = true;

}

[Client]
private void StreamFragmentReceived ( int transmissionId, byte[] data )
{
}

And there it is the transmitter code, same as a script publish in an old post about sending big data.

public class Transmitter : NetworkBehaviour
{

private static readonly string LOG_PREFIX = "[" + typeof(Transmitter).Name + "]: ";
public const int RELIABLE_SEQUENCED_CHANNEL = 0;
private static int defaultBufferSize = 1024; //max ethernet MTU is ~1400

private class TransmissionData
{
    public int curDataIndex; //current position in the array of data already received.
    public byte[] data;

    public TransmissionData(byte[] _data)
    {
        curDataIndex = 0;
        data = _data;
    }
}

//list of transmissions currently going on. a transmission id is used to uniquely identify to which transmission a received byte[] belongs to.
List<int> serverTransmissionIds = new List<int>();

//maps the transmission id to the data being received.
Dictionary<int, TransmissionData> clientTransmissionData = new Dictionary<int, TransmissionData>();

//callbacks which are invoked on the respective events. int = transmissionId. byte[] = data sent or received.
public event UnityAction<int, byte[]> OnDataComepletelySent;
public event UnityAction<int, byte[]> OnDataFragmentSent;
public event UnityAction<int, byte[]> OnDataFragmentReceived;
public event UnityAction<int, byte[]> OnDataCompletelyReceived;

[Server]
public void SendBytesToClients(int transmissionId, byte[] data)
{
    Debug.Assert(!serverTransmissionIds.Contains(transmissionId));
    StartCoroutine(SendBytesToClientsRoutine(transmissionId, data));
}

[Server]
public IEnumerator SendBytesToClientsRoutine(int transmissionId, byte[] data)
{

    Debug.Assert(!serverTransmissionIds.Contains(transmissionId));
    Debug.Log(LOG_PREFIX + "SendBytesToClients processId=" + transmissionId + " | datasize=" + data.Length);

    //tell client that he is going to receive some data and tell him how much it will be.
    RpcPrepareToReceiveBytes(transmissionId, data.Length);
    yield return null;

    //begin transmission of data. send chunks of 'bufferSize' until completely transmitted.
    serverTransmissionIds.Add(transmissionId);
    TransmissionData dataToTransmit = new TransmissionData(data);
    int bufferSize = defaultBufferSize;
    while (dataToTransmit.curDataIndex < dataToTransmit.data.Length - 1)
    {
        //determine the remaining amount of bytes, still need to be sent.
        int remaining = dataToTransmit.data.Length - dataToTransmit.curDataIndex;
        if (remaining < bufferSize)
            bufferSize = remaining;

        //prepare the chunk of data which will be sent in this iteration
        byte[] buffer = new byte[bufferSize];
        System.Array.Copy(dataToTransmit.data, dataToTransmit.curDataIndex, buffer, 0, bufferSize);

        //send the chunk
        RpcReceiveBytes(transmissionId, buffer);
        dataToTransmit.curDataIndex += bufferSize;

        yield return null;

        if (null != OnDataFragmentSent)
            OnDataFragmentSent.Invoke(transmissionId, buffer);
    }

    //transmission complete.
    serverTransmissionIds.Remove(transmissionId);

    if (null != OnDataComepletelySent)
        OnDataComepletelySent.Invoke(transmissionId, dataToTransmit.data);
}

[ClientRpc]
private void RpcPrepareToReceiveBytes(int transmissionId, int expectedSize)
{
    if (clientTransmissionData.ContainsKey(transmissionId))
        return;

    //prepare data array which will be filled chunk by chunk by the received data
    TransmissionData receivingData = new TransmissionData(new byte[expectedSize]);
    clientTransmissionData.Add(transmissionId, receivingData);
}

//use reliable sequenced channel to ensure bytes are sent in correct order
[ClientRpc(channel = RELIABLE_SEQUENCED_CHANNEL)]
private void RpcReceiveBytes(int transmissionId, byte[] recBuffer)
{
    //already completely received or not prepared?
    if (!clientTransmissionData.ContainsKey(transmissionId))
        return;

    //copy received data into prepared array and remember current dataposition
    TransmissionData dataToReceive = clientTransmissionData[transmissionId];
    System.Array.Copy(recBuffer, 0, dataToReceive.data, dataToReceive.curDataIndex, recBuffer.Length);
    dataToReceive.curDataIndex += recBuffer.Length;

    if (null != OnDataFragmentReceived)
        OnDataFragmentReceived(transmissionId, recBuffer);

    if (dataToReceive.curDataIndex < dataToReceive.data.Length - 1)
        //current data not completely received
        return;

    //current data completely received
    Debug.Log(LOG_PREFIX + "Completely Received Data at transmissionId=" + transmissionId);
    clientTransmissionData.Remove(transmissionId);

    if (null != OnDataCompletelyReceived)
        OnDataCompletelyReceived.Invoke(transmissionId, dataToReceive.data);
}

Thanks for reading ! :slight_smile:
and thanks in advance for any clue or solution !

I recommended you to try this plugin. As Unet will be depreciated in future, we use basic TCP solution for streaming.

FM Exhibition Tool Pack | Forum

Provides demos of in-game view/live video, audio streaming solution. All codes are written in C# and you can freely customise for your own project.

Supported Platforms: iOS/Android/Mac/PC