WebRTC: how to stream audio

Unity Engine
, ,
1

Hi all,

I’d like to stream audio from Unity to a webrtc client. I am using the gstreamer signalling server, and Unity package webrtc 3.0.0-pre6.
Unity makes the SDP offer. The connection between the two peers is ok but I cannot get any sound from the client. It is like no sound is sent.
Here’s what it looks like. Basically WebRTCBase handles the sigalling part.

using System.Collections;
using System.Collections.Generic;
using System.Collections.Concurrent;
using UnityEngine;
using System;

using UnityEngine.UI;
using Unity.WebRTC;


[RequireComponent(typeof(AudioSource))]
public class WebRTCAudioSender : WebRTCBase
{
    [SerializeField] private AudioSource inputAudioSource;

    private MediaStream _sendStream;

    private AudioStreamTrack m_audioTrack;

    const int sampleRate = 48000;

    private AudioClip m_clipInput;

    private RTCRtpSender _sender = null;

    protected override void Start()
    {
        base.Start();
        int m_lengthSeconds = 1;

        string m_deviceName = Microphone.devices[0];
        Debug.Log("Microphone: " + m_deviceName);
        Microphone.GetDeviceCaps(m_deviceName, out int minFreq, out int maxFreq);
        m_clipInput = Microphone.Start(m_deviceName, true, m_lengthSeconds, sampleRate);
        // set the latency to “0” samples before the audio starts to play.
        while (!(Microphone.GetPosition(m_deviceName) > 0)) { }

        inputAudioSource = GetComponent<AudioSource>();
        inputAudioSource.loop = true;
        inputAudioSource.clip = m_clipInput;

        m_audioTrack = new AudioStreamTrack(inputAudioSource);
        m_audioTrack.Loopback = false;
    }


    protected override void WebRTCCall()
    {
        base.WebRTCCall();
        _sendStream = new MediaStream();

        if (_pc != null)
        {
       
            inputAudioSource.Play();
            m_audioTrack = new AudioStreamTrack(inputAudioSource);
            m_audioTrack.Loopback = true;
            Debug.Log("[WebRTCAudioSender] After AudioStreamTrack");
            _sender = _pc.AddTrack(m_audioTrack, _sendStream);
       
        }
    }

    protected override void WebRTCHangUp()
    {
        if (_pc != null && _sender != null)
            _pc.RemoveTrack(_sender);

        //m_audioTrack?.Stop();
        m_audioTrack?.Dispose();
        _sendStream?.Dispose();
        inputAudioSource.Stop();

        base.WebRTCHangUp();
    }

}

That’s the offer I’m sending. I’ve also tested to force only the opus codec (#96).

v=0
o=- 2696995438896112006 2 IN IP4 127.0.0.1
s=-
t=0 0
a=group:BUNDLE 0
a=extmap-allow-mixed
a=msid-semantic: WMS 2d7771d1-b05a-4903-a77c-f29a33ab7e5e
m=audio 54396 UDP/TLS/RTP/SAVPF 96 97 98 99 102 9 0 8 100 101 107 108 109 114 106 105 13 110 112 113 126
c=IN IP4 10.0.1.32
a=rtcp:9 IN IP4 0.0.0.0
a=candidate:3029103603 1 udp 2122262783 2a00:1028:8388:1986:ec33:a025:499:598e 48269 typ host generation 0 network-id 4 network-cost 50
a=candidate:1596389547 1 udp 2122194687 10.0.1.32 54396 typ host generation 0 network-id 3 network-cost 50
a=ice-ufrag:SSzt
a=ice-pwd:hIJWYSG2DJPRq+0GtI0mkSBL
a=ice-options:trickle
a=fingerprint:sha-256 40:51:7A:25:94:95:A3:AE:82:F4:01:22:70:FC:4A:5D:49:5B:2C:66:30:5E:BD:28:8A:6D:94:08:90:48:BD:6D
a=setup:actpass
a=mid:0
a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level
a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid
a=sendrecv
a=msid:2d7771d1-b05a-4903-a77c-f29a33ab7e5e 98e001e7-34bc-4dc7-9ec8-018a4a299a5c
a=rtcp-mux
a=rtpmap:96 opus/48000/2
a=rtcp-fb:96 transport-cc
a=fmtp:96 minptime=10;sprop-stereo=1;stereo=1;useinbandfec=1
a=rtpmap:97 red/48000/2
a=fmtp:97 96/96
a=rtpmap:98 multiopus/48000/6
a=fmtp:98 channel_mapping=0,4,1,2,3,5;coupled_streams=2;minptime=10;num_streams=4;useinbandfec=1
a=rtpmap:99 multiopus/48000/8
a=fmtp:99 channel_mapping=0,6,1,2,3,4,5,7;coupled_streams=3;minptime=10;num_streams=5;useinbandfec=1
a=rtpmap:102 ILBC/8000
a=rtpmap:9 G722/8000
a=rtpmap:0 PCMU/8000
a=rtpmap:8 PCMA/8000
a=rtpmap:100 L16/8000
a=rtpmap:101 L16/16000
a=rtpmap:107 L16/32000
a=rtpmap:108 L16/8000/2
a=rtpmap:109 L16/16000/2
a=rtpmap:114 L16/32000/2
a=rtpmap:106 CN/32000
a=rtpmap:105 CN/16000
a=rtpmap:13 CN/8000
a=rtpmap:110 telephone-event/48000
a=rtpmap:112 telephone-event/32000
a=rtpmap:113 telephone-event/16000
a=rtpmap:126 telephone-event/8000
a=ssrc:330584810 cname:p6EZqE41wmYOKUjs
a=ssrc:330584810 msid:2d7771d1-b05a-4903-a77c-f29a33ab7e5e 98e001e7-34bc-4dc7-9ec8-018a4a299a5c

If I am using this web client I’ve got no sound. That’s what I can see in the chrome webrtc inspector:

9286768--1301605--upload_2023-9-11_17-53-17.png
9286768--1301605--upload_2023-9-11_17-53-17.png988×368 61.9 KB
9286768--1301608--upload_2023-9-11_17-55-46.png
9286768--1301608--upload_2023-9-11_17-55-46.png1189×315 84.1 KB

9286768--1301605--upload_2023-9-11_17-53-17.png
9286768--1301605--upload_2023-9-11_17-53-17.png988×368 61.9 KB

This client works well with the provided streamer example (gst-launch-1.0 webrtcsink name=ws meta="meta,name=robot" pulsesrc ! opusenc ! audio/x-opus, rate=48000, channels=2 ! ws.). An interesting difference is that I can see a codec line there.

9286768--1301608--upload_2023-9-11_17-55-46.png
9286768--1301608--upload_2023-9-11_17-55-46.png1189×315 84.1 KB

So my questions are, is there something to specify on the unity side regarding the codec? is there something to respect during the signalling (SDP before ice candidates exhange…)?

Thanks

2

Please check the “Change Codecs” scene in the package sample.

3

Yes, I’ve already tried this:

           ....
            _sender = _pc.AddTrack(m_audioTrack, _sendStream);
            var codecs = RTCRtpSender.GetCapabilities(TrackKind.Audio).codecs;

            List<RTCRtpCodecCapability> availableCodecs = new List<RTCRtpCodecCapability>();
            foreach (var codec in codecs)
            {
                if (codec.mimeType.Contains("audio/opus"))
                {
                    availableCodecs.Add(codec);
                }
            }

            var transceiver1 = _pc.GetTransceivers().First();
            var error = transceiver1.SetCodecPreferences(availableCodecs.ToArray());
            if (error != RTCErrorType.None)
                Debug.LogError(error);

So the following offer is generated, with opus only. But it is still no producing any sound on the client side

v=0
o=- 8535911314100857197 2 IN IP4 127.0.0.1
s=-
t=0 0
a=group:BUNDLE 0
a=extmap-allow-mixed
a=msid-semantic: WMS ba10a5c6-a05d-40f0-b942-3a69ccd14177
m=audio 40509 UDP/TLS/RTP/SAVPF 96
c=IN IP4 10.0.1.32
a=rtcp:9 IN IP4 0.0.0.0
a=candidate:1412432050 1 udp 2122262783 2a00:1028:8388:1986:ec33:a025:499:598e 50101 typ host generation 0 network-id 4 network-cost 50
a=candidate:829125631 1 udp 2122194687 10.0.1.32 40509 typ host generation 0 network-id 3 network-cost 50
a=ice-ufrag:Ias0
a=ice-pwd:nSrgD/b246OtF8r+zBXgbGp7
a=ice-options:trickle
a=fingerprint:sha-256 1A:2A:4E:F4:B8:81:F3:E7:65:B2:22:5A:ED:smile:A:16:1E:4E:B7:38:43:57:2D:E9:6F:smile:D:A3:94:BA:5F:9E:44:31
a=setup:actpass
a=mid:0
a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level
a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid
a=sendonly
a=msid:ba10a5c6-a05d-40f0-b942-3a69ccd14177 1fadbd9b-3d97-4bd8-ac2b-685f086c3a40
a=rtcp-mux
a=rtpmap:96 opus/48000/2
a=rtcp-fb:96 transport-cc
a=fmtp:96 minptime=10;sprop-stereo=1;stereo=1;useinbandfec=1
a=ssrc:2297734260 cname:IPqP8QlD+K5GMsrv
a=ssrc:2297734260 msid:ba10a5c6-a05d-40f0-b942-3a69ccd14177 1fadbd9b-3d97-4bd8-ac2b-685f086c3a40
4

It is my mistake. I was sending the sdp offer but not processing the answer. So the connection was enabled but not completed.