@Franz909 @Whlam Hello, I’m also trying to port it over to the Hololens I’m receiving the same issue. Have you been able to find a solution? If not I’ve been able to send UDP packets to control a servo from the esp8266 Wi-Fi module and using the ArduCam OV2640 2MP Camera to capture video using this code in Unity. I will attempt to use the same coding structure and hopefully, that will bypass the compilation errors but still run on the HoloLens as it still can access libraries from the .NET framework. You can follow my progress on my blog www.arvibe.com.
Update: @Franz909 @Whlam
I’ve posted this as an answer to my question back on Dec 2 of 2018 but it’s awaiting moderation… don’t know why? But here you go!
I was working on creating my own memory stream and stream.ReadByte() since I can’t seem to get it to build for the HoloLens. I’m also using UnitWebRequest and custom DownloadHandlerScript. There are also some links in the resources that may already achieve this as well but some have not had much success with implementation.
But for a simpler answer for IP-cameras that are just sending JPEGS. I’m not sure if this is the best method but it works on the HoloLens and any other device that has build issues using what’s previously mentioned.
- image-based IP cameras only (ImageStream.cs)
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
//stream for image-based IP cameras
//Open Global IP's to try
//Busy Parking lot
//http://107.144.24.100:88/record/current.jpg?rand=509157
//Beach
//http://107.144.24.100/record/current.jpg?rand=291145
public class ImageStream : MonoBehaviour {
public string uri = "http://107.144.24.100/record/current.jpg?rand=291145";
public RawImage frame;
private Texture2D texture;
// Use this for initialization
void Start () {
texture = new Texture2D(2, 2);
StartCoroutine(GetImage());
}
IEnumerator GetImage()
{
yield return new WaitForEndOfFrame();
using(WWW www = new WWW(uri))
{
yield return www;
www.LoadImageIntoTexture(texture);
frame.texture = texture;
Start();
}
}
// Update is called once per frame
void Update()
{
}
}
Then you have IP cameras that continuously sending bytes of data through an open stream. Here I use UnityWebRequest instead of WWW in a MonoBehavior script and a custom DownloadHandlerScript to access the downloaded content and convert the bytes to get the ‘content-length’ and read the JPEG beginning byte “FFD8” and ending byte “FFD9” and load it into a RawImage Texture.
- Streaming IP cameras (WebStream.cs)
using UnityEngine;
using System.Collections;
using UnityEngine.Networking;
using UnityEngine.UI;
using System;
//Open IP-camera address for nice beach in Spain
// http://212.170.100.189:80/mjpg/video.mjpg
public class WebStream : MonoBehaviour
{
//Custom download handler script
CustomWebRequest customWebRequest;
//Web request
UnityWebRequest webRequest;
[Tooltip("Set this to view the video")]
public RawImage frame;
[Tooltip("Set this to change the url of the streaming IP camera")]
public string url = "http://212.170.100.189:80/mjpg/video.mjpg";
[Tooltip("Set this to change the servo connection")]
public string servoURL = "SERVO-IP-ADDRESS";
[Tooltip("Set this to change the received data buffer increases lag")]
public int BufferSize = 100000;
[Tooltip("Connect button to connect to device")]
public Button ConnectButton;
[Tooltip("Connecting button status")]
public Button ConnectingButton;
[Tooltip("Disconnect button to disconnect from device")]
public Button DisconnectButton;
[Tooltip("IP address input field")]
public InputField InputIPAddress;
[Tooltip("Pass rate field decreases lag increases distortion")]
public InputField InputPassRate;
[Tooltip("Buffer rate input by user during runtime")]
public InputField InputBufferRate;
[Tooltip("Received data pass rate input by user during runtime")]
public InputField InputRxDataRate;
[Tooltip("Text to display pass percentage")]
public Text PercentageDisplay;
[Tooltip("Information panel to display results")]
public Image InfoPanel;
public Vector3 rightLeft;
public Vector3 upDown;
private string moveData;
byte[] byteArray;
byte[] bytes = new byte[100000];
//Connect button pressed
public void onConnect()
{
bytes = new byte[BufferSize];
if (InputIPAddress.textComponent.text == "")
{
return;
}
url = InputIPAddress.textComponent.text;
Debug.Log(url);
ConnectButton.gameObject.SetActive(false);
InputBufferRate.gameObject.SetActive(false);
InputRxDataRate.gameObject.SetActive(true);
customWebRequest = new CustomWebRequest(bytes);
GetVideo();
StartCoroutine(initConnecting());
}
//Waits 6 seconds for the first image to be downloaded then continuous streaming
IEnumerator initConnecting()
{
ConnectingButton.gameObject.SetActive(true);
yield return new WaitForSeconds(6);
if(customWebRequest.Connected == true)
{
ConnectingButton.gameObject.SetActive(false);
DisconnectButton.gameObject.SetActive(true);
}
else
{
ConnectingButton.gameObject.SetActive(false);
onDisconnect();
PercentageDisplay.text = "Please check connection!!!";
}
}
//Disconnects from source and dispose of request
public void onDisconnect()
{
InputIPAddress.textComponent.text = "";
DisconnectButton.gameObject.SetActive(false);
ConnectButton.gameObject.SetActive(true);
InputBufferRate.gameObject.SetActive(true);
InputRxDataRate.gameObject.SetActive(false);
PercentageDisplay.text = "change buffer rate only when disconnected";
webRequest.Dispose();
}
//Checks if bytes length is certain percentage of content-length to display image
public void onPassRateEdit(string passRate)
{
if (InputPassRate.text == "")
{
return;
}
float updateRate = float.Parse("." + InputPassRate.text);
customWebRequest.ImagePassPercent = updateRate;
}
//Sets buffer rate
public void onBufferRate(string BufferRate)
{
if (InputBufferRate.text == "")
{
return;
}
BufferSize = Int32.Parse(InputBufferRate.text);
customWebRequest.bufferSize = BufferSize; ;
}
//Sets received data rate
public void onRxDataRate(string passRate)
{
if (InputRxDataRate.text == "")
{
return;
}
float RxDataRate = float.Parse("." + InputRxDataRate.text);
customWebRequest.RxDataPassRate = RxDataRate; ;
}
//initialization
public void Start()
{
InputIPAddress.text = url;
}
//moves servos not (needed this was for IoT device)
public void move(string postData)
{
moveData = postData;
string data = "?move=" + postData;
UnityWebRequest www = new UnityWebRequest(servoURL + data);
www.Send();
if (www.isNetworkError || www.isHttpError)
{
Debug.Log(www.error);
}
else
{
// Debug.Log("Upload complete!");
}
}
//connect to URL and assign custom download handler script to the connections download handler ;
public void GetVideo()
{
webRequest = new UnityWebRequest(url);
webRequest.downloadHandler = customWebRequest;
webRequest.Send();
}
//update every frame
private void Update()
{ //update the information panel
if(!ConnectButton.IsActive())
{
PercentageDisplay.text = "Pass Rate: " + customWebRequest.ImagePassPercent * 100 + "%" +
"
Rx Pass Rate: " + customWebRequest.RxDataPassRate * 100 + “%”+
"
Connected: " + customWebRequest.Connected +
"
Content-Length: " + customWebRequest.contentLength +
"
Rx Data: " + customWebRequest.RxDataLength +
"
JPEG: " + customWebRequest.JPEGsize +
"
Buffer: " + customWebRequest.bufferSize +
"
moving: " + moveData;
}
//moves camera attached to servo's on IOT ip camera
if (Input.GetKey(KeyCode.A))
move("left");
if (Input.GetKey(KeyCode.D))
move("right");
if (Input.GetKey(KeyCode.W))
move("up");
if (Input.GetKey(KeyCode.S))
move("down");
if (Input.GetKey(KeyCode.LeftShift))
move("center");
/*
#elif UNITY_IOS
rightLeft = new Vector3(0, Input.acceleration.x, 0);
upDown = new Vector3(Input.acceleration.y, 0, 0);
if (Input.acceleration.x > .2f)
move("right");
if (Input.acceleration.x < -.2f)
move("left");
if (Input.acceleration.y < -.95f)
move("up");
if (Input.acceleration.y > -.75f)
move("down");
#endif
*/
}
//Close the connection if the application is closed
private void OnApplicationQuit()
{
if(webRequest!=null){
webRequest.Dispose();
}
}
//On focus display information panel to screen view.
private void OnApplicationFocus(bool focus)
{
if (focus == false)
{
InputPassRate.gameObject.SetActive(false);
InfoPanel.gameObject.SetActive(false);
InputBufferRate.gameObject.SetActive(false);
}
else
{
InputPassRate.gameObject.SetActive(true);
InfoPanel.gameObject.SetActive(true);
InputBufferRate.gameObject.SetActive(true);
}
}
}
- Custom DownloadHanderScript (CustomeWebRequest.cs)
using System;
using System.Text;
using UnityEngine;
using UnityEngine.Networking;
using UnityEngine.UI;
//custom download handler request
public class CustomWebRequest : DownloadHandlerScript
{
/////////////////////////////////////////////////////////
/// Standard scripted download handler -
/// allocates memory on each ReceiveData callback.
//////////////////////////////////////////////////////////////////
GameObject Image;
public float ImagePassPercent = .50f;
public float RxDataPassRate = .50f;
public int contentLength;
public int JPEGsize;
public int bufferSize;
public int RxDataLength;
public byte[] RxData;
public bool Connected = false;
WebStream webStream;
Texture2D camTexture = new Texture2D(2, 2);
public float timeElapsed;
public int prevLength;
public CustomWebRequest() : base()
{
}
////////////////////////////////////////////////////////////////////
/// Pre-allocated scripted download handler reuses
/// the supplied byte array to deliver
/// data and eliminates memory allocation.
/////////////////////////////////////////////////////////////
public CustomWebRequest(byte[] buffer) : base(buffer)
{
}
////////////////////////////////////////////////////
/// Required by DownloadHandler base class.
/// Called when you address the 'bytes' property.
///////////////////////////////////////////////////
protected override byte[] GetData() { return null; }
///////////////////////////////////////////////////
/// Called once per frame when data
/// has been received from the network.
//////////////////////////////////////////////////
protected override bool ReceiveData(byte[] byteFromCamera, int dataLength)
{
RxData = byteFromCamera;
RxDataLength = dataLength; //return dataLength to webstream to display on screen.
bufferSize = byteFromCamera.Length; //return bufferSize to be displayed on screen.
if (byteFromCamera == null || byteFromCamera.Length < 1)
{
Debug.Log("CustomWebRequest :: ReceiveData - received a null/empty buffer");
return false;
}
contentLength = FindLength(byteFromCamera); // find the length of the JPEG
Debug.Log(contentLength);
if (contentLength == 0)
{
//Debug.Log("Not enough Bytes to read!");
byteFromCamera = new byte[bufferSize];
return true;
}
//string inputStream = Encoding.Default.GetString(byteFromCamera);
//inputStream = ToHexString(inputStream);
string inputStream = BitConverter.ToString(byteFromCamera).Replace("-", "");
//receieving array same size as bytes from camera
byte[] RecievedImageByte = new byte[inputStream.Length / 2];
//find the begining of the JPEG and trim excess
inputStream = inputStream.Substring(inputStream.IndexOf("FFD8")).Trim();
//if bytes contains end of JPEG find get begining and end
//else return true and new buffer
if (inputStream.Contains("FFD9"))
{
inputStream = inputStream.Substring(inputStream.IndexOf("FFD8"), inputStream.IndexOf("FFD9") + 4);
}
else
{
byteFromCamera = new byte[bufferSize];
return true;
}
//convert string to byte committing changes
if (StringToByteArray(inputStream) != null)
{
RecievedImageByte = StringToByteArray(inputStream);
}
else { return true; }
//We want the beginning of the JPEG file
if (inputStream.IndexOf("FFD8") == 0)
{
//Check to see if the stream and length are readable (Needs tuning)
if (imageReadable(contentLength, inputStream))
{
//create the image array with the length of the inputStream
byte[] completeImageByte = new byte[inputStream.Length];
DisplayImage(RecievedImageByte, completeImageByte, inputStream);
RecievedImageByte = new byte[inputStream.Length / 2];
completeImageByte = new byte[bufferSize];
inputStream = "";
}
else
{
Debug.Log("Image not readable!");
}
}
//return a new buffer
byteFromCamera = new byte[bufferSize];
return true;
}
//Call from WebStream to get content-length
public int ReceivedCameraData()
{
return contentLength;
}
////////////////////////////////////////////////////
/// Returns if the image is readable
///////////////////////////////////////////////////
public bool imageReadable(int contentLength, string inputStream)
{
float passRate;
float dataPassRate;
passRate = contentLength * ImagePassPercent;
dataPassRate = contentLength * RxDataPassRate;
if (inputStream.Length / 2 > contentLength || inputStream.Length / 2 < passRate || RxDataLength < dataPassRate)
{
return false;
}
return true;
}
//////////////////////////////
/// Display Image
/////////////////////////////
public bool DisplayImage(byte[] RecievedImageByte, byte[] completeImageByte, string inputStream)
{
//BlockCopy in place of Memory Stream
Buffer.BlockCopy(RecievedImageByte, inputStream.IndexOf("FFD8"), completeImageByte, 0, inputStream.Length / 2);
//We have to use GameObject.Find
//unless we instantiate the a prefab I'll add it later
Image = GameObject.Find("RawImage");
WebStream webStream = Image.GetComponent<WebStream>();
RawImage screenDisplay = webStream.frame;
JPEGsize = inputStream.Length / 2;
//Load bytes into texture
camTexture.LoadImage(completeImageByte);
Connected = true;
//Assign the texture the RawImage GameObject
screenDisplay.color = Color.white;
screenDisplay.texture = camTexture;
//return that it was successful
return true;
}
//////////////////////////////////////////////////////
/// Called when all data has been received
/// from the server and delivered via ReceiveData.
//////////////////////////////////////////////////////////////
protected override void CompleteContent()
{
Debug.Log("CustomWebRequest :: CompleteContent - DOWNLOAD COMPLETE!");
}
////////////////////////////////////////////
/// Called when a Content-Length
/// header is received from the server.
/////////////////////////////////////////////////
protected override void ReceiveContentLength(int contentLength)
{
Debug.Log(string.Format("CustomWebRequest :: ReceiveContentLength - length {0}", contentLength));
}
/////////////////////////////////////////////////////////////////////
/// Finds the length of the JPEG image received
///////////////////////////////////////////////////////////////////
public int FindLength(byte[] bytesReceived)
{
int position = 1;
int content = 0;
string inputStream = "";
//convert stream to string hex
inputStream = BitConverter.ToString(bytesReceived).Replace("-", "");
//find "h: " of "content-length: " and 6 postions after
//for content-length size
position = inputStream.IndexOf("683A20") + 6;
int contentToRead = inputStream.IndexOf("FFD8") - position;
if (inputStream.Contains("683A20"))
{
if (contentToRead > 0)
{
string contentLength = inputStream.Substring(position, contentToRead);
if (contentLength.Length > 32) { return 0; }
content = Int32.Parse(FromHexString(contentLength));
}
}
return content;
}
/////////////////////////////////////////////////////////////
/// Converts String back to a Byte Array.
///////////////////////////////////////////////////////////
public static byte[] StringToByteArray(string hex)
{
int count = 0;
while (hex.Length % 2 != 0)
{
hex = hex.Insert(hex.Length - 7, "0");
count++;
if (count == 5) { return null; }
}
int NumberChars = hex.Length;
byte[] bytes = new byte[NumberChars / 2];
for (int i = 0; i < NumberChars; i += 2)
bytes[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16);
return bytes;
}
//////////////////////////////////////////////////////////////
/// Converts String back to a Byte Array.
////////////////////////////////////////////////////////////
public static string FromHexString(string hexString)
{
var bytes = new byte[hexString.Length / 2];
for (var i = 0; i < bytes.Length; i++)
{
bytes _= Convert.ToByte(hexString.Substring(i * 2, 2), 16);_
}
return Encoding.UTF8.GetString(bytes); // returns content length
}
public static string ToHexString(string str)
{
var sb = new StringBuilder();
var bytes = Encoding.Default.GetBytes(str);
foreach (var t in bytes)
{
sb.Append(t.ToString(“X2”));
}
return sb.ToString(); // returns: “48656C6C6F20776F726C64” for “Hello world”
}
}
----------
## Some of the resources that helped in guiding development:
----------
> ### How to understand webresponse:
>c# - HttpWebResponse with MJPEG and multipart/x-mixed-replace; boundary=--myboundary response content type from security camera not working - Stack Overflow>replace-boundary-myboundary?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa
----------
> ### How to stream live footage from camera to unity3d:
>c# - Streaming live footage from camera to Unity3D - Stack Overflow
----------
> ### How to convert a byte array to a hexadecimal string, and vice versa:
>c# - How do you convert a byte array to a hexadecimal string, and vice versa? - Stack Overflow
----------
> ### Show Video from IP camera source:
> Show Video from IP camera source. - Unity Answers
----------
>### Unity3D Project displaying video from Mjpg stream:
>SampleUnityMjpegViewer/Assets/Scripts/MjpegProcessor.cs at master · DanielArnett/SampleUnityMjpegViewer · GitHub
----------
>### Streaming live footage from camera to Unity3D:
>c# - Streaming live footage from camera to Unity3D - Stack Overflow
----------
I really hope this helps others in future developments with HoloLens, Unity3d, and IP cameras.
You can view the code and setup instructions on my Github or blog @ www.arvibe.com:
GitHub - decoderzhub/IP-Camera-To-HoloLens: View any IP camera from your HoloLens using Unity3D
