Hi,
We are using the TestCameraImage.cs script from ARFoundation as a base for an iOS app. The idea is to take the camera’s input, run some computer vision on it, and then display the output on the screen.
As a first step, I noticed the image was rotated 90* ccw, so I implemented the following below to rotate it 90* cw every frame. It works but performance is terrible - as slow as 1 FPS per Xcode’s CoreAnimation profiler.
Here is the rotation code. Using the Memory Leak profiler, it seems there are no leaks - memory usage is constant during the app’s run.
private Texture2D Rotate90(Texture2D orig)
{
Debug.Log("doing rotate90");
Color32[] origpix = orig.GetPixels32(0);
Color32[] newpix = new Color32[orig.width * orig.height];
for (int c = 0; c < orig.height; c++)
{
for (int r = 0; r < orig.width; r++)
{
newpix[orig.width * orig.height - (orig.height * r + orig.height) + c] =
origpix[orig.width * orig.height - (orig.width * c + orig.width) + r];
}
}
Texture2D newtex = new Texture2D(orig.height, orig.width, orig.format, false);
newtex.SetPixels32(newpix, 0);
newtex.Apply();
Destroy(orig);
return newtex;
}
Here is the main part of the code - basically the TestCameraImage.cs with a few modifications
unsafe void OnCameraFrameReceived(ARCameraFrameEventArgs eventArgs)
{
// Attempt to get the latest camera image. If this method succeeds,
// it acquires a native resource that must be disposed (see below).
XRCameraImage image;
if (!cameraManager.TryGetLatestImage(out image))
{
return;
}
// Display some information about the camera image
m_ImageInfo.text = string.Format(
"Image info:\n\twidth: {0}\n\theight: {1}\n\tplaneCount: {2}\n\ttimestamp: {3}\n\tformat: {4}",
image.width, image.height, image.planeCount, image.timestamp, image.format);
// Once we have a valid XRCameraImage, we can access the individual image "planes"
// (the separate channels in the image). XRCameraImage.GetPlane provides
// low-overhead access to this data. This could then be passed to a
// computer vision algorithm. Here, we will convert the camera image
// to an RGBA texture and draw it on the screen.
// Choose an RGBA format.
// See XRCameraImage.FormatSupported for a complete list of supported formats.
var format = TextureFormat.RGBA32;
if (m_Texture == null || m_Texture.width != image.width || m_Texture.height != image.height)
{
m_Texture = new Texture2D(image.width, image.height, format, false);
}
// Convert the image to format, flipping the image across the Y axis.
// We can also get a sub rectangle, but we'll get the full image here.
var conversionParams = new XRCameraImageConversionParams(image, format,
CameraImageTransformation.None);
// Texture2D allows us write directly to the raw texture data
// This allows us to do the conversion in-place without making any copies.
var rawTextureData = m_Texture.GetRawTextureData<byte>();
try
{
image.Convert(conversionParams, new IntPtr(rawTextureData.GetUnsafePtr()), rawTextureData.Length);
}
finally
{
// We must dispose of the XRCameraImage after we're finished
// with it to avoid leaking native resources.
image.Dispose();
}
// Apply the updated texture data to our texture
m_Texture.Apply();
Texture2D m_rotated = RotateRawImageDependingOnDeviceScreenOrientation(m_Texture);
Texture2D oldTexture;
if(m_RawImage.texture != null)
{
oldTexture = m_RawImage.texture;
}
// Set the RawImage's texture so we can visualize it.
m_RawImage.texture = m_rotated;
if(oldTexture != null)
{
Destroy(oldTexture);
}
}
There’s another function that is a passthrough at the moment but eventually will call the right rotating function depending on the original screen orientation
Texture2D RotateRawImageDependingOnDeviceScreenOrientation(Texture2D texture)
{
Debug.Log(Screen.orientation);
return Rotate90(texture);
//switch (Screen.orientation)
//{
// case ScreenOrientation.Portrait:
// Rotate90(texture);
// break;
// case ScreenOrientation.LandscapeLeft:
// break;
// case ScreenOrientation.LandscapeRight:
// break;
// case ScreenOrientation.PortraitUpsideDown:
// break;
//}
}