Portrait mode

Unity Engine
, ,
1

Hi,

I’m working on project with image recognition. I use: Unity 2020.2.4 and Barracuda 1.3.0.

I have a problem with portrait mode in application. In landscape mode everything works, preview image on fullscreen is displaying correctly, frames are in correct places and have correct size. When I change device orientation to portrait, then my neural network doesn’t work and preview is in wrong orientation (black bars on top and bottom of the screen and image is distorted).

I use WebCamTexture to get image from iPhone back camera to display preview on RawImage. I use same texture and pass it to tensor. I think I should rotate texture, but I don’t know how to do this.

private void InitiateCameraDevice()
{
    var webCamDevices = WebCamTexture.devices;
           
    if (webCamDevices.Length == 0)
    {
        Debug.Log("No camera detected");
        return;
    }

    foreach (var device in webCamDevices)
    {
        if (device.isFrontFacing == false)
        {
            _webCamTexture = new WebCamTexture(device.name, Screen.width, Screen.height);
            break;
        }
    }

    if (_webCamTexture == null)
    {
        Debug.Log("Unable to find back camera");
        return;
    }
           
    _webCamTexture.Play();
    _previewUI.texture = _webCamTexture;
}

private void Update()
{
    if (Source == VideoSourceForDetector.Camera && _webCamTexture == null)
    {
        return;
    }

    var texture = (Source == VideoSourceForDetector.Camera) ? ProcessCameraImage() : ProcessVideoPlayerOutput();
           
    if (texture.width <= 16)
    {
        return;
    }

   // pass texture to detector
    _detector.ProcessImage(texture, _scoreThreshold, _overlapThreshold);

    var i = 0;
    foreach (var box in _detector.DetectedObjects)
    {
        if (i == _markers.Length)
        {
            break;
        }

        _markers[i++].SetAttributes(box, _webCamTexture.videoRotationAngle, ShowFrames);
    }

    for (; i < _markers.Length; i++)
    {
        _markers[i].Hide();
    }
}

private Texture ProcessCameraImage()
{
    var ratio = (float)Screen.width / (float)Screen.height;
    _aspectRatioFitter.aspectRatio = ratio;
    _markersAspectRatioFitter.aspectRatio = ratio;
           
    var scaleY = _webCamTexture.videoVerticallyMirrored ? -1f : 1f;
    _previewUI.rectTransform.localScale = new Vector3(1, scaleY, 1);

    return _webCamTexture;
}

private Texture ProcessVideoPlayerOutput()
{
    return _videoPlayeRenderTexture;
}
2

I also tried to use ARFoundation XRCpuImage. Now preview image is ok, but image recognition still doesn’t work.

unsafe private void OnDrameReceivedFromArCamera(ARCameraFrameEventArgs obj)
{
        XRCpuImage image;
        if (_cameraManager.TryAcquireLatestCpuImage(out image) == false)
        {
            return;
        }

        var format = TextureFormat.RGBA32;

        if (_texture == null || _texture.width != image.width || _texture.height != image.height)
        {
            _texture = new Texture2D(image.width, image.height, format, false);
        }

        var conversionParams = new XRCpuImage.ConversionParams(image, format, XRCpuImage.Transformation.None);
        int size = image.GetConvertedDataSize(conversionParams);
        var buffer = new NativeArray<byte>(size, Allocator.Temp);

        try
        {
            image.Convert(conversionParams, new IntPtr(buffer.GetUnsafePtr()), buffer.Length);
        }
        catch
        {

        }
        finally
        {
            image.Dispose();
        }

        _texture.LoadRawTextureData(buffer);
        _texture.Apply();
        _previewImage.texture = _texture;
        _detector.ProcessImage(_texture, _scoreThreshold, _overlapThreshold);
        UpdateMarkers();
        buffer.Dispose();
}
3

@michalekmarcin could you provide your model?
My guess would be that it has something to do with the image from the webcam having different dimensions in landscape vs portrait.
Is that the case?
If so, remember to recreate the input Tensor if the image dimension changes. (and to dispose the previous one)
Also do check if your model is dimension independent. So that it works with both input resolution.

Let me know