I’m trying to use Firebase’s MLKit with unity, trying to use the face detection in my project. While sending a single image works, there is a high ping associated with the plugin call from unity and the time by which the callback actually returns the value. Is there any way to achieve a low latency solution for this, so that a reliable connection can be made?
Unity Code
public void AndroidRender() {
tex = new Texture2D(webcamTexture.width, webcamTexture.height);
textureHolder.GetComponent<RawImage>().texture = tex;
tex.SetPixels32(webcamTexture.GetPixels32());
tex.Apply();
byteObject.Call("passByteArray", colorBytes);
coordinates = byteObject.Call<string>("getData");
}
Java code
public passByteArray(byte[] byteArray) {
if(gotData) {
return; //create new values only if previous one's have been consumed
}
//create bitmap
BitmapFactory.Options bitmapOptions = new BitmapFactory.Options();
bitmapOptions.inMutable = true;
Bitmap bmp = BitmapFactory.decodeByteArray(byteArray, 0, byteArray.length, bitmapOptions);
bmp = rotateBitmap(bmp, 90);
FirebaseVisionImage image = FirebaseVisionImage.fromBitmap(bmp);
//start detection
FirebaseVisionFaceDetector detector = FirebaseVision.getInstance()
.getVisionFaceDetector(options);
Task<List<FirebaseVisionFace>> result =
detector.detectInImage(image)
.addOnSuccessListener(
new OnSuccessListener<List<FirebaseVisionFace>>() {
@Override
public void onSuccess(List<FirebaseVisionFace> faces) {
Log.i(TAG, "detecting faces");
for (FirebaseVisionFace face : faces) {
nose = face.getLandmark(6);
leftEye = face.getLandmark(4);
rightEye = face.getLandmark(10);
if (nose != null) {
FirebaseVisionPoint point = nose.getPosition();
x_nose = point.getX();
y_nose = point.getY();
}
if (leftEye != null) {
FirebaseVisionPoint point = leftEye.getPosition();
x_lEye = point.getX();
y_rEye = point.getY();
}
if (rightEye != null) {
FirebaseVisionPoint point = rightEye.getPosition();
x_rEye = point.getX();
y_rEye = point.getY();
}
}
if(faces.size() == 0) {
coordinates = "0.0-0.0-0.0-0.0-0.0-0.0";
} else {
coordinates = "" + x_nose + "-" + y_nose + "-" + x_lEye + "-" + y_lEye + "-" + x_rEye + "-" + y_rEye; //actual value to be returned
}
gotData = true;
//Log.i(TAG,faces.size() + " -> " + x + "-" + y);
}
})
.addOnFailureListener(
new OnFailureListener() {
@Override
public void onFailure(@NonNull Exception e) {
Log.i(TAG, "some error");
}
});
}
The boolean gotData at the start of Java plugin is used to wait till the callback actually returns value before passing back to Unity