When transforming the Handles.matrix, it looks like the rotation handle is being depth tested in the wrong view space. It behaves correctly, and draws the colors in the right orientation, but culls the wrong sides.
Here’s a little editor script showing this behavior. First, draw the rotation handle by passing rotation and translation in the Handles.RotationHandle() function. Next, render it in the same position, but instead transform the Handles matrix and pass identity and zero to the RotationHandle method. The latter is in the correct place with the right orientation, but renders half missing in the wrong direction.

using UnityEngine;
using UnityEditor;
using System.Collections;
[CustomEditor(typeof(HandlesMatrix))]
public class HandlesMatrixEditor : Editor
{
bool useMatrix = true;
void OnSceneGUI()
{
Handles.BeginGUI();
if(GUILayout.Button("Toggle Matrix", GUILayout.MaxWidth(128)))
useMatrix = !useMatrix;
Handles.EndGUI();
Quaternion right = Quaternion.LookRotation(Vector3.up, Vector3.right);
/**
* If not modifying the Handles.matrix, things render properly.
*/
if(!useMatrix)
{
Handles.RotationHandle(right, Vector3.one);
Handles.PositionHandle(Vector3.one, right);
}
else
{
/**
* Change the handles matrix, and the rotation handle renders incorrectly (position is fine, but the little plane grabbies are off)
*/
Matrix4x4 mat = Matrix4x4.TRS(Vector3.one, right, Vector3.one);
Handles.matrix = mat;
Handles.RotationHandle(Quaternion.identity, Vector3.zero);
Handles.PositionHandle(Vector3.zero, Quaternion.identity);
Handles.matrix = Matrix4x4.identity;
}
}
}