Camera Limit Rotations Problem

I use this code to limit the rotation on the X and Y axis of my camera.

public void LimitRotation()
	{
		Quaternion axisRotation = Quaternion.AngleAxis(t.localRotation.eulerAngles.x, Vector3.right);
		float angleFromMin = Quaternion.Angle( axisRotation, lowerRotationLimit );
		float angleFromMax = Quaternion.Angle( axisRotation, upperRotationLimit );
			
		if ( angleFromMin > rotationRange || angleFromMax > rotationRange )
		{
			Vector3 euler = t.localRotation.eulerAngles;
			euler.x = (angleFromMin > angleFromMax) ? upperRotationLimit.eulerAngles.x : lowerRotationLimit.eulerAngles.x;
			t.localEulerAngles = euler;		
		}
	}

But if i move the camera up and down really fast the camera freaks out and stays
in a weird rotated position.
Full Script:

public class CameraPivot : MonoBehaviour
{
	public Transform target; //Target to follow
	public Vector3 offset; //Offset from target
	public float horizontalRotationSpeed; //Horizontal rotation speed of the camera pivot
	public float verticalRotationSpeed; //Vertical rotation speed of the camera pivot
	public Vector2 rotationLimit; //Limit for the vertical rotation of the camera. X is higer limit, Y is lower limit.
	public bool invertY; //Invert vertical rotation
	
	//Rotation values used for limiting rotation of the camera pivot
	private Quaternion upperRotationLimit;
	private Quaternion lowerRotationLimit;
	private float rotationRange;
	private Transform t; //The target the camera is looking at
	
	public void Awake()
	{
		t = transform; //Cache transform component
		//Calculate final upper and lower rotation limits and the rotation range
		lowerRotationLimit = t.localRotation * Quaternion.AngleAxis( rotationLimit.y, Vector3.right );
		upperRotationLimit = t.localRotation * Quaternion.AngleAxis( rotationLimit.x, Vector3.right );
		rotationRange = rotationLimit.x - rotationLimit.y;
	}
	
	public void LateUpdate()
	{
		//Follow player
		t.position = target.position + offset;
		//Update rotation
		t.Rotate(0, Input.GetAxis("Mouse X") * horizontalRotationSpeed * Time.deltaTime, 0, Space.World);
		t.Rotate(((invertY) ? -1 : 1) * Input.GetAxis("Mouse Y") * verticalRotationSpeed * Time.deltaTime, 0, 0);
		//Limit rotation
		LimitRotation();
	}
	
	public void LimitRotation()
	{
		Quaternion axisRotation = Quaternion.AngleAxis(t.localRotation.eulerAngles.x, Vector3.right);
		float angleFromMin = Quaternion.Angle( axisRotation, lowerRotationLimit );
		float angleFromMax = Quaternion.Angle( axisRotation, upperRotationLimit );
			
		if ( angleFromMin > rotationRange || angleFromMax > rotationRange )
		{
			Vector3 euler = t.localRotation.eulerAngles;
			euler.x = (angleFromMin > angleFromMax) ? upperRotationLimit.eulerAngles.x : lowerRotationLimit.eulerAngles.x;
			t.localEulerAngles = euler;		
		}
	}
}
public class CameraPivot : MonoBehaviour
{
	public Transform target; //Target to follow
	public Vector3 offset; //Offset from target
	public float horizontalRotationSpeed; //Horizontal rotation speed of the camera pivot
	public float verticalRotationSpeed; //Vertical rotation speed of the camera pivot
	public Vector2 rotationLimit; //Limit for the vertical rotation of the camera. X is higer limit, Y is lower limit.
	public bool invertY; //Invert vertical rotation

	private Transform _transform;
	private Quaternion initialRotation;
	private float currentXrotation = 0f;
	private float currentYrotation = 0f;

	void Awake()
	{
		_transform = transform; //Cache transform component
	}

	void Start()
	{
		//store initial rotation
		initialRotation = transform.rotation;
	}

	void LateUpdate()
	{
		//Follow player
		_transform.position = target.position + offset;

		//read mouse movements
		currentXrotation += Input.GetAxis("Mouse Y") * horizontalRotationSpeed * Time.deltaTime;
		currentYrotation += ((invertY) ? -1 : 1) * Input.GetAxis("Mouse X") * verticalRotationSpeed * Time.deltaTime;

		//limit rotations
		currentYrotation = Mathf.Repeat(currentYrotation, 360f);
		currentXrotation = Mathf.Clamp(currentXrotation, rotationLimit.x, rotationLimit.y);

		//rotate
		_transform.rotation = initialRotation * Quaternion.Euler(currentXrotation, currentYrotation, 0f);
	}
}