I know there was a lot if question like this. But usually there are about player movement and simple camera restriction by clamping bounds in the inspector.
Here is what I am trying to do. I want the centre (Vector3.zero) was always inside camera frustum at specific distance. I am using Perspective camera.
So I managed to find width and height of the camera frustum at specific distance from the camera. And I can detect when target ( centre of the world is outside the frustum)
The question I have is to how to clamp camera movement?
To move the camera I am basically moving parent game object using Vector3.Lerp. It looks good 
Here is my Camera Pan:
/ Moves camera controller rig object
public void CameraPan(Vector3 zeroTouchPosition, Vector3 firstTouchPosition)
{
// Find mid point between two touches
twoTouchesMidPoint = (zeroTouchPosition + firstTouchPosition) / 2;
// Get touch end position
twoTouchesEnd = Utils.GetMouseWorldPosition(twoTouchesMidPoint);
// Calculate touch vector direction
Vector3 touchDirection = twoTouchesStart - twoTouchesEnd;
// Pan by moving camera rig
transform.position = Vector3.Lerp(transform.position, transform.position + touchDirection, panSpeed * Time.deltaTime);
}
And this how I detect if object is outside Camera Frustum:
private void CameraMovementWithRestriction()
{
Vector3 worldCentre = Vector3.zero;
//Finding a frustum plane that include object position point
//The vector from camera to frustum plane
Vector3 cameraToFrustum = mainCamera.transform.forward;
//Debug.DrawRay(mainCamera.transform.position, cameraToFrustum * 10000, Color.red, 10000);
//Vector between camera and object
Vector3 cameraToObject = worldCentre - Camera.main.transform.position;
// Debug.DrawRay(worldCentre, cameraToObject, Color.green, 10000);
//The angle dot product between center of frustum and object postion
float angleDot = Vector3.Dot(cameraToFrustum, cameraToObject) / (cameraToFrustum.magnitude * cameraToObject.magnitude);
//Debug.Log("Dot producte between Frustum centre and target: " + angleDot);
//Distance between camera and frustum
float distanceCameraToFrustum = cameraToObject.magnitude * angleDot;
//Debug.Log("Distance: " + distanceCameraToFrustum);
//Center of frustum plane
Vector3 frustumPlaneCentre = mainCamera.transform.position + distanceCameraToFrustum * (cameraToFrustum / cameraToFrustum.magnitude);
//Debug.Log("Frustum Centre: " + frustumPlaneCentre);
//Vector between center of frustum and object
Vector3 FrustumToObject = worldCentre - frustumPlaneCentre;
//Debug.DrawRay(frustumPlaneCentre, FrustumToObject, Color.blue, 10000);
//Width And Height of the distance between center of frustum and object in the screen
float width = Vector3.Dot(FrustumToObject, mainCamera.transform.right);
float height = Vector3.Dot(FrustumToObject, mainCamera.transform.up);
//Frustum Width and Height
float frustumHeight = 2.0f * distanceCameraToFrustum * Mathf.Tan(mainCamera.fieldOfView * 0.5f * Mathf.Deg2Rad);
float frustumWidth = frustumHeight * mainCamera.aspect;
//Check if the object is out of the screen
if (((Mathf.Abs(width) + panBoundsBufferRadius) >= frustumWidth * 0.5f) || ((Mathf.Abs(height) + panBoundsBufferRadius) >= frustumHeight * 0.5f))
{
//Debug.Log("OUTSIDE");
}
}