How to make a half Rts half top down camera?

Im trying to make a camera that follows the player, the player will always face towards the cursor and when moving the cursor close to the screens edges, the camera will move in same direction, but no more than the player is clearly visable in the other side of the screen.

I already got most of the code done, I just need to make the Rts part work together with the top down part.
Right now I can toggle between camera behaviors and I still need to clamp the camera.

public class CameraController : MonoBehaviour {

// Follow Player
public Transform target;
private float smoothSpeed = 10f;
public Vector3 offset;

// Look At and Toggle
public bool followPlayerOn = true;
public bool followPlayerOff;
private Camera mainCamera;

// Move Camera
public float panSpeed = 10f;
private float panBorderThickness = 15f;
Vector2 screenHalfSize;

void Start () {
    mainCamera = FindObjectOfType<Camera>();
    screenHalfSize = new Vector2(Camera.main.aspect * Camera.main.orthographicSize, Camera.main.orthographicSize);
}

void FixedUpdate() {

    // Camera Follow Player
    if (followPlayerOn == true) {
        Vector3 desiredPosition = target.position + offset;
        Vector3 smoothedPosition = Vector3.Lerp(transform.position, desiredPosition, smoothSpeed * Time.deltaTime);
        transform.position = smoothedPosition;
    }
}

void Update() {

    // Toggle Follow Player
    if (followPlayerOff && Input.GetKeyDown(KeyCode.Space)) {
        followPlayerOn = true;
        followPlayerOff = false; }

    else if (followPlayerOn && Input.GetKeyDown(KeyCode.Space)) {
        followPlayerOn = false;
        followPlayerOff = true; }

    // Move Camera
    if (followPlayerOn == false) {

        Vector3 pos = transform.position;

        if (Input.mousePosition.y >= Screen.height - panBorderThickness)
            pos.z += panSpeed * Time.deltaTime;

        if (Input.mousePosition.y <= panBorderThickness)
            pos.z -= panSpeed * Time.deltaTime;

        if (Input.mousePosition.x >= Screen.width - panBorderThickness)
            pos.x += panSpeed * Time.deltaTime;

        if (Input.mousePosition.x <= panBorderThickness)
            pos.x -= panSpeed * Time.deltaTime;

        transform.position = pos;
    }

    // Player Look Direktion
    Ray cameraRay = mainCamera.ScreenPointToRay(Input.mousePosition);
    Plane groundPlane = new Plane(Vector3.up, Vector3.zero);
    float rayLength;

    if (groundPlane.Raycast(cameraRay, out rayLength)) {
        Vector3 pointToLook = cameraRay.GetPoint(rayLength);
        Debug.DrawLine(cameraRay.origin, pointToLook, Color.blue);

        target.transform.LookAt(new Vector3(pointToLook.x, target.transform.position.y, pointToLook.z));
    }
}

}

Toggle both behaviors on at the same time, kinda does what I want it to do, but it’s very jerky to look at since it is trying to focus on the player and move to the side at the same time.