Character instantly falls off ledges?

Movement and jumps are fine but if he walks off a ledge he doesn’t fall slowly to the next one he just instantly sinks to the bottom of the map.

using UnityEngine;
using System.Collections;

public class GoBoy : MonoBehaviour
{
// Moving fields
[SerializeField] // This will make the variable below appear in the inspector
float speed = 6;
[SerializeField]
float jumpSpeed = 8;
[SerializeField]
float gravity = 20;
Vector3 moveDirection = Vector3.zero;
CharacterController controller;
//bool isJumping; // “controller.isGrounded” can be used instead
[SerializeField]
int nrOfAlowedDJumps = 1; // New vairable
int dJumpCounter = 0; // New variable

void Start()
{
    controller = GetComponent<CharacterController>();
}
void Update()
{
    moveDirection.x = Input.GetAxis("Horizontal") * speed;
    moveDirection.z = Input.GetAxis("Vertical") * speed;

    if (Input.GetButtonDown("Jump"))
    {
        if (controller.isGrounded)
        {
            moveDirection.y = jumpSpeed;    
            dJumpCounter = 0;
        }
        if (!controller.isGrounded && dJumpCounter < nrOfAlowedDJumps)
        {
            moveDirection.y = jumpSpeed;
            dJumpCounter++;
        }
    }

    moveDirection.y -= gravity * Time.deltaTime;
    controller.Move(moveDirection * Time.deltaTime);
}

}

From line 25 is constantly trying to move the player downwards at a set speed. Use isGrounded to stop this from happening and reset moveDirection.y when it lands because the -= is taking away the gravity value every frame.

if(!controller.isGrounded){
    moveDirection.y -= gravity * Time.deltaTime;
    controller.Move(moveDirection * Time.deltaTime);
}
else{
   moveDirection.y = 0;
}

Can you check the your gravity in Edit > Project Settings > Physics? It should be -9.81 for the y axis