Player sticking to wall (non-Character Controller) 3D

Unity Engine
1

I’ve been having an issue where the player can stick to the wall when moving forward and I know what is causing the problem but I am unsure of what I should do to prevent it.
I am also not using a character controller due to issues with other movement controls

This code is constantly running and is what makes the player stick to the wall:

void HandleMovement()
    {
        float moveHorizontal = Input.GetAxis("Horizontal");
        float moveVertical = Input.GetAxis("Vertical");

        // Stop player from moving backwards and forwards when on a ledge
        if(!verticalMovement)
        {
            moveVertical = 0f;
        }
       
        // Moves the Player
        Vector3 moveDirection = transform.forward * moveVertical + transform.right * moveHorizontal;
        rb.velocity = new Vector3(moveDirection.x * walkingSpeed, rb.velocity.y, moveDirection.z * walkingSpeed);
        Debug.Log(rb.velocity);
    }

Here is the whole script:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Cinemachine;

public class MovementControls : MonoBehaviour
{

    // Variables for movement
    public float walkingSpeed = 5f;
    public float sprintSpeed = 7f;
    public Transform player;
    public Rigidbody rb;
    public float jumpSpeed;
    public Transform wallCheck;
    public LayerMask wallMask;



    private float currentSpeed;
    private bool verticalMovement = true;

    [HideInInspector] public bool hanging;
    bool canGrab = true;
    bool isGrounded = true;
    public Transform groundCheck;
    public float groundDistance = 0.4f;
    public LayerMask groundMask;
    Vector3 velocity;

    bool isTouchingWall = false;


    void Start()
    {
        currentSpeed = walkingSpeed;
    }

   

    void FixedUpdate()
    {
        isGrounded = Physics.CheckSphere(groundCheck.position, groundDistance, groundMask);

        // Handle player movement
        if (Input.GetKey(KeyCode.LeftShift))
        {
            currentSpeed = sprintSpeed;
        }

        if (Input.GetKeyUp(KeyCode.LeftShift))
        {
            currentSpeed = walkingSpeed;
        }

        HandleMovement();
       
        if(canGrab) LedgeGrab();


        if(Input.GetKeyDown(KeyCode.Space))
        {
            if(hanging)
            {
                hanging = false;
                rb.useGravity = true;
                rb.velocity = new Vector3(rb.velocity.x, jumpSpeed + 2f, rb.velocity.z);
                isGrounded = false;
               
                StartCoroutine(EnableCanMove(0.25f));
            }
            else
            {
                rb.velocity = new Vector3(rb.velocity.x, jumpSpeed, rb.velocity.z);
                isGrounded = false;
            }
        }

        if(Input.GetKeyDown(KeyCode.S))
        {
            canGrab = false;
            hanging = false;
            rb.useGravity = true;

            StartCoroutine(EnableCanMove(0.7f));
        }

       
        isTouchingWall = Physics.CheckSphere(wallCheck.position, 0.5f, wallMask);

        if(isGrounded)
        {
            canGrab = true;
        }

       
    }

    IEnumerator EnableCanMove(float waitTime)
    {
        yield return new WaitForSeconds(waitTime);
        verticalMovement = true;
    }

    void HandleMovement()
    {
        float moveHorizontal = Input.GetAxis("Horizontal");
        float moveVertical = Input.GetAxis("Vertical");

       
        // Stop player from moving backwards and forwards when on a ledge
        if(!verticalMovement)
        {
            moveVertical = 0f;
        }
       
        // Moves the Player
        Vector3 moveDirection = transform.forward * moveVertical + transform.right * moveHorizontal;
        rb.velocity = new Vector3(moveDirection.x * walkingSpeed, rb.velocity.y, moveDirection.z * walkingSpeed);
        Debug.Log(rb.velocity);
    }

    void LedgeGrab()
    {
        if(rb.velocity.y < 0 && !hanging)
        {
            RaycastHit downHit;
            Vector3 lineDownStart = (transform.position + Vector3.up * 1.7f) + transform.forward;
            Vector3 LineDownEnd = (transform.position + Vector3.up * 1f) + transform.forward;
            Physics.Linecast(lineDownStart, LineDownEnd, out downHit, LayerMask.GetMask("Ledge"));
            Debug.DrawLine(lineDownStart, LineDownEnd);

            if(downHit.collider != null)
            {
                RaycastHit fwdHit;
                Vector3 lineFwdStart = new Vector3(transform.position.x, downHit.point.y - 0.1f, transform.position.z);
                Vector3 LineFwdEnd = new Vector3(transform.position.x, downHit.point.y - 0.1f, transform.position.z) + transform.forward;
                Physics.Linecast(lineFwdStart, LineFwdEnd, out fwdHit, LayerMask.GetMask("Ledge"));
                Debug.DrawLine(lineFwdStart, LineFwdEnd);

                if(fwdHit.collider != null)
                {
                    verticalMovement = false;
                    currentSpeed = 2f;
                    rb.useGravity = false;
                    rb.velocity = Vector3.zero;

                    hanging = true;

                    Vector3 hangPos = new Vector3(fwdHit.point.x, downHit.point.y, fwdHit.point.z);
                    Vector3 offset = transform.forward * -0.1f + transform.up * -1f;
                    hangPos += offset;
                    transform.position = hangPos;
                    transform.forward = -fwdHit.normal;


                }
            }
        }
    }
   
}

I’ve tried using CheckSphere() to detect when colliding with a wall but this prevents the player from moving backwards off of it

I’ve also tried creating a raycast but this has the same issues as using CheckSphere()