Raycast Inefficiency

I have been recently trying to make a gravity reverser, and here is the code I have so far:
switchGravity.cs

using UnityEngine;

public class switchGravity : MonoBehaviour
{
public Rigidbody rb;

public bool gravity = true;

public void Switch()
{
    if (Input.GetMouseButtonDown(0))
    {
        if(gravity == true)
        {
            rb.useGravity = false;
            rb.velocity = new Vector3(rb.velocity.x, 15f, rb.velocity.z);
            gravity = false;
        }
        else if(gravity == false)
        {
            rb.useGravity = true;
            rb.velocity = new Vector3(rb.velocity.x, 0f, rb.velocity.z);
            gravity = true;
        }

    }
}

}

RaycastManager.cs

using UnityEngine;

public class RaycastManager : MonoBehaviour
{

public LayerMask reverseLayer;

public switchGravity switchGravity;

public GameObject cam;

void FixedUpdate()
{
    Debug.DrawRay(cam.transform.position, cam.transform.TransformDirection(Vector3.forward));
    if (Physics.Raycast(cam.transform.position, cam.transform.TransformDirection(Vector3.forward), Mathf.Infinity, reverseLayer))
    {
        switchGravity.Switch();
    }
}

}

This code sort of works, but the raycasts do not work well, it can be triggered through walls (which I don’t want). Is there some way I can fix these problems?

put your walls on a seperate layer