Mesh Cutout with SDF Shader

[sAmmY17]

Hi Unity shader fans,

I´m trying to implement a shader that allows cutting away parts of a mesh with Signed Distance Fields without converting the mesh into an SDF. My idea is to first render only the front parts that are not cut away in a first pass. Then in a second pass, I change the depth buffer to have the maximum depth wherever I have previously rendered the object and the depth of the SDF in the parts that were not rendered. In a final pass that uses ZTest GEqual it should only render the parts of the cutout, but if the depth of the SDF is still bigger than the backside of the mesh, then nothing will be rendered representing holes that were cut by the SDF.
I have written the following shader to test my idea:

Shader "Unlit/Cutout"
{
    SubShader
    {
        Tags { "RenderType"="Opaque" }

        CGINCLUDE
        #include "UnityCG.cginc"

        #define ITERATIONS 120

        struct appdata
        {
            float4 vertex : POSITION;
        };

        struct v2f
        {
            float4 vertex : SV_POSITION;
            float3 world : TEXCOORD0;
        };

        float sphereDistance(float3 p, float3 c, float r)
        {
            return distance(p, c) - r;
        }

        float map(float3 p)
        {
            return sphereDistance(p, float3(0.6, 0, 0), 0.8);
        }

        int raymarch(inout float3 position, float3 direction)
        {
            for (int i = 0; i < ITERATIONS; i++)
            {
                float distance = -map(position);
                position += distance * direction;
                if (distance < 0.001) return i;
            }
            return -1;
        }

        v2f vert(appdata v)
        {
            v2f o;
            o.vertex = UnityObjectToClipPos(v.vertex);
            o.world = mul(unity_ObjectToWorld, v.vertex).xyz;
            return o;
        }

        ENDCG

        Pass
        {
            ZWrite Off
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            fixed4 frag (v2f i) : SV_Target
            {
                if (map(i.world) <= 0.0) discard;
                return fixed4(1, 0, 0, 1);
            }
            ENDCG
        }

        Pass
        {
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            float frag(v2f i) : SV_Depth
            {
                float depth = 0.0;

                if (map(i.world) <= 0.0)
                {
                    float3 origin = i.world;
                    float3 direction = normalize(origin - _WorldSpaceCameraPos.xyz);
                    int n = raymarch(origin, direction);

                    if (n >= 0)
                    {
                        float3 local = mul(unity_WorldToObject, origin);
                        float4 clippos = UnityObjectToClipPos(float4(local, 1.0));
                        depth = clippos.z / clippos.w;
                    }
                }

                return depth;
            }
            ENDCG
        }

        Pass
        {
            Cull Off
            ZTest GEqual
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            fixed4 frag(v2f i) : SV_Target
            {
                return fixed4(0,0,1,1);
            }
            ENDCG
        }
    }
}

This works quite well as long as the object is at the origin of world space as you can see in these screenshots.

But as soon as I move the object to a different location strange artifacts are rendered. They also change just by changing the view direction.

My first thought was that there is some error with object and world space conversion, but I can´t find any and it also makes no sense that the artifacts change with the view direction. Maybe my depth calculation is wrong?

Can anyone spot an error I made or has any idea what could cause these problems?

Thanks for your help!

[sAmmY17]

Can someone confirm that this is the right way to write the depth of some world position to the depth buffer with SV_Depth?

float3 local = mul(unity_WorldToObject, origin);
float4 clippos = UnityObjectToClipPos(float4(local, 1.0));
depth = clippos.z / clippos.w;

[bgolus]

It’ll work. Though it’d be easier to use:

float4 clippos = UnityWorldToClipPos(origin);
depth = clippos.z / clippos.w;

I’m honestly not entirely sure what’s wrong with your above shader, but it’s something with your ray marching that’s not quite right. Replacing it with a ray sphere intersection shows the expected result. (Plus doing a much more simplified 2 pass setup.)

Shader "Unlit/sAmmY17_Cutout"
{
    Properties {
        _Sphere ("Sphere (XYZ pos W rad)", Vector) = (0.6, 0.0, 0.0, 0.8)
    }

    SubShader
    {
        Tags { "RenderType"="Opaque" }

        CGINCLUDE
        #include "UnityCG.cginc"

        float4 _Sphere;

        struct appdata
        {
            float4 vertex : POSITION;
        };

        struct v2f
        {
            float4 pos : SV_POSITION;
            float3 world : TEXCOORD0;
        };

        float sphereDistance(float3 p, float3 c, float r)
        {
            return distance(p, c) - r;
        }

        // sphere intersection, cribbed from iq
        // https://www.iquilezles.org/www/articles/spherefunctions/spherefunctions.htm
        float sphereIntersect( float3 position, float3 direction, float4 sphere )
        {
            float3 oc = position - sphere.xyz;
            float b = dot( oc, direction );
            float c = dot( oc, oc ) - sphere.w*sphere.w;
            float h = b*b - c;
            if( h<0.0 ) return -1.0;
            h = sqrt( h );
            return -b - h;
        }

        float map(float3 p)
        {
            return sphereDistance(p, _Sphere.xyz, _Sphere.w);
        }

        v2f vert(appdata v)
        {
            v2f o;
            o.pos = UnityObjectToClipPos(v.vertex);
            o.world = mul(unity_ObjectToWorld, float4(v.vertex.xyz, 1.0)).xyz;
            return o;
        }
        ENDCG

        Pass
        {
            // draw to stencil
            Stencil {
                Ref 1
                Pass Replace
                // draw to stencil even if occluded
                ZFail Replace
            }

            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            fixed4 frag (v2f i) : SV_Target
            {
                // clip(x); == if (x < 0.0) discard;
                clip(map(i.world));

                return fixed4(1, 0, 0, 1);
            }
            ENDCG
        }

        Pass
        {
            // draw only where the stencil wasn't set
            Stencil {
                Ref 1
                Comp NotEqual
            }

            // draw only back faces
            Cull Front

            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            fixed4 frag(v2f i, out float out_depth : SV_Depth) : SV_Target
            {
                clip(map(i.world));

                float3 viewDir = normalize(i.world - _WorldSpaceCameraPos.xyz);

                // we're tracing from the back face towards the camera to hit the sphere's back surface
                float dist = sphereIntersect(i.world, -viewDir, _Sphere);

                clip(dist);

                // calculate world space hit position
                float3 hitPos = i.world - viewDir * dist;

                // set output depth
                float4 clippos = UnityWorldToClipPos(hitPos);
                out_depth = clippos.z / clippos.w;

                // quick normal to help with visualization
                float3 sphereNormal = normalize(_Sphere.xyz - hitPos);
                return fixed4(0,0, sphereNormal.y * 0.5 + 0.5,1);
            }
            ENDCG
        }
    }
}

[sAmmY17]

Thanks bgolus, you are the best!
Your shader works perfectly, I will figure out what´s wrong with my raymarching.

Thanks again