This reply was perfect. SV_Depth was exactly what I needed.
I’ve implemented the second option (with a depth clear in the middle) and it’s almost perfect. I’m using 4 different shaders.
WriteDepthOnly
This shader renders nothing (ColorMask 0) and writes to the Z buffer (ZWrite on) and is queued at “Geometry-100” so it render first. It is on all objects and masks.
WriteDepthOnly build a set of ZBufferValues to correctly build stencil buffer values with.
WriteDepthOnly.shader
Shader "Stencils/WriteDepthOnly"
{
SubShader
{
Tags
{
"RenderType" = "Opaque"
"Queue" = "Geometry-100"
}
ColorMask 0
ZWrite on
Pass
{
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
struct appdata
{
float4 vertex : POSITION;
};
struct v2f
{
float4 pos : SV_POSITION;
};
v2f vert(appdata v)
{
v2f o;
o.pos = mul(UNITY_MATRIX_MVP, v.vertex);
return o;
}
half4 frag(v2f i) : COLOR
{
return half4(1,1,0,1);
}
ENDCG
}
}
}
StencilSetValue
This shader renders nothing (ColorMask 0) and does not write to the Z buffer (ZWrite off) and is queued at “Geometry-50” so it render second. If it passes the ZTest it sets the stencil buffer to the value of its property _StencilMask. It is on all masks.
using the Z buffer values build by WriteDepthOnly, if the ZTest passes, set your value in the Stencil Buffer so that the appropriate objects can be rendered behind the mask.
StencilSetValue.shader
Shader "Stencils/StencilSetValue"
{
Properties
{
_StencilMask("Stencil Mask", Int) = 0
}
SubShader
{
Tags
{
"RenderType" = "Opaque"
"Queue" = "Geometry-50"
}
ColorMask 0
ZWrite Off
Stencil
{
Ref[_StencilMask]
Comp always
Pass replace
//ZFail replace
}
Pass
{
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
struct appdata
{
float4 vertex : POSITION;
};
struct v2f
{
float4 pos : SV_POSITION;
};
v2f vert(appdata v)
{
v2f o;
o.pos = mul(UNITY_MATRIX_MVP, v.vertex);
return o;
}
half4 frag(v2f i) : COLOR
{
return half4(1,1,0,1);
}
ENDCG
}
}
}
ClearDepthBuffer
This shader renders nothing (ColorMask 0) and writes to the Z buffer (ZWrite on) and is queued at “Geometry-25” so it render third. It is on one quad in front of the camera.
ClearDepthBuffer sets the depth buffer value to 0, leaving it empty before normal rendering begins. this prevents the masks depth buffer entries from preventing the objects behind them from being rendered.
ClearDepthBuffer.shader
Shader "Stencils/ClearDepthBuffer"
{
//from http://wiki.popcornfx.com/index.php/PostFx_Workaround_(Unity_Plugin)
Properties
{
_MainTex("Texture", 2D) = "white" {}
}
SubShader
{
// No culling or depth
ZTest Always
ZWrite On
// Writes to a single-component texture (TextureFormat.Depth)
Pass
{
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
struct appdata
{
float4 vertex : POSITION;
float2 uv : TEXCOORD0;
};
struct v2f
{
fixed2 uv : TEXCOORD0;
float4 vertex : SV_POSITION;
};
sampler2D _MainTex;
v2f vert(appdata v)
{
v2f o;
o.vertex = mul(UNITY_MATRIX_MVP, v.vertex);
o.uv = v.uv;
return o;
}
#if !defined(SHADER_API_D3D9) && !defined(SHADER_API_D3D11_9X)
fixed frag(v2f i) : SV_Depth
{
fixed4 col = tex2D(_MainTex, i.uv);
return 0;
}
#else
void frag(v2f i, out float4 dummycol:COLOR, out float depth : DEPTH)
{
fixed4 col = tex2D(_MainTex, i.uv);
dummycol = col;
depth = 0;
}
#endif
ENDCG
}
}
}
StencilRevealByReference
This is a surface shader that renders only if the value in the stencil buffer matches its _StencilMask property.
It is on all objects that should be masked.
ClearDepthBuffer.shader
Shader "Stencil/StencilRevealByReference"
{
Properties{
_StencilMask("Stencil Mask", Int) = 0
_Color("Color", Color) = (1,1,1,1)
_MainTex("Albedo (RGB)", 2D) = "white" {}
_Glossiness("Smoothness", Range(0,1)) = 0.5
_Metallic("Metallic", Range(0,1)) = 0.0
}
SubShader{
Tags
{
"RenderType" = "Opaque"
"Queue" = "Geometry"
}
LOD 200
Stencil{
Ref[_StencilMask]
Comp equal
Pass keep
}
CGPROGRAM
// Physically based Standard lighting model, and enable shadows on all light types
#pragma surface surf Standard fullforwardshadows
// Use shader model 3.0 target, to get nicer looking lighting
#pragma target 3.0
sampler2D _MainTex;
struct Input {
float2 uv_MainTex;
};
half _Glossiness;
half _Metallic;
fixed4 _Color;
void surf(Input IN, inout SurfaceOutputStandard o) {
// Albedo comes from a texture tinted by color
fixed4 c = tex2D(_MainTex, IN.uv_MainTex) * _Color;
o.Albedo = c.rgb;
// Metallic and smoothness come from slider variables
o.Metallic = _Metallic;
o.Smoothness = _Glossiness;
o.Alpha = c.a;
}
ENDCG
}
FallBack "Diffuse"
}
There are 2 main bugs I’m dealing with now.
- There is Z fighting between the masks entries into the zBuffer and its ZTest. This is causing the patterns on these cubes. I feel like there might be some clever way to offset the ZBuffer entry of the masks. Or I can hack it together by putting the WriteDepthOnly shader on a different mesh with a small offset from the mask. The masks are all quads. Is there a difference between 1 mesh being rendered twice with 2 materials and 2 meshes being rendered once?
- If an object that is meant to be masked, sits in front of the mask object, then it blocks the mask from setting the stencil buffer values around it, preventing it from being rendered, as desired. However, this also prevents any objects which are behind the forward object and the mask from being rendered, essentially turning any object in front of its mask into a sort of ‘null mask’. This problem seems systemic and would require a different technique entirely to fix. I can manage it in this project with careful level design.