Transparent shader with shadows showing texture bounds

[Danil0v3s]

Hello guys,

I’m working on a game where the players clothes colors are saved in a palette format with a 256x1 texture using indexed8 and the base texture of the clothes being a single channel image, monsters and NPCs however don’t have palettes other than the default so I pack everything into a single texture.

I’ve been trying to make units receive and cast shadows dynamically. I could get them to receive shadows with RenderMode = ForwardBase plus some other tricks I’ve found around this forum and the docs.

Now to make them cast shadows using a trick Bgolus said (Using alphablend prop and fallback to “Particles/Standard Surface”) I could get only the monsters and npcs working because their textures have the alpha channel but as players don’t, I can see the rest of “square” of the texture/quad. Like in the picture below.

Is there any way I can get around this?

Here’s the full shader

Shader "UnityRO/BillboardSpriteShader"
{
    Properties
    {
        _MainTex("Texture", 2D) = "white" {}
        _PaletteTex("Texture", 2D) = "white" {}
        _Alpha("Alpha", Range(0.0, 1.0)) = 1.0
        _UsePalette("Use Palette", Float) = 0

        [Toggle(_ALPHABLEND_ON)] _ALPHABLEND_ON ("Enable Dithered Shadows", Float) = 1
        //_Color("Color", Color) = (0.0,0.0,0.0,0.0)
    }

    SubShader
    {
        Tags
        {
            "LightMode" = "ForwardBase"
            "Queue" = "AlphaTest+50"
            "RenderType" = "Geometry"
        }

        ZWrite Off
        Blend SrcAlpha OneMinusSrcAlpha

        Pass
        {
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            // compile shader into multiple variants, with and without shadows
            // (we don't care about any lightmaps yet, so skip these variants)
            #pragma multi_compile_fwdbase nolightmap nodirlightmap nodynlightmap novertexlight
            // shadow helper functions and macros
            #include "AutoLight.cginc"

            #include "UnityCG.cginc"
            #include "Lighting.cginc"

            struct appdata
            {
                float4 vertex : POSITION;
                float2 uv : TEXCOORD0;
            };

            struct v2f
            {
                float2 uv : TEXCOORD0;
                SHADOW_COORDS(1) // put shadows data into TEXCOORD1
                fixed3 diff : COLOR0;
                fixed3 ambient : COLOR1;
                float4 pos : SV_POSITION;
            };

            sampler2D _MainTex;
            sampler2D _PaletteTex;

            float4 _MainTex_TexelSize;
            float _Alpha;
            float _UsePalette;

            float rayPlaneIntersection(float3 rayDir, float3 rayPos, float3 planeNormal, float3 planePos)
            {
                float denom = dot(planeNormal, rayDir);
                denom = max(denom, 0.000001);
                float3 diff = planePos - rayPos;
                return dot(diff, planeNormal) / denom;
            }

            fixed4 bilinearSample(sampler2D indexT, sampler2D LUT, float2 uv, float4 indexT_TexelSize)
            {
                float2 TextInterval = 1.0 / indexT_TexelSize.zw;

                float tlLUT = tex2D(indexT, uv).x;
                float trLUT = tex2D(indexT, uv + float2(TextInterval.x, 0.0)).x;
                float blLUT = tex2D(indexT, uv + float2(0.0, TextInterval.y)).x;
                float brLUT = tex2D(indexT, uv + TextInterval).x;

                float4 transparent = float4(0.0, 0.0, 0.0, 0.0);

                float4 tl = tlLUT == 0.0 ? transparent : float4(tex2D(LUT, float2(tlLUT, 1.0)).rgb, 1.0);
                float4 tr = trLUT == 0.0 ? transparent : float4(tex2D(LUT, float2(trLUT, 1.0)).rgb, 1.0);
                float4 bl = blLUT == 0.0 ? transparent : float4(tex2D(LUT, float2(blLUT, 1.0)).rgb, 1.0);
                float4 br = brLUT == 0.0 ? transparent : float4(tex2D(LUT, float2(brLUT, 1.0)).rgb, 1.0);

                float2 f = frac(uv.xy * indexT_TexelSize.zw);
                float4 tA = lerp(tl, tr, f.x);
                float4 tB = lerp(bl, br, f.x);

                return lerp(tA, tB, f.y);
            }

            v2f vert(appdata_base v)
            {
                v2f o;
                o.uv = v.texcoord;

                half3 worldNormal = UnityObjectToWorldNormal(v.normal);
                half nl = max(0, dot(worldNormal, _WorldSpaceLightPos0.xyz));
                o.diff = nl * _LightColor0.rgb;
                o.ambient = ShadeSH9(half4(worldNormal, 1));

                // billboard mesh towards camera
                float3 vpos = mul((float3x3)unity_ObjectToWorld, v.vertex.xyz);
                float4 worldCoord = float4(unity_ObjectToWorld._m03_m13_m23, 1);
                float4 viewPivot = mul(UNITY_MATRIX_V, worldCoord);

                // Temporary ignoring shaders billboard rotation, handled by cs script until we join all quads sprites in one
                float4 viewPos = float4(viewPivot + mul(vpos, (float3x3)unity_ObjectToWorld), 1.0);
                o.pos = UnityObjectToClipPos(v.vertex);

                // calculate distance to vertical billboard plane seen at this vertex's screen position
                const float3 planeNormal = normalize(
                    (_WorldSpaceCameraPos.xyz - unity_ObjectToWorld._m03_m13_m23) * float3(1, 0, 1));
                const float3 planePoint = unity_ObjectToWorld._m03_m13_m23;
                const float3 rayStart = _WorldSpaceCameraPos.xyz;
                const float3 rayDir = -normalize(mul(UNITY_MATRIX_I_V, float4(viewPos.xyz, 1.0)).xyz - rayStart);
                // convert view to world, minus camera pos
                const float dist = rayPlaneIntersection(rayDir, rayStart, planeNormal, planePoint);

                // calculate the clip space z for vertical plane
                float4 planeOutPos = mul(UNITY_MATRIX_VP, float4(rayStart + rayDir * dist, 1.0));
                float newPosZ = planeOutPos.z / planeOutPos.w * o.pos.w;

                // use the closest clip space z
                #if defined(UNITY_REVERSED_Z)
                o.pos.z = max(o.pos.z, newPosZ);
                #else
                    o.pos.z = min(o.pos.z, newPosZ);
                #endif

                UNITY_TRANSFER_FOG(o, o.vertex);
                // compute shadows data
                TRANSFER_SHADOW(o)

                return o;
            }

            fixed4 frag(v2f i) : SV_Target
            {
                // compute shadow attenuation (1.0 = fully lit, 0.0 = fully shadowed)
                fixed shadow = SHADOW_ATTENUATION(i);
                // darken light's illumination with shadow, keep ambient intact
                fixed3 lighting = i.diff * shadow + i.ambient;

                fixed4 col;
                if (_UsePalette)
                {
                    col = bilinearSample(_MainTex, _PaletteTex, i.uv, _MainTex_TexelSize);
                }
                else
                {
                    col = tex2D(_MainTex, i.uv);
                }
                col.rgb *= lighting;

                if (col.a == 0.0) discard;
                col.a *= _Alpha;

                // UNITY_APPLY_FOG(i.fogCoord, tex);

                return col;
            }
            ENDCG
        }
    }

    Fallback "Particles/Standard Surface"
}

[bgolus]

The way around this is to write a custom shadow caster pass that calculates the appropriate alpha.

Here’s the alpha tested shadow caster pass that most alpha tested shaders end up falling back to (the particle shader has its own that handles things like particle color and instancing that you aren’t making use of).

    Pass {
        Name "Caster"
        Tags { "LightMode" = "ShadowCaster" }

CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#pragma target 2.0
#pragma multi_compile_shadowcaster
#pragma multi_compile_instancing // allow instanced shadow pass for most of the shaders
#include "UnityCG.cginc"

struct v2f {
    V2F_SHADOW_CASTER;
    float2  uv : TEXCOORD1;
    UNITY_VERTEX_OUTPUT_STEREO
};

uniform float4 _MainTex_ST;

v2f vert( appdata_base v )
{
    v2f o;
    UNITY_SETUP_INSTANCE_ID(v);
    UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
    TRANSFER_SHADOW_CASTER_NORMALOFFSET(o)
    o.uv = TRANSFORM_TEX(v.texcoord, _MainTex);
    return o;
}

uniform sampler2D _MainTex;
uniform fixed _Cutoff;
uniform fixed4 _Color;

float4 frag( v2f i ) : SV_Target
{
    fixed4 texcol = tex2D( _MainTex, i.uv );
    clip( texcol.a*_Color.a - _Cutoff );

    SHADOW_CASTER_FRAGMENT(i)
}
ENDCG

    }

You should be able to basically just copy this into your existing shader, and then add in the bilinearSample() function and related lines to the fragment shader of that pass and have it “just work”. You can also move shared code (like that function, and _UsePalette, _Alpha, etc.) into a CGINCLUDE / ENDCG block just in the SubShader rather than in a Pass.

[Danil0v3s]

@bgolus

The way around this is to write a custom shadow caster pass that calculates the appropriate alpha.

Here’s the alpha tested shadow caster pass that most alpha tested shaders end up falling back to (the particle shader has its own that handles things like particle color and instancing that you aren’t making use of).

    Pass {
        Name "Caster"
        Tags { "LightMode" = "ShadowCaster" }

CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#pragma target 2.0
#pragma multi_compile_shadowcaster
#pragma multi_compile_instancing // allow instanced shadow pass for most of the shaders
#include "UnityCG.cginc"

struct v2f {
    V2F_SHADOW_CASTER;
    float2  uv : TEXCOORD1;
    UNITY_VERTEX_OUTPUT_STEREO
};

uniform float4 _MainTex_ST;

v2f vert( appdata_base v )
{
    v2f o;
    UNITY_SETUP_INSTANCE_ID(v);
    UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
    TRANSFER_SHADOW_CASTER_NORMALOFFSET(o)
    o.uv = TRANSFORM_TEX(v.texcoord, _MainTex);
    return o;
}

uniform sampler2D _MainTex;
uniform fixed _Cutoff;
uniform fixed4 _Color;

float4 frag( v2f i ) : SV_Target
{
    fixed4 texcol = tex2D( _MainTex, i.uv );
    clip( texcol.a*_Color.a - _Cutoff );

    SHADOW_CASTER_FRAGMENT(i)
}
ENDCG

    }

You should be able to basically just copy this into your existing shader, and then add in the bilinearSample() function and related lines to the fragment shader of that pass and have it “just work”. You can also move shared code (like that function, and _UsePalette, _Alpha, etc.) into a CGINCLUDE / ENDCG block just in the SubShader rather than in a Pass.

Hey!

I’ve tried placing the said Pass after my current Pass but nothing seems to have changed.

Any ideas?

Shader "UnityRO/BillboardSpriteShader"
{
Properties
{
_MainTex("Texture", 2D) = "white" {}
_PaletteTex("Texture", 2D) = "white" {}
_Alpha("Alpha", Range(0.0, 1.0)) = 1.0
_UsePalette("Use Palette", Float) = 0

[Toggle(_ALPHABLEND_ON)] _ALPHABLEND_ON ("Enable Dithered Shadows", Float) = 1
_Color("Color", Color) = (1,1,1,1)
_Cutoff("Cutoff", float) = 0.5
}

SubShader
{
Tags
{
"LightMode" = "ForwardBase"
"Queue" = "AlphaTest+50"
"RenderType" = "Geometry"
}

ZWrite Off
Blend SrcAlpha OneMinusSrcAlpha

CGINCLUDE
#include "UnityCG.cginc"

sampler2D _MainTex;
sampler2D _PaletteTex;

float4 _MainTex_TexelSize;
float _Alpha;
float _UsePalette;

half4 bilinearSample(sampler2D indexT, sampler2D LUT, float2 uv, float4 indexT_TexelSize)
{
float2 TextInterval = 1.0 / indexT_TexelSize.zw;

float tlLUT = tex2D(indexT, uv).x;
float trLUT = tex2D(indexT, uv + float2(TextInterval.x, 0.0)).x;
float blLUT = tex2D(indexT, uv + float2(0.0, TextInterval.y)).x;
float brLUT = tex2D(indexT, uv + TextInterval).x;

float4 transparent = float4(0.0, 0.0, 0.0, 0.0);

float4 tl = tlLUT == 0.0 ? transparent : float4(tex2D(LUT, float2(tlLUT, 1.0)).rgb, 1.0);
float4 tr = trLUT == 0.0 ? transparent : float4(tex2D(LUT, float2(trLUT, 1.0)).rgb, 1.0);
float4 bl = blLUT == 0.0 ? transparent : float4(tex2D(LUT, float2(blLUT, 1.0)).rgb, 1.0);
float4 br = brLUT == 0.0 ? transparent : float4(tex2D(LUT, float2(brLUT, 1.0)).rgb, 1.0);

float2 f = frac(uv.xy * indexT_TexelSize.zw);
float4 tA = lerp(tl, tr, f.x);
float4 tB = lerp(bl, br, f.x);

return lerp(tA, tB, f.y);
}
ENDCG

Pass
{
CGPROGRAM
#pragma vertex vert
#pragma fragment frag

// compile shader into multiple variants, with and without shadows
// (we don't care about any lightmaps yet, so skip these variants)
#pragma multi_compile_fwdbase nolightmap nodirlightmap nodynlightmap novertexlight
// shadow helper functions and macros
#include "AutoLight.cginc"

#include "Lighting.cginc"

struct appdata
{
float4 vertex : POSITION;
float2 uv : TEXCOORD0;
};

struct v2f
{
float2 uv : TEXCOORD0;
SHADOW_COORDS(1) // put shadows data into TEXCOORD1
fixed3 diff : COLOR0;
fixed3 ambient : COLOR1;
float4 pos : SV_POSITION;
};

float rayPlaneIntersection(float3 rayDir, float3 rayPos, float3 planeNormal, float3 planePos)
{
float denom = dot(planeNormal, rayDir);
denom = max(denom, 0.000001);
float3 diff = planePos - rayPos;
return dot(diff, planeNormal) / denom;
}

v2f vert(appdata_base v)
{
v2f o;
o.uv = v.texcoord;

half3 worldNormal = UnityObjectToWorldNormal(v.normal);
half nl = max(0, dot(worldNormal, _WorldSpaceLightPos0.xyz));
o.diff = nl * _LightColor0.rgb;
o.ambient = ShadeSH9(half4(worldNormal, 1));


// billboard mesh towards camera
float3 vpos = mul((float3x3)unity_ObjectToWorld, v.vertex.xyz);
float4 worldCoord = float4(unity_ObjectToWorld._m03_m13_m23, 1);
float4 viewPivot = mul(UNITY_MATRIX_V, worldCoord);

// Temporary ignoring shaders billboard rotation, handled by cs script until we join all quads sprites in one
float4 viewPos = float4(viewPivot + mul(vpos, (float3x3)unity_ObjectToWorld), 1.0);
o.pos = UnityObjectToClipPos(v.vertex);

// calculate distance to vertical billboard plane seen at this vertex's screen position
const float3 planeNormal = normalize(
(_WorldSpaceCameraPos.xyz - unity_ObjectToWorld._m03_m13_m23) * float3(1, 0, 1));
const float3 planePoint = unity_ObjectToWorld._m03_m13_m23;
const float3 rayStart = _WorldSpaceCameraPos.xyz;
const float3 rayDir = -normalize(mul(UNITY_MATRIX_I_V, float4(viewPos.xyz, 1.0)).xyz - rayStart);
// convert view to world, minus camera pos
const float dist = rayPlaneIntersection(rayDir, rayStart, planeNormal, planePoint);

// calculate the clip space z for vertical plane
float4 planeOutPos = mul(UNITY_MATRIX_VP, float4(rayStart + rayDir * dist, 1.0));
float newPosZ = planeOutPos.z / planeOutPos.w * o.pos.w;

// use the closest clip space z
#if defined(UNITY_REVERSED_Z)
o.pos.z = max(o.pos.z, newPosZ);
#else
o.pos.z = min(o.pos.z, newPosZ);
#endif

//UNITY_TRANSFER_FOG(o, o.vertex);
// compute shadows data
TRANSFER_SHADOW(o)

return o;
}

fixed4 frag(v2f i) : SV_Target
{
// compute shadow attenuation (1.0 = fully lit, 0.0 = fully shadowed)
fixed shadow = SHADOW_ATTENUATION(i);
// darken light's illumination with shadow, keep ambient intact
fixed3 lighting = i.diff * shadow + i.ambient;

fixed4 col = _UsePalette
? bilinearSample(_MainTex, _PaletteTex, i.uv, _MainTex_TexelSize)
: tex2D(_MainTex, i.uv);

col.rgb *= lighting;

if (col.a == 0.0) discard;
col.a *= _Alpha;

// UNITY_APPLY_FOG(i.fogCoord, tex);

return col;
}
ENDCG
}

Pass
{
Name "Caster"
Tags
{
"LightMode" = "ShadowCaster"
}

CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#pragma target 2.0
#pragma multi_compile_shadowcaster
#pragma multi_compile_instancing // allow instanced shadow pass for most of the shaders

struct v2f
{
V2F_SHADOW_CASTER;
float2 uv : TEXCOORD1;
UNITY_VERTEX_OUTPUT_STEREO
};

uniform float4 _MainTex_ST;

v2f vert(appdata_base v)
{
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
TRANSFER_SHADOW_CASTER_NORMALOFFSET(o);
o.uv = TRANSFORM_TEX(v.texcoord, _MainTex);
return o;
}

uniform fixed _Cutoff;
uniform fixed4 _Color;

float4 frag(v2f i) : SV_Target
{
fixed4 col = _UsePalette
? bilinearSample(_MainTex, _PaletteTex, i.uv, _MainTex_TexelSize)
: tex2D(_MainTex, i.uv);

clip(col.a * _Color.a - _Cutoff);

SHADOW_CASTER_FRAGMENT(i)
}
ENDCG
}
}
}

[c0d3_m0nk3y]

@Daniloleemes

Any ideas?

My guess: The vertex shader of the shadow pass doesn’t match the vertex shader of the color pass at all. Usually, you’d want them to be more or less the same (sometimes you can remove a few things) but for billboards it’s a bit more complicated, I think. You don’t want the billboard to align with the shadow map camera - it should always be aligned with the main camera.

[Danil0v3s]

@c0d3_m0nk3y

My guess: The vertex shader of the shadow pass doesn’t match the vertex shader of the color pass at all. Usually, you’d want them to be more or less the same (sometimes you can remove a few things) but for billboards it’s a bit more complicated, I think. You don’t want the billboard to align with the shadow map camera - it should always be aligned with the main camera.

Hey, so I’ve removed everything from the first pass vertex but the lighting calculation

 v2f vert(appdata_base v)
            {
                v2f o;
                o.uv = TRANSFORM_TEX(v.texcoord, _MainTex);
                o.pos = UnityObjectToClipPos(v.vertex);

                half3 worldNormal = UnityObjectToWorldNormal(v.normal);
                half nl = max(0, dot(worldNormal, _WorldSpaceLightPos0.xyz));
                o.diff = nl * _LightColor0.rgb;
                o.ambient = ShadeSH9(half4(worldNormal, 1));

                TRANSFER_SHADOW(o);

                return o;
            }

And v2f is like this

            struct v2f
            {
                float2 uv : TEXCOORD0;
                SHADOW_COORDS(1) // put shadows data into TEXCOORD1
                fixed3 diff : COLOR0;
                fixed3 ambient : COLOR1;
                float4 pos : SV_POSITION;
            };

But no dice =(

[bgolus]

You’re both overthinking the problem.

You have ZWrite Off and the blend mode defined in the SubShader, those should be in the main Pass as it’s disabling it for the shadow caster as well.

Also, you should never use a LightMode tag in the SubShader, it’ll be ignored. You want that in the Pass as well.

[Danil0v3s]

@bgolus

You’re both overthinking the problem.

You have ZWrite Off and the blend mode defined in the SubShader, those should be in the main Pass as it’s disabling it for the shadow caster as well.

Also, you should never use a LightMode tag in the SubShader, it’ll be ignored. You want that in the Pass as well.

There we go, thank you both!