Trouble combining sprite shaders

Unity Engine
1

Hey there!

I’m trying to create a shader for sprites that would allow me to change the color of their outlines (or set it invisible), have them react to lighting and be able to change their tints.

I’ve spent some time trying to combine a sprite outline shader with Unity’s default sprite/diffuse shader.

Right now it kind of works. Only problems are that if I set the output.alpha on the surface shader then the outlines are always visible and if the tint-property’s rgb values are anything other than 0,0,0, I start getting strange artifacts on the sprites. Any idea what am I doing wrong here?

Also note that I’m a complete idiot when it comes to shader coding, so if there are some horrible mistakes here, feel free to let me know!

Here’s the original outline shader:

Shader "Custom/SpriteOutline"
{
    Properties {
        _MainTex ("Base (RGB)", 2D) = "white" {}
        _Color ("Color", Color) = (1, 1, 1, 1)
    }
    SubShader {
        Tags {"Queue"="Transparent" "RenderType"="Transparent"}
        Cull Off
        Blend One OneMinusSrcAlpha
      
        Pass {
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            #include "UnityCG.cginc"
            sampler2D _MainTex;
            struct v2f {
                float4 pos : SV_POSITION;
                half2 uv : TEXCOORD0;
            };
            v2f vert(appdata_base v) {
                v2f o;
                o.pos = UnityObjectToClipPos(v.vertex);
                o.uv = v.texcoord;
                return o;
            }
            fixed4 _Color;
            float4 _MainTex_TexelSize;
            fixed4 frag(v2f i) : COLOR
            {
                half4 c = tex2D(_MainTex, i.uv);
                c.rgb *= c.a;
                half4 outlineC = _Color;
                outlineC.a *= ceil(c.a);
                outlineC.rgb *= outlineC.a;
                fixed alpha_up = tex2D(_MainTex, i.uv + fixed2(0, _MainTex_TexelSize.y)).a;
                fixed alpha_down = tex2D(_MainTex, i.uv - fixed2(0, _MainTex_TexelSize.y)).a;
                fixed alpha_right = tex2D(_MainTex, i.uv + fixed2(_MainTex_TexelSize.x, 0)).a;
                fixed alpha_left = tex2D(_MainTex, i.uv - fixed2(_MainTex_TexelSize.x, 0)).a;
                return lerp(outlineC, c, ceil(alpha_up * alpha_down * alpha_right * alpha_left));
            } 
            ENDCG
        }
    }
    FallBack "Diffuse"
}

And of course Unity’s sprite/diffuse

Shader "Sprites/Diffuse"
{
    Properties
    {
        [PerRendererData] _MainTex ("Sprite Texture", 2D) = "white" {}
        _Color ("Tint", Color) = (1,1,1,1)
        [MaterialToggle] PixelSnap ("Pixel snap", Float) = 0
        [HideInInspector] _RendererColor ("RendererColor", Color) = (1,1,1,1)
        [HideInInspector] _Flip ("Flip", Vector) = (1,1,1,1)
        [PerRendererData] _AlphaTex ("External Alpha", 2D) = "white" {}
        [PerRendererData] _EnableExternalAlpha ("Enable External Alpha", Float) = 0
    }

    SubShader
    {
        Tags
        {
            "Queue"="Transparent"
            "IgnoreProjector"="True"
            "RenderType"="Transparent"
            "PreviewType"="Plane"
            "CanUseSpriteAtlas"="True"
        }

        Cull Off
        Lighting Off
        ZWrite Off
        Blend One OneMinusSrcAlpha

        CGPROGRAM
        #pragma surface surf Lambert vertex:vert nofog nolightmap nodynlightmap keepalpha noinstancing
        #pragma multi_compile_local _ PIXELSNAP_ON
        #pragma multi_compile _ ETC1_EXTERNAL_ALPHA
        #include "UnitySprites.cginc"

        struct Input
        {
            float2 uv_MainTex;
            fixed4 color;
        };

        void vert (inout appdata_full v, out Input o)
        {
            v.vertex = UnityFlipSprite(v.vertex, _Flip);

            #if defined(PIXELSNAP_ON)
            v.vertex = UnityPixelSnap (v.vertex);
            #endif

            UNITY_INITIALIZE_OUTPUT(Input, o);
            o.color = v.color * _Color * _RendererColor;
        }

        void surf (Input IN, inout SurfaceOutput o)
        {
            fixed4 c = SampleSpriteTexture (IN.uv_MainTex) * IN.color;
            o.Albedo = c.rgb * c.a;
            o.Alpha = c.a;
        }
        ENDCG
    }

Fallback "Transparent/VertexLit"
}

And here is the combined result:

Shader "Custom/SpriteOutline"
{
    Properties {
        _MainTex ("Base (RGB)", 2D) = "white" {}
        _Color ("Tint", Color) = (1, 1, 1, 1)
        _Color2 ("Outline color", Color) = (1, 1, 1, 1)
    }
    SubShader {
        Tags {"Queue"="Transparent" "RenderType"="Transparent"}
        Cull Off
        Blend One OneMinusSrcAlpha
        Pass {
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            #include "UnityCG.cginc"
            sampler2D _MainTex;
            struct v2f {
                float4 pos : SV_POSITION;
                half2 uv : TEXCOORD0;
            };
            v2f vert(appdata_base v) {
                v2f o;
                o.pos = UnityObjectToClipPos(v.vertex);
                o.uv = v.texcoord;
                return o;
            }
            fixed4 _Color2;
            float4 _MainTex_TexelSize;
            fixed4 frag(v2f i) : COLOR
            {
                half4 c = tex2D(_MainTex, i.uv);
                c.rgb *= c.a;
                half4 outlineC = _Color2;
                outlineC.a *= ceil(c.a);
                outlineC.rgb *= outlineC.a;
                fixed alpha_up = tex2D(_MainTex, i.uv + fixed2(0, _MainTex_TexelSize.y)).a;
                fixed alpha_down = tex2D(_MainTex, i.uv - fixed2(0, _MainTex_TexelSize.y)).a;
                fixed alpha_right = tex2D(_MainTex, i.uv + fixed2(_MainTex_TexelSize.x, 0)).a;
                fixed alpha_left = tex2D(_MainTex, i.uv - fixed2(_MainTex_TexelSize.x, 0)).a;
                return lerp(outlineC, c, ceil(alpha_up * alpha_down * alpha_right * alpha_left));
            }
            ENDCG
        }


        //Surface shader starts here

        Cull Off
        Lighting Off
        ZWrite Off
        Blend One OneMinusSrcAlpha

        CGPROGRAM
        #pragma surface surf Lambert vertex:vert fragment:frag nofog nolightmap nodynlightmap keepalpha noinstancing
        #pragma multi_compile_local _ PIXELSNAP_ON
        #pragma multi_compile _ ETC1_EXTERNAL_ALPHA
        #include "UnitySprites.cginc"
       
        struct Input
        {
            float2 uv_MainTex;
            fixed4 color;
        };

        void vert (inout appdata_full v, out Input o)
        {
            v.vertex = UnityFlipSprite(v.vertex, _Flip);

            #if defined(PIXELSNAP_ON)
            v.vertex = UnityPixelSnap (v.vertex);
            #endif

            UNITY_INITIALIZE_OUTPUT(Input, o);
            o.color = v.color * _Color * _RendererColor;
        }        

        void surf (Input IN, inout SurfaceOutput o)
        {
            fixed4 c = SampleSpriteTexture (IN.uv_MainTex) * IN.color;
            o.Albedo = tex2D (_MainTex, IN.uv_MainTex) * IN.color;
            //If I disable alpha then outlines are hidden properly, why?
            //o.Alpha = c.a;
        }
        ENDCG
    }
   
    FallBack "Diffuse"
}
2

Oof.

Okay, your attempt to combine them is really just putting both shaders in one shader file. They’re still totally separate. That’ll produce the same effect as having two separate sprites with the pre-existing shaders applied to them.

What you really want to do is copy the code from the outline shader into the surf function of the second.

You’ll need to add these lines outside of the surf function (similar to how they’re just above and outside of the frag function):
fixed4 _Color2;
float4 _MainTex_TexelSize;

And you may want to use SampleSpriteTexture(uv).a instead of tex2D(_MainTex, uv).a for the “alpha_direction” values.

3

Ok so a small question, I don’t understand what I’m supposed to do with these new alpha values. The original shader had a fragment shader that returned a lerped fixed4, but here I have no idea what it does.

Shader "Custom/SpriteOutline"
{
    Properties {
        _MainTex ("Base (RGB)", 2D) = "white" {}
        _Color ("Tint", Color) = (1, 1, 1, 1)
        _Color2 ("Outline color", Color) = (1, 1, 1, 1)
        [HideInInspector] _RendererColor ("RendererColor", Color) = (1,1,1,1)
    }
    SubShader {
        Tags {"Queue"="Transparent" "RenderType"="Transparent"}
        Cull Off
        Lighting Off
        ZWrite Off
        Blend One OneMinusSrcAlpha

        CGPROGRAM
        #pragma surface surf Lambert vertex:vert fragment:frag nofog nolightmap nodynlightmap keepalpha noinstancing
        #pragma multi_compile_local _ PIXELSNAP_ON
        #pragma multi_compile _ ETC1_EXTERNAL_ALPHA
        #include "UnitySprites.cginc"
       
        struct Input
        {
            float2 uv_MainTex;
            fixed4 color;
            float4 pos : SV_POSITION;
            half2 uv : TEXCOORD0;

        };

        void vert (inout appdata_full v, out Input o)
        {
            v.vertex = UnityFlipSprite(v.vertex, _Flip);

            #if defined(PIXELSNAP_ON)
            v.vertex = UnityPixelSnap (v.vertex);
            #endif

            UNITY_INITIALIZE_OUTPUT(Input, o);
            o.color = v.color * _Color * _RendererColor;


            o.pos = UnityObjectToClipPos(v.vertex);
            o.uv = v.texcoord;
        }        


        fixed4 _Color2;
        float4 _MainTex_TexelSize;


        void surf (Input IN, inout SurfaceOutput o)
        {
            half4 c = SampleSpriteTexture(IN.uv);
            c.rgb *= c.a;
            half4 outlineC = _Color2;
            outlineC.a *= ceil(c.a);
            outlineC.rgb *= outlineC.a;
            fixed alpha_up = SampleSpriteTexture(IN.uv + fixed2(0, _MainTex_TexelSize.y)).a;
            fixed alpha_down = SampleSpriteTexture(IN.uv - fixed2(0, _MainTex_TexelSize.y)).a;
            fixed alpha_right = SampleSpriteTexture(IN.uv + fixed2(_MainTex_TexelSize.x, 0)).a;
            fixed alpha_left = SampleSpriteTexture(IN.uv - fixed2(_MainTex_TexelSize.x, 0)).a;

            fixed4 alphaTotal = lerp(outlineC, c, ceil(alpha_up * alpha_down * alpha_right * alpha_left));       
           
            fixed4 x = SampleSpriteTexture (IN.uv_MainTex) * IN.color;

            o.Albedo = x.rgb * x.a;
            o.Alpha = x.a;
        }

        ENDCG
    }
   
    FallBack "Diffuse"
}

Is this even close to something workable or should I just give up? I’ve spent so many days already reading about shaders and still understand basically nothing.

4

You’re very close.

The so called “alphaTotal” value isn’t an alpha total value, it’s the output color value. For an unlit shader, it’s the color and alpha the fragment shader outputs. For a lit shader, like a surface shader it’d be the albedo and alpha.

The lerp() function is a linear interpolation; it blends between two values based on a third float. The third value in this case is the max of the alpha from 4 offset positions that make the outline. So that line of code is blending between the normal sprite texture’s color and an outline color. So the lines after that should just be removed, and then assign the o.Albedo and o.Alpha to the RGB and A values of the output of the lerp.