Sprite with stencil mask and outline

Hi, I’m trying to combine two shaders on the same sprite but unfortunately after a lot of trial and error I can’t seem to figure it out.

Basically I have two shaders, one to mask a sprite using the stencil buffer, and another that applies an outline to a sprite.

Shader "Sprites/Custom/SpriteMasked"
{
   Properties
   {
      [PerRendererData] _MainTex("Sprite Texture", 2D) = "white" {}
      _Color("Tint", Color) = (1,1,1,1)
      [MaterialToggle] PixelSnap("Pixel snap", Float) = 0
      [HideInInspector] _RendererColor ("RendererColor", Color) = (1,1,1,1)
      [HideInInspector] _Flip ("Flip", Vector) = (1,1,1,1)
      [PerRendererData] _AlphaTex ("External Alpha", 2D) = "white" {}
      [PerRendererData] _EnableExternalAlpha ("Enable External Alpha", Float) = 0

      // Add values to determine if outlining is enabled and outline color.
      [PerRendererData] _Outline("Outline", Float) = 0
      [PerRendererData] _OutlineColor("Outline Color", Color) = (1,1,1,1)
      [PerRendererData] _OutlineSize("Outline Size", int) = 1
   }

    SubShader
   {
      Tags
        {
            "Queue" = "Transparent"
            "IgnoreProjector" = "True"
            "RenderType" = "Transparent"
            "PreviewType" = "Plane"
            "CanUseSpriteAtlas" = "True"
        }

      Cull Off
      Lighting Off
      ZWrite Off
      Fog{ Mode Off }
      Blend One OneMinusSrcAlpha

      Pass
       {
         Stencil
         {
                Ref 2
                Comp equal
                Pass replace
          }

            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            #pragma multi_compile DUMMY PIXELSNAP_ON
            #include "UnityCG.cginc"
    
            struct appdata_t
            {
                float4 vertex   : POSITION;
                float4 color    : COLOR;
                float2 texcoord : TEXCOORD0;
            };
    
            struct v2f
            {
                float4 vertex   : SV_POSITION;
                fixed4 color : COLOR;
                half2 texcoord  : TEXCOORD0;
            };
    
            fixed4 _Color;
    
            v2f vert(appdata_t IN)
            {
                v2f OUT;
                OUT.vertex = UnityObjectToClipPos(IN.vertex);
                OUT.texcoord = IN.texcoord;
                OUT.color = IN.color * _Color;
                #ifdef PIXELSNAP_ON
                OUT.vertex = UnityPixelSnap(OUT.vertex);
                #endif
        
                return OUT;
            }
    
            sampler2D _MainTex;
    
            fixed4 frag(v2f IN) : SV_Target
            {
                fixed4 c = tex2D(_MainTex, IN.texcoord) * IN.color;
                c.rgb *= c.a;
                return c;
            }
            ENDCG
       }
   }
}
Shader "Sprites/Custom/SpriteMaskedOutline"
{
   Properties
   {
      [PerRendererData] _MainTex ("Sprite Texture", 2D) = "white" {}
      _Color ("Tint", Color) = (1,1,1,1)
      [MaterialToggle] PixelSnap ("Pixel snap", Float) = 0
      [HideInInspector] _RendererColor ("RendererColor", Color) = (1,1,1,1)
      [HideInInspector] _Flip ("Flip", Vector) = (1,1,1,1)
      [PerRendererData] _AlphaTex ("External Alpha", 2D) = "white" {}
      [PerRendererData] _EnableExternalAlpha ("Enable External Alpha", Float) = 0

      // Add values to determine if outlining is enabled and outline color.
      _Outline("Outline", Float) = 1
      _OutlineColor("Outline Color", Color) = (1,1,1,1)
      _OutlineSize("Outline Size", int) = 10
   }

   SubShader
   {
      Tags
      { 
         "Queue"="Transparent" 
         "IgnoreProjector"="True" 
         "RenderType"="Transparent" 
         "PreviewType"="Plane"
         "CanUseSpriteAtlas"="True"
      }

      Cull Off
      Lighting Off
      ZWrite Off
      Fog{ Mode Off }
      Blend One OneMinusSrcAlpha

      Pass
      {
        CGPROGRAM
         #pragma vertex SpriteVert
         #pragma fragment frag
         #pragma target 2.0
         #pragma multi_compile_instancing
         #pragma multi_compile _ PIXELSNAP_ON
         #pragma multi_compile _ ETC1_EXTERNAL_ALPHA
         #include "UnitySprites.cginc"

         float _Outline;
         fixed4 _OutlineColor;
         int _OutlineSize;
         float4 _MainTex_TexelSize;

         fixed4 frag(v2f IN) : SV_Target
         {
            fixed4 c = SampleSpriteTexture(IN.texcoord) * IN.color;
                
            // If outline is enabled and there is a pixel, try to draw an outline.
            if (_Outline > 0 && c.a != 0) {
               float totalAlpha = 1.0;

               [unroll(16)]
               for (int i = 1; i < _OutlineSize + 1; i++) {
                  fixed4 pixelUp = tex2D(_MainTex, IN.texcoord + fixed2(0, i * _MainTex_TexelSize.y));
                  fixed4 pixelDown = tex2D(_MainTex, IN.texcoord - fixed2(0,i *  _MainTex_TexelSize.y));
                  fixed4 pixelRight = tex2D(_MainTex, IN.texcoord + fixed2(i * _MainTex_TexelSize.x, 0));
                  fixed4 pixelLeft = tex2D(_MainTex, IN.texcoord - fixed2(i * _MainTex_TexelSize.x, 0));

                  totalAlpha = totalAlpha * pixelUp.a * pixelDown.a * pixelRight.a * pixelLeft.a;
               }

               if (totalAlpha == 0) {
                  c.rgba = fixed4(1, 1, 1, 1) * _OutlineColor;
               }
            }

            c.rgb *= c.a;

            return c;
         }
      ENDCG
      }
   }
}

The image attached shows what I’m getting and what I’d like to get.

So basically my question is, how can I apply the outline shader with the results that I get from the stencil shader (combining the two)?

1 Like

Bump! I’ve also included an UnityPackage that has a barebones sample scene that includes both shaders and should make it easier to visualize what I’m going for. Any hints?

4520752–418438–outline-test.unitypackage (14.2 KB)