Color replacement vertex shader with multiple passes

Hi there, I am trying to make a shader that will take 3 specified colors and replaced them with 3 other specified colors. I figured I would do each color change in its own pass (maybe that’s wrong).
I got a basic shader set up, but my issue is that only results from the last pass take effect, others are ignored. I imagine it’s because all three passes sample the original texture, which means first two changes are ignored. I am guessing i should use GrabPass and pass a texture from pass to pass, but not sure how to do that correctly. Long story short, here is what I got so far:

Shader "Custom/ColorChanger"
{
    Properties
    {
        [PerRendererData] _MainTex ("Sprite Texture", 2D) = "white" {}
        _Color ( "Tint", Color ) = ( 1, 1, 1, 1 )
        _ColorToReplace1 ( "Color To Replace 1", Color ) = ( 1, 1, 1, 1 )
        _ColorToReplace2 ( "Color To Replace 2", Color ) = ( 1, 0, 0, 1 )
        _ColorToReplace3 ( "Color To Replace 3", Color ) = ( 0, 1, 0, 1 )
        _ColorToReplaceWith1 ( "Color To Replace With 1", Color ) = ( 1, 0, 0, 1 )
        _ColorToReplaceWith2 ( "Color To Replace With 2", Color ) = ( 0, 1, 0, 1 )
        _ColorToReplaceWith3 ( "Color To Replace With 3", Color ) = ( 0, 0, 1, 1 )
        [MaterialToggle] PixelSnap ("Pixel snap", Float) = 0
    }
 
    SubShader
    {
        Tags
        {
            "Queue"="Transparent"
            "IgnoreProjector"="True"
            "RenderType"="Transparent"
            "PreviewType"="Plane"
            "CanUseSpriteAtlas"="True"
        }
 
        Cull Off
        Lighting Off
        ZWrite Off
        Fog { Mode Off }
        Blend SrcAlpha OneMinusSrcAlpha
        
        
        GrabPass { "_MainTex" }
        
        
        Pass
        {
        CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            //#pragma multi_compile DUMMY PIXELSNAP_ON
            #include "UnityCG.cginc"
            
            
            struct appdata_t
            {
                float4 vertex   : POSITION;
                float4 color    : COLOR;
                float2 texcoord : TEXCOORD0;
            };
 
            struct v2f
            {
                float4 vertex   : SV_POSITION;
                fixed4 color    : COLOR;
                half2 texcoord  : TEXCOORD0;
            };
           
            fixed4 _Color;
            fixed4 _ColorToReplace1;
            fixed4 _ColorToReplace2;
            fixed4 _ColorToReplace3;
            fixed4 _ColorToReplaceWith1;
            fixed4 _ColorToReplaceWith2;
            fixed4 _ColorToReplaceWith3;
           
 
            v2f vert(appdata_t IN)
            {
                v2f OUT;
                OUT.vertex = mul(UNITY_MATRIX_MVP, IN.vertex);
                OUT.texcoord = IN.texcoord;
                OUT.color = IN.color * _Color;
                #ifdef PIXELSNAP_ON
                OUT.vertex = UnityPixelSnap (OUT.vertex);
                #endif
                return OUT;
            }
 
            sampler2D _MainTex;
 
            fixed4 frag(v2f IN) : COLOR
            {
	            fixed4 result = tex2D(_MainTex, IN.texcoord); // * IN.color;
	            float dist = distance(result.rgb, _ColorToReplace1);
	            float factor = saturate(1.0 - dist / 0.5);
	            result.rgb = lerp(result.rgb, _ColorToReplaceWith1, factor);
	            return result;
            }
        ENDCG
        }
        
        Pass
        {
        CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            //#pragma multi_compile DUMMY PIXELSNAP_ON
            #include "UnityCG.cginc"
            
            
            struct appdata_t
            {
                float4 vertex   : POSITION;
                float4 color    : COLOR;
                float2 texcoord : TEXCOORD0;
            };
 
            struct v2f
            {
                float4 vertex   : SV_POSITION;
                fixed4 color    : COLOR;
                half2 texcoord  : TEXCOORD0;
            };
           
            fixed4 _Color;
            fixed4 _ColorToReplace1;
            fixed4 _ColorToReplace2;
            fixed4 _ColorToReplace3;
            fixed4 _ColorToReplaceWith1;
            fixed4 _ColorToReplaceWith2;
            fixed4 _ColorToReplaceWith3;
           
 
            v2f vert(appdata_t IN)
            {
                v2f OUT;
                OUT.vertex = mul(UNITY_MATRIX_MVP, IN.vertex);
                OUT.texcoord = IN.texcoord;
                OUT.color = IN.color * _Color;
                #ifdef PIXELSNAP_ON
                OUT.vertex = UnityPixelSnap (OUT.vertex);
                #endif
                return OUT;
            }
 
            sampler2D _MainTex;
 
            fixed4 frag(v2f IN) : COLOR
            {
	            fixed4 result = tex2D(_MainTex, IN.texcoord); // * IN.color;
	            float dist = distance(result.rgb, _ColorToReplace2);
	            float factor = saturate(1.0 - dist / 0.5);
	            result.rgb = lerp(result.rgb, _ColorToReplaceWith2, factor);
	            return result;
            }
        ENDCG
        }
        
        Pass
        {
        CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            //#pragma multi_compile DUMMY PIXELSNAP_ON
            #include "UnityCG.cginc"
           
           
            struct appdata_t
            {
                float4 vertex   : POSITION;
                float4 color    : COLOR;
                float2 texcoord : TEXCOORD0;
            };
 
            struct v2f
            {
                float4 vertex   : SV_POSITION;
                fixed4 color    : COLOR;
                half2 texcoord  : TEXCOORD0;
            };
           
            fixed4 _Color;
            fixed4 _ColorToReplace1;
            fixed4 _ColorToReplace2;
            fixed4 _ColorToReplace3;
            fixed4 _ColorToReplaceWith1;
            fixed4 _ColorToReplaceWith2;
            fixed4 _ColorToReplaceWith3;

 
            v2f vert(appdata_t IN)
            {
                v2f OUT;
                OUT.vertex = mul(UNITY_MATRIX_MVP, IN.vertex);
                OUT.texcoord = IN.texcoord;
                OUT.color = IN.color * _Color;
                #ifdef PIXELSNAP_ON
                OUT.vertex = UnityPixelSnap (OUT.vertex);
                #endif
                return OUT;
            }
 
            sampler2D _MainTex;
 
            fixed4 frag(v2f IN) : COLOR
            {
	            fixed4 result = tex2D(_MainTex, IN.texcoord); // * IN.color;
	            float dist = distance(result.rgb, _ColorToReplace3);
	            float factor = saturate(1.0 - dist / 0.5);
	            result.rgb = lerp(result.rgb, _ColorToReplaceWith3, factor);
	            return result;
            }
        ENDCG
        }
    }
}

Each pass is identical, just uses different color values. Any suggestions on how to fix this?

I thought about doing this in one pass as well, but for some reason something similiar is happening - only first portion is applied, but not the following two:

Shader "Custom/ColorChanger"
{
    Properties
    {
        [PerRendererData] _MainTex ("Sprite Texture", 2D) = "white" {}
        _Color ("Tint", Color) = (1,1,1,1)
        _ColorToReplace1 ( "Color To Replace 1", Color ) = ( 1, 1, 1, 1 )
        _ColorToReplace2 ( "Color To Replace 2", Color ) = ( 1, 0, 0, 1 )
        _ColorToReplace3 ( "Color To Replace 3", Color ) = ( 0, 1, 0, 1 )
        _ColorToReplaceWith1 ( "Color To Replace With 1", Color ) = ( 1, 0, 0, 1 )
        _ColorToReplaceWith2 ( "Color To Replace With 2", Color ) = ( 0, 1, 0, 1 )
        _ColorToReplaceWith3 ( "Color To Replace With 3", Color ) = ( 0, 0, 1, 1 )
        _Threashold ( "Color Threshold", Range ( 0.02, 0.9 ) ) = 0.5
        [MaterialToggle] PixelSnap ("Pixel snap", Float) = 0
    }
 
    SubShader
    {
        Tags
        {
            "Queue"="Transparent"
            "IgnoreProjector"="True"
            "RenderType"="Transparent"
            "PreviewType"="Plane"
            "CanUseSpriteAtlas"="True"
        }
 
        Cull Off
        Lighting Off
        ZWrite Off
        Fog { Mode Off }
        Blend SrcAlpha OneMinusSrcAlpha
 
        Pass
        {
        CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            #pragma multi_compile DUMMY PIXELSNAP_ON
            #include "UnityCG.cginc"
           
            struct appdata_t
            {
                float4 vertex   : POSITION;
                float4 color    : COLOR;
                float2 texcoord : TEXCOORD0;
            };
 
            struct v2f
            {
                float4 vertex   : SV_POSITION;
                fixed4 color    : COLOR;
                half2 texcoord  : TEXCOORD0;
            };
           
            fixed4 _Color;
            fixed4 _ColorToReplace1;
            fixed4 _ColorToReplace2;
            fixed4 _ColorToReplace3;
            fixed4 _ColorToReplaceWith1;
            fixed4 _ColorToReplaceWith2;
            fixed4 _ColorToReplaceWith3;
            float _Threashold;
 
            v2f vert(appdata_t IN)
            {
                v2f OUT;
                OUT.vertex = mul(UNITY_MATRIX_MVP, IN.vertex);
                OUT.texcoord = IN.texcoord;
                OUT.color = IN.color * _Color;
                #ifdef PIXELSNAP_ON
                OUT.vertex = UnityPixelSnap (OUT.vertex);
                #endif
                return OUT;
            }
 
            sampler2D _MainTex;
 
            fixed4 frag(v2f IN) : COLOR
            {
	            fixed4 result = tex2D(_MainTex, IN.texcoord); // * IN.color;
	            
	            float dist = distance(result.rgb, _ColorToReplace1);
	            float factor = saturate(1.0 - dist / _Threashold);
	            result.rgb = lerp(result.rgb, _ColorToReplaceWith1, factor);
	            
	            dist = distance(result.rgb, _ColorToReplace2);
	            factor = saturate(1.0 - dist / _Threashold);
	            result.rgb = lerp(result.rgb, _ColorToReplaceWith2, factor);
	            
	            dist = distance(result.rgb, _ColorToReplace3);
	            factor = saturate(1.0 - dist / _Threashold);
	            result.rgb = lerp(result.rgb, _ColorToReplaceWith3, factor);
	            
	            return result;
            }
        ENDCG
        }
    }
 }

Any ideas why would that happen? I think I am not understanding a core concept here…