Convolution Shader not working. Please help.

Unity Engine
1

I am trying to make a convolution Shader where the current pixel and the surrounding ones (a total of 9) are multiplied by a kernel matrix. This is my shader, all I am getting is pink…

Shader "Test/Convolution"
{
    Properties
    {
        _MainTex ("Texture", 2D) = "white" {}
    }
    SubShader
    {
        // No culling or depth
        Cull Off ZWrite Off ZTest Always

        Pass
        {
            CGPROGRAM
// Upgrade NOTE: excluded shader from DX11 because it uses wrong array syntax (type[size] name)
#pragma exclude_renderers d3d11
            #pragma vertex vert
            #pragma fragment frag
           
            #include "UnityCG.cginc"

            struct appdata
            {
                float4 vertex : POSITION;
                float2 uv : TEXCOORD0;
            };

            struct v2f
            {
                float2 uv : TEXCOORD0;
                float4 vertex : SV_POSITION;
            };

            v2f vert (appdata v)
            {
                v2f o;
                o.vertex = UnityObjectToClipPos(v.vertex);
                o.uv = v.uv;
                return o;
            }
           
            sampler2D _MainTex;

            //gets all pixels around current pixel
            float3[9] GetMatrix(sampler2D tex, float2 uv, float4 size)
            {
                float3[9] c = float3(
                tex2D(tex, uv + float2(-size.x, -size.y)).rgb,
                tex2D(tex, uv + float2(0, -size.y)).rgb,
                tex2D(tex, uv + float2(size.x, -size.y)).rgb,
                tex2D(tex, uv + float2(-size.x, 0)).rgb,
                tex2D(tex, uv + float2(0, 0)).rgb,
                tex2D(tex, uv + float2(size.x, 0)).rgb,
                tex2D(tex, uv + float2(-size.x, size.y)).rgb,
                tex2D(tex, uv + float2(0, size.y)).rgb,
                tex2D(tex, uv + float2(size.x, size.y)).rgb
                );   
                return c;
            }
            fixed4  Covolution(float3[9] c, float kernal)
            {
                fixed4 r;
                r.r = c[0].r * kernal[8];
                r.r += c[1].r * kernal[7];
                r.r += c[2].r * kernal[6];
                r.r += c[3].r * kernal[5];
                r.r += c[4].r * kernal[4];
                r.r += c[5].r * kernal[3];
                r.r += c[6].r * kernal[2];
                r.r += c[7].r * kernal[1];
                r.r += c[9].r * kernal[0];

                r.g = c[0].g * kernal[8];
                r.g += c[1].g * kernal[7];
                r.g += c[2].g * kernal[6];
                r.g += c[3].g * kernal[5];
                r.g += c[4].g * kernal[4];
                r.g += c[5].g * kernal[3];
                r.g += c[6].g * kernal[2];
                r.g += c[7].g * kernal[1];
                r.g += c[8].g * kernal[0];

                r.b = c[0].b * kernal[8;
                r.b += c[1].b * kernal[7];
                r.b += c[2].b * kernal[6];
                r.b += c[3].b * kernal[5];
                r.b += c[4].b * kernal[4];
                r.b += c[5].b * kernal[3];
                r.b += c[6].b * kernal[2];
                r.b += c[7].b * kernal[1];
                r.b += c[8].b * kernal[0];

                r[3]=1; //alpha
                return r;
            }
            fixed4 frag (v2f i) : SV_Target
            {
                float[9] kernal = float[9](0,0,0,0,1,0,0,0,0); //identity matrix
                float3[9] pixels = GetMatrix(_MainTex, i.uv, _MainTex_TexelSize);
                fixed4 c =Covolution(pixels,kernal);

                return c;
            }
            ENDCG
        }
    }
}
2

The error seems to be because of the way you’re declaring arrays:
float3[N]
while the correct syntax is
float3 [N]

e.g. “float3[9] c” is wrong, it would be “float3 c[9]”

1 Like
3

Thanks, I’ll give that a try. While waiting for a reply I tried a different approach. I got rid of the arrays and used float3x3. This one compiles but all I get is black. It is supposed to do the same thing as above. But a little cleaner code wise. I basically want to get these matrix’s to work from this link Kernel (image processing) - Wikipedia
I think I am getting close. I’m just dumb when it comes to shaders. :?/

Shader "Convolution"
{
    Properties
    {
        _MainTex ("Texture", 2D) = "white" {}
    }
    SubShader
    {
        // No culling or depth
        Cull Off ZWrite Off ZTest Always

        Pass
        {
            CGPROGRAM

            #pragma vertex vert
            #pragma fragment frag
           
            #include "UnityCG.cginc"

            struct appdata
            {
                float4 vertex : POSITION;
                float2 uv : TEXCOORD0;
            };

            struct v2f
            {
                float2 uv : TEXCOORD0;
                float4 vertex : SV_POSITION;
            };

            v2f vert (appdata v)
            {
                v2f o;
                o.vertex = UnityObjectToClipPos(v.vertex);
                o.uv = v.uv;
                return o;
            }
           
            sampler2D _MainTex;
            float4 _MainTex_TexelSize;

            float3x3 GetData(int channel, sampler2D tex, float2 uv, float4 size)
            {
                float3x3 mat;
                for (int y=-1; y<2; y++)
                {  
                    for(int x=-1; x<2; x++)
                    {       
                        mat[x+1,y+1]=tex2D(tex, uv + float2(x*size.x, y*size.y))[channel];
                    }               
                }
                return mat;
            }
            float3x3 GetMean(float3x3 matr, float3x3 matg, float3x3 matb)
            {
                float3x3 mat;
                for (int y=0; y<3; y++)
                {  
                    for(int x=0; x<3; x++)
                    {
                        mat[x,y] = (matr[x,y] + matg[x,y] + matb[x,y]) / 3.0;
                    }
                }
                return mat;
            }

            float Convolve(float3x3 kernel, float3x3 pixels, float denom, float offset)
            {
                float res = 0.0;
                for (int y=0; y<3; y++)
                {  
                    for(int x=0; x<3; x++)
                    { 
                        res += kernel[2-x,2-y]*pixels[x,y];
                    }
                }
                return  clamp(res/denom + offset,0.0,1.0);
            }

            fixed4 frag (v2f i) : SV_Target
            {
                float3x3 kerIdentity = float3x3 (    0.0,  0.0,  0.0,
                                                    0.0,  1.0,  0.0,
                                                    0.0,  0.0,  0.0);

               float3x3 kerEmboss = float3x3 (2.0,  0.0,  0.0,
                                             0.0, -1.0,  0.0,
                                             0.0,  0.0, -1.0);

               float3x3 kerSharpness = float3x3 (-1.0, -1.0, -1.0,
                                                -1.0,  9.0, -1.0,
                                                -1.0, -1.0, -1.0);

               float3x3 kerGausBlur = float3x3 (1.0, 2.0, 1.0,
                                                2.0, 4.0, 2.0,
                                                1.0, 2.0, 1.0);

               float3x3 kerEdgeDetect = float3x3 (-1.0/8.0, -1.0/8.0, -1.0/8.0,
                                                 -1.0/8.0,  1.0    , -1.0/8.0,
                                                 -1.0/8.0, -1.0/8.0, -1.0/8.0);

                float3x3 matr = GetData(0, _MainTex, i.uv, _MainTex_TexelSize);
                float3x3 matg = GetData(1, _MainTex, i.uv, _MainTex_TexelSize);
                float3x3 matb = GetData(2, _MainTex, i.uv, _MainTex_TexelSize);
                float3x3 mata = GetMean(matr, matg, matb);

               
                // Sharpness kernel
               float4 gl_FragColor = float4(Convolve(kerIdentity,matr,1.0,0.0),
                                            Convolve(kerIdentity,matg,1.0,0.0),
                                            Convolve(kerIdentity,matb,1.0,0.0),
                                            1.0);

//               float4 gl_FragColor = float4(Convolve(kerSharpness,matr,1.0,0.0),
//                                            Convolve(kerSharpness,matg,1.0,0.0),
//                                            Convolve(kerSharpness,matb,1.0,0.0),
//                                            1.0);

               // Gaussian blur kernel
               //gl_FragColor = vec4(Convolve(kerGausBlur,matr,16.,0.),
               //                    Convolve(kerGausBlur,matg,16.,0.),
               //                    Convolve(kerGausBlur,matb,16.,0.),1.0);

               // Edge Detection kernel
               //gl_FragColor = vec4(Convolve(kerEdgeDetect,mata,0.1,0.),
               //                    Convolve(kerEdgeDetect,mata,0.1,0.),
               //                    Convolve(kerEdgeDetect,mata,0.1,0.),1.0);

               // Emboss kernel
               //float4 gl_FragColor = float4(Convolve(kerEmboss,mata,1.,1./2.),
                //                            Convolve(kerEmboss,mata,1.,1./2.),
                //                            Convolve(kerEmboss,mata,1.,1./2.),1.0);


                return gl_FragColor;
            }
            ENDCG
        }
    }
}
4

“mat[x+1,y+1]” is also kinda wrong notation
what you probably meant was “mat[x+1][y+1]”

2 Likes
5

You rock my friend! That was exactly the problem. Once corrected my convolution shader works!
Thank you so, much! Here is the working code in case anyone else in the world wants to play with it.

Thanks again!

Shader "Convolution"
{
    Properties
    {
        _MainTex ("Texture", 2D) = "white" {}
    }
    SubShader
    {
        // No culling or depth
        Cull Off ZWrite Off ZTest Always

        Pass
        {
            CGPROGRAM

            #pragma vertex vert
            #pragma fragment frag
           
            #include "UnityCG.cginc"

            struct appdata
            {
                float4 vertex : POSITION;
                float2 uv : TEXCOORD0;
            };

            struct v2f
            {
                float2 uv : TEXCOORD0;
                float4 vertex : SV_POSITION;
            };

            v2f vert (appdata v)
            {
                v2f o;
                o.vertex = UnityObjectToClipPos(v.vertex);
                o.uv = v.uv;
                return o;
            }
           
            sampler2D _MainTex;
            float4 _MainTex_TexelSize;

            float3x3 GetData(int channel, sampler2D tex, float2 uv, float4 size)
            {
                float3x3 mat;
                for (int y=-1; y<2; y++)
                {  
                    for(int x=-1; x<2; x++)
                    {       
                        mat[x+1][y+1]=tex2D(tex, uv + float2(x*size.x, y*size.y))[channel];
                    }               
                }
                return mat;
            }
            float3x3 GetMean(float3x3 matr, float3x3 matg, float3x3 matb)
            {
                float3x3 mat;
                for (int y=0; y<3; y++)
                {  
                    for(int x=0; x<3; x++)
                    {
                        mat[x][y] = (matr[x][y] + matg[x][y] + matb[x][y]) / 3.0;
                    }
                }
                return mat;
            }

            float Convolve(float3x3 kernel, float3x3 pixels, float denom, float offset)
            {
                float res = 0.0;
                for (int y=0; y<3; y++)
                {  
                    for(int x=0; x<3; x++)
                    { 
                        res += kernel[2-x][2-y]*pixels[x][y];
                    }
                }
               
                return  res;
            }

            fixed4 frag (v2f i) : SV_Target
            {
                float3x3 kerIdentity = float3x3 (    -1.0,  -1.0,  -1.0,
                                                    -1.0,  8.0,  -1.0,
                                                    -1.0,  -1.0,  -1.0);

               float3x3 kerEmboss = float3x3 (2.0,  0.0,  0.0,
                                             0.0, -1.0,  0.0,
                                             0.0,  0.0, -1.0);

               float3x3 kerSharpness = float3x3 (-1.0, -1.0, -1.0,
                                                -1.0,  9.0, -1.0,
                                                -1.0, -1.0, -1.0);

               float3x3 kerGausBlur = float3x3 (1.0, 2.0, 1.0,
                                                2.0, 4.0, 2.0,
                                                1.0, 2.0, 1.0);

                float3x3 kerEdgeDetect = float3x3 (    -1.0,  -1.0,  -1.0,
                                                    -1.0,  8.0,  -1.0,
                                                    -1.0,  -1.0,  -1.0);
   
                float3x3 matr = GetData(0, _MainTex, i.uv, _MainTex_TexelSize);
                float3x3 matg = GetData(1, _MainTex, i.uv, _MainTex_TexelSize);
                float3x3 matb = GetData(2, _MainTex, i.uv, _MainTex_TexelSize);
                float3x3 mata = GetMean(matr, matg, matb);

               
                // kernel
               float4 gl_FragColor = float4(Convolve(kerEdgeDetect,matr,1.0,0.0),
                                            Convolve(kerEdgeDetect,matg,1.0,0.0),
                                            Convolve(kerEdgeDetect,matb,1.0,0.0),
                                            1.0);

                return gl_FragColor;
            }
            ENDCG
        }
    }
}
2 Likes
6

567×581 128 KB

Shader "Smkgames/Convolution"
{
Properties
{
_MainTex ("Texture", 2D) = "white" {}
        [Enum(kerEdgeDetectionA,1,kerEdgeDetectionB,2,kerEdgeDetectionC,3,kerSharpen,4,kerBoxBlur,5)]
        _Kernel("Kernel", Float) = 1
}
SubShader
{
// No culling or depth
Cull Off ZWrite Off ZTest Always

Pass
{
CGPROGRAM

#pragma vertex vert
#pragma fragment frag

#include "UnityCG.cginc"

struct appdata
{
float4 vertex : POSITION;
float2 uv : TEXCOORD0;
};

struct v2f
{
float2 uv : TEXCOORD0;
float4 vertex : SV_POSITION;
};

v2f vert (appdata v)
{
v2f o;
o.vertex = UnityObjectToClipPos(v.vertex);
o.uv = v.uv;
return o;
}

sampler2D _MainTex;
float4 _MainTex_TexelSize;

float3x3 GetData(int channel, sampler2D tex, float2 uv, float4 size)
{
float3x3 mat;
for (int y=-1; y<2; y++)
{
for(int x=-1; x<2; x++)
{
mat[x+1][y+1]=tex2D(tex, uv + float2(x*size.x, y*size.y))[channel];
}
}
return mat;
}
float3x3 GetMean(float3x3 matr, float3x3 matg, float3x3 matb)
{
float3x3 mat;
for (int y=0; y<3; y++)
{
for(int x=0; x<3; x++)
{
mat[x][y] = (matr[x][y] + matg[x][y] + matb[x][y]) / 3.0;
}
}
return mat;
}

float Convolve(float3x3 kernel, float3x3 pixels, float denom, float offset)
{
float res = 0.0;
for (int y=0; y<3; y++)
{
for(int x=0; x<3; x++)
{
res += kernel[2-x][2-y]*pixels[x][y];
}
}

return res;
}

            float _Kernel;

fixed4 frag (v2f i) : SV_Target
{


float3x3 kerEdgeDetectionA = float3x3 ( 0.0, 0, -1.0,
1.0, 0, -1.0,
0.0, 1.0, 0.0);

float3x3 kerEdgeDetectionB = float3x3 (0.0, 1.0, 0.0,
1.0, -4.0, 1.0,
0.0, 1.0, 0.0);

float3x3 kerEdgeDetectionC = float3x3 (-1.0, -1.0, -1.0,
-1.0, 8.0, -1.0,
-1.0, -1.0, -1.0);

float3x3 kerSharpen = float3x3 (0.0, -1.0, 0.0,
-1.0, 5.0, -1.0,
0.0, -1.0, 0.0);



float3x3 kerBoxBlur = (1.0/9.0)*float3x3 ( 1.0, 1.0, 1.0,
1.0, 1.0, 1.0,
1.0, 1.0, 1.0);


  

                float3x3 kernelSelection;

                if(_Kernel == 1){
                kernelSelection = kerEdgeDetectionA;
                }else if(_Kernel == 2){
                kernelSelection = kerEdgeDetectionB;  
                }else if(_Kernel == 3){
                kernelSelection = kerEdgeDetectionC;
                }else if(_Kernel == 4){
                kernelSelection = kerSharpen;  
                }else if(_Kernel == 5){
                kernelSelection = kerBoxBlur;
                }

float3x3 matr = GetData(0, _MainTex, i.uv, _MainTex_TexelSize);
float3x3 matg = GetData(1, _MainTex, i.uv, _MainTex_TexelSize);
float3x3 matb = GetData(2, _MainTex, i.uv, _MainTex_TexelSize);
float3x3 mata = GetMean(matr, matg, matb);


// kernel
float4 gl_FragColor = float4(Convolve(kernelSelection,matr,1.0,0.0),
Convolve(kernelSelection,matg,1.0,0.0),
Convolve(kernelSelection,matb,1.0,0.0),
1.0);

return gl_FragColor;
}
ENDCG
}
}
}