Tessellation, Geometry Shader Error

Just wondering if anyone knows if its possible to create a vertex shader that has both tessellation and geometry components within it? Looking at the shader pipeline (vertex → tessellation (hull → domain) → geometry → fragment), it seems like this should be possible. I found a vertex based geometry shader and a tessellation one and have tried to combine them, but I get a really weird error:

GLSL Compilation failed: ERROR: 0.6: Identifier name gl_Position cannot start with gl_

Anyone know where this may be coming from? There are no errors in the disassembly other than that no shader variant can be compiled. Thanks for any help!

This is the code I’ve been working on:

Shader "Custom/Water" {

    Properties
    {
        _Color("Color", Color) = (1,0,0,1)
        _TessEdge ("Edge Tess", Range(1,64)) = 2
        _SpecColor("Specular Material Color", Color) = (1,1,1,1)
        _Shininess("Shininess", Float) = 1.0
        _WaveLength("Wave length", Float) = 0.5
        _WaveHeight("Wave height", Float) = 0.5
        _WaveSpeed("Wave speed", Float) = 1.0
        _RandomHeight("Random height", Float) = 0.5
        _RandomSpeed("Random Speed", Float) = 0.5
    }
    SubShader
    {
   
        Tags { "Queue"="Transparent" "IgnoreProjector"="True" "RenderType"="Transparent" }
        Blend SrcAlpha OneMinusSrcAlpha

        Pass
        {

            CGPROGRAM
            #include "UnityCG.cginc"
            #pragma vertex vert
            #pragma geometry geom
            #pragma fragment frag
            #pragma hull tess
            #pragma domain dom

            float rand(float3 co)
            {
                return frac(sin(dot(co.xyz ,float3(12.9898,78.233,45.5432))) * 43758.5453);
            }

            float rand2(float3 co)
            {
                return frac(sin(dot(co.xyz ,float3(19.9128,75.2,34.5122))) * 12765.5213);
            }

            float _WaveLength;
            float _WaveHeight;
            float _WaveSpeed;
            float _RandomHeight;
            float _RandomSpeed;
            float _TessEdge;
           
            uniform float4 _LightColor0;

            uniform float4 _Color;
            uniform float4 _SpecColor;
            uniform float _Shininess;

            struct v2h
            {
                float4  pos : SV_POSITION;
                float3    norm : NORMAL;
                 float2  uv : TEXCOORD0;
            };
           
            struct g2f
            {
                float4  pos : SV_POSITION;
                float3  norm : NORMAL;
                float2  uv : TEXCOORD0;           
                float3 diffuseColor : TEXCOORD1;
                float3 specularColor : TEXCOORD2;
            };

            struct hsc
            {
                   float TessFactor[3]    : SV_TessFactor;
                float InsideTessFactor : SV_InsideTessFactor;
            };

            struct h2d
            {
                float3 pos    : POS;
            };

            struct d2g
            {
                float4 pos   : SV_POSITION;
                 float2  uv : TEXCOORD0;

            };

            v2h vert(appdata_full v)
            {
                float3 v0 = mul(unity_ObjectToWorld, v.vertex).xyz;

                float phase0 = (_WaveHeight)* sin((_Time[1] * _WaveSpeed) + (v0.x * _WaveLength) + (v0.z * _WaveLength) + rand2(v0.xzz));
                float phase0_1 = (_RandomHeight)*sin(cos(rand(v0.xzz) * _RandomHeight * cos(_Time[1] * _RandomSpeed * sin(rand(v0.xxz)))));
               
                v0.y += phase0 + phase0_1;

                v.vertex.xyz = mul((float3x3)unity_WorldToObject, v0);

                v2h OUT;
                OUT.pos = v.vertex;
                OUT.norm = v.normal;
                OUT.uv = v.texcoord;
                return OUT;
            }

            hsc hullconst( InputPatch<v2h, 3> Input )
            {
                hsc Output = (hsc)0;
                Output.TessFactor[0] = Output.TessFactor[1] = Output.TessFactor[2] = _TessEdge;
                Output.InsideTessFactor = _TessEdge;   
                return Output;
            }

            [domain("tri")]
            [partitioning("integer")]
            [outputtopology("triangle_cw")]
            [patchconstantfunc("hullconst")]
            [outputcontrolpoints(3)]
            h2d tess( InputPatch<v2h, 3> Input, uint uCPID : SV_OutputControlPointID )
            {
                h2d Output = (h2d)0;
                Output.pos = Input[uCPID].pos.xyz;
                return Output;
            }
   
            [domain("tri")]
            d2g dom( hsc HSConstantData,
                        const OutputPatch<h2d, 3> Input,
                        float3 BarycentricCoords : SV_DomainLocation)
            {
                d2g Output = (d2g)0;
    
                float fU = BarycentricCoords.x;
                float fV = BarycentricCoords.y;
                float fW = BarycentricCoords.z;
      
                float3 pos = Input[0].pos * fU + Input[1].pos * fV + Input[2].pos * fW;
     
                Output.pos = mul (UNITY_MATRIX_MVP, float4(pos.xyz,1.0));
          
                return Output;
            }
    
            [maxvertexcount(3)]
            void geom(triangle d2g IN[3], inout TriangleStream<g2f> triStream)
            {
                float3 v0 = IN[0].pos.xyz;
                float3 v1 = IN[1].pos.xyz;
                float3 v2 = IN[2].pos.xyz;

                float3 centerPos = (v0 + v1 + v2) / 3.0;

                float3 vn = normalize(cross(v1 - v0, v2 - v0));
               
                float4x4 modelMatrix = unity_ObjectToWorld;
                float4x4 modelMatrixInverse = unity_WorldToObject;

                float3 normalDirection = normalize(
                    mul(float4(vn, 0.0), modelMatrixInverse).xyz);
                float3 viewDirection = normalize(_WorldSpaceCameraPos
                    - mul(modelMatrix, float4(centerPos, 0.0)).xyz);
                float3 lightDirection = normalize(_WorldSpaceLightPos0.xyz);
                float attenuation = 1.0;

                float3 ambientLighting =
                    UNITY_LIGHTMODEL_AMBIENT.rgb * _Color.rgb;

                float3 diffuseReflection =
                    attenuation * _LightColor0.rgb * _Color.rgb
                    * max(0.0, dot(normalDirection, lightDirection));

                float3 specularReflection;
                if (dot(normalDirection, lightDirection) < 0.0)
                {
                    specularReflection = float3(0.0, 0.0, 0.0);
                }
                else
                {
                    specularReflection = attenuation * _LightColor0.rgb
                        * _SpecColor.rgb * pow(max(0.0, dot(
                            reflect(-lightDirection, normalDirection),
                            viewDirection)), _Shininess);
                }

                g2f OUT;
                OUT.pos = mul(UNITY_MATRIX_MVP, IN[0].pos);
                OUT.norm = vn;
                OUT.uv = IN[0].uv;
                OUT.diffuseColor = ambientLighting + diffuseReflection;
                OUT.specularColor = specularReflection;
                triStream.Append(OUT);

                OUT.pos = mul(UNITY_MATRIX_MVP, IN[1].pos);
                OUT.norm = vn;
                OUT.uv = IN[1].uv;
                OUT.diffuseColor = ambientLighting + diffuseReflection;
                OUT.specularColor = specularReflection;
                triStream.Append(OUT);

                OUT.pos = mul(UNITY_MATRIX_MVP, IN[2].pos);
                OUT.norm = vn;
                OUT.uv = IN[2].uv;
                OUT.diffuseColor = ambientLighting + diffuseReflection;
                OUT.specularColor = specularReflection;
                triStream.Append(OUT);
            }
           
            half4 frag(g2f IN) : COLOR
            {
                return float4(IN.specularColor +
                IN.diffuseColor, _Color.a);
            }
           
            ENDCG

        }
    }
}

I actually thought this wasn’t possible as I couldn’t find an example or info in the docs and failed while back. But just tested it and it seems you can.

Looking at your shader, I believe you should use “SV_POSITION” only in your g2f struct. The other attributes should use a TEXCOORD semantic.

Interesting, I’ll have to try that. I actually did get this shader working a while back tho, ended up having to change the tess and hullconst functions. Tessellation with geometry is quite a cool combination actually!

Hey! I’m also trying to figure out how to make tessellation works with geometry. Do you mind post what you have at the end so that I can learn from it? Thanks!

Tessellation with geometry shader - simple example:

Shader "Tessellation with Geometry"
{
    Properties
    {   
        _TessellationFactor ("Tessellation Factor", Range(0, 64)) = 16
        _QuadSize ("Quad Size", Float) = 0.05
    }
    SubShader
    {
        Pass
        {   
            CGPROGRAM

            #pragma vertex VSMain
            #pragma hull HSMain
            #pragma domain DSMain
            #pragma geometry GSMain
            #pragma fragment PSMain
            #pragma target 5.0

            float _TessellationFactor, _QuadSize;

            struct CONTROL_POINT
            {
                float4 position : SV_POSITION;
            };
           
            CONTROL_POINT VSMain (float4 vertex:POSITION)
            {
                CONTROL_POINT vs;
                vs.position = vertex;
                return vs;
            }
            void constantsHS (InputPatch<CONTROL_POINT,3> V, out float edge[3]:SV_TessFactor, out float inside:SV_InsideTessFactor)
            {
                edge[0] = edge[1] = edge[2] = _TessellationFactor;
                inside = _TessellationFactor; 
            }

            [domain("tri")]
            [partitioning("integer")]
            [outputtopology("triangle_cw")]
            [patchconstantfunc("constantsHS")]
            [outputcontrolpoints(3)]
           
            CONTROL_POINT HSMain (InputPatch<CONTROL_POINT,3> V, uint ID : SV_OutputControlPointID)
            {
                return V[ID];
            }

            [domain("tri")]
            CONTROL_POINT DSMain ( float edge[3]:SV_TessFactor, float inside:SV_InsideTessFactor, const OutputPatch<CONTROL_POINT,3> P, float3 K : SV_DomainLocation)
            {
                CONTROL_POINT ds;
                ds.position =  float4(P[0].position.xyz*K.x + P[1].position.xyz*K.y + P[2].position.xyz*K.z, 1.0);
                return ds;
            }

            [maxvertexcount(64)]
            void GSMain( triangle CONTROL_POINT patch[3], inout TriangleStream<CONTROL_POINT> stream )
            {
                CONTROL_POINT GS;
                float3 delta = float3 (_QuadSize, 0.00, 0.00);
                float3 center = float3((patch[0].position.xyz + patch[1].position.xyz + patch[2].position.xyz) / 3);
                GS.position = UnityObjectToClipPos(center + delta.yyy);
                stream.Append(GS);
                GS.position = UnityObjectToClipPos(center + delta.yyx);
                stream.Append(GS);
                GS.position = UnityObjectToClipPos(center + delta.xyy);
                stream.Append(GS);
                GS.position = UnityObjectToClipPos(center + delta.xyy);
                stream.Append(GS);
                GS.position = UnityObjectToClipPos(center + delta.xyx);
                stream.Append(GS);
                GS.position = UnityObjectToClipPos(center + delta.yyx);
                stream.Append(GS);
                stream.RestartStrip();
            }
           
            float4 PSMain (CONTROL_POINT ps) : SV_Target
            {
                return 0;
            }
            ENDCG
        }
    }

}

Geometry shader creates quad in the center of input tessellation triangle.

2 Likes