Possible to pass surface shader tessellation output to vert/frag?

Let me start off by saying I’m a beginner at this stuff

Right now I have a shader setup I was trying that’s like below (I edited it just to make it simpler to look over the structure)

I basically tried to do a surface pass just to get the tessellation, and then do my vert/frag for the lighting. The result is two separate objects overlapping, my original object and a solid black tessellated version of that object.

Is there some way I’m supposed to pass my output from the surface shader TO my vert/frag or is this not even possible?

Shader "TestCustom"
{
    Properties
    {    
        _Tess ("Tessellation", Range(1,32)) = 4
        _Displacement ("Displacement", Range(0, 1.0)) = 0.3
    }
 
    SubShader {
      // Tags { "Queue"="Transparent" "RenderType"="Transparent" }
       //Blend One OneMinusSrcAlpha made ocean colored even with 0 alph
       Blend SrcAlpha OneMinusSrcAlpha
           Cull Off
            Tags { "Queue"="Transparent-1" "IgnoreProjector"="True" "RenderType"="Transparent"}
            Fog {Mode Off}
        LOD 2
   
        GrabPass { "_RefractionNoDistort" }  
   
       
            CGPROGRAM
            #pragma surface surf BlinnPhong addshadow fullforwardshadows vertex:disp tessellate:tessFixed nolightmap
            #pragma target 5.0

            struct appdata {
                float4 vertex : POSITION;
                float4 tangent : TANGENT;
                float3 normal : NORMAL;
                float2 texcoord : TEXCOORD0;
            };

            float _Tess;

            float4 tessFixed()
            {
                return _Tess;
            }

            sampler2D _Bump;
            float _Displacement;

            void disp (inout appdata v)
            {
                float d = tex2Dlod(_Bump, float4(v.texcoord.xy,0,0)).r * _Displacement;
                v.vertex.xyz += v.normal * d;
            }

            struct Input {
                float2 uv_MainTex;
            };

            sampler2D _MainTex;
            sampler2D _NormalMap;
            fixed4 _Color;

            void surf (Input IN, inout SurfaceOutput o) {
                half4 c = tex2D (_MainTex, IN.uv_MainTex) * _Color;
                o.Albedo = c.rgb;
                o.Specular = 0.2;
                o.Gloss = 1.0;
                o.Normal = UnpackNormal(tex2D(_NormalMap, IN.uv_MainTex));
            }
            ENDCG
   
   
   
        Pass {

            CGPROGRAM
            //#pragma exclude_renderers xbox360       
            #pragma vertex vert
            #pragma fragment frag
            #pragma target 3.0
     

            struct v2f
            {
           
           
                   
            };

    

   
            v2f vert (appdata_tan v)
       
            {       
           
           
                 
                return o;
            }

        

            half4 frag (v2f i) : COLOR
            {          
         
           
                return half4;
            }
            ENDCG
       
       
        }
   
   
   
   
    }
}

If this isn’t possible, I would like to see how these two surface shaders could be combined:

Unity - Manual: Surface Shaders with DX11 / OpenGL Core Tessellation - “Distance based tessellation”

Unity - Manual: Surface Shader examples - “Custom data computed per-vertex”

I’m willing to try and port my shader over to surface if it means I can have tessellation, but I also need to be able to have a vertex output

I’d really appreciate an example like this!

Sadly, it is not possible. To utilize custom data, you will need to write a hull/domain shader and add the tessellation.

If you are interested, Shader Forge can add tessellation and will compute the vert/frag shader. Or, if you’re reasonably comfortable with modifying code yourself, here’s some of my code below! I’ve removed my portions of it. If you already have a vertex/frag shader though then you can probably integrate it right together.

I keep seeing questions about this posted, so for future reference to anyone who sees this, here’s a tessellation shader where you can modify the vert and frag easily.

To give proper credit though, Shader Forge did give me the quick start on understanding shaders and this is primarily built from that until I reached the limits of node-based programming (then again, I’m on the limits of GPU’s and Unity’s shader compiler… tex2Darray would be glorious right now). So, I’d recommend it if you have any intention of doing work with shaders. It’ll simplify a lot of processes and make prototyping easy until you want to optimize it. Hopefully this helps anyone get the basics!

Edit: Made a few touch-ups since I’m posting it to the public… lol. Plus a few clarifications. The code is commented. It’s almost drop-in and use if you copy the vert/frag sections from a shader in.

SubShader {
        Tags {
            "RenderType"="Opaque"
        }

        Pass {
            Name "DEFERRED"
            Tags {
                "LightMode"="Deferred"
            }

            CGPROGRAM
            #pragma hull hull
            #pragma domain domain
            #pragma vertex tessvert
            #pragma fragment frag
            #define UNITY_PASS_DEFERRED
            #include "UnityCG.cginc"
            #include "Tessellation.cginc"
            #include "UnityPBSLighting.cginc"
            #include "UnityStandardBRDF.cginc"
            #pragma fragmentoption ARB_precision_hint_fastest
            #pragma multi_compile_shadowcaster
            #pragma exclude_renderers xbox360 ps3
            #pragma target 5.0
        
            // If you modify the input, either by removing or adding,
            // you have to modify the domain portion of the shader below.
            // You may also have to modify the TessVertex portion, too.
            struct VertexInput {
                float4 vertex : POSITION;
                float3 normal : NORMAL;
                float4 tangent : TANGENT;
                float2 texcoord0 : TEXCOORD0;
                float2 texcoord1 : TEXCOORD1;
                float2 texcoord2 : TEXCOORD2;
                float2 texcoord3 : TEXCOORD3;
                float4 vertexColor : COLOR;
            };
        
            struct VertexOutput {
                float4 pos : SV_POSITION;
                float2 uv0 : TEXCOORD0;
                float2 uv1 : TEXCOORD1;
                float2 uv2 : TEXCOORD2;
                float2 uv3 : TEXCOORD3;
                float4 posWorld : TEXCOORD4;
                float3 normalDir : TEXCOORD5;
                float4 vertexColor : COLOR;
            };
        
            // Vertex portion goes here!
            VertexOutput vert (VertexInput v) {
                VertexOutput o = (VertexOutput)0;
                o.uv0 = v.texcoord0;
                o.uv1 = v.texcoord1;
                o.uv2 = v.texcoord2;
                o.uv3 = v.texcoord3;
                o.vertexColor = v.vertexColor;
                o.normalDir = UnityObjectToWorldNormal(v.normal);
                o.posWorld = mul(_Object2World, v.vertex);
                o.pos = mul(UNITY_MATRIX_MVP, v.vertex);
                return o;
            }
        
            #ifdef UNITY_CAN_COMPILE_TESSELLATION
                struct TessVertex {
                    float4 vertex : INTERNALTESSPOS;
                    float3 normal : NORMAL;
                    float4 tangent : TANGENT;
                    float2 texcoord0 : TEXCOORD0;
                    float2 texcoord1 : TEXCOORD1;
                    float2 texcoord2 : TEXCOORD2;
                    float2 texcoord3 : TEXCOORD3;
                    float4 vertexColor : COLOR;
                };
            
                struct OutputPatchConstant {
                    float edge[3]         : SV_TessFactor;
                    float inside          : SV_InsideTessFactor;
                    // I previously had other components here, but they aren't needed for this.
                };
            
                TessVertex tessvert (VertexInput v) {
                    TessVertex o;
                    o.vertex = v.vertex;
                    o.normal = v.normal;
                    o.tangent = v.tangent;
                    o.texcoord0 = v.texcoord0;
                    o.texcoord1 = v.texcoord1;
                    o.texcoord2 = v.texcoord2;
                    o.texcoord3 = v.texcoord3;
                    o.vertexColor = v.vertexColor;
                    return o;
                }
            
                void displacement (inout VertexInput v)
                {
                    // compute displacement here!
                    // Sampling a texture requires sampling with tex2Dlod(...)
                    // http://http.developer.nvidia.com/Cg/tex2Dlod.html
                    // Example: float h = tex2Dlod(_HeightTex, float4(TRANSFORM_TEX(v.texcoord, _HeightTex), 0, 0));
                
                    // My final setup (after getting the height value)
                    // v.vertex.xyz += (v.normal*(((1.0 - h)*2.0+-1.0)*((-1.0)*_Depth)));
                }
            
                float4 Tessellation(TessVertex v, TessVertex v1, TessVertex v2){
                    // You can also do UnityEdgeLengthBasedTessCull(...)
                    // Or you can return a flat constant for uniform tessellation
                    return UnityEdgeLengthBasedTess(v.vertex, v1.vertex, v2.vertex, _Tessellation);
                }
            
                OutputPatchConstant hullconst (InputPatch<TessVertex,3> v) {
                    OutputPatchConstant o;
                    float4 ts = Tessellation( v[0], v[1], v[2] );
                    o.edge[0] = ts.x;
                    o.edge[1] = ts.y;
                    o.edge[2] = ts.z;
                    o.inside = ts.w;
                    return o;
                }
            
                [domain("tri")]
                [partitioning("fractional_odd")]
                [outputtopology("triangle_cw")]
                [patchconstantfunc("hullconst")]
                [outputcontrolpoints(3)]
                TessVertex hull (InputPatch<TessVertex,3> v, uint id : SV_OutputControlPointID) {
                    return v[id];
                }
            
                [domain("tri")]
                VertexOutput domain (OutputPatchConstant tessFactors, const OutputPatch<TessVertex,3> vi, float3 bary : SV_DomainLocation) {
                    VertexInput v = (VertexInput)0;
                    v.vertex = vi[0].vertex*bary.x + vi[1].vertex*bary.y + vi[2].vertex*bary.z;
                    v.normal = vi[0].normal*bary.x + vi[1].normal*bary.y + vi[2].normal*bary.z;
                    v.tangent = vi[0].tangent*bary.x + vi[1].tangent*bary.y + vi[2].tangent*bary.z;
                    v.texcoord0 = vi[0].texcoord0*bary.x + vi[1].texcoord0*bary.y + vi[2].texcoord0*bary.z;
                    v.texcoord1 = vi[0].texcoord1*bary.x + vi[1].texcoord1*bary.y + vi[2].texcoord1*bary.z;
                    v.texcoord2 = vi[0].texcoord2*bary.x + vi[1].texcoord2*bary.y + vi[2].texcoord2*bary.z;
                    v.texcoord3 = vi[0].texcoord3*bary.x + vi[1].texcoord3*bary.y + vi[2].texcoord3*bary.z;
                    v.vertexColor = vi[0].vertexColor*bary.x + vi[1].vertexColor*bary.y + vi[2].vertexColor*bary.z;
                    displacement(v);
                    VertexOutput o = vert(v);
                    return o;
                }
            
            #endif

            // This also works!
            // fixed4 frag( VertexOutput i ) : COLOR
            // {
            //     //frag shader goes here!
            // }

            void frag(
                VertexOutput i,
                out half4 outDiffuse : SV_Target0,
                out half4 outSpecSmoothness : SV_Target1,
                out half4 outNormal : SV_Target2,
                out half4 outEmission : SV_Target3 )
            {
                //frag shader goes here!
            }
            ENDCG
        }
3 Likes

Hey thanks for sharing this it helps a lot!
I just have one issue and I wonder if you would be able to help…
I want to use this with GrabPass and distort what is behind the surface. It works fine with forward and deferred, but as soon as I turn on HDR I get some funky stuff as you can see in this screenshot:

Looks like the shadow gets distorted but you can still see the original shape of the object. happens only in Deferred + HDR with tesellation, regular frag shader doesn’t do that.

here’s the shader code:

Shader "DX11 - Deferred"{
    Properties {
        _Tess ("Tessellation", Range(1,32)) = 4
        _MainTex ("Base (RGB)", 2D) = "white" {}
        _DispTex ("Disp Texture", 2D) = "gray" {}
        _BumpMap ("Normalmap", 2D) = "bump" {}
        _Displacement ("Displacement", Range(0, 1.0)) = 0.3
        _Color ("Color", color) = (1,1,1,0)
        _SpecColor ("Spec color", color) = (0.5,0.5,0.5,0.5)
        _Distortion ("Distortion", Range(0, 128)) = 32
    }

    SubShader {

        Tags { "Queue"="Transparent" "RenderType"="Opaque" }

         GrabPass
        {
            Name "BASE"
            Tags {
                "LightMode" = "Always"
            }
        }

        Pass {
               Tags {"LightMode" = "ForwardBase"}
            Name "FORWARD"
            ZWrite Off
            Blend SrcAlpha OneMinusSrcAlpha
            CGPROGRAM
            #pragma hull hull
            #pragma domain domain
            #pragma vertex tessvert
            #pragma fragment frag
            #define UNITY_PASS_DEFERRED
            #include "UnityCG.cginc"
            #include "Tessellation.cginc"
            #include "UnityPBSLighting.cginc"
            #include "UnityStandardBRDF.cginc"
            #pragma fragmentoption ARB_precision_hint_fastest
            #pragma exclude_renderers xbox360 ps3
            #pragma target 5.0

            sampler2D _GrabTexture;
            half4 _GrabTexture_TexelSize;
            sampler2D _MainTex;
            sampler2D _BumpMap;
            float _Tess;
            float _Distortion;
            fixed4 _Color;

            struct VertexInput {
                float4 vertex : POSITION;
                float2 texcoord0 : TEXCOORD0;
            };
     
            struct VertexOutput {
                float4 pos : SV_POSITION;
                float2 uv0 : TEXCOORD0;
                float4 GrabUV : TEXCOORD3;
            };
     
            // Vertex portion goes here!
            VertexOutput vert (VertexInput v) {
                VertexOutput o = (VertexOutput)0;
                o.uv0 = v.texcoord0;
                o.pos = mul(UNITY_MATRIX_MVP, v.vertex);
                o.GrabUV = ComputeGrabScreenPos(o.pos);

                return o;
            }
     
            #ifdef UNITY_CAN_COMPILE_TESSELLATION
                struct TessVertex {
                    float4 vertex : INTERNALTESSPOS;
                    float2 texcoord0 : TEXCOORD0;
                };
         
                struct OutputPatchConstant {
                    float edge[3]         : SV_TessFactor;
                    float inside          : SV_InsideTessFactor;
                };
         
                TessVertex tessvert (VertexInput v) {
                    TessVertex o;
                    o.vertex = v.vertex;
                    return o;
                }
         
                void displacement (inout VertexInput v)
                {
                   
                }
         
                float4 Tessellation(TessVertex v, TessVertex v1, TessVertex v2){
                   
                    return UnityEdgeLengthBasedTess(v.vertex, v1.vertex, v2.vertex, _Tess);
                }
         
                OutputPatchConstant hullconst (InputPatch<TessVertex,3> v) {
                    OutputPatchConstant o;
                    float4 ts = Tessellation( v[0], v[1], v[2] );
                    o.edge[0] = ts.x;
                    o.edge[1] = ts.y;
                    o.edge[2] = ts.z;
                    o.inside = ts.w;
                    return o;
                }
         
                [domain("tri")]
                [partitioning("fractional_odd")]
                [outputtopology("triangle_cw")]
                [patchconstantfunc("hullconst")]
                [outputcontrolpoints(3)]
                TessVertex hull (InputPatch<TessVertex,3> v, uint id : SV_OutputControlPointID) {
                    return v[id];
                }
         
                [domain("tri")]
                VertexOutput domain (OutputPatchConstant tessFactors, const OutputPatch<TessVertex,3> vi, float3 bary : SV_DomainLocation) {
                    VertexInput v = (VertexInput)0;
                    v.vertex = vi[0].vertex*bary.x + vi[1].vertex*bary.y + vi[2].vertex*bary.z;
                    v.texcoord0 = vi[0].texcoord0*bary.x + vi[1].texcoord0*bary.y + vi[2].texcoord0*bary.z;
                  
                    displacement(v);
                    VertexOutput o = vert(v);
                    return o;
                }
         
            #endif

// Just want forwardMode but still be usable with deferred
             fixed4 frag( VertexOutput i ) : SV_Target
             {

                half2 bump = UnpackNormal(tex2D(_BumpMap, i.uv0)).rg;
                float2 offset1 = bump * _Distortion * _GrabTexture_TexelSize.xy;
                i.GrabUV.xy = offset1 * i.GrabUV.z + i.GrabUV.xy;

                half4 col = tex2Dproj( _GrabTexture, UNITY_PROJ_COORD(i.GrabUV));

                 return col * _Color;
       
             }
            ENDCG
        }
    }
}

Thanks!