How to receive shadows in custom vert/frag shader

I wrote a vert-frag shader using unitycookie’s tutorials but I have problems with it. It won’t receive shadows. Read that I should add AutoLight.cginc or LIGHT_ATTENUATION(i) but I’ve set atten to 1/distance^2 in my code. I get declaration of “x” conflicts with previous declaration at (53) at line 47 for multiple properties when I try and include AutoLight.cginc. I tried to see how shadows are cast in AutoLight but I can’t figure out what part of the code or how to port it into my shader.

Also I would like to convert this to half Lambert lighting but I don’t know what I need to do to SNdotL.

here’s my shader code ( not targeted for mobile )

Shader"custom/Skin"

{


properties

{

//_Ramp("BlendingEffects", 2D) = "white" {}

//_Range("Gloss",range(0,20))= 5

//_AmbGlo("AmbiantLighting",range(0,50))= 10

//_Occl("Oclusion",range(0,50))= 10

_MainTex("Texture", 2D) = "white" {}



_Smap("SpecMap", 2D) = "white" {}

_Shininess ("Specularity",range(0.005,5))= 2

_SpecColor ("specTint",color)= (1.0,1.0,1.0,1.0)

_RimColor ("Rim Color", Color) = (1.0,1.0,1.0,1.0)

_RimPower ("Rim Power", Range(0.1,10.0)) = 3.0

_Normals ("Normal", 2D)="white" {}

_NorRange ("Dephs",range(1,5))= 3



// above is good



}

SubShader

{

Pass{

Tags{"LightMode" = "ForwardBase"}



CGINCLUDE

// #include "AutoLight.cginc"



CGPROGRAM

#pragma vertex vertsA

#pragma fragment fragsA  

#pragma target 3.0

#pragma only_renderers d3d9  debug



// no simcs on the pragmas



//user variables and unity variables

uniform half4 _Color;

uniform half4 _SpecColor;

uniform half _Shininess;

uniform half4 _LightColor0;

uniform sampler2D _MainTex;

uniform half4 _MainTex_ST;

uniform sampler2D _Normals;

uniform half4 _Normals_ST;

uniform half _NorRange;

uniform sampler2D _Smap;

uniform half4 _Smap_ST;

uniform fixed4 _RimColor;

uniform half _RimPower;



half4x4 _Object2World;

half4x4 _World2Object;

half4 _WorldSpaceLightPos0; 

half3 _WorldSpaceCameraPos;



//input structs


struct VertInput1 {   // vertex imput info

half4 vertex : POSITION;

half3 normal : NORMAL;

half4 texcoord : TEXCOORD0; 

half4 tangent : TANGENT;

};



//output structs

struct VertOutput1 { // vertex output info

half4 pos : SV_POSITION;

half4 colur: COLOR;

half4 sktex : TEXCOORD0; 

half4 lightDir : TEXCOORD1;

half3 posNormal : TEXCOORD2;

half3 tans : TEXCOORD3;

half3 binoms : TEXCOORD4;

half3 cameraView :TEXCOORD5;

half4 spTex : TEXCOORD6;



};



// vertex



VertOutput1 vertsA(VertInput1 g)

{ 

VertOutput1 h;

// direction variables



h.posNormal =  normalize(mul(half4(g.normal,0.0),_World2Object).xyz);

h.tans = normalize( mul(_Object2World, g.tangent).xyz);

h.binoms = normalize(cross(h.posNormal,h.tans)* g.tangent.w);



half4 posMain = mul(_Object2World, g.vertex);

h.pos = mul(UNITY_MATRIX_MVP, g.vertex);

h.sktex = g.texcoord;


h.cameraView = normalize (_WorldSpaceCameraPos.xyz -posMain.xyz); //tweakable



// light physics

half3 facetolite= _WorldSpaceLightPos0.xyz -posMain.xyz;

half distance = length(facetolite);



h.lightDir = half4(normalize(lerp(_WorldSpaceLightPos0.xyz,facetolite,_WorldSpaceLightPos0.w)),

lerp(1.0,(1.0/(distance*distance)),_WorldSpaceLightPos0.w));



return h;



}



// fragment



half4 fragsA(VertOutput1 i) : COLOR

{


//maps



half4 sktex = tex2D(_MainTex, i.sktex.xy + (_MainTex_ST.xy * _MainTex_ST.zw));

half4 tans = tex2D(_Normals, i.sktex.xy + (_Normals_ST.xy * _Normals_ST.zw));

half4 spTex = tex2D(_Smap, i.sktex.xy + (_Smap_ST.xy *_Smap_ST.zw));



//normals

half3 localCoords = half3(2.0 * tans.ag - half2(1.0,1.0),_NorRange);



half3x3 local2WorldTranspose = float3x3( i.tans, i.binoms, i.posNormal);

half3 normalDirection =  normalize(mul(localCoords, local2WorldTranspose));




//Lambert lighting calc 



half3 SNdotL=  saturate( dot( normalDirection,i.lightDir.xyz));

half3 thelighting = i.lightDir.w * _LightColor0.xyz *2* SNdotL;



//Rim Lighting

fixed rim = 1 - SNdotL;

fixed3 rimLighting = SNdotL * _RimColor.xyz * _LightColor0.xyz * pow( rim, _RimPower );



//Specularity calc (0.7*normalDirection,0.7*mainlightDir for wetness, specularity and shine,

half3 theshine = i.lightDir.w * spTex.rgb* max( 0.0,dot(0.7*normalDirection,0.7* i.lightDir.xyz)) 

* pow(max(0.0, dot(reflect(-i.lightDir.xyz, normalDirection),i.cameraView )),_Shininess);

half3 matcaplight = thelighting + (UNITY_LIGHTMODEL_AMBIENT.xyz)+ rimLighting;





// messy overlay highlights

half3 anotherT = lerp((1-(1-sktex.rgb)),((1-2*sktex.rgb)*sktex.rgb*matcaplight),step(8.9*matcaplight, 1));

half3 overlayfun = lerp((1-2*(1-sktex.rgb)),(2*sktex.rgb* matcaplight),step(matcaplight, 0.2 ));



half3 supertest=(overlayfun)/(1-anotherT*0.3)*(theshine*0.2) + (0.2*theshine* (_SpecColor.rgb * _LightColor0.xyz ));




//final lighting

return half4 (sktex.rgb*matcaplight+supertest,1.0);





}



ENDCG







}

Just using the LIGHT_ATTENUATION() macro will not do the trick, you have to “feed” the data to it first by specifying the light coords, the LIGHT_ATTENUATION() macro is not set by code.

first of all, specify #pragma multi_compile_fwdbase (if you want to support more lights, this will become #pragma multi_compile_fwdadd)

In your vertex output struct, you will need to specify LIGHTING_COORDS(x, x) where x are the next available texcoord semantics (7 and 8 in your case)

In the vertex shader at the very end (but before the return) you will need to specify TRANSFER_VERTEX_TO_FRAGMENT(output) (or h in your case).

LIGHT_ATTENUATION is now filled with both the shadowing and attenuation values (combined) which you can use.

Will adding LIGHT_ATTENUATION conflict with my lightDir which defines the attenuation for this shader?

Considering that LIGHT_ATTENUTATION contains the combined shadow and attenuation factor you might get conflicts. Unless you’re doing something different with your attenuation than what unity does, you will not need it anymore. Only one way to find out though!

If you really want to keep your own calculation, you will need to dig into the include files unity provides to see how you can directly sample the shadow maps. I haven’t done this myself, but I did see some relevant macros passing by that makes me think it is possible.

my atten is 1/distance^2. It’s set like that because it reduces the flickering I get from point lights when I move away from them. I tried checking AutoLight.cginc because I think that is where the shadow calcs are but I don’t understand what’s in there the variables seem to be connected to some other functions.Also there seems to be calculations for cubes and pills but not for general objects.
do you think anyone has tried this before?

I don’t know exactly how it works in the Forward Lighting path, but with the Deferred Lighting path, the built-in ‘Internal-PrePassLighting’ shader calls ComputeShadow, which gives you most of the information you need. Normally it’s combined with lighting information, so if you want to use it to get the stand-alone shadow information, you still need to do some extra work.

It’s something I had to change to create dynamic shadows on lightmaps. The source code for that project is available, so it might help you figure out how to create the effect you’re looking for.

I looked at you code and I’m not sure if it is usable in my application

as for computeShadow I found a reference listed in complex-prepasslight-in

But I don’t know what this is. It looks like am incomplete shader or something:

inline half unitySampleShadow ( float3 vec , float mydist )
{
float z = 1.0 / 128.0 ;
float4 shadowVals ;
shadowVals . x = SampleCubeDistance ( vec + float3 ( z , z , z ) ) ;
shadowVals . y = SampleCubeDistance ( vec + float3 ( - z , - z , z ) ) ;
shadowVals . z = SampleCubeDistance ( vec + float3 ( - z , z , - z ) ) ;
shadowVals . w = SampleCubeDistance ( vec + float3 ( z , - z , - z ) ) ;
half4 shadows = ( shadowVals < mydist . xxxx ) ? _LightShadowData . rrrr : 1.0f ;
return dot ( shadows , 0.25 ) ;
}
half ComputeShadow ( float3 vec , float z , float2 uv )
{
float fade = z * _LightShadowData . z + _LightShadowData . w ;
fade = saturate ( fade ) ;
float mydist = length ( vec ) * _LightPositionRange . w ;
mydist *= 0.97 ;
return unitySampleShadow ( vec , mydist ) ;
return 1.0 ;
}

SampleCubeDistance seems to have come from AutoLight.cginc:

inline float unityCubeShadow (float3 vec)
{
	float mydist = length(vec) * _LightPositionRange.w;
	mydist *= 0.97; // bias

	#if defined (SHADOWS_SOFT)
	float z = 1.0/128.0;
	float4 shadowVals;
	shadowVals.x = SampleCubeDistance (vec+float3( z, z, z));
	shadowVals.y = SampleCubeDistance (vec+float3(-z,-z, z));
	shadowVals.z = SampleCubeDistance (vec+float3(-z, z,-z));
	shadowVals.w = SampleCubeDistance (vec+float3( z,-z,-z));
	half4 shadows = (shadowVals < mydist.xxxx) ? _LightShadowData.rrrr : 1.0f;
	return dot(shadows,0.25);
	#else
	float dist = SampleCubeDistance (vec);
	return dist < mydist ? _LightShadowData.r : 1.0;
	#endif
}

But I don’t know what’s going on here. maybe it’s someone’s to be a modded AutoLight.cginc file

I’m going to assume that both codes are from one of unities files.

You’re looking at the code that’s used for shadows in the Deferred lighting path.

They contain code from Internal-PrePassLighting, like I mentioned above, but they’re different from to the version used in Unity 4.3.4. They’re both missing some bits. In the case of ComputeShadow you can see they forgot to remove the return 1.0f, which was only relevant before the #if blocks were removed. You’re better off looking at the original file from the built-in shaders zip.

My example project was based on the same file (Albeit one from an older version of Unity), and #pragma debug versions of surface shaders.

What version is the code I posted?
Did Deferred lighting and Forward Lighting use the same shadow code in older versions of unity?
No matter if it did or not I guess I have to figure out where Forward Lighting’s shadow code is located.

I don’t know. I assume it’s modified by someone else, because it removes a lot of shadow types. You can find a lot of old versions in the archive, if you really want to find out which one it’s most similar to.

I don’t know, but I doubt it. The code for shadows in deferred lighting is different and supports way more types of shadows. Forward Rendering only supports one directional light.

To chase down the location of shadow code for Forward lighting, you can start by looking at the shader code generated for a surface shader using #pragma debug. In Forward Rendering, only the first lighting pass can have shadows, so you’re looking for "LightMode" = "ForwardBase".
In the generated fragment shader you’ll find a reference to LIGHT_ATTENUATION(IN)
Several versions of which are defined in AutoLight.cginc (in the \Data\CGIncludes folder). From what I can tell, all of those call SHADOW_ATTENUATION(a)
Some versions are simply defined as a float #define SHADOW_ATTENUATION(a) 1.0
But there are others that actually call other methods for shadow calculation.

#define SHADOW_ATTENUATION(a) unitySampleShadow(a._ShadowCoord)
#define SHADOW_ATTENUATION(a) unityCubeShadow(a._ShadowCoord)

You can ignore the second one, because the comment above says:// ---- Point light shadows
(Comments never lie, right?)
And according to the documentation, those are only supported with Deferred Lighting.
Which leaves you with three versions of unitySampleShadow. One of them is for mobile, one is only used if SPOT is defined (so I’m guessing it’s only for Deferred Lighting, again), and one of them is really simple:

inline fixed unitySampleShadow (float4 shadowCoord)
{
	fixed shadow = tex2Dproj( _ShadowMapTexture, UNITY_PROJ_COORD(shadowCoord) ).r;
	return shadow;
}

So, there you go. I got a bit carried away tracking down the code you’re looking for, but I guess it might be useful for others too.

Oh thank you. I will go back to working on my shader.

Sorry for the necro post but I’m still working on this shader. I have actually gotten it to be a good looking skin shader but I was still using the forward rendering path.
I happen to run into a shader which is a vert/frag with shadows using deferred rendering.
Kyle Halladay’s shader

Shader "Specular-Deferred"
{
Properties {
    _MainTex ("Base (RGB)", 2D) = "white"{}
    _SpecColor ("Specular Color", Color) = (0.5,0.5,0.5,1)
     _Shininess ("Shininess", Range(0.01,1)) = 0.078125
}
SubShader{
    Pass{Tags {"LightMode" = "PrePassBase"}
    CGPROGRAM
    #pragma vertex vert
    #pragma fragment frag       
    uniform float _Shininess;

    struct vIN
    {
        float4 vertex : POSITION;
        float3 normal : NORMAL;
    };
    struct vOUT
    {
        float4 pos : SV_POSITION;
        float3 wNorm : TEXCOORD0;
    };
    vOUT vert(vIN v)
    {
        vOUT o;
        o.pos = mul(UNITY_MATRIX_MVP, v.vertex);
        o.wNorm = mul((float3x3)_Object2World, v.normal);
        return o;
    }

    float4 frag(vOUT i) : COLOR
    {
        float3 norm = (i.wNorm * 0.5) + 0.5;
        return float4(norm, _Shininess);
    }
    ENDCG

    }





    Pass{Tags{"LightMode" = "PrePassFinal"}
    ZWrite off
    CGPROGRAM
    #pragma vertex vert
    #pragma fragment frag

    sampler2D _MainTex;
    uniform float4 _SpecColor;
    uniform sampler2D _LightBuffer;

    struct vIN
    {
        float4 vertex : POSITION;
        float2 texcoord : TEXCOORD0;
    };

    struct vOUT
    {
        float4 pos : SV_POSITION;
        float2 uv : TEXCOORD0;
        float4 uvProj : TEXCOORD1;
    };
   
vOUT vert(vIN v)
{
    vOUT o;
    o.pos = mul(UNITY_MATRIX_MVP, v.vertex);
   
    float4 posHalf = o.pos * 0.5;
    posHalf.y *= _ProjectionParams.x;
   
    o.uvProj.xy = posHalf.xy + float2(posHalf.w, posHalf.w);
    o.uvProj.zw = o.pos.zw;
    o.uv = v.texcoord;
   
    return o;
}



float4 frag(vOUT i) : COLOR
{
    float4 light = tex2Dproj(_LightBuffer, i.uvProj);
    float4 logLight = -(log2(max(light, float4(0.001,0.001,0.001,0.001))));
    float4 texCol = tex2D(_MainTex, i.uv);
    return float4((texCol.xyz * light.xyz) + float3(_SpecColor.xyz) * light.w, texCol.w);
}

}



    Pass{


Name "ShadowCaster"
    Tags { "LightMode" = "ShadowCaster" }
   
    Fog {Mode Off}
    ZWrite On ZTest LEqual Cull Off
    Offset 1, 1

    CGPROGRAM
    #pragma vertex vert
    #pragma fragment frag
    #pragma multi_compile_shadowcaster
    #include "UnityCG.cginc"

    struct v2f {
        V2F_SHADOW_CASTER;
    };

    v2f vert( appdata_base v )
    {
        v2f o;
        TRANSFER_SHADOW_CASTER(o)
        return o;
    }

    float4 frag( v2f i ) : SV_Target
    {
        SHADOW_CASTER_FRAGMENT(i)
    }
    ENDCG
    }
}
}

but I’m confused as how to convert my own shader because I’m using using madcaps and highlights. Also I’ve never heard or seen float4 logLight = -(log2(max(light, float4(0.001,0.001,0.001,0.001)))); and I have no idea what it does in his code.
My shader

_MainTex("Texture", 2D) = "white" {}
_Smap("SpecMap", 2D) = "white" {}
_Shininess ("Specularity",range(0.005,2.5))= 0.05
_MatCap ("MatCap (RGB)", 2D) = "white" {}
_CapPower("matPower",range(1.2,1.7))= 1.4
_Normals ("Normal", 2D)="white" {}
_NorRange ("Dephs",range(0.95,8))= 3

}
SubShader
{
Pass{
Tags{"LightMode" = "ForwardBase"}

CGPROGRAM
#pragma vertex vertsA
#pragma fragment fragsA 
#pragma target 3.0
#pragma only_renderers d3d9  debug


//user variables 
uniform half4 _Color;
uniform half _Shininess;
uniform half4 _LightColor0;
uniform sampler2D _MainTex;
uniform half4 _MainTex_ST;
uniform sampler2D _Normals;
uniform half4 _Normals_ST;
uniform half _NorRange;
uniform half _CapPower;
uniform sampler2D _Smap;
uniform half4 _Smap_ST;
uniform sampler2D _MatCap;
uniform float2 matcapUV;
half4x4 _Object2World;
half4x4 _World2Object;
half4 _WorldSpaceLightPos0;
half3 _WorldSpaceCameraPos;

//input structs


struct VertInput1 {   // vertex imput info
half4 vertex : POSITION;
half3 normal : NORMAL;
half4 texcoord : TEXCOORD0;
half4 tangent : TANGENT;
};

//output structs
struct VertOutput1 { // vertex output info
half4 pos : SV_POSITION;
half4 colur: COLOR;
half4 sktex : TEXCOORD0;
half4 lightDir : TEXCOORD1;
half3 posNormal : TEXCOORD2;
half3 tans : TEXCOORD3;
half3 binoms : TEXCOORD4;
half3 cameraView :TEXCOORD5;
half4 spTex : TEXCOORD6;
float2 cap  : TEXCOORD7;

};

// vertex

VertOutput1 vertsA(VertInput1 g)
{
VertOutput1 h;
//matcap
  half2 capCoord;
capCoord.x = dot(UNITY_MATRIX_IT_MV[0].xyz,g.normal);
capCoord.y = dot(UNITY_MATRIX_IT_MV[1].xyz,g.normal);
h.cap = capCoord * 0.5 + 0.5;


// direction variables
h.posNormal =  normalize(mul(half4(g.normal,0.0),_World2Object).xyz); //normalworld
h.tans = normalize( mul(_Object2World, g.tangent).xyz); //tangentworld
h.binoms = normalize(cross(h.posNormal,h.tans)* g.tangent.w);//binormalWorld

half4 posMain = mul(_Object2World, g.vertex);
h.pos = mul(UNITY_MATRIX_MVP, g.vertex);
h.sktex = g.texcoord;

h.cameraView = normalize (_WorldSpaceCameraPos.xyz -posMain.xyz); //tweakable



//multilight and light physics
half3 facetolite= _WorldSpaceLightPos0.xyz -posMain.xyz;
half distance = length(facetolite);

h.lightDir = half4(normalize(lerp(_WorldSpaceLightPos0.xyz,facetolite,_WorldSpaceLightPos0.w)),
lerp(1.0,(1.0/(distance*distance)),_WorldSpaceLightPos0.w));


return h;

}



// fragment

half4 fragsA(VertOutput1 i) : COLOR
{


//maps

half4 sktexf = tex2D(_MainTex, i.sktex.xy + (_MainTex_ST.xy * _MainTex_ST.zw));
half4 tans = tex2D(_Normals, i.sktex.xy + (_Normals_ST.xy * _Normals_ST.zw));
half4 spTex = tex2D(_Smap, i.sktex.xy + (_Smap_ST.xy *_Smap_ST.zw));

//normals
half3 localCoords = half3(2.0 * tans.ag - half2(1.0,1.0),_NorRange);

half3x3 local2WorldTranspose = float3x3( i.tans, i.binoms, i.posNormal);
half3 normalDirection =  normalize(mul(localCoords, local2WorldTranspose));

//matcap
fixed4 mc = tex2D(_MatCap,i.cap);

//Half-Lambert lighting calc.

half3 SNdotL=  saturate( dot( normalDirection,i.lightDir.xyz));
half3 thelighting = i.lightDir.w * _LightColor0.xyz *2* SNdotL;


//Specularty calc (0.7*normalDirection,0.7*mainlightDir for wetness, specularity and shine,
half3 theshine = i.lightDir.w * 0.8*spTex.rgb*(0.4*max( 0.0,dot(0.7*normalDirection,0.7* i.lightDir.xyz)))
* pow(max(0.0, dot(reflect(-i.lightDir.xyz, normalDirection),i.cameraView )),_Shininess);
//spec highlights messy code
half3 thehigh = i.lightDir.w * max( 0.0,dot(normalDirection, i.lightDir.xyz))
*pow(max(0.0, dot(reflect(-i.lightDir.xyz, normalDirection),i.cameraView )),15);
half3 thebrightness = theshine+(0.06*thehigh);
//combine the lights
half3 lights = (mc*_CapPower)*(thelighting)+(thebrightness);//(theshine*0.6);// + (2*UNITY_LIGHTMODEL_AMBIENT.xyz);



//final lighting

return half4 (sktexf.rgb*lights,1.0);

}

ENDCG

}

I’m alittle worried that I might have too much going on in the frag but that’s not my main issues.
I am not sure what parts of my shader should go in the base pass in deferred lighting. I know the “direction variables” need to be in the first pass but what about the “matcap” and the maps; I’m not use to separating stuff like this. Lastly can I make my own lightDirection/attenuation in deferred lighting or should I just edit AutoLight.cginc.

I’m planning on releasing the skin shader code when I’m done so I appreciate the help.

As it turned out I was able to get shadows using forward rendering by using by adding
#include “AutoLight.cginc”
#include “UnityCG.cginc”
SHADOW_COORDS(x) to output struct
TRANSFER_SHADOW(o) right above return. It has to be o so name your vertex out put o

I had to delete
float4x4 _Object2World;
float4x4 _World2Object;
float4 _WorldSpaceLightPos0;
float3 _WorldSpaceCameraPos;

I’m getting ready to release the shader