LayerMask on SSAO

I made some modifications on the SSAO script, my objetive is simple: The shader only should draw occlusions on the selected objects in a layermask...

My code basically show a layermask in the inspector, change the layermask on the camera, render the occlusion and restores the layermask in the camera...

But not works...

Oclussion is in all objects of all layers...

Is possible to make that??

Sorry for my English and Thanks ;)

P.D:

If you want to take a look in the code...

using UnityEngine;

[ExecuteInEditMode] [RequireComponent (typeof(Camera))] [AddComponentMenu("Image Effects/SSAO")] public class SSAOEffect : MonoBehaviour { public enum SSAOSamples { Low = 0, Medium = 1, High = 2, }

public float m_Radius = 0.4f;
public SSAOSamples m_SampleCount = SSAOSamples.Medium;
public float m_OcclusionIntensity = 1.5f;
public int m_Blur = 2;
public int m_Downsampling = 2;
public float m_OcclusionAttenuation = 1.0f;
public float m_MinZ = 0.01f;

//LayerMask para SSAO
public bool m_UseLayers = false;
public LayerMask m_SsaoLayers = -1;
private LayerMask OldLayerMask;

public Shader m_SSAOShader;
private Material m_SSAOMaterial;

public Texture2D m_RandomTexture;

private bool m_Supported;
private bool m_IsOpenGL;

private static Material CreateMaterial (Shader shader)
{
    if (!shader)
        return null;
    Material m = new Material (shader);
    m.hideFlags = HideFlags.HideAndDontSave;
    return m;
}
private static void DestroyMaterial (Material mat)
{
    if (mat)
    {
        DestroyImmediate (mat);
        mat = null;
    }
}

void OnDisable()
{
    DestroyMaterial (m_SSAOMaterial);
}

void Start()
{
    if (!SystemInfo.supportsImageEffects || !SystemInfo.SupportsRenderTextureFormat (RenderTextureFormat.Depth))
    {
        m_Supported = false;
        enabled = false;
        return;
    }

    CreateMaterials ();
    if (!m_SSAOMaterial || m_SSAOMaterial.passCount != 5)
    {
        m_Supported = false;
        enabled = false;
        return;
    }

    //CreateRandomTable (26, 0.2f);

    camera.depthTextureMode = DepthTextureMode.DepthNormals;
    m_Supported = true;
    m_IsOpenGL = SystemInfo.graphicsDeviceVersion.StartsWith("OpenGL");
}

private void CreateMaterials ()
{
    if (!m_SSAOMaterial && m_SSAOShader.isSupported)
    {
        m_SSAOMaterial = CreateMaterial (m_SSAOShader);
        m_SSAOMaterial.SetTexture ("_RandomTexture", m_RandomTexture);
    }
}

void OnRenderImage (RenderTexture source, RenderTexture destination)
{
    if (!m_Supported || !m_SSAOShader.isSupported) {
        enabled = false;
        return;
    }
    CreateMaterials ();

    m_Downsampling = Mathf.Clamp (m_Downsampling, 1, 6);
    m_Radius = Mathf.Clamp (m_Radius, 0.05f, 1.0f);
    m_MinZ = Mathf.Clamp (m_MinZ, 0.00001f, 0.5f);
    m_OcclusionIntensity = Mathf.Clamp (m_OcclusionIntensity, 0.5f, 4.0f);
    m_OcclusionAttenuation = Mathf.Clamp (m_OcclusionAttenuation, 0.2f, 2.0f);
    m_Blur = Mathf.Clamp (m_Blur, 0, 4);

    //SSAO Layers...
    if (m_UseLayers)
    {

        OldLayerMask = camera.cullingMask;
        camera.cullingMask = m_SsaoLayers;
    }

    // Render SSAO term into a smaller texture
    RenderTexture rtAO = RenderTexture.GetTemporary (source.width / m_Downsampling, source.height / m_Downsampling, 0);
    float fovY = camera.fieldOfView;
    float far = camera.farClipPlane;
    float y = Mathf.Tan (fovY * Mathf.Deg2Rad * 0.5f) * far;
    float x = y * camera.aspect;
    m_SSAOMaterial.SetVector ("_FarCorner", new Vector3(x,y,far));
    int noiseWidth, noiseHeight;
    if (m_RandomTexture) {
        noiseWidth = m_RandomTexture.width;
        noiseHeight = m_RandomTexture.height;
    } else {
        noiseWidth = 1; noiseHeight = 1;
    }
    m_SSAOMaterial.SetVector ("_NoiseScale", new Vector3 ((float)rtAO.width / noiseWidth, (float)rtAO.height / noiseHeight, 0.0f));
    m_SSAOMaterial.SetVector ("_Params", new Vector4(
        m_Radius,
        m_MinZ,
        1.0f / m_OcclusionAttenuation,
        m_OcclusionIntensity));

    bool doBlur = m_Blur > 0;
    Graphics.Blit (doBlur ? null : source, rtAO, m_SSAOMaterial, (int)m_SampleCount);

    if (doBlur)
    {
        // Blur SSAO horizontally
        RenderTexture rtBlurX = RenderTexture.GetTemporary (source.width, source.height, 0);
        m_SSAOMaterial.SetVector ("_TexelOffsetScale",
            m_IsOpenGL ?
            new Vector4 (m_Blur,0,1.0f/m_Downsampling,0) :
            new Vector4 ((float)m_Blur / source.width, 0,0,0));
        m_SSAOMaterial.SetTexture ("_SSAO", rtAO);
        Graphics.Blit (null, rtBlurX, m_SSAOMaterial, 3);
        RenderTexture.ReleaseTemporary (rtAO); // original rtAO not needed anymore

        // Blur SSAO vertically
        RenderTexture rtBlurY = RenderTexture.GetTemporary (source.width, source.height, 0);
        m_SSAOMaterial.SetVector ("_TexelOffsetScale",
            m_IsOpenGL ?
            new Vector4 (0, m_Blur, 1,0) :
            new Vector4 (0, (float)m_Blur/source.height, 0,0));
        m_SSAOMaterial.SetTexture ("_SSAO", rtBlurX);
        Graphics.Blit (source, rtBlurY, m_SSAOMaterial, 3);
        RenderTexture.ReleaseTemporary (rtBlurX); // blurX RT not needed anymore

        rtAO = rtBlurY; // AO is the blurred one now
    }

    // Modulate scene rendering with SSAO
    m_SSAOMaterial.SetTexture ("_SSAO", rtAO);
    Graphics.Blit (source, destination, m_SSAOMaterial, 4);

    RenderTexture.ReleaseTemporary (rtAO);
    //SSAO Layers...
    if (m_UseLayers)
    {
        camera.cullingMask = OldLayerMask;
    }
}

/*
private void CreateRandomTable (int count, float minLength)
{
    Random.seed = 1337;
    Vector3[] samples = new Vector3[count];
    // initial samples
    for (int i = 0; i < count; ++i)
        samples *= Random.onUnitSphere;*
 *// energy minimization: push samples away from others*
 *int iterations = 100;*
 *while (iterations-- > 0) {*
 *for (int i = 0; i < count; ++i) {*
 _Vector3 vec = samples*;*_
 _*Vector3 res = Vector3.zero;*_
 _*// minimize with other samples*_
 _*for (int j = 0; j < count; ++j) {*_
 _*Vector3 force = vec - samples[j];*_
 _*float fac = Vector3.Dot (force, force);*_
 _*if (fac > 0.00001f)*_
 <em>_res += force * (1.0f / fac);_</em>
 _*}*_
 <em><em>samples <em>= (samples _+ res * 0.5f).normalized;_</em></em></em>
 <em><em>_*}*_</em></em>
 <em><em>_*}*_</em></em>
 <em><em>_*// now scale samples between minLength and 1.0*_</em></em>
 <em><em>_*for (int i = 0; i < count; ++i) {*_</em></em>
 <em><em><em><em>samples <em>= samples _* Random.Range (minLength, 1.0f);_</em></em></em></em></em>
 <em><em><em><em>_*}*_</em></em></em></em> 
 <em><em><em><em><em>*string table = string.Format ("#define SAMPLE_COUNT {0}
", count);*</em></em></em></em></em>
 <em><em><em><em><em>*table += "const float3 RAND_SAMPLES[SAMPLE_COUNT] = {
";*</em></em></em></em></em>
 <em><em><em><em>_*for (int i = 0; i < count; ++i) {*_</em></em></em></em>
 <em><em><em><em><em>_Vector3 v = samples*;*_</em></em></em></em></em>
 <em><em><em><em><em>_*table += string.Format("	float3({0},{1},{2}),
", v.x, v.y, v.z);*_</em></em></em></em></em>
 <em><em><em><em><em>_*}*_</em></em></em></em></em>
 <em><em><em><em><em>_*table += "};
";*_</em></em></em></em></em>
 <em><em><em><em><em>_*Debug.Log (table);*_</em></em></em></em></em>
<em><em><em><em><em>_*}*_</em></em></em></em></em>
<em><em><em><em><em><em>_*/_</em></em></em></em></em></em>
<em><em><em><em><em>_*```*_</em></em></em></em></em>
<em><em><em><em><em>_*<p>}</p>*_</em></em></em></em></em>

I don't think it's possible using the standar SSAO script. When OnRenderImage is called the full scene has been already rendered and is placed at the source RenderTexture. The effect is applied in several "passes" (look at the Blit calls) each one affecting the entire image.

A possible way of achieving what you want is using two cameras: first one renders the objects without SSAO. The second one then renders the objects with SSAO without clearing the backbuffer (camera.clearFlags = CameraClearFlags.Nothing). Use camera.depth to specify the order each camera is rendered.