Render White Pixels as Transparent

[Joey-The-Great]

I have a camera in a 2D scene that renders black and white. I’d like to overlay it on top of another camera’s output, but only the black pixels, ignoring the white ones. How do I do this?

Edit: an alternative solution could be to have a sprite renderer that is invisible wherever there is no shadows.

[Eno-Khaon]

Well, here’s a bit to get you started. I kind’ve forget the “ideal” way to set this sort of thing up, since I just adapted a much more complex full screen shader for this example as a proof of concept. This is also why the shaders are formatted the way they are; the original versions had more passes that reused functions.

I apologize in advance if I overlooked anything significant, since I made these conversions fairly hastily.

MaskRegion.cs – Attach to your main camera

using UnityEngine;

[RequireComponent(typeof(Camera))]
public class MaskRegion : MonoBehaviour
{
	public GameObject camObj;
	public Camera cam;
	Material mat;
	Shader s;
	RenderTexture rt;
	int maskTexID;

	Camera thisCam;

	bool initialized = false;

	// If you don't need full resolution, reducing
	// maskScale will be a meaningful workload reduction
	public float maskScale = 0.5f;

	int lastWidth;
	int lastHeight;

	private void Start()
	{
		// If you already have this camera created,
		// you can skip these and just assign
		// the camera instead
		camObj = new GameObject("Mask Camera");
		cam = camObj.AddComponent<Camera>();
		thisCam = GetComponent<Camera>();
		// In this camera-cloning example, this matches clipping planes and FOV and such
		cam.CopyFrom(thisCam);
		// Change the cullingMask to use your layer(s) for your overlay
		cam.cullingMask = thisCam.cullingMask;
		// You only potentially need to link the camera's transform
		// to your main camera if it's going to see anything the main camera will as well
		camObj.SetActive(false);
		camObj.transform.parent = transform;
		// ----------------------------------------

		// Ensure that the camera won't draw anything unnecessary. The clearing will be handled by your main camera instead.
		cam.clearFlags = CameraClearFlags.Nothing;
		// Safety net in case of window/screen resizing... I think these variables' use covers it adequately?
		lastWidth = Screen.width;
		lastHeight = Screen.height;
		rt = new RenderTexture((int)(lastWidth * maskScale), (int)(lastHeight * maskScale), 16, RenderTextureFormat.Default);
		rt.name = "Mask RenderTexture";
		// Your choice of filterMode, using Point as a hard-edged example
		rt.filterMode = FilterMode.Point;
		cam.targetTexture = rt;
		s = Shader.Find("EK/MaskSilhouette");
		mat = new Material(Shader.Find("EK/MaskRegion"));
		maskTexID = Shader.PropertyToID("_MaskTex");

		initialized = true;
	}

	private void OnApplicationFocus(bool focus)
	{
		if(initialized)
		{
			if(lastWidth != Screen.width || lastHeight != Screen.height)
			{
				lastWidth = Screen.width;
				lastHeight = Screen.height;
				rt = new RenderTexture((int)(lastWidth * maskScale), (int)(lastHeight * maskScale), 16, RenderTextureFormat.Default);
				rt.filterMode = FilterMode.Bilinear;
				cam.targetTexture = rt;
			}
		}
	}

	private void OnRenderImage(RenderTexture source, RenderTexture destination)
	{
		// Prepare current render to use the alternate camera
		RenderTexture rtActive = RenderTexture.active;
		RenderTexture.active = rt;

		// Clear background, most importantly, 0 alpha
		GL.Clear(true, true, Color.clear);

		cam.RenderWithShader(s, "");
		// SetTexture needs to be used consistently because
		// RenderTextures break at the drop of a hat
		mat.SetTexture(maskTexID, rt);

		// Return regular rendering to your main camera
		RenderTexture.active = rtActive;

		// Merge with original, passing in the original (Color) and masking (low res) RenderTextures
		Graphics.Blit(source, destination, mat);
	}
}

---

MaskSilhouette.shader -- This draws any objects visible to the camera in white.

More importantly, objects have alpha of 1, while background has alpha of 0.

Shader "EK/MaskSilhouette"
{
	Properties
	{
		_MainTex ("Texture", 2D) = "white" {}
	}

	CGINCLUDE
		#include "UnityCG.cginc"

		struct appdata
		{
			float4 vertex : POSITION;
			float2 uv : TEXCOORD0;
		};

		struct v2f
		{
			float2 uv : TEXCOORD0;
			float4 vertex : SV_POSITION;
		};

		sampler2D _MainTex;
		float4 _MainTex_ST;

		v2f vert (appdata v)
		{
			v2f o;
			o.vertex = UnityObjectToClipPos(v.vertex);
			o.uv = TRANSFORM_TEX(v.uv, _MainTex);
			return o;
		}
			
		fixed4 simpleColor (v2f i) : SV_Target
		{
			// Return fixed4(1,1,1,1), solid white
			return 1.0;
		}
	ENDCG

	SubShader
	{
		// Pass 0, draw visible objects in white
		Pass
		{
			CGPROGRAM

			#pragma vertex vert
			#pragma fragment simpleColor

			ENDCG
		}
	}
}

---

MaskRegion.shader -- Using the texture obtained from MaskSilhouette, draw ONLY the occluded regions: `(1.0 - silhouette.a)`.

Shader "EK/MaskRegion"
{
	Properties
	{
		_MainTex ("Texture", 2D) = "white" {}
	}

	CGINCLUDE
		#include "UnityCG.cginc"

		struct appdata
		{
			float4 vertex : POSITION;
			float2 uv : TEXCOORD0;
		};

		struct v2f
		{
			float2 uv : TEXCOORD0;
			float4 vertex : SV_POSITION;
		};

		sampler2D _MainTex;
		float4 _MainTex_ST;
		float4 _MainTex_TexelSize;

		sampler2D _MaskTex;
			
		v2f vert(appdata v)
		{
			v2f o;
			o.vertex = UnityObjectToClipPos(v.vertex);
			o.uv = TRANSFORM_TEX(v.uv, _MainTex);
			return o;
		}

		fixed4 frag(v2f i) : SV_Target
		{
			fixed4 baseColor = tex2D(_MainTex, i.uv);
			fixed4 silhouette = tex2D(_MaskTex, i.uv);

			return lerp(silhouette, baseColor, silhouette.a);
		}
	ENDCG
		
	SubShader
	{
		// Pass 0
		Pass
		{
			CGPROGRAM

			#pragma vertex vert
			#pragma fragment frag

			ENDCG
		}
	}
}

Again, I apologize if I made any mistakes in here; I think I removed all of my personal script calls, and didn’t put this in a new project for testing or anything.

Also, let me know if there’s anything you’d like clarified more. Most of this conversion process was spent on comments (including reminding myself what I wrote here), but I was also just trying to make sure I gutted what was no longer necessary.

[Joey-The-Great]

Bumping, because this has become a serious roadblock.