Physics2D.Linecast + queriesStartInColliders false = hit inner triangles of polygon

[mitaywalle]

What Should I change in Collider/Global Settings/Filter/Casting Call to not get this result?

Maybe add duplicative collider in outline mode and check if OutlineCollider2D.ClosestPoint(hit.point) is same?

Code To reproduce

using System.Collections.Generic;
using UnityEditor;
using UnityEngine;

namespace Game
{
	public class TestCast : MonoBehaviour
	{
		[SerializeField] private ContactFilter2D _filter;
		[SerializeField] private eOverrideBool _queriesStartInColliders;
		[SerializeField] private Transform _origin;
		private List<RaycastHit2D> _hits = new();

		private void OnDrawGizmos()
		{
			if (!_origin) return;
			bool cached = Physics2D.queriesStartInColliders;
			if (_queriesStartInColliders != eOverrideBool.Default)
			{
				Physics2D.queriesStartInColliders = _queriesStartInColliders == eOverrideBool.Enabled;
			}

			#if UNITY_EDITOR
			Handles.Label(transform.position, "target");
			Handles.Label(_origin.position, "origin");
			#endif

			Physics2D.Linecast(_origin.position, transform.position, _filter, _hits);

			Gizmos.color = Color.white / 5;
			Gizmos.DrawLine(transform.position, _origin.position);

			if (_hits.Count > 0)
			{
				Gizmos.color = Color.green;
				Gizmos.DrawLine(transform.position, _hits[0].point);
			}
			Physics2D.queriesStartInColliders = cached;
		}

		private void Reset()
		{
			_origin = new GameObject("TestCast Origin").transform;
		}

		public enum eOverrideBool
		{
			Default,
			Enabled,
			Disabled,
		}
	}
}

[mitaywalle]

Idea in post worked:

• duplicate PolygonCollider2D
• place duplicate at child of original
• Add CompositeCollider2D to duplicate
• set duplicate to layer IgnoreRaycast
• bool hitOuterSide = CompositeCollider2D.ClosestPoint(_hits[0].point) == _hits[0].point;

Fixed script

using System.Collections.Generic;
using UnityEditor;
using UnityEngine;

namespace Game
{
	public class TestCast : MonoBehaviour
	{
		[SerializeField] private ContactFilter2D _filter;
		[SerializeField] private eOverrideBool _queriesStartInColliders;
		[SerializeField] private Transform _origin;
		private List<RaycastHit2D> _hits = new();

		private void OnDrawGizmos()
		{
			if (!_origin) return;
			bool cached = Physics2D.queriesStartInColliders;
			if (_queriesStartInColliders != eOverrideBool.Default)
			{
				Physics2D.queriesStartInColliders = _queriesStartInColliders == eOverrideBool.Enabled;
			}

			#if UNITY_EDITOR
			Handles.Label(transform.position, "target");
			Handles.Label(_origin.position, "origin");
			#endif

			Physics2D.Linecast(_origin.position, transform.position, _filter, _hits);

			Gizmos.color = Color.white / 5;
			Gizmos.DrawLine(transform.position, _origin.position);

			if (_hits.Count > 0)
			{
				CompositeCollider2D comp = _hits[0].collider.GetComponentInChildren<CompositeCollider2D>();
				bool hitOuterSide = true;
				if (comp)
				{
					hitOuterSide = comp.ClosestPoint(_hits[0].point) == _hits[0].point;
				}
				
				if (hitOuterSide)
				{
					Gizmos.color = Color.green;
					Gizmos.DrawLine(transform.position, _hits[0].point);
				}
			}
			Physics2D.queriesStartInColliders = cached;
		}

		private void Reset()
		{
			_origin = new GameObject("TestCast Origin").transform;
		}

		public enum eOverrideBool
		{
			Default,
			Enabled,
			Disabled,
		}
	}
}