Cutout quality improvement in HDRP

Hey everyone, we are currently working on a project, and our cutout materials are flickering a lot. We’ve tried all the material settings and post-processing anti-aliasing settings but couldn’t find a solution. Do you have any suggestions? Decals look nice, why is the cutout experiencing flickering :confused:

it’s because alpha clipping is on a per pixel level, so it creates it’s own aliasing.

i’d read this, but it will take some shader writing.
Anti-aliased Alpha Test: The Esoteric Alpha To Coverage | by Ben Golus | Medium

1 Like

We had seen this article before, but since it was published in 2017 and we are using HDRP, we couldn’t use the shaders mentioned in it. Yesterday, we tried some things with the help of ChatGPT, but we still haven’t noticed any difference.

Shader "Custom/AlphaToCoverage"
{
    Properties
    {
        _MainTex ("Texture", 2D) = "white" {}
        _Cutoff ("Alpha cutoff", Range(0,1)) = 0.5
    }
    SubShader
    {
        Tags {"Queue"="AlphaTest" "RenderType"="TransparentCutout"}
        LOD 200

        Pass
        {
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            #include "UnityCG.cginc"

            struct appdata_t
            {
                float4 vertex : POSITION;
                float2 uv : TEXCOORD0;
            };

            struct v2f
            {
                float2 uv : TEXCOORD0;
                float4 vertex : SV_POSITION;
            };

            sampler2D _MainTex;
            float4 _MainTex_ST;
            float _Cutoff;

            v2f vert (appdata_t v)
            {
                v2f o;
                o.vertex = UnityObjectToClipPos(v.vertex);
                o.uv = TRANSFORM_TEX(v.uv, _MainTex);
                return o;
            }

            half4 frag (v2f i) : SV_Target
            {
                half4 tex = tex2D(_MainTex, i.uv);
                clip(tex.a - _Cutoff);
                return tex;
            }
            ENDCG
        }
    }
    FallBack "Diffuse"
}

maybe try this:

o.vertex = UnityPixelSnap(UnityObjectToClipPos(v.vertex));

not sure if this function is in hdrp though

The code is compatible with HDRP but they are much weirder and sharper :smile: When I examine similar cutouts in Valorant, I see that there is no flicker

New Shader;

Shader "Unlit/CustomAlphaToCoverage"
{
    Properties
    {
        _MainTex("Texture", 2D) = "white" {}
        _Cutoff("Alpha cutoff", Range(0,1)) = 0.5
    }
        SubShader
        {
            Tags {"Queue" = "AlphaTest" "RenderType" = "TransparentCutout"}
            LOD 200

            Pass
            {
                CGPROGRAM
                #pragma vertex vert
                #pragma fragment frag
                #include "UnityCG.cginc"

                struct appdata_t
                {
                    float4 vertex : POSITION;
                    float2 uv : TEXCOORD0;
                };

                struct v2f
                {
                    float2 uv : TEXCOORD0;
                    float4 vertex : SV_POSITION;
                };

                sampler2D _MainTex;
                float4 _MainTex_ST;
                float _Cutoff;

                v2f vert(appdata_t v)
                {
                    v2f o;
                    o.vertex = UnityPixelSnap(UnityObjectToClipPos(v.vertex));
                    o.uv = TRANSFORM_TEX(v.uv, _MainTex);
                    return o;
                }

                half4 frag(v2f i) : SV_Target
                {
                    half4 tex = tex2D(_MainTex, i.uv);
                    // Apply smooth edge using alpha cutoff and smoothstep
                    float alpha = smoothstep(_Cutoff - 0.05, _Cutoff + 0.05, tex.a);
                    if (alpha < 0.01)
                    {
                        discard;
                    }
                    return half4(tex.rgb, alpha);
                }
                ENDCG
            }
        }
            FallBack "Diffuse"
}

Although it is not a complete solution, it has solved my problem for now, now we are having problems with Decals. When sorting by draw order, it creates a border mask. How can I reduce this?


9884277--1426191--upload_2024-6-11_13-18-19.png

try messing with the offset command on the shaders of both, but it can cause overlapping with too high of values
Unity - Manual: ShaderLab command: Offset (unity3d.com)

I tried this code in the Passes and SubShader section, it did not make any changes and the code deleted them when compiling.

it goes right before cgprogram.

Yes, but this is HDRP’s default decal shader, where HLSLPROGRAM is used instead of CGPROGRAM.

Shader "HDRP/Decal"
{
    Properties
    {
        _BaseColor("_BaseColor", Color) = (1,1,1,1)
        _BaseColorMap("BaseColorMap", 2D) = "white" {}
        _NormalMap("NormalMap", 2D) = "bump" {}     // Tangent space normal map
        _MaskMap("MaskMap", 2D) = "white" {}
        _DecalBlend("_DecalBlend", Range(0.0, 1.0)) = 0.5
        _NormalBlendSrc("_NormalBlendSrc", Float) = 0.0
        _MaskBlendSrc("_MaskBlendSrc", Float) = 1.0
        [Enum(Depth Bias, 0, View Bias, 1)] _DecalMeshBiasType("_DecalMeshBiasType", Int) = 0
        _DecalMeshDepthBias("_DecalMeshDepthBias", Float) = 0.0
        _DecalMeshViewBias("_DecalMeshViewBias", Float) = 0.0
        _DrawOrder("_DrawOrder", Int) = 0
        [HDR] _EmissiveColor("EmissiveColor", Color) = (0, 0, 0)
        // Used only to serialize the LDR and HDR emissive color in the material UI,
        // in the shader only the _EmissiveColor should be used
        [HideInInspector] _EmissiveColorLDR("EmissiveColor LDR", Color) = (0, 0, 0)
        [HDR][HideInInspector] _EmissiveColorHDR("EmissiveColor HDR", Color) = (0, 0, 0)
        _EmissiveColorMap("EmissiveColorMap", 2D) = "white" {}
        _EmissiveIntensityUnit("Emissive Mode", Int) = 0
        [ToggleUI] _UseEmissiveIntensity("Use Emissive Intensity", Int) = 0
        _EmissiveIntensity("Emissive Intensity", Float) = 1
        _EmissiveExposureWeight("Emissive Pre Exposure", Range(0.0, 1.0)) = 1.0

        // Remapping
        _MetallicRemapMin("_MetallicRemapMin", Range(0.0, 1.0)) = 0.0
        _MetallicRemapMax("_MetallicRemapMax", Range(0.0, 1.0)) = 1.0
        _SmoothnessRemapMin("SmoothnessRemapMin", Float) = 0.0
        _SmoothnessRemapMax("SmoothnessRemapMax", Float) = 1.0
        _AORemapMin("AORemapMin", Float) = 0.0
        _AORemapMax("AORemapMax", Float) = 1.0

        // scaling
        _DecalMaskMapBlueScale("_DecalMaskMapBlueScale", Range(0.0, 1.0)) = 1.0

        // Alternative when no mask map is provided
        _Smoothness("_Smoothness",  Range(0.0, 1.0)) = 0.5
        _Metallic("_Metallic",  Range(0.0, 1.0)) = 0.0
        _AO("_AO",  Range(0.0, 1.0)) = 1.0

        [ToggleUI]_AffectAlbedo("Boolean", Float) = 1
        [ToggleUI]_AffectNormal("Boolean", Float) = 1
        [ToggleUI]_AffectAO("Boolean", Float) = 0
        [ToggleUI]_AffectMetal("Boolean", Float) = 1
        [ToggleUI]_AffectSmoothness("Boolean", Float) = 1
        [ToggleUI]_AffectEmission("Boolean", Float) = 0

        // Stencil state
        [HideInInspector] _DecalStencilRef("_DecalStencilRef", Int) = 16
        [HideInInspector] _DecalStencilWriteMask("_DecalStencilWriteMask", Int) = 16

        // Decal color masks
        [HideInInspector]_DecalColorMask0("_DecalColorMask0", Int) = 0
        [HideInInspector]_DecalColorMask1("_DecalColorMask1", Int) = 0
        [HideInInspector]_DecalColorMask2("_DecalColorMask2", Int) = 0
        [HideInInspector]_DecalColorMask3("_DecalColorMask3", Int) = 0

        // TODO: Remove when name garbage is solve (see IsHDRenderPipelineDecal)
        // This marker allow to identify that a Material is a HDRP/Decal
        [HideInInspector]_Unity_Identify_HDRP_Decal("_Unity_Identify_HDRP_Decal", Float) = 1.0
    }

    HLSLINCLUDE

    #pragma target 4.5
    #pragma only_renderers d3d11 playstation xboxone xboxseries vulkan metal switch
    //#pragma enable_d3d11_debug_symbols

    //-------------------------------------------------------------------------------------
    // Variant
    //-------------------------------------------------------------------------------------
    #pragma shader_feature_local_fragment _COLORMAP
    #pragma shader_feature_local_fragment _MASKMAP
    #pragma shader_feature_local _NORMALMAP
    #pragma shader_feature_local_fragment _EMISSIVEMAP

    #pragma shader_feature_local_fragment _MATERIAL_AFFECTS_ALBEDO
    #pragma shader_feature_local_fragment _MATERIAL_AFFECTS_NORMAL
    #pragma shader_feature_local_fragment _MATERIAL_AFFECTS_MASKMAP

    #pragma multi_compile_instancing

    //-------------------------------------------------------------------------------------
    // Include
    //-------------------------------------------------------------------------------------

    #include "Packages/com.unity.render-pipelines.core/ShaderLibrary/Common.hlsl"
    #include "Packages/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/ShaderPass/FragInputs.hlsl"
    #include "Packages/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/ShaderPass/ShaderPass.cs.hlsl"

    // All our shaders use same name for entry point
    #pragma vertex Vert
    #pragma fragment Frag

    ENDHLSL

    SubShader
    {
        Tags{ "RenderPipeline" = "HDRenderPipeline"}

        // c# code relies on the order in which the passes are declared, any change will need to be reflected in
        // DecalSystem.cs - enum MaterialDecalPass
        // DecalSubTarget.cs  - class SubShaders
        // Caution: passes stripped in builds (like the scene picking pass) need to be put last to have consistent indices

        Pass // 0
        {
            Name "DBufferProjector"
            Tags{"LightMode" = "DBufferProjector"} // Metalness

            Stencil
            {
                WriteMask[_DecalStencilWriteMask]
                Ref[_DecalStencilRef]
                Comp Always
                Pass Replace
            }

            // back faces with zfail, for cases when camera is inside the decal volume
            Cull Front
            ZWrite Off
            ZTest Greater

            // using alpha compositing https://developer.nvidia.com/gpugems/GPUGems3/gpugems3_ch23.html
            Blend 0 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
            Blend 1 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
            Blend 2 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
            Blend 3 Zero OneMinusSrcColor

            ColorMask [_DecalColorMask0]
            ColorMask [_DecalColorMask1] 1
            ColorMask [_DecalColorMask2] 2
            ColorMask [_DecalColorMask3] 3

            HLSLPROGRAM

            #pragma multi_compile_fragment DECALS_3RT DECALS_4RT
            #pragma multi_compile_fragment _ DECAL_SURFACE_GRADIENT
            #define SHADERPASS SHADERPASS_DBUFFER_PROJECTOR
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/ShaderLibrary/ShaderVariables.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProperties.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/Decal.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/ShaderPass/DecalSharePass.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalData.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/ShaderPass/ShaderPassDecal.hlsl"

            ENDHLSL
        }

        Pass // 1
        {
            Name "DecalProjectorForwardEmissive"
            Tags{ "LightMode" = "DecalProjectorForwardEmissive" }

            Stencil
            {
                WriteMask[_DecalStencilWriteMask]
                Ref[_DecalStencilRef]
                Comp Always
                Pass Replace
            }
            // back faces with zfail, for cases when camera is inside the decal volume
            Cull Front
            ZWrite Off
            ZTest Greater

            // additive
            Blend 0 SrcAlpha One

            HLSLPROGRAM

            #define _MATERIAL_AFFECTS_EMISSION
            #define SHADERPASS SHADERPASS_FORWARD_EMISSIVE_PROJECTOR
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/ShaderLibrary/ShaderVariables.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProperties.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/Decal.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/ShaderPass/DecalSharePass.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalData.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/ShaderPass/ShaderPassDecal.hlsl"

            ENDHLSL
        }

        Pass // 2
        {
            Name "DBufferMesh"
            Tags{"LightMode" = "DBufferMesh"}

            Stencil
            {
                WriteMask [_DecalStencilWriteMask]
                Ref [_DecalStencilRef]
                Comp Always
                Pass Replace
            }

            ZWrite Off
            ZTest LEqual

            // using alpha compositing https://developer.nvidia.com/gpugems/GPUGems3/gpugems3_ch23.html
            Blend 0 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
            Blend 1 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
            Blend 2 SrcAlpha OneMinusSrcAlpha, Zero OneMinusSrcAlpha
            Blend 3 Zero OneMinusSrcColor

            ColorMask [_DecalColorMask0]
            ColorMask [_DecalColorMask1] 1
            ColorMask [_DecalColorMask2] 2
            ColorMask [_DecalColorMask3] 3

            HLSLPROGRAM

            #pragma multi_compile_fragment DECALS_3RT DECALS_4RT
            #pragma multi_compile_fragment _ DECAL_SURFACE_GRADIENT
            #pragma multi_compile _ DOTS_INSTANCING_ON
            // enable dithering LOD crossfade
            #pragma multi_compile _ LOD_FADE_CROSSFADE

            #define SHADERPASS SHADERPASS_DBUFFER_MESH
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/ShaderLibrary/ShaderVariables.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProperties.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/Decal.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/ShaderPass/DecalSharePass.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalData.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/ShaderPass/ShaderPassDecal.hlsl"

            ENDHLSL
        }

        Pass // 3
        {
            Name "DecalMeshForwardEmissive"
            Tags{ "LightMode" = "DecalMeshForwardEmissive" }

            Stencil
            {
                WriteMask[_DecalStencilWriteMask]
                Ref[_DecalStencilRef]
                Comp Always
                Pass Replace
            }
            // back faces with zfail, for cases when camera is inside the decal volume
            ZWrite Off
            ZTest LEqual

            // additive
            Blend 0 SrcAlpha One

            HLSLPROGRAM
            #pragma multi_compile _ DOTS_INSTANCING_ON
            // enable dithering LOD crossfade
            #pragma multi_compile _ LOD_FADE_CROSSFADE

            #define _MATERIAL_AFFECTS_EMISSION
            #define SHADERPASS SHADERPASS_FORWARD_EMISSIVE_MESH
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/ShaderLibrary/ShaderVariables.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProperties.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/Decal.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/ShaderPass/DecalSharePass.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalData.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/ShaderPass/ShaderPassDecal.hlsl"

            ENDHLSL
        }

        Pass // 4
        {
            Name "ScenePickingPass"
            Tags { "LightMode" = "Picking" }

            Cull Back

            HLSLPROGRAM

            #pragma only_renderers d3d11 playstation xboxone xboxseries vulkan metal switch

            //enable GPU instancing support
            #pragma instancing_options renderinglayer
            #pragma multi_compile _ DOTS_INSTANCING_ON
            // enable dithering LOD crossfade
            #pragma multi_compile _ LOD_FADE_CROSSFADE

            // Note: Require _SelectionID variable

            // We reuse depth prepass for the scene selection, allow to handle alpha correctly as well as tessellation and vertex animation
            #define SHADERPASS SHADERPASS_DEPTH_ONLY
            #define SCENEPICKINGPASS
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/ShaderLibrary/ShaderVariables.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/ShaderLibrary/PickingSpaceTransforms.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProperties.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/Decal.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/ShaderPass/DecalSharePass.hlsl"
            #include "Packages/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/ShaderPass/ShaderPassDecal.hlsl"

            #pragma editor_sync_compilation

            ENDHLSL
        }

    }
    CustomEditor "Rendering.HighDefinition.DecalUI"
}

I tried to use the code here, where exactly should I use it?

that’s where it goes in cg.