0
\$\begingroup\$

I am currently trying to edit a Vertex Shader of a game and want to add inverted hull outlines to the desired objects, but there are a lot of tutorials on the inverted hull for Unity, Blender, and Unreal but not a single one for HLSL language.

So, as a beginner, I think it must be good to get some tips from the experts, so any HLSL shader expert advice will be appreciated.

Edit:

so I am adding some code here as well:
Vertex Shader for outline according to RenderDoc:

// ---- Created with 3Dmigoto v1.2.45 on Fri Oct  7 11:10:15 2022

cbuffer vs_matrix_cb : register(b0)
{
  float4x4 g_mWVP_VS : packoffset(c0);
  float4x4 g_mVP_VS : packoffset(c4);
  float4x4 g_mWVP_Prev_VS : packoffset(c8);
  float4x4 g_mWIT_VS : packoffset(c12);
  float4x4 g_mWLP_SM_VS : packoffset(c16);
  float4x4 g_mWLPB_SM_VS : packoffset(c20);
  float4x4 g_mWLP_PM_VS : packoffset(c24);
  float4x4 g_mWLPB_PM_VS : packoffset(c28);
  float4x3 g_mWV_VS : packoffset(c32);
  float4 g_mWV_VS_padding : packoffset(c35);
  float4x3 g_mW_VS : packoffset(c36);
  float4 g_mW_VS_padding : packoffset(c39);
}



// 3Dmigoto declarations
#define cmp -
Texture1D<float4> IniParams : register(t120);
Texture2D<float4> StereoParams : register(t125);


void main( 
  float3 v0 : POSITION0,
  float2 v1 : TEXCOORD0,
  out float4 o0 : SV_POSITION0,
  out float2 o1 : TEXCOORD0)
{
  float4 r0;
  uint4 bitmask, uiDest;
  float4 fDest;

  r0.xyz = v0.xyz;
  r0.w = 1;
  o0.x = dot(r0.xyzw, g_mWVP_VS._m00_m10_m20_m30);
  o0.y = dot(r0.xyzw, g_mWVP_VS._m01_m11_m21_m31);
  o0.z = dot(r0.xyzw, g_mWVP_VS._m02_m12_m22_m32);
  o0.w = dot(r0.xyzw, g_mWVP_VS._m03_m13_m23_m33);
  o1.xy = v1.xy;
  return;
}

and this is the pixel shader for the same:

// ---- Created with 3Dmigoto v1.2.45 on Fri Oct  7 13:33:48 2022

cbuffer ps_versatile_cb : register(b4)
{
  float4 g_vColor0_PS : packoffset(c0);
  float4 g_vColor1_PS : packoffset(c1);
  float4 g_vColor2_PS : packoffset(c2);
  float4 g_vColor3_PS : packoffset(c3);
  float4 g_vColor4_PS : packoffset(c4);
  float4 g_vColor5_PS : packoffset(c5);
  float4 g_vColor6_PS : packoffset(c6);
  float4 g_vColor7_PS : packoffset(c7);
  float4 g_vColor8_PS : packoffset(c8);
  float4 g_vColor9_PS : packoffset(c9);
  float4 g_vColor10_PS : packoffset(c10);
  float4 g_vColor11_PS : packoffset(c11);
  float4 g_vColor12_PS : packoffset(c12);
  float4 g_vColor13_PS : packoffset(c13);
  float4 g_vColor14_PS : packoffset(c14);
  float4 g_vColor15_PS : packoffset(c15);
  float4 g_vParam0_PS : packoffset(c16);
  float4 g_vParam1_PS : packoffset(c17);
  float4 g_vParam2_PS : packoffset(c18);
  float4 g_vParam3_PS : packoffset(c19);
  float4 g_vParam4_PS : packoffset(c20);
  float4 g_vParam5_PS : packoffset(c21);
  float4 g_vParam6_PS : packoffset(c22);
  float4 g_vParam7_PS : packoffset(c23);
  float4 g_vParam8_PS : packoffset(c24);
  float4 g_vParam9_PS : packoffset(c25);
  float4 g_vParam10_PS : packoffset(c26);
  float4 g_vParam11_PS : packoffset(c27);
  float4 g_vParam12_PS : packoffset(c28);
  float4 g_vParam13_PS : packoffset(c29);
  float4 g_vParam14_PS : packoffset(c30);
  float4 g_vParam15_PS : packoffset(c31);
  float4 g_vThreshold0_PS : packoffset(c32);
  float4 g_vThreshold1_PS : packoffset(c33);
  float4 g_vThreshold2_PS : packoffset(c34);
  float4 g_vThreshold3_PS : packoffset(c35);
}

SamplerState State_ImageSampler0_s : register(s0);
Texture2D<float4> Texture_ImageSampler0 : register(t0);


// 3Dmigoto declarations
#define cmp -
Texture1D<float4> IniParams : register(t120);
Texture2D<float4> StereoParams : register(t125);


void main( 
  float4 v0 : SV_POSITION0,
  float2 v1 : TEXCOORD0,
  out float4 o0 : SV_TARGET0,
  out float4 o1 : SV_TARGET1,
  out float4 o2 : SV_TARGET2)
{
  float4 r0,r1;
  uint4 bitmask, uiDest;
  float4 fDest;

  r0.xy = -g_vParam1_PS.xy + v1.xy;
  r0.xyz = Texture_ImageSampler0.Sample(State_ImageSampler0_s, r0.xy).xyz;
  r1.xy = g_vParam1_PS.xy + v1.xy;
  r1.xyz = Texture_ImageSampler0.Sample(State_ImageSampler0_s, r1.xy).xyz;
  r0.xyz = -r1.xyz + r0.xyz;
  r0.x = dot(r0.xyz, r0.xyz);
  r1.xyzw = g_vParam1_PS.xyxy * float4(1,-1,-1,1) + v1.xyxy;
  r0.yzw = Texture_ImageSampler0.Sample(State_ImageSampler0_s, r1.xy).xyz;
  r1.xyz = Texture_ImageSampler0.Sample(State_ImageSampler0_s, r1.zw).xyz;
  r0.yzw = -r1.xyz + r0.yzw;
  r0.y = dot(r0.yzw, r0.yzw);
  r0.x = r0.x + r0.y;
  r0.x = r0.x * 16 + -g_vParam0_PS.w;
  r0.y = 1 + -g_vParam0_PS.w;
  r0.y = 1 / r0.y;
  r0.x = saturate(r0.x * r0.y);
  r0.y = r0.x * -2 + 3;
  r0.x = r0.x * r0.x;
  r0.x = r0.y * r0.x;
  r0.y = Texture_ImageSampler0.Sample(State_ImageSampler0_s, v1.xy).z;
  r0.y = saturate(r0.y + r0.y);
  r0.y = log2(r0.y);
  r0.y = g_vParam0_PS.z * r0.y;
  r0.y = exp2(r0.y);
  r0.y = g_vParam0_PS.y * r0.y;
  r0.x = r0.y * r0.x;
  r0.x = saturate(1.42857146 * r0.x);
  r0.x = log2(r0.x);
  r0.x = 10 * r0.x;
  o0.w = exp2(r0.x);
  o0.xyz = float3(0,0,0);
  o1.xyzw = float4(1,1,1,1);
  o2.xyzw = float4(1,1,1,1);
  return;
}

and in the end here is the reference image of the outline looking right now:

image of renderDoc

\$\endgroup\$
8
  • \$\begingroup\$ There is no pure HLSL implementation of this effect. HLSL is used to write the shader that executes during a draw call, but not to issue that draw call in the first place, or to set the rendering state flags like reversing the culling direction — both of which you need for this outlining effect, to issue a second draw call with the winding reversed. Do you have access to other parts of the game, such as CG Effect files which can specify these extra steps, the asset/material definitions, or the rendering code? \$\endgroup\$ Commented Oct 7, 2022 at 12:21
  • \$\begingroup\$ link got the motivation from here as it is almost HLSL but in Unity and the second thing yes I can able to access the other models and effects of this game. \$\endgroup\$ Commented Oct 7, 2022 at 12:27
  • \$\begingroup\$ Yeah, Unity uses an extra language wrapped around HLSL called ShaderLab, so the extra pass and culling changes are set in ShaderLab, not in HLSL code. Can you edit your question to show us how the effects in this game are set up? There are multiple conventions so we'd need to see which one your game is using to give you a compatible answer. \$\endgroup\$ Commented Oct 7, 2022 at 12:32
  • \$\begingroup\$ so I have try to add the vertex shader and pixel shader causing the outline in the game using depth techniques like sobel \$\endgroup\$ Commented Oct 7, 2022 at 12:40
  • \$\begingroup\$ A sobel filter will work, if you have a full-screen pass with access to the G-buffers or depth and normals from the rendered scene that you can edit. But then you should change your question and its title to not require the solution use the "inverted hull" technique. \$\endgroup\$ Commented Oct 7, 2022 at 12:52

0

Reset to default

You must log in to answer this question.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.