From b3aac984a717e83e082263a4e5a0b334a558984e Mon Sep 17 00:00:00 2001
From: Jack <jack@jackmchristensen.com>
Date: Wed, 28 Aug 2024 23:07:40 -0400
Subject: [PATCH] Correctly calculate normals with dynamic tessellation

---
 shaders/tessellation_ctrl.glsl |  7 +++++--
 shaders/tessellation_eval.glsl | 21 ++++++++++++++-------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/shaders/tessellation_ctrl.glsl b/shaders/tessellation_ctrl.glsl
index 0577fc4..defab8c 100644
--- a/shaders/tessellation_ctrl.glsl
+++ b/shaders/tessellation_ctrl.glsl
@@ -24,6 +24,7 @@ in VS_OUT {
 out TC_OUT {
     vec3 position;
     vec2 texCoord;
+    float tessLevel;
 } tc_out[];
 
 float getHeight(vec2 uv) {
@@ -39,8 +40,8 @@ float calculateSlope() {
     float hUp = getHeight(tc_in[gl_InvocationID].texCoord + vec2(0.0f, step));
     float hDown = getHeight(tc_in[gl_InvocationID].texCoord + vec2(0.0f, -step));
 
-    float slopeX = abs(hLeft - hRight);
-    float slopeY = abs(hUp - hDown);
+    float slopeX = abs(hLeft - hRight) * displacementScale;
+    float slopeY = abs(hUp - hDown) * displacementScale;
 
     float maxSlope = max(slopeX, slopeY);
 
@@ -54,6 +55,8 @@ void main() {
 
     float slope = calculateSlope();
     float tessLevel = tessellationFactor * (1.0f + slope * 10.0f);
+    tessLevel = max(tessLevel, maxTessellation);
+    tc_out[gl_InvocationID].tessLevel = tessLevel;
 
     // Set tessellation levels
     if (gl_InvocationID == 0) {
diff --git a/shaders/tessellation_eval.glsl b/shaders/tessellation_eval.glsl
index e3a197f..40a7065 100644
--- a/shaders/tessellation_eval.glsl
+++ b/shaders/tessellation_eval.glsl
@@ -18,6 +18,7 @@ layout(std140, binding = 4) uniform GeoUniforms
 in TC_OUT {
     vec3 position;
     vec2 texCoord;
+    float tessLevel;
 } te_in[];
 
 out TE_OUT {
@@ -30,19 +31,25 @@ float getHeight(vec2 uv) {
     return texture(heightTexture, uv).r * displacementScale;
 }
 
+float getNeighbor(float tessFactor) {
+    return (float(gridDensity) * tessFactor + 1.0f);
+}
+
 vec3 calculateSmoothNormal(vec3 pos, vec2 coord) {
     // Currently hardcoded to match patch density
-    float step = 1.0f / (float(gridDensity) * maxTessellation + 1.0f);
+    //float step = 1.0f / (float(gridDensity) * tessLevel + 1.0f);
+    float stepU = 1.0f / (float(gridDensity) * gl_TessLevelInner[0]);
+    float stepV = 1.0f / (float(gridDensity) * gl_TessLevelInner[1]);
 
     // Find surrounding heights
-    float hLeft = getHeight(coord + vec2(-step, 0.0f));
-    float hRight = getHeight(coord + vec2(step, 0.0f));
-    float hUp = getHeight(coord + vec2(0.0f, step));
-    float hDown = getHeight(coord + vec2(0.0f, -step));
+    float hLeft = getHeight(coord + vec2(-stepU, 0.0f));
+    float hRight = getHeight(coord + vec2(stepU, 0.0f));
+    float hUp = getHeight(coord + vec2(0.0f, stepV));
+    float hDown = getHeight(coord + vec2(0.0f, -stepV));
     
     // Calculate tangents
-    vec3 tangentX = normalize(vec3(step, hRight - hLeft, 0.0));
-    vec3 tangentY = normalize(vec3(0.0, hUp - hDown, step));
+    vec3 tangentX = normalize(vec3(stepU, hRight - hLeft, 0.0));
+    vec3 tangentY = normalize(vec3(0.0, hUp - hDown, stepV));
 
     return normalize(cross(tangentY, tangentX));
 }