Cycles: Add reflection fix to Bump and Normal Map nodes

While changing the shading normal is a great way to add additional detail to a model, there are some problems with it.
One of them is that at grazing angles and/or strong changes to the normal, the reflected ray can end up pointing into the actual geometry, which results in a black spot.

This patch helps avoid this by automatically reducing the strength of the bump/normal map if the reflected direction would end up too shallow or inside the geometry.

Differential Revision: https://developer.blender.org/D2574

intern/cycles/kernel/shaders/node_bump.osl

@@ -64,5 +64,7 @@ surface node_bump(
 	if (use_object_space) {
 		NormalOut = normalize(transform("object", "world", NormalOut));
 	}
+
+	NormalOut = ensure_valid_reflection(Ng, I, NormalOut);
 }
 

intern/cycles/kernel/shaders/node_normal_map.osl

@@ -88,5 +88,7 @@ shader node_normal_map(
 
 	if (Strength != 1.0)
 		Normal = normalize(NormalIn + (Normal - NormalIn) * max(Strength, 0.0));
+
+	Normal = ensure_valid_reflection(Ng, I, Normal);
 }
 

intern/cycles/kernel/shaders/stdosl.h

@@ -282,6 +282,36 @@ point rotate (point p, float angle, point a, point b)
     return transform (M, p-a) + a;
 }
 
+normal ensure_valid_reflection(normal Ng, vector I, normal N)
+{
+    float sqr(float x) { return x*x; }
+
+    vector R = 2*dot(N, I)*N - I;
+    if (dot(Ng, R) >= 0.05) {
+        return N;
+    }
+
+    /* Form coordinate system with Ng as the Z axis and N inside the X-Z-plane.
+     * The X axis is found by normalizing the component of N that's orthogonal to Ng.
+     * The Y axis isn't actually needed.
+     */
+    vector X = normalize(N - dot(N, Ng)*Ng);
+
+    /* Calculate N.z and N.x in the local coordinate system. */
+    float Ix = dot(I, X), Iz = dot(I, Ng);
+    float Ix2 = sqr(dot(I, X)), Iz2 = sqr(dot(I, Ng));
+    float Ix2Iz2 = Ix2 + Iz2;
+
+    float a = sqrt(Ix2*(Ix2Iz2 - sqr(0.05)));
+    float b = Iz*0.05 + Ix2Iz2;
+    float c = (a + b > 0.0)? (a + b) : (-a + b);
+
+    float Nz = sqrt(0.5 * c * (1.0 / Ix2Iz2));
+    float Nx = sqrt(1.0 - sqr(Nz));
+
+    /* Transform back into global coordinates. */
+    return Nx*X + Nz*Ng;
+}
 
 
 // Color functions