Fix T70476: Sculpting with Subsurf on top produces artifacts

The issue was caused by crazy space distortion orientation happening
for subsurf modifier.

Solved by making it so subsurf only deforms the surface but keeps
matrices as-is. This is not fully mathematically correct, but is better
that the fall-back solution which was doing wrong matrices anyway.
Also, this is closer to have subsurf was handled prior to the
related changes.

Reviewed By: brecht, pablodp606

Differential Revision: https://developer.blender.org/D5991

source/blender/blenkernel/intern/crazyspace.c

@@ -355,7 +355,7 @@ static void crazyspace_init_verts_and_matrices(const Mesh *mesh,
   BLI_assert(num_verts == mesh->totvert);
 }
 
-static bool crazyspace_modifier_supports_deform(ModifierData *md)
+static bool crazyspace_modifier_supports_deform_matrices(ModifierData *md)
 {
   if (ELEM(md->type, eModifierType_Subsurf, eModifierType_Multires)) {
     return true;
@@ -364,6 +364,12 @@ static bool crazyspace_modifier_supports_deform(ModifierData *md)
   return (mti->type == eModifierTypeType_OnlyDeform);
 }
 
+static bool crazyspace_modifier_supports_deform(ModifierData *md)
+{
+  const ModifierTypeInfo *mti = modifierType_getInfo(md->type);
+  return (mti->type == eModifierTypeType_OnlyDeform);
+}
+
 int BKE_sculpt_get_first_deform_matrices(struct Depsgraph *depsgraph,
                                          Scene *scene,
                                          Object *object,
@@ -391,13 +397,12 @@ int BKE_sculpt_get_first_deform_matrices(struct Depsgraph *depsgraph,
   md = modifiers_getVirtualModifierList(&object_eval, &virtualModifierData);
 
   for (; md; md = md->next) {
-    const ModifierTypeInfo *mti = modifierType_getInfo(md->type);
-
     if (!modifier_isEnabled(scene, md, eModifierMode_Realtime)) {
       continue;
     }
 
-    if (mti->type == eModifierTypeType_OnlyDeform) {
+    if (crazyspace_modifier_supports_deform_matrices(md)) {
+      const ModifierTypeInfo *mti = modifierType_getInfo(md->type);
       if (defmats == NULL) {
         /* NOTE: Evaluated object si re-set to its original undeformed
          * state. */

source/blender/modifiers/intern/MOD_multires.c

@@ -228,16 +228,22 @@ static Mesh *applyModifier(ModifierData *md, const ModifierEvalContext *ctx, Mes
   return result;
 }
 
-static void deformVerts(ModifierData *md,
-                        const ModifierEvalContext *UNUSED(ctx),
-                        Mesh *mesh,
-                        float (*vertex_cos)[3],
-                        int num_verts)
+static void deformMatrices(ModifierData *md,
+                           const ModifierEvalContext *UNUSED(ctx),
+                           Mesh *mesh,
+                           float (*vertex_cos)[3],
+                           float (*deform_matrices)[3][3],
+                           int num_verts)
+
 {
 #if !defined(WITH_OPENSUBDIV)
   modifier_setError(md, "Disabled, built without OpenSubdiv");
   return;
 #endif
+
+  /* Subsurf does not require extra space mapping, keep matrices as is. */
+  (void)deform_matrices;
+
   MultiresModifierData *mmd = (MultiresModifierData *)md;
   SubdivSettings subdiv_settings;
   BKE_multires_subdiv_settings_init(&subdiv_settings, mmd);
@@ -268,8 +274,8 @@ ModifierTypeInfo modifierType_Multires = {
 
     /* copyData */ copyData,
 
-    /* deformVerts */ deformVerts,
-    /* deformMatrices */ NULL,
+    /* deformVerts */ NULL,
+    /* deformMatrices */ deformMatrices,
     /* deformVertsEM */ NULL,
     /* deformMatricesEM */ NULL,
     /* applyModifier */ applyModifier,

source/blender/modifiers/intern/MOD_subsurf.c

@@ -241,16 +241,21 @@ static Mesh *applyModifier(ModifierData *md, const ModifierEvalContext *ctx, Mes
   return result;
 }
 
-static void deformVerts(ModifierData *md,
-                        const ModifierEvalContext *UNUSED(ctx),
-                        Mesh *mesh,
-                        float (*vertex_cos)[3],
-                        int num_verts)
+static void deformMatrices(ModifierData *md,
+                           const ModifierEvalContext *UNUSED(ctx),
+                           Mesh *mesh,
+                           float (*vertex_cos)[3],
+                           float (*deform_matrices)[3][3],
+                           int num_verts)
 {
 #if !defined(WITH_OPENSUBDIV)
   modifier_setError(md, "Disabled, built without OpenSubdiv");
   return;
 #endif
+
+  /* Subsurf does not require extra space mapping, keep matrices as is. */
+  (void)deform_matrices;
+
   SubsurfModifierData *smd = (SubsurfModifierData *)md;
   SubdivSettings subdiv_settings;
   subdiv_settings_init(&subdiv_settings, smd);
@@ -281,8 +286,8 @@ ModifierTypeInfo modifierType_Subsurf = {
 
     /* copyData */ copyData,
 
-    /* deformVerts */ deformVerts,
-    /* deformMatrices */ NULL,
+    /* deformVerts */ NULL,
+    /* deformMatrices */ deformMatrices,
     /* deformVertsEM */ NULL,
     /* deformMatricesEM */ NULL,
     /* applyModifier */ applyModifier,