Edit Mode: Rework display to differentiate selection modes

This removes the large edges and instead use colors to hint in which
selection mode the user is.

The component in each individual selection mode is more prominent to add
more emphasis. The other components are less prominent and dimmed.

A minor default theme changes needed to be made to keep clarity in all
combinations.

Forcing old behavior (no selection mode hit) could be done quite easily
by just making the 2 booleans true (selectFaces and selectEdges).

Reviewers: campbellbarton, billreynish

Reviewed By: campbellbarton, billreynish

Subscribers: ThinkingPolygons

Maniphest Tasks: T1234

Differential Revision: https://developer.blender.org/D4526

release/datafiles/userdef/userdef_default_theme.c

@@ -312,18 +312,18 @@ const bTheme U_theme_default = {
 		.active = RGBA(0xffa028ff),
 		.transform = RGBA(0xffffffff),
 		.vertex = RGBA(0x000000ff),
-		.vertex_select = RGBA(0xff8500ff),
+		.vertex_select = RGBA(0xff7a00ff),
 		.vertex_bevel = RGBA(0x00a5ffff),
 		.edge = RGBA(0x000000ff),
-		.edge_select = RGBA(0xffa000ff),
+		.edge_select = RGBA(0xff7200ff),
 		.edge_seam = RGBA(0xdb2512ff),
 		.edge_sharp = RGBA(0x00ffffff),
 		.edge_facesel = RGBA(0x4b4b4bff),
 		.edge_crease = RGBA(0xcc0099ff),
 		.edge_bevel = RGBA(0x00a5ffff),
 		.face = RGBA(0x00000012),
-		.face_select = RGBA(0xff85004d),
+		.face_select = RGBA(0xff990073),
-		.face_dot = RGBA(0xff8500ff),
+		.face_dot = RGBA(0xff8a00ff),
 		.extra_edge_len = RGBA(0x150806ff),
 		.extra_edge_angle = RGBA(0x4d4d00ff),
 		.extra_face_angle = RGBA(0x0000ccff),
@@ -338,7 +338,7 @@ const bTheme U_theme_default = {
 		.time_keyframe = RGBA(0xddd700ff),
 		.time_gp_keyframe = RGBA(0xb5e61dff),
 		.freestyle_edge_mark = RGBA(0x7fff7fff),
-		.freestyle_face_mark = RGBA(0x7fff7f33),
+		.freestyle_face_mark = RGBA(0x7fff7f4d),
 		.nurb_uline = RGBA(0x909000ff),
 		.nurb_vline = RGBA(0x803060ff),
 		.act_spline = RGBA(0xdb2512ff),

source/blender/draw/modes/edit_mesh_mode.c

@@ -336,8 +336,8 @@ static DRWPass *edit_mesh_create_overlay_pass(float *face_alpha,
   DRW_shgroup_uniform_block(grp, "globalsBlock", G_draw.block_ubo);
   DRW_shgroup_uniform_float(grp, "faceAlphaMod", face_alpha, 1);
   DRW_shgroup_uniform_ivec4(grp, "dataMask", data_mask, 1);
-  DRW_shgroup_uniform_bool_copy(grp, "doEdges", do_edges);
   DRW_shgroup_uniform_float_copy(grp, "ofs", 0.0f);
+  DRW_shgroup_uniform_bool_copy(grp, "selectFaces", select_face);
   if (rv3d->rflag & RV3D_CLIPPING) {
     DRW_shgroup_world_clip_planes_from_rv3d(grp, rv3d);
   }
@@ -354,7 +354,8 @@ static DRWPass *edit_mesh_create_overlay_pass(float *face_alpha,
   DRW_shgroup_uniform_ivec4(grp, "dataMask", data_mask, 1);
   DRW_shgroup_uniform_bool_copy(grp, "doEdges", do_edges);
   DRW_shgroup_uniform_float_copy(grp, "ofs", depth_ofs);
-  DRW_shgroup_uniform_float_copy(grp, "edgeScale", show_wide_edge ? 1.75f : 1.0f);
+  DRW_shgroup_uniform_bool_copy(grp, "selectEdges", select_edge);
+
   DRW_shgroup_state_enable(grp, DRW_STATE_OFFSET_NEGATIVE);
   /* To match blender loop structure. */
   DRW_shgroup_state_enable(grp, DRW_STATE_FIRST_VERTEX_CONVENTION);
@@ -422,9 +423,6 @@ static void EDIT_MESH_cache_init(void *vedata)
         stl->g_data->do_faces = false;
         stl->g_data->do_zbufclip = false;
       }
-      if ((tsettings->selectmode & SCE_SELECT_FACE) == 0) {
-        stl->g_data->data_mask[0] &= ~VFLAG_FACE_ACTIVE;
-      }
       if ((v3d->overlay.edit_flag & V3D_OVERLAY_EDIT_SEAMS) == 0) {
         stl->g_data->data_mask[1] &= ~VFLAG_EDGE_SEAM;
       }
@@ -552,6 +550,7 @@ static void EDIT_MESH_cache_init(void *vedata)
 
     /* however we loose the front faces value (because we need the depth of occluded wires and
      * faces are alpha blended ) so we recover them in a new pass. */
+    bool select_face = (tsettings->selectmode & SCE_SELECT_FACE) != 0;
     psl->facefill_occlude = DRW_pass_create(
         "Front Face Color", DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_LESS_EQUAL | DRW_STATE_BLEND);
     stl->g_data->facefill_occluded_shgrp = DRW_shgroup_create(sh_data->overlay_facefill,

source/blender/draw/modes/shaders/edit_mesh_overlay_common_lib.glsl

@@ -1,5 +1,7 @@
 
 uniform bool doEdges = true;
+uniform bool selectFaces = true;
+uniform bool selectEdges = true;
 
 vec4 EDIT_MESH_edge_color_outer(int edge_flag, int face_flag, float crease, float bweight)
 {
@@ -15,8 +17,8 @@ vec4 EDIT_MESH_edge_color_outer(int edge_flag, int face_flag, float crease, floa
 vec4 EDIT_MESH_edge_color_inner(int edge_flag)
 {
   vec4 color = colorWireEdit;
-  color = ((edge_flag & EDGE_SELECTED) != 0) ? colorEdgeSelect : color;
+  color = (doEdges && ((edge_flag & EDGE_SELECTED) != 0)) ? colorEdgeSelect : color;
-  color = ((edge_flag & EDGE_ACTIVE) != 0) ? colorEditMeshActive : color;
+  color = (doEdges && ((edge_flag & EDGE_ACTIVE) != 0)) ? colorEditMeshActive : color;
   return color;
 }
 
@@ -43,18 +45,15 @@ vec4 EDIT_MESH_vertex_color(int vertex_flag)
 
 vec4 EDIT_MESH_face_color(int face_flag)
 {
-  if ((face_flag & FACE_ACTIVE) != 0) {
+  vec4 color = colorFace;
-    return mix(colorFaceSelect, colorEditMeshActive, 0.5);
+  color = ((face_flag & FACE_FREESTYLE) != 0) ? colorFaceFreestyle : color;
-  }
+  color = ((face_flag & FACE_SELECTED) != 0) ? colorFaceSelect : color;
-  else if ((face_flag & FACE_SELECTED) != 0) {
+  color = ((face_flag & FACE_ACTIVE) != 0) ? mix(colorFaceSelect, colorEditMeshActive, 0.5) :
-    return colorFaceSelect;
+                                             color;
-  }
+  color.a *= ((face_flag & (FACE_FREESTYLE | FACE_SELECTED | FACE_ACTIVE)) == 0 || selectFaces) ?
-  else if ((face_flag & FACE_FREESTYLE) != 0) {
+                 1.0 :
-    return colorFaceFreestyle;
+                 0.5;
-  }
+  return color;
-  else {
-    return colorFace;
-  }
 }
 
 vec4 EDIT_MESH_facedot_color(float facedot_flag)

source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl

@@ -12,8 +12,6 @@
 #define GRID_LINE_SMOOTH_START (0.5 - DISC_RADIUS)
 #define GRID_LINE_SMOOTH_END (0.5 + DISC_RADIUS)
 
-uniform float edgeScale;
-
 flat in vec4 finalColorOuter_f;
 in vec4 finalColor_f;
 noperspective in float edgeCoord_f;
@@ -22,8 +20,8 @@ out vec4 FragColor;
 
 void main()
 {
-  float dist = abs(edgeCoord_f) - max(sizeEdge * edgeScale - 0.5, 0.0);
+  float dist = abs(edgeCoord_f) - max(sizeEdge - 0.5, 0.0);
-  float dist_outer = dist - max(sizeEdge * edgeScale, 1.0);
+  float dist_outer = dist - max(sizeEdge, 1.0);
 #ifdef USE_SMOOTH_WIRE
   float mix_w = smoothstep(GRID_LINE_SMOOTH_START, GRID_LINE_SMOOTH_END, dist);
   float mix_w_outer = smoothstep(GRID_LINE_SMOOTH_START, GRID_LINE_SMOOTH_END, dist_outer);
@@ -33,4 +31,5 @@ void main()
 #endif
   FragColor = mix(finalColorOuter_f, finalColor_f, 1.0 - mix_w * finalColorOuter_f.a);
   FragColor.a *= 1.0 - (finalColorOuter_f.a > 0.0 ? mix_w_outer : mix_w);
+  FragColor.a *= (selectEdges) ? 1.0 : 0.4;
 }

source/blender/draw/modes/shaders/edit_mesh_overlay_geom.glsl

@@ -4,7 +4,6 @@ layout(triangle_strip, max_vertices = 4) out;
 
 uniform vec2 viewportSize;
 uniform vec2 viewportSizeInv;
-uniform float edgeScale;
 
 in vec4 finalColor[2];
 in vec4 finalColorOuter[2];
@@ -57,9 +56,9 @@ void main()
   line = abs(line) * viewportSize;
 
   finalColorOuter_f = finalColorOuter[0];
-  float half_size = sizeEdge * edgeScale;
+  float half_size = sizeEdge;
   /* Enlarge edge for flag display. */
-  half_size += (finalColorOuter_f.a > 0.0) ? max(sizeEdge * edgeScale, 1.0) : 0.0;
+  half_size += (finalColorOuter_f.a > 0.0) ? max(sizeEdge, 1.0) : 0.0;
 
 #ifdef USE_SMOOTH_WIRE
   /* Add 1 px for AA */

source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl

@@ -74,22 +74,17 @@ void main()
 
 #endif
 
-#if !defined(FACE) && !defined(EDGE_DECORATION)
+#if !defined(FACE)
   /* Facing based color blend */
   vec4 vpos = ModelViewMatrix * vec4(pos, 1.0);
   vec3 view_normal = normalize(NormalMatrix * vnor + 1e-4);
   vec3 view_vec = (ProjectionMatrix[3][3] == 0.0) ? normalize(vpos.xyz) : vec3(0.0, 0.0, 1.0);
   float facing = dot(view_vec, view_normal);
-  facing = 1.0 - abs(facing) * 0.3;
+  facing = 1.0 - abs(facing) * 0.2;
 
   finalColor = mix(colorEditMeshMiddle, finalColor, facing);
   finalColor.a = 1.0;
 
-#  if defined(EDGE) && !defined(FLAT)
-  /* Hack to blend color in pixel shader in case of overide.  */
-  finalColor.a = facing;
-#  endif
-
 #endif
 
 #ifdef USE_WORLD_CLIP_PLANES