Custom Properties: allow changing the property UI to color picker.

To fully support storing colors as a custom property, it is necessary
to allow switching the property UI to the standard color picker button.
That means in effect supporting custom property subtype values.

Change RNA_property_subtype to look for a 'subtype' string field
in _RNA_UI and parse it as an enum value. To minimize performance
impact, only do it if the property is an array; also, don't use
the custom subtype during RNA path parsing.

On the python side, allow setting some most useful seeming values
from the custom property settings editor.

Also, since some color picker code seems to run into a risk of
buffer overruns if the array size is wrong, check the size in
the UI layout code to be safe.

Reviewers: campbellbarton

Differential Revision: https://developer.blender.org/D5475

release/scripts/startup/bl_operators/wm.py

@@ -1085,6 +1085,14 @@ rna_is_overridable_library = BoolProperty(
     default=False,
 )
 
+# Most useful entries of rna_enum_property_subtype_items for number arrays:
+rna_vector_subtype_items = (
+    ('NONE', "Plain Data", "Data values without special behavior"),
+    ('COLOR', "Linear Color", "Color in the linear space"),
+    ('COLOR_GAMMA', "Gamma-Corrected Color", "Color in the gamma corrected space"),
+    ('EULER', "Euler Angles", "Euler rotation angles in radians"),
+    ('QUATERNION', "Quaternion Rotation", "Quaternion rotation (affects NLA blending)"),
+)
 
 class WM_OT_properties_edit(Operator):
     bl_idname = "wm.properties_edit"
@@ -1105,6 +1113,23 @@ class WM_OT_properties_edit(Operator):
     description: StringProperty(
         name="Tooltip",
     )
+    subtype: EnumProperty(
+        name="Subtype",
+        items=lambda self,ctx: WM_OT_properties_edit.subtype_items,
+    )
+
+    subtype_items = rna_vector_subtype_items
+
+    def init_subtype(self, prop_type, is_array, subtype):
+        subtype = subtype or 'NONE'
+        subtype_items = rna_vector_subtype_items
+
+        # Add a temporary enum entry to preserve unknown subtypes
+        if not any(subtype == item[0] for item in subtype_items):
+            subtype_items += ((subtype, subtype, ""),)
+
+        WM_OT_properties_edit.subtype_items = subtype_items
+        self.subtype = subtype
 
     def _cmp_props_get(self):
         # Changing these properties will refresh the UI
@@ -1193,6 +1218,11 @@ class WM_OT_properties_edit(Operator):
                 prop_ui["soft_min"] = prop_type(self.min)
                 prop_ui["soft_max"] = prop_type(self.max)
 
+        if prop_type == float and is_array and self.subtype != 'NONE':
+            prop_ui["subtype"] = self.subtype
+        else:
+            prop_ui.pop("subtype", None)
+
         prop_ui["description"] = self.description
 
         rna_idprop_ui_prop_default_set(item, prop, default_eval)
@@ -1235,7 +1265,11 @@ class WM_OT_properties_edit(Operator):
         return {'FINISHED'}
 
     def invoke(self, context, _event):
-        from rna_prop_ui import rna_idprop_ui_prop_get, rna_idprop_value_to_python
+        from rna_prop_ui import (
+            rna_idprop_ui_prop_get,
+            rna_idprop_value_to_python,
+            rna_idprop_value_item_type
+        )
 
         data_path = self.data_path
 
@@ -1252,7 +1286,7 @@ class WM_OT_properties_edit(Operator):
         self.is_overridable_library = bool(eval(exec_str))
 
         # default default value
-        prop_type = type(self.get_value_eval())
+        prop_type, is_array = rna_idprop_value_item_type(self.get_value_eval())
         if prop_type in {int, float}:
             self.default = str(prop_type(0))
         else:
@@ -1276,6 +1310,12 @@ class WM_OT_properties_edit(Operator):
                 self.max != self.soft_max
             )
 
+            subtype = prop_ui.get("subtype", None)
+        else:
+            subtype = None
+
+        self.init_subtype(prop_type, is_array, subtype)
+
         # store for comparison
         self._cmp_props = self._cmp_props_get()
 
@@ -1341,6 +1381,9 @@ class WM_OT_properties_edit(Operator):
         row.prop(self, "soft_max", text="Soft Max")
         layout.prop(self, "description")
 
+        if is_array and proptype == float:
+            layout.prop(self, "subtype")
+
 
 class WM_OT_properties_add(Operator):
     bl_idname = "wm.properties_add"