resolve some issues with curve resolution calculaction

- resolution could become so high that it would wrap around  to a negative number, now check for small numbers before doing float division.
- resolution was being calculated in some cases when it already met the clamp value - now this is skipped.

source/blender/blenkernel/BKE_mask.h

@@ -64,19 +64,19 @@ void BKE_mask_layer_copy_list(struct ListBase *masklayers_new, struct ListBase *
 /* splines */
 struct MaskSpline *BKE_mask_spline_add(struct MaskLayer *masklay);
 
-int BKE_mask_spline_resolution(struct MaskSpline *spline, int width, int height);
+unsigned int BKE_mask_spline_resolution(struct MaskSpline *spline, int width, int height);
-int BKE_mask_spline_feather_resolution(struct MaskSpline *spline, int width, int height);
+unsigned int BKE_mask_spline_feather_resolution(struct MaskSpline *spline, int width, int height);
 
-int BKE_mask_spline_differentiate_calc_total(const struct MaskSpline *spline, const int resol);
+int BKE_mask_spline_differentiate_calc_total(const struct MaskSpline *spline, const unsigned int resol);
 
 float (*BKE_mask_spline_differentiate(struct MaskSpline *spline, int *tot_diff_point))[2];
 float (*BKE_mask_spline_feather_differentiated_points(struct MaskSpline *spline, int *tot_feather_point))[2];
 
 float (*BKE_mask_spline_differentiate_with_resolution_ex(struct MaskSpline *spline, int *tot_diff_point,
-                                                         const int resol))[2];
+                                                         const unsigned int resol))[2];
 float (*BKE_mask_spline_differentiate_with_resolution(struct MaskSpline *spline, int width, int height, int *tot_diff_point))[2];
 float (*BKE_mask_spline_feather_differentiated_points_with_resolution_ex(struct MaskSpline *spline, int *tot_feather_point,
-                                                                         const int resol, const int do_collapse))[2];
+                                                                         const unsigned int resol, const int do_collapse))[2];
 float (*BKE_mask_spline_feather_differentiated_points_with_resolution(struct MaskSpline *spline, int width, int height, int *tot_feather_point))[2];
 
 float (*BKE_mask_spline_feather_points(struct MaskSpline *spline, int *tot_feather_point))[2];
@@ -98,16 +98,18 @@ void BKE_mask_point_handle(struct MaskSplinePoint *point, float handle[2]);
 void BKE_mask_point_set_handle(struct MaskSplinePoint *point, float loc[2], int keep_direction,
                                float orig_handle[2], float orig_vec[3][3]);
 
-float *BKE_mask_point_segment_diff(struct MaskSpline *spline, struct MaskSplinePoint *point, int *tot_diff_point);
+float *BKE_mask_point_segment_diff(struct MaskSpline *spline, struct MaskSplinePoint *point,
+                                   unsigned int *tot_diff_point);
 float *BKE_mask_point_segment_feather_diff(struct MaskSpline *spline, struct MaskSplinePoint *point,
-                                           int *tot_feather_point);
+                                           unsigned int *tot_feather_point);
 
 float *BKE_mask_point_segment_diff_with_resolution(struct MaskSpline *spline, struct MaskSplinePoint *point,
-                                                   int width, int height, int *tot_diff_point);
+                                                   int width, int height,
+                                                   unsigned int *tot_diff_point);
 
 float *BKE_mask_point_segment_feather_diff_with_resolution(struct MaskSpline *spline, struct MaskSplinePoint *point,
                                                            int width, int height,
-                                                           int *tot_feather_point);
+                                                           unsigned int *tot_feather_point);
 
 void BKE_mask_point_segment_co(struct MaskSpline *spline, struct MaskSplinePoint *point, float u, float co[2]);
 void BKE_mask_point_normal(struct MaskSpline *spline, struct MaskSplinePoint *point,

source/blender/blenkernel/intern/mask.c

@@ -265,10 +265,10 @@ MaskSpline *BKE_mask_spline_add(MaskLayer *masklay)
 	return spline;
 }
 
-int BKE_mask_spline_resolution(MaskSpline *spline, int width, int height)
+unsigned int BKE_mask_spline_resolution(MaskSpline *spline, int width, int height)
 {
 	float max_segment = 0.01f;
-	int i, resol = 1;
+	unsigned int i, resol = 1;
 
 	if (width != 0 && height != 0) {
 		if (width >= height)
@@ -281,7 +281,7 @@ int BKE_mask_spline_resolution(MaskSpline *spline, int width, int height)
 		MaskSplinePoint *point = &spline->points[i];
 		BezTriple *bezt, *bezt_next;
 		float a, b, c, len;
-		int cur_resol;
+		unsigned int cur_resol;
 
 		bezt = &point->bezt;
 		bezt_next = mask_spline_point_next_bezt(spline, spline->points, point);
@@ -298,24 +298,27 @@ int BKE_mask_spline_resolution(MaskSpline *spline, int width, int height)
 		cur_resol = len / max_segment;
 
 		resol = MAX2(resol, cur_resol);
+
+		if (resol >= MASK_RESOL_MAX) {
+			break;
+		}
 	}
 
-	BLI_assert(resol > 0);
+	return CLAMPIS(resol, 1, MASK_RESOL_MAX);
-
-	if (resol > MASK_RESOL_MAX) {
-		resol = MASK_RESOL_MAX;
-	}
-
-	return resol;
 }
 
-int BKE_mask_spline_feather_resolution(MaskSpline *spline, int width, int height)
+unsigned int BKE_mask_spline_feather_resolution(MaskSpline *spline, int width, int height)
 {
 	const float max_segment = 0.005;
-	int resol = BKE_mask_spline_resolution(spline, width, height);
+	unsigned int resol = BKE_mask_spline_resolution(spline, width, height);
 	float max_jump = 0.0f;
 	int i;
 
+	/* avoid checking the featrher if we already hit the maximum value */
+	if (resol >= MASK_RESOL_MAX) {
+		return MASK_RESOL_MAX;
+	}
+
 	for (i = 0; i < spline->tot_point; i++) {
 		MaskSplinePoint *point = &spline->points[i];
 		float prev_u, prev_w;
@@ -325,9 +328,16 @@ int BKE_mask_spline_feather_resolution(MaskSpline *spline, int width, int height
 		prev_w = point->bezt.weight;
 
 		for (j = 0; j < point->tot_uw; j++) {
-			float jump = fabsf((point->uw[j].w - prev_w) / (point->uw[j].u - prev_u));
+			const float w_diff = (point->uw[j].w - prev_w);
+			const float u_diff = (point->uw[j].u - prev_u);
 
-			max_jump = MAX2(max_jump, jump);
+			/* avoid divide by zero and very high values,
+			 * though these get clamped eventually */
+			if (u_diff > FLT_EPSILON) {
+				float jump = fabsf(w_diff / u_diff);
+
+				max_jump = MAX2(max_jump, jump);
+			}
 
 			prev_u = point->uw[j].u;
 			prev_w = point->uw[j].w;
@@ -336,35 +346,22 @@ int BKE_mask_spline_feather_resolution(MaskSpline *spline, int width, int height
 
 	resol += max_jump / max_segment;
 
-	BLI_assert(resol > 0);
+	return CLAMPIS(resol, 1, MASK_RESOL_MAX);
-
-	if (resol > MASK_RESOL_MAX) {
-		resol = MASK_RESOL_MAX;
-	}
-
-	return resol;
 }
 
-int BKE_mask_spline_differentiate_calc_total(const MaskSpline *spline, const int resol)
+int BKE_mask_spline_differentiate_calc_total(const MaskSpline *spline, const unsigned int resol)
 {
-	int len;
-
-	/* count */
-	len = (spline->tot_point - 1) * resol;
-
 	if (spline->flag & MASK_SPLINE_CYCLIC) {
-		len += resol;
+		return spline->tot_point * resol;
 	}
 	else {
-		len++;
+		return ((spline->tot_point - 1) * resol) + 1;
 	}
-
-	return len;
 }
 
 float (*BKE_mask_spline_differentiate_with_resolution_ex(MaskSpline *spline,
                                                          int *tot_diff_point,
-                                                         const int resol
+                                                         const unsigned int resol
                                                          ))[2]
 {
 	MaskSplinePoint *points_array = BKE_mask_spline_point_array(spline);
@@ -425,7 +422,7 @@ float (*BKE_mask_spline_differentiate_with_resolution(MaskSpline *spline, int wi
                                                       int *tot_diff_point
                                                       ))[2]
 {
-	int resol = BKE_mask_spline_resolution(spline, width, height);
+	int unsigned resol = BKE_mask_spline_resolution(spline, width, height);
 
 	return BKE_mask_spline_differentiate_with_resolution_ex(spline, tot_diff_point, resol);
 }
@@ -705,7 +702,7 @@ static void spline_feather_collapse_inner_loops(MaskSpline *spline, float (*feat
  */
 float (*BKE_mask_spline_feather_differentiated_points_with_resolution_ex(MaskSpline *spline,
                                                                          int *tot_feather_point,
-                                                                         const int resol,
+                                                                         const unsigned int resol,
                                                                          const int do_collapse
                                                                          ))[2]
 {
@@ -779,7 +776,7 @@ float (*BKE_mask_spline_feather_differentiated_points_with_resolution_ex(MaskSpl
 float (*BKE_mask_spline_feather_differentiated_points_with_resolution(MaskSpline *spline, int width, int height,
                                                                       int *tot_feather_point))[2]
 {
-	int resol = BKE_mask_spline_feather_resolution(spline, width, height);
+	unsigned int resol = BKE_mask_spline_feather_resolution(spline, width, height);
 
 	return BKE_mask_spline_feather_differentiated_points_with_resolution_ex(spline, tot_feather_point, resol, FALSE);
 }
@@ -1056,10 +1053,11 @@ void BKE_mask_point_set_handle(MaskSplinePoint *point, float loc[2], int keep_di
 
 float *BKE_mask_point_segment_feather_diff_with_resolution(MaskSpline *spline, MaskSplinePoint *point,
                                                            int width, int height,
-                                                           int *tot_feather_point)
+                                                           unsigned int *tot_feather_point)
 {
 	float *feather, *fp;
-	int i, resol = BKE_mask_spline_feather_resolution(spline, width, height);
+	unsigned int resol = BKE_mask_spline_feather_resolution(spline, width, height);
+	unsigned int i;
 
 	feather = fp = MEM_callocN(2 * resol * sizeof(float), "mask point spline feather diff points");
 
@@ -1080,13 +1078,13 @@ float *BKE_mask_point_segment_feather_diff_with_resolution(MaskSpline *spline, M
 	return feather;
 }
 
-float *BKE_mask_point_segment_feather_diff(MaskSpline *spline, MaskSplinePoint *point, int *tot_feather_point)
+float *BKE_mask_point_segment_feather_diff(MaskSpline *spline, MaskSplinePoint *point, unsigned int *tot_feather_point)
 {
 	return BKE_mask_point_segment_feather_diff_with_resolution(spline, point, 0, 0, tot_feather_point);
 }
 
 float *BKE_mask_point_segment_diff_with_resolution(MaskSpline *spline, MaskSplinePoint *point,
-                                                   int width, int height, int *tot_diff_point)
+                                                   int width, int height, unsigned int *tot_diff_point)
 {
 	MaskSplinePoint *points_array = BKE_mask_spline_point_array_from_point(spline, point);
 
@@ -1115,7 +1113,7 @@ float *BKE_mask_point_segment_diff_with_resolution(MaskSpline *spline, MaskSplin
 	return diff_points;
 }
 
-float *BKE_mask_point_segment_diff(MaskSpline *spline, MaskSplinePoint *point, int *tot_diff_point)
+float *BKE_mask_point_segment_diff(MaskSpline *spline, MaskSplinePoint *point, unsigned int *tot_diff_point)
 {
 	return BKE_mask_point_segment_diff_with_resolution(spline, point, 0, 0, tot_diff_point);
 }

source/blender/blenkernel/intern/mask_rasterize.c

@@ -577,9 +577,9 @@ void BKE_maskrasterize_handle_init(MaskRasterHandle *mr_handle, struct Mask *mas
 			float (*diff_feather_points)[2];
 			int tot_diff_feather_points;
 
-			const int resol_a = BKE_mask_spline_resolution(spline, width, height) / 4;
+			const unsigned int resol_a = BKE_mask_spline_resolution(spline, width, height) / 4;
-			const int resol_b = BKE_mask_spline_feather_resolution(spline, width, height) / 4;
+			const unsigned int resol_b = BKE_mask_spline_feather_resolution(spline, width, height) / 4;
-			const int resol = CLAMPIS(MAX2(resol_a, resol_b), 4, 512);
+			const unsigned int resol = CLAMPIS(MAX2(resol_a, resol_b), 4, 512);
 
 			diff_points = BKE_mask_spline_differentiate_with_resolution_ex(
 			                  spline, &tot_diff_point, resol);

source/blender/editors/mask/mask_add.c

@@ -91,13 +91,14 @@ static int find_nearest_diff_point(const bContext *C, Mask *mask, const float no
 			     i++, cur_point++)
 			{
 				float *diff_points;
-				int tot_diff_point;
+				unsigned int tot_diff_point;
 
 				diff_points = BKE_mask_point_segment_diff_with_resolution(spline, cur_point, width, height,
 				                                                          &tot_diff_point);
 
 				if (diff_points) {
-					int i, tot_feather_point, tot_point;
+					int i, tot_point;
+					unsigned int tot_feather_point;
 					float *feather_points = NULL, *points;
 
 					if (feather) {