StyleBoxFlat scale down corners when oveflowing to prevent glitchy overlapping shapes.

scene/resources/style_box_flat.cpp

@@ -234,6 +234,79 @@ inline void set_inner_corner_radius(const Rect2 style_rect, const Rect2 inner_re
 	inner_corner_radius[3] = MAX(corner_radius[3] - MIN(border_bottom, border_left), 0); // Bottom left.
 }
 
+inline void set_corner_scale(const Rect2 &style_rect, const Rect2 &inner_rect, const real_t corner_radius[4], Point2 *inner_scale) {
+	real_t border_left = inner_rect.position.x - style_rect.position.x;
+	real_t border_top = inner_rect.position.y - style_rect.position.y;
+	real_t border_right = style_rect.size.width - inner_rect.size.width - border_left;
+	real_t border_bottom = style_rect.size.height - inner_rect.size.height - border_top;
+
+	// Amount of overflow along an edge.
+	// Ex. SIDE_LEFT edge is the overflow between top_left and bottom_left corners.
+	// MIN(0,) is to ignore underflow, and negating is to make values positive.
+	real_t edge_overflow[4] = {
+		-MIN(0, inner_rect.size.y - corner_radius[CORNER_TOP_LEFT] - corner_radius[CORNER_BOTTOM_LEFT]),
+		-MIN(0, inner_rect.size.x - corner_radius[CORNER_TOP_LEFT] - corner_radius[CORNER_TOP_RIGHT]),
+		-MIN(0, inner_rect.size.y - corner_radius[CORNER_TOP_RIGHT] - corner_radius[CORNER_BOTTOM_RIGHT]),
+		-MIN(0, inner_rect.size.x - corner_radius[CORNER_BOTTOM_LEFT] - corner_radius[CORNER_BOTTOM_RIGHT])
+	};
+
+	// Sums of borders.
+	real_t hb_sum = border_left + border_right;
+	real_t vb_sum = border_top + border_bottom;
+
+	// Ratio of each side to the sum of itself and opposite side.
+	// Since overflow only happens with opposite borders, you only need to get the ratio of each border relative to the sum of involved borders.
+	real_t ratios[4] = {
+		// Prevent divide by 0 errors.
+		hb_sum > 0 ? (border_left / hb_sum) : 0,
+		vb_sum > 0 ? (border_top / vb_sum) : 0,
+		hb_sum > 0 ? (border_right / hb_sum) : 0,
+		vb_sum > 0 ? (border_bottom / vb_sum) : 0
+	};
+
+	// Raw amount each corner should shrink.
+	Point2 corner_reduction[4] = {
+		Point2(edge_overflow[SIDE_TOP] * ratios[SIDE_LEFT], edge_overflow[SIDE_LEFT] * ratios[SIDE_TOP]),
+		Point2(edge_overflow[SIDE_TOP] * ratios[SIDE_RIGHT], edge_overflow[SIDE_RIGHT] * ratios[SIDE_TOP]),
+		Point2(edge_overflow[SIDE_BOTTOM] * ratios[SIDE_RIGHT], edge_overflow[SIDE_RIGHT] * ratios[SIDE_BOTTOM]),
+		Point2(edge_overflow[SIDE_BOTTOM] * ratios[SIDE_LEFT], edge_overflow[SIDE_LEFT] * ratios[SIDE_BOTTOM]),
+	};
+
+	// Corner Radii as Point2s.
+	Point2 pcr[4] = {
+		Point2(corner_radius[0], corner_radius[0]),
+		Point2(corner_radius[1], corner_radius[1]),
+		Point2(corner_radius[2], corner_radius[2]),
+		Point2(corner_radius[3], corner_radius[3]),
+	};
+
+	// If corner radii are too small, they won't shrink the full amount.
+	// Adjacent corners will have to shrink the leftovers if they can.
+	// Minf(0) is to ignore non-leftovers, and negating is to make values positive.
+	Point2 leftovers[4] = {
+		-((pcr[0] - corner_reduction[0]).minf(0)),
+		-((pcr[1] - corner_reduction[1]).minf(0)),
+		-((pcr[2] - corner_reduction[2]).minf(0)),
+		-((pcr[3] - corner_reduction[3]).minf(0)),
+	};
+
+	// New shrunken radii after distributing the leftovers.
+	Point2 distributed[4] = {
+		((pcr[0] - corner_reduction[0] - leftovers[3] - leftovers[1]).maxf(0)),
+		((pcr[1] - corner_reduction[1] - leftovers[0] - leftovers[2]).maxf(0)),
+		((pcr[2] - corner_reduction[2] - leftovers[1] - leftovers[3]).maxf(0)),
+		((pcr[3] - corner_reduction[3] - leftovers[2] - leftovers[0]).maxf(0)),
+	};
+
+	// How much the curve should scale to achieve the shrunken radii.
+	for (int i = 0; i < 4; i++) {
+		// Unshrinkable is how much is still left over, even after distributing leftovers.
+		// Exclude it from the final scale.
+		Point2 unshrinkable = (leftovers[(i + 1) % 4] + leftovers[(i + 4 - 1) % 4] - distributed[i]).maxf(0);
+		inner_scale[i] = distributed[i] / (pcr[i] - unshrinkable).maxf(FLT_EPSILON);
+	}
+}
+
 inline void draw_rounded_rectangle(Vector<Vector2> &verts, Vector<int> &indices, Vector<Color> &colors, const Rect2 &style_rect, const real_t corner_radius[4],
 		const Rect2 &ring_rect, const Rect2 &inner_rect, const Color &inner_color, const Color &outer_color, const int corner_detail, const Vector2 &skew, bool is_filled = false) {
 	int vert_offset = verts.size();
@@ -244,23 +317,30 @@ inline void draw_rounded_rectangle(Vector<Vector2> &verts, Vector<int> &indices,
 	real_t ring_corner_radius[4];
 	set_inner_corner_radius(style_rect, ring_rect, corner_radius, ring_corner_radius);
 
+	Point2 ring_scale[4];
+	set_corner_scale(style_rect, ring_rect, ring_corner_radius, ring_scale);
+
 	// Corner radius center points.
 	Vector<Point2> outer_points = {
-		ring_rect.position + Vector2(ring_corner_radius[0], ring_corner_radius[0]), //tl
-		Point2(ring_rect.position.x + ring_rect.size.x - ring_corner_radius[1], ring_rect.position.y + ring_corner_radius[1]), //tr
-		ring_rect.position + ring_rect.size - Vector2(ring_corner_radius[2], ring_corner_radius[2]), //br
-		Point2(ring_rect.position.x + ring_corner_radius[3], ring_rect.position.y + ring_rect.size.y - ring_corner_radius[3]) //bl
+		ring_rect.position + Vector2(ring_corner_radius[0], ring_corner_radius[0]) * ring_scale[0], //tl
+		Point2(ring_rect.position.x + ring_rect.size.x - ring_corner_radius[1] * ring_scale[1].x, ring_rect.position.y + ring_corner_radius[1] * ring_scale[1].y), //tr
+		ring_rect.position + ring_rect.size - Vector2(ring_corner_radius[2], ring_corner_radius[2]) * ring_scale[2], //br
+		Point2(ring_rect.position.x + ring_corner_radius[3] * ring_scale[3].x, ring_rect.position.y + ring_rect.size.y - ring_corner_radius[3] * ring_scale[3].y) //bl
 	};
 
 	real_t inner_corner_radius[4];
 	set_inner_corner_radius(style_rect, inner_rect, corner_radius, inner_corner_radius);
 
+	Point2 inner_scale[4];
+	set_corner_scale(style_rect, inner_rect, inner_corner_radius, inner_scale);
+
 	Vector<Point2> inner_points = {
-		inner_rect.position + Vector2(inner_corner_radius[0], inner_corner_radius[0]), //tl
-		Point2(inner_rect.position.x + inner_rect.size.x - inner_corner_radius[1], inner_rect.position.y + inner_corner_radius[1]), //tr
-		inner_rect.position + inner_rect.size - Vector2(inner_corner_radius[2], inner_corner_radius[2]), //br
-		Point2(inner_rect.position.x + inner_corner_radius[3], inner_rect.position.y + inner_rect.size.y - inner_corner_radius[3]) //bl
+		inner_rect.position + Vector2(inner_corner_radius[0], inner_corner_radius[0]) * inner_scale[0], //tl
+		Point2(inner_rect.position.x + inner_rect.size.x - inner_corner_radius[1] * inner_scale[1].x, inner_rect.position.y + inner_corner_radius[1] * inner_scale[1].y), //tr
+		inner_rect.position + inner_rect.size - Vector2(inner_corner_radius[2], inner_corner_radius[2]) * inner_scale[2], //br
+		Point2(inner_rect.position.x + inner_corner_radius[3] * inner_scale[3].x, inner_rect.position.y + inner_rect.size.y - inner_corner_radius[3] * inner_scale[3].y) //bl
 	};
+
 	// Calculate the vertices.
 
 	// If the center is filled, we do not draw the border and directly use the inner ring as reference. Because all calls to this
@@ -289,8 +369,8 @@ inline void draw_rounded_rectangle(Vector<Vector2> &verts, Vector<int> &indices,
 			const real_t angle_sine = Math::sin(pt_angle);
 
 			{
-				const real_t x = inner_corner_radius[corner_idx] * angle_cosine + inner_points[corner_idx].x;
-				const real_t y = inner_corner_radius[corner_idx] * angle_sine + inner_points[corner_idx].y;
+				const real_t x = inner_corner_radius[corner_idx] * angle_cosine * inner_scale[corner_idx].x + inner_points[corner_idx].x;
+				const real_t y = inner_corner_radius[corner_idx] * angle_sine * inner_scale[corner_idx].y + inner_points[corner_idx].y;
 				const float x_skew = -skew.x * (y - style_rect_center.y);
 				const float y_skew = -skew.y * (x - style_rect_center.x);
 				verts_ptr[verts_size + idx_ofs] = Vector2(x + x_skew, y + y_skew);
@@ -298,8 +378,8 @@ inline void draw_rounded_rectangle(Vector<Vector2> &verts, Vector<int> &indices,
 			}
 
 			if (draw_border) {
-				const real_t x = ring_corner_radius[corner_idx] * angle_cosine + outer_points[corner_idx].x;
-				const real_t y = ring_corner_radius[corner_idx] * angle_sine + outer_points[corner_idx].y;
+				const real_t x = ring_corner_radius[corner_idx] * angle_cosine * ring_scale[corner_idx].x + outer_points[corner_idx].x;
+				const real_t y = ring_corner_radius[corner_idx] * angle_sine * ring_scale[corner_idx].y + outer_points[corner_idx].y;
 				const float x_skew = -skew.x * (y - style_rect_center.y);
 				const float y_skew = -skew.y * (x - style_rect_center.x);
 				verts_ptr[verts_size + idx_ofs + 1] = Vector2(x + x_skew, y + y_skew);