Hopefully fixed snapping extremes/artifacts

src/PixiEditor/Views/Overlays/TransformOverlay/TransformHelper.cs

@@ -301,4 +301,27 @@ internal static class TransformHelper
             _ => throw new ArgumentException($"{capturedAnchor} is not a corner or a side"),
         };
     }
+
+    public static Anchor GetOppositeAnchor(Anchor anchor)
+    {
+        return anchor switch
+        {
+            Anchor.TopLeft => Anchor.BottomRight,
+            Anchor.TopRight => Anchor.BottomLeft,
+            Anchor.BottomLeft => Anchor.TopRight,
+            Anchor.BottomRight => Anchor.TopLeft,
+            Anchor.Top => Anchor.Bottom,
+            Anchor.Bottom => Anchor.Top,
+            Anchor.Left => Anchor.Right,
+            Anchor.Right => Anchor.Left,
+            _ => throw new ArgumentException($"{anchor} is not a corner or a side"),
+        };
+    }
+
+    public static bool RotationIsAlmostCardinal(double radians, double threshold = 0.03)
+    {
+        double normalized = Math.Abs(radians % (2 * Math.PI));
+        double[] cardinals = { 0, Math.PI / 2, Math.PI, 3 * Math.PI / 2, 2 * Math.PI };
+        return cardinals.Any(cardinal => Math.Abs(normalized - cardinal) < threshold);
+    }
 }

src/PixiEditor/Views/Overlays/TransformOverlay/TransformOverlay.cs

@@ -227,7 +227,9 @@ internal class TransformOverlay : Overlay
     private bool rotationCursorActive = false;
 
     private VecD lastPointerPos;
-
+    private InfoBox infoBox;
+    private VecD lastSize;
+    
     public TransformOverlay()
     {
         topLeftHandle = new AnchorHandle(this);
@@ -286,7 +288,7 @@ internal class TransformOverlay : Overlay
 
     public override void RenderOverlay(Canvas drawingContext, RectD canvasBounds)
     {
-        DrawOverlay(drawingContext, new(Bounds.Width, Bounds.Height), Corners, InternalState.Origin, (float)ZoomScale);
+        DrawOverlay(drawingContext, canvasBounds.Size, Corners, InternalState.Origin, (float)ZoomScale);
 
         if (capturedAnchor is null)
             UpdateRotationCursor(lastPointerPos);
@@ -318,6 +320,7 @@ internal class TransformOverlay : Overlay
     private void DrawOverlay
         (Canvas context, VecD size, ShapeCorners corners, VecD origin, float zoomboxScale)
     {
+        lastSize = size;
         // draw transparent background to enable mouse input
         DrawMouseInputArea(context, size);
 
@@ -685,6 +688,11 @@ internal class TransformOverlay : Overlay
             VecD targetPos = originalAnchorPos + pos - mousePosOnStartAnchorDrag;
 
             VecD projected = targetPos.ProjectOntoLine(originalAnchorPos, InternalState.Origin);
+            if (projected.IsNaNOrInfinity())
+            {
+                projected = targetPos;
+            }
+            
             VecD anchorRelativeDelta = projected - originalAnchorPos;
 
             var adjacentAnchors = TransformHelper.GetAdjacentAnchors((Anchor)capturedAnchor);
@@ -709,12 +717,26 @@ internal class TransformOverlay : Overlay
             }
             else
             {
-                snapped = FindProjectedAnchorSnap(projected);
-                if (snapped.Delta == VecI.Zero)
+                // If rotation is almost cardinal, projecting snapping points result in extreme values when perpendicular to the axis
+                if (!TransformHelper.RotationIsAlmostCardinal(cornersOnStartAnchorDrag.RectRotation))
+                {
+                    snapped = FindProjectedAnchorSnap(projected);
+                    if (snapped.Delta == VecI.Zero)
+                    {
+                        snapped = FindAdjacentCornersSnap(adjacentAnchors, anchorRelativeDelta);
+                    }
+                }
+                else
                 {
-                    snapped = FindAdjacentCornersSnap(adjacentAnchors, anchorRelativeDelta);
+                    snapped = TrySnapAnchor(targetPos);
                 }
             }
+            
+            VecD potentialPos = targetPos + snapped.Delta;
+            if(potentialPos.X < 0 || potentialPos.Y < 0 || potentialPos.X > lastSize.X || potentialPos.Y > lastSize.Y)
+            {
+                snapped = new SnapData();
+            }
 
             ShapeCorners? newCorners = TransformUpdateHelper.UpdateShapeFromSide
             ((Anchor)capturedAnchor, SideFreedom, InternalState.ProportionalAngle1,
@@ -754,7 +776,13 @@ internal class TransformOverlay : Overlay
 
         VecD snapOrigin = TransformHelper.GetAnchorPosition(cornersOnStartAnchorDrag, adjacent) + anchorRelativeDelta;
         var snapped = TrySnapAnchorAlongLine(adjacentAnchorPos, snapOrigin);
-
+        double maxDistance = GetSizeToOppositeSide(cornersOnStartAnchorDrag, capturedAnchor.Value) / 8f;
+        
+        if(snapped.Delta.Length > maxDistance)
+        {
+            snapped = new SnapData();
+        }
+        
         if (snapped.Delta == VecI.Zero)
         {
             adjacentAnchorPos = TransformHelper.GetAnchorPosition(cornersOnStartAnchorDrag, adjacentAnchors.Item2) +
@@ -764,10 +792,22 @@ internal class TransformOverlay : Overlay
             snapOrigin = TransformHelper.GetAnchorPosition(cornersOnStartAnchorDrag, adjacent) + anchorRelativeDelta;
 
             snapped = TrySnapAnchorAlongLine(adjacentAnchorPos, snapOrigin);
+            if(snapped.Delta.Length > maxDistance)
+            {
+                snapped = new SnapData();
+            }
         }
 
         return snapped;
     }
+    
+    private double GetSizeToOppositeSide(ShapeCorners corners, Anchor anchor1)
+    {
+        Anchor opposite = TransformHelper.GetOppositeAnchor(anchor1);
+        VecD oppositePos = TransformHelper.GetAnchorPosition(corners, opposite);
+        VecD anchorPos = TransformHelper.GetAnchorPosition(corners, anchor1);
+        return (oppositePos - anchorPos).Length;
+    }
 
     private SnapData FindProjectedAnchorSnap(VecD projected)
     {