Catch a case when the simplex algorithm converges to a false point

Sometimes the algorithm converges to a false point, even though error
walue is still hight. It makes the transformation bounding box jump
from place to place. Just ignore this transformation if the solution
has not been found.

CCBUG:426949

plugins/tools/tool_transform2/kis_free_transform_strategy.cpp

@@ -517,6 +517,10 @@ void KisFreeTransformStrategy::continuePrimaryAction(const QPointF &mousePos,
                                  movingPoint,
                                  dist);
 
+        if (!result.isValid) {
+            break;
+        }
+
         if (shiftModifierActive ||  m_d->currentArgs.keepAspectRatio()) {
             qreal aspectRatio = m_d->clickArgs.scaleX() / m_d->clickArgs.scaleY();
             m_d->currentArgs.setScaleX(aspectRatio * result.scale);
@@ -567,6 +571,10 @@ void KisFreeTransformStrategy::continuePrimaryAction(const QPointF &mousePos,
                                  movingPoint,
                                  dist);
 
+        if (!result.isValid) {
+            break;
+        }
+
         if (shiftModifierActive  ||  m_d->currentArgs.keepAspectRatio()) {
             qreal aspectRatio = m_d->clickArgs.scaleY() / m_d->clickArgs.scaleX();
             m_d->currentArgs.setScaleY(aspectRatio * result.scale);

plugins/tools/tool_transform2/kis_free_transform_strategy_gsl_helpers.cpp

@@ -151,6 +151,15 @@ namespace GSL
             size = gsl_multimin_fminimizer_size (s);
             status = gsl_multimin_test_size (size, 1e-6);
 
+            /**
+             * Sometimes the algorithm may converge to a wrond point,
+             * they just try to force it search better or return invalid
+             * result.
+             */
+            if (status == GSL_SUCCESS && scaleError1D<Strategy>(s->x, &p) > 0.5) {
+                status = GSL_CONTINUE;
+            }
+
             if (status == GSL_SUCCESS)
             {
                 // dbgKrita << "*******Converged to minimum";
@@ -162,6 +171,7 @@ namespace GSL
                 result.transformedCenter =
                     QPointF(gsl_vector_get (s->x, 1),
                             gsl_vector_get (s->x, 2));
+                result.isValid = true;
             }
         }
         while (status == GSL_CONTINUE && iter < 10000);

plugins/tools/tool_transform2/kis_free_transform_strategy_gsl_helpers.h

@@ -30,6 +30,7 @@ namespace GSL
 
         QPointF transformedCenter;
         qreal scale;
+        bool isValid = false;
     };
 
     ScaleResult1D calculateScaleX(const ToolTransformArgs &args,