Fixed crashes in Observer mode related to infinite bounding rects

Bug fixed by avoiding uniting or subtracting QPolygonF with potentially
infinite coordinates.

The LiveSelectionIndicator now uses a QGraphicsRectItem rather than a
QGraphicsPolygonItem and displays only the boundaries of selected
objects, not including their children.

The SubcomponentMaskLayerItem now works with rectangles and uses a
QRegion to determine the area around the current context, converting
this to a polygon only as a last step.

Reviewed-by: Kai Koehne
Task-number: QTCREATORBUG-4559
Change-Id: I266f5387fa67017fc50215282a95b4ee6498be6d
(cherry picked from commit d03065da2999b8539d8c5160b58d56dd94373d6f)

3 files changed, 38 insertions, 67 deletions

diff --git a/src/plugins/qmltooling/declarativeobserver/editor/liveselectionindicator.cpp b/src/plugins/qmltooling/declarativeobserver/editor/liveselectionindicator.cpp
index 44167e3294..2752957314 100644
--- a/src/plugins/qmltooling/declarativeobserver/editor/liveselectionindicator.cpp
+++ b/src/plugins/qmltooling/declarativeobserver/editor/liveselectionindicator.cpp
@@ -44,20 +44,17 @@
 #include "../qdeclarativeviewobserver_p_p.h"
 #include "../qmlobserverconstants_p.h"
 
-#include <QtCore/QDebug>
-
-#include <QtGui/QGraphicsPolygonItem>
+#include <QtGui/QGraphicsRectItem>
 #include <QtGui/QGraphicsObject>
 #include <QtGui/QGraphicsScene>
 #include <QtGui/QPen>
 
-#include <cmath>
-
 QT_BEGIN_NAMESPACE
 
-LiveSelectionIndicator::LiveSelectionIndicator(QDeclarativeViewObserver *editorView,
-                                       QGraphicsObject *layerItem)
-    : m_layerItem(layerItem), m_view(editorView)
+LiveSelectionIndicator::LiveSelectionIndicator(QDeclarativeViewObserver *viewObserver,
+                                               QGraphicsObject *layerItem)
+    : m_layerItem(layerItem)
+    , m_view(viewObserver)
 {
 }
 
@@ -68,24 +65,23 @@ LiveSelectionIndicator::~LiveSelectionIndicator()
 
 void LiveSelectionIndicator::show()
 {
-    foreach (QGraphicsPolygonItem *item, m_indicatorShapeHash.values())
+    foreach (QGraphicsRectItem *item, m_indicatorShapeHash)
         item->show();
 }
 
 void LiveSelectionIndicator::hide()
 {
-    foreach (QGraphicsPolygonItem *item, m_indicatorShapeHash.values())
+    foreach (QGraphicsRectItem *item, m_indicatorShapeHash)
         item->hide();
 }
 
 void LiveSelectionIndicator::clear()
 {
     if (!m_layerItem.isNull()) {
-        QHashIterator<QGraphicsItem*, QGraphicsPolygonItem *> iter(m_indicatorShapeHash);
-        while (iter.hasNext()) {
-            iter.next();
-            m_layerItem.data()->scene()->removeItem(iter.value());
-            delete iter.value();
+        QGraphicsScene *scene = m_layerItem.data()->scene();
+        foreach (QGraphicsRectItem *item, m_indicatorShapeHash) {
+            scene->removeItem(item);
+            delete item;
         }
     }
 
@@ -93,56 +89,29 @@ void LiveSelectionIndicator::clear()
 
 }
 
-QPolygonF LiveSelectionIndicator::addBoundingRectToPolygon(QGraphicsItem *item, QPolygonF &polygon)
-{
-    // ### remove this if statement when QTBUG-12172 gets fixed
-    if (item->boundingRect() != QRectF(0,0,0,0)) {
-        QPolygonF bounding = item->mapToScene(item->boundingRect());
-        if (bounding.isClosed()) //avoid crashes if there is an infinite scale.
-            polygon = polygon.united(bounding);
-    }
-
-    foreach (QGraphicsItem *child, item->childItems()) {
-        if (!QDeclarativeViewObserverPrivate::get(m_view)->isEditorItem(child))
-            addBoundingRectToPolygon(child, polygon);
-    }
-    return polygon;
-}
-
 void LiveSelectionIndicator::setItems(const QList<QWeakPointer<QGraphicsObject> > &itemList)
 {
     clear();
 
-    // set selections to also all children if they are not editor items
-
     foreach (const QWeakPointer<QGraphicsObject> &object, itemList) {
         if (object.isNull())
             continue;
 
         QGraphicsItem *item = object.data();
 
-        QGraphicsPolygonItem *newSelectionIndicatorGraphicsItem
-                = new QGraphicsPolygonItem(m_layerItem.data());
         if (!m_indicatorShapeHash.contains(item)) {
-            m_indicatorShapeHash.insert(item, newSelectionIndicatorGraphicsItem);
-
-            QPolygonF boundingShapeInSceneSpace;
-            addBoundingRectToPolygon(item, boundingShapeInSceneSpace);
-
-            QRectF boundingRect
-                    = m_view->adjustToScreenBoundaries(boundingShapeInSceneSpace.boundingRect());
-            QPolygonF boundingRectInLayerItemSpace = m_layerItem.data()->mapFromScene(boundingRect);
-
-            QPen pen;
-            pen.setColor(QColor(108, 141, 221));
-            newSelectionIndicatorGraphicsItem->setData(Constants::EditorItemDataKey,
-                                                       QVariant(true));
-            newSelectionIndicatorGraphicsItem->setFlag(QGraphicsItem::ItemIsSelectable, false);
-            newSelectionIndicatorGraphicsItem->setPolygon(boundingRectInLayerItemSpace);
-            newSelectionIndicatorGraphicsItem->setPen(pen);
+            QGraphicsRectItem *selectionIndicator = new QGraphicsRectItem(m_layerItem.data());
+            m_indicatorShapeHash.insert(item, selectionIndicator);
+
+            const QRectF boundingRect = m_view->adjustToScreenBoundaries(item->mapRectToScene(item->boundingRect()));
+            const QRectF boundingRectInLayerItemSpace = m_layerItem.data()->mapRectFromScene(boundingRect);
+
+            selectionIndicator->setData(Constants::EditorItemDataKey, true);
+            selectionIndicator->setFlag(QGraphicsItem::ItemIsSelectable, false);
+            selectionIndicator->setRect(boundingRectInLayerItemSpace);
+            selectionIndicator->setPen(QColor(108, 141, 221));
         }
     }
 }
 
 QT_END_NAMESPACE
-
diff --git a/src/plugins/qmltooling/declarativeobserver/editor/liveselectionindicator_p.h b/src/plugins/qmltooling/declarativeobserver/editor/liveselectionindicator_p.h
index 6d2b545be5..efd2c5f488 100644
--- a/src/plugins/qmltooling/declarativeobserver/editor/liveselectionindicator_p.h
+++ b/src/plugins/qmltooling/declarativeobserver/editor/liveselectionindicator_p.h
@@ -47,7 +47,7 @@
 
 QT_BEGIN_NAMESPACE
 class QGraphicsObject;
-class QGraphicsPolygonItem;
+class QGraphicsRectItem;
 class QGraphicsItem;
 class QPolygonF;
 QT_END_NAMESPACE
@@ -63,7 +63,7 @@ class QDeclarativeViewObserver;
 class LiveSelectionIndicator
 {
 public:
-    LiveSelectionIndicator(QDeclarativeViewObserver* editorView, QGraphicsObject *layerItem);
+    LiveSelectionIndicator(QDeclarativeViewObserver *viewObserver, QGraphicsObject *layerItem);
     ~LiveSelectionIndicator();
 
     void show();
@@ -74,13 +74,9 @@ public:
     void setItems(const QList<QWeakPointer<QGraphicsObject> > &itemList);
 
 private:
-    QPolygonF addBoundingRectToPolygon(QGraphicsItem *item, QPolygonF &polygon);
-
-private:
-    QHash<QGraphicsItem*, QGraphicsPolygonItem *> m_indicatorShapeHash;
+    QHash<QGraphicsItem*, QGraphicsRectItem *> m_indicatorShapeHash;
     QWeakPointer<QGraphicsObject> m_layerItem;
     QDeclarativeViewObserver *m_view;
-
 };
 
 QT_END_NAMESPACE
diff --git a/src/plugins/qmltooling/declarativeobserver/editor/subcomponentmasklayeritem.cpp b/src/plugins/qmltooling/declarativeobserver/editor/subcomponentmasklayeritem.cpp
index da6e82b826..27e20791d6 100644
--- a/src/plugins/qmltooling/declarativeobserver/editor/subcomponentmasklayeritem.cpp
+++ b/src/plugins/qmltooling/declarativeobserver/editor/subcomponentmasklayeritem.cpp
@@ -86,6 +86,13 @@ static QRectF resizeRect(const QRectF &newRect, const QRectF &oldRect)
     return result;
 }
 
+static QPolygonF regionToPolygon(const QRegion &region)
+{
+    QPainterPath path;
+    foreach (const QRect &rect, region.rects())
+        path.addRect(rect);
+    return path.toFillPolygon();
+}
 
 void SubcomponentMaskLayerItem::setCurrentItem(QGraphicsItem *item)
 {
@@ -95,25 +102,24 @@ void SubcomponentMaskLayerItem::setCurrentItem(QGraphicsItem *item)
     if (!m_currentItem)
         return;
 
-    QPolygonF viewPoly(QRectF(m_observer->declarativeView()->rect()));
-    viewPoly = m_observer->declarativeView()->mapToScene(viewPoly.toPolygon());
+    QRect viewRect = m_observer->declarativeView()->rect();
+    viewRect = m_observer->declarativeView()->mapToScene(viewRect).boundingRect().toRect();
 
     QRectF itemRect = item->boundingRect() | item->childrenBoundingRect();
-    QPolygonF itemPoly(itemRect);
-    itemPoly = item->mapToScene(itemPoly);
+    itemRect = item->mapRectToScene(itemRect);
 
     // if updating the same item as before, resize the rectangle only bigger, not smaller.
     if (prevItem == item && prevItem != 0) {
-        m_itemPolyRect = resizeRect(itemPoly.boundingRect(), m_itemPolyRect);
+        m_itemPolyRect = resizeRect(itemRect, m_itemPolyRect);
     } else {
-        m_itemPolyRect = itemPoly.boundingRect();
+        m_itemPolyRect = itemRect;
     }
     QRectF borderRect = m_itemPolyRect;
     borderRect.adjust(-1, -1, 1, 1);
     m_borderRect->setRect(borderRect);
 
-    itemPoly = viewPoly.subtracted(QPolygonF(m_itemPolyRect));
-    setPolygon(itemPoly);
+    const QRegion externalRegion = QRegion(viewRect).subtracted(m_itemPolyRect.toRect());
+    setPolygon(regionToPolygon(externalRegion));
 }
 
 QGraphicsItem *SubcomponentMaskLayerItem::currentItem() const