QmlDesigner: Render effects on document level

Rendering effects requires actually rendering another item.
The effect is defined by another item that is rendered instead
of the original item.

For items that have an effect we do not render the children, since
the effect is only applied to the layer not the individual items.

We set layer.enabled temporarily to false, this ensures the effect
is not rendered as part of its parent item.

To detect the effect we use the source property an check if it points
to the ShaderEffectSource.

If one of the children inside the effect is transformed we have
to update the effect item.

If layer.enabled or layer.effect is changed we set the dirty flag
on the item and all children.

Change-Id: Iff61ef950e62a7a598b4bfa181ea70cb144368f3
Reviewed-by: Miikka Heikkinen <miikka.heikkinen@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>

share/qtcreator/qml/qmlpuppet/qml2puppet/instances/qt5nodeinstanceserver.cpp

@@ -335,6 +335,44 @@ QImage Qt5NodeInstanceServer::grabWindow()
     return  {};
 }
 
+static bool hasEffect(QQuickItem *item)
+{
+    QQuickItemPrivate *pItem = QQuickItemPrivate::get(item);
+    return pItem && pItem->layer() && pItem->layer()->enabled() && pItem->layer()->effect();
+}
+
+QQuickItem *Qt5NodeInstanceServer::parentEffectItem(QQuickItem *item)
+{
+    QQuickItem *parent = item->parentItem();
+    while (parent) {
+        if (hasEffect(parent))
+            return parent;
+        parent = parent->parentItem();
+    }
+    return nullptr;
+}
+
+static bool isEffectItem(QQuickItem *item, QQuickShaderEffectSource *sourceItem)
+{
+    QQuickItemPrivate *pItem = QQuickItemPrivate::get(sourceItem);
+
+    if (!pItem || !pItem->layer())
+        return false;
+
+    const auto propName = pItem->layer()->name();
+
+    QQmlProperty prop(item, QString::fromLatin1(propName));
+    if (!prop.isValid())
+        return false;
+
+    return prop.read().value<QQuickShaderEffectSource *>() == sourceItem;
+}
+
+static bool isLayerEnabled(QQuickItemPrivate *item)
+{
+    return item && item->layer() && item->layer()->enabled();
+}
+
 QImage Qt5NodeInstanceServer::grabItem(QQuickItem *item)
 {
     QImage renderImage;
@@ -343,10 +381,29 @@ QImage Qt5NodeInstanceServer::grabItem(QQuickItem *item)
         return {};
 
     QQuickItemPrivate *pItem = QQuickItemPrivate::get(item);
+
+    const bool renderEffects = qEnvironmentVariableIsSet("QMLPUPPET_RENDER_EFFECTS");
+
+    if (renderEffects) {
+        if (parentEffectItem(item))
+            return renderImage;
+
+        // Effects are actually implemented as a separate item we have to find first
+        if (hasEffect(item)) {
+            if (auto parent = item->parentItem()) {
+                const auto siblings = parent->childItems();
+                for (auto sibling : siblings) {
+                    if (isEffectItem(sibling, pItem->layer()->effectSource()))
+                        return grabItem(sibling);
+                }
+            }
+        }
+    }
+
+    if (!isLayerEnabled(pItem))
         pItem->refFromEffectItem(false);
 
     ServerNodeInstance instance = instanceForObject(item);
-    const auto childInstances = instance.childItems();
 
     // Setting layer enabled to false messes up the bounding rect.
     // Therefore we calculate it upfront.
@@ -358,13 +415,22 @@ QImage Qt5NodeInstanceServer::grabItem(QQuickItem *item)
 
     // Hide immediate children that have instances and are QQuickItems so we get only
     // the parent item's content, as compositing is handled on creator side.
+    QSet<QQuickItem *> layerChildren;
+
+    if (instance.isValid()) { //Not valid for effect
+        const auto childInstances = instance.childItems();
         for (const auto &childInstance : childInstances) {
             QQuickItem *childItem = qobject_cast<QQuickItem *>(childInstance.internalObject());
             if (childItem) {
                 QQuickItemPrivate *pChild = QQuickItemPrivate::get(childItem);
+                if (pChild->layer() && pChild->layer()->enabled()) {
+                    layerChildren.insert(childItem);
+                    pChild->layer()->setEnabled(false);
+                }
                 pChild->refFromEffectItem(true);
             }
         }
+    }
 
     m_viewData.renderControl->polishItems();
     m_viewData.renderControl->beginFrame();
@@ -406,15 +472,24 @@ QImage Qt5NodeInstanceServer::grabItem(QQuickItem *item)
 
     m_viewData.renderControl->endFrame();
 
+    if (instance.isValid()) { //Not valid for effect
+        const auto childInstances = instance.childItems();
+
         // Restore visibility of immediate children that have instances and are QQuickItems
         for (const auto &childInstance : childInstances) {
             QQuickItem *childItem = qobject_cast<QQuickItem *>(childInstance.internalObject());
             if (childItem) {
                 QQuickItemPrivate *pChild = QQuickItemPrivate::get(childItem);
                 pChild->derefFromEffectItem(true);
+                if (pChild->layer() && layerChildren.contains(childItem))
+                    pChild->layer()->setEnabled(true);
             }
         }
+    }
+
+    if (!isLayerEnabled(pItem))
         pItem->derefFromEffectItem(false);
+
 #else
     Q_UNUSED(item)
 #endif

share/qtcreator/qml/qmlpuppet/qml2puppet/instances/qt5nodeinstanceserver.h

@@ -71,6 +71,8 @@ public:
     QImage grabWindow() override;
     QImage grabItem(QQuickItem *item) override;
 
+    static QQuickItem *parentEffectItem(QQuickItem *item);
+
 protected:
     void initializeView() override;
     void resizeCanvasToRootItem() override;

share/qtcreator/qml/qmlpuppet/qml2puppet/instances/qt5rendernodeinstanceserver.cpp

@@ -89,8 +89,20 @@ void Qt5RenderNodeInstanceServer::collectItemChangesAndSendChangeCommands()
                         if (hasInstanceForObject(item)) {
                             if (DesignerSupport::isDirty(item, DesignerSupport::ContentUpdateMask))
                                 m_dirtyInstanceSet.insert(instanceForObject(item));
+                            if (QQuickItem *effectParent = parentEffectItem(item)) {
+                                if ((DesignerSupport::isDirty(
+                                        item,
+                                        DesignerSupport::DirtyType(
+                                            DesignerSupport::TransformUpdateMask
+                                            | DesignerSupport::Visible
+                                            | DesignerSupport::ContentUpdateMask)))
+                                    && hasInstanceForObject(effectParent)) {
+                                    m_dirtyInstanceSet.insert(instanceForObject(effectParent));
+                                }
+                            }
                         } else if (DesignerSupport::isDirty(item, DesignerSupport::AllMask)) {
-                            ServerNodeInstance ancestorInstance = findNodeInstanceForItem(item->parentItem());
+                            ServerNodeInstance ancestorInstance = findNodeInstanceForItem(
+                                item->parentItem());
                             if (ancestorInstance.isValid())
                                 m_dirtyInstanceSet.insert(ancestorInstance);
                         }

share/qtcreator/qml/qmlpuppet/qml2puppet/instances/quickitemnodeinstance.cpp

@@ -612,6 +612,18 @@ void QuickItemNodeInstance::updateAllDirtyNodesRecursive(QQuickItem *parentItem)
     updateDirtyNode(parentItem);
 }
 
+void QuickItemNodeInstance::setAllNodesDirtyRecursive(QQuickItem *parentItem) const
+{
+#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
+    Q_UNUSED(parentItem)
+#else
+    const QList<QQuickItem *> children = parentItem->childItems();
+    for (QQuickItem *childItem : children)
+        setAllNodesDirtyRecursive(childItem);
+    DesignerSupport::addDirty(parentItem, QQuickDesignerSupport::Content);
+#endif
+}
+
 static inline bool isRectangleSane(const QRectF &rect)
 {
     return rect.isValid() && (rect.width() < 10000) && (rect.height() < 10000);
@@ -813,6 +825,9 @@ void QuickItemNodeInstance::setPropertyVariant(const PropertyName &name, const Q
     if (name == "y")
         m_y = value.toDouble();
 
+    if (name == "layer.enabled" || name == "layer.effect")
+        setAllNodesDirtyRecursive(quickItem());
+
     ObjectNodeInstance::setPropertyVariant(name, value);
 
     refresh();
@@ -882,6 +897,9 @@ void QuickItemNodeInstance::resetProperty(const PropertyName &name)
     if (name == "y")
         m_y = 0.0;
 
+    if (name == "layer.enabled" || name == "layer.effect")
+        setAllNodesDirtyRecursive(quickItem());
+
     DesignerSupport::resetAnchor(quickItem(), QString::fromUtf8(name));
 
     if (name == "anchors.fill") {

share/qtcreator/qml/qmlpuppet/qml2puppet/instances/quickitemnodeinstance.h

@@ -118,6 +118,7 @@ protected:
     Qt5NodeInstanceServer *qt5NodeInstanceServer() const;
     void updateDirtyNodesRecursive(QQuickItem *parentItem) const;
     void updateAllDirtyNodesRecursive(QQuickItem *parentItem) const;
+    void setAllNodesDirtyRecursive(QQuickItem *parentItem) const;
     QRectF boundingRectWithStepChilds(QQuickItem *parentItem) const;
     void resetHorizontal();
     void resetVertical();

src/plugins/qmldesigner/designercore/instances/puppetcreator.cpp

@@ -493,6 +493,7 @@ QProcessEnvironment PuppetCreator::processEnvironment() const
     environment.set("QML_BAD_GUI_RENDER_LOOP", "true");
     environment.set("QML_PUPPET_MODE", "true");
     environment.set("QML_DISABLE_DISK_CACHE", "true");
+    environment.set("QMLPUPPET_RENDER_EFFECTS", "true");
 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
     if (!environment.hasKey("QT_SCREEN_SCALE_FACTORS") && !environment.hasKey("QT_SCALE_FACTOR")
             && QApplication::testAttribute(Qt::AA_EnableHighDpiScaling))