From 4e0c1e0b8263bcd125f4a9c0a8a995ddc79d6aa2 Mon Sep 17 00:00:00 2001
From: Miikka Heikkinen <miikka.heikkinen@qt.io>
Date: Fri, 1 Sep 2023 15:59:14 +0300
Subject: [PATCH] QmlDesigner: Fix issues in 3D view when parent node has
 transform

1. Hiding the selection box resets the parent transform we calculate
   during selection box geometry generation. Worked around this issue
   by hiding the boxes by using null material instead of visible
   property.

2. Aligning cameras to views and vice versa didn't consider the
   possibility that the camera might be a child node with a parent node
   that has transform.

Fixes: QDS-10502
Change-Id: Ibf4af71424b8b673a42042dbc1840754967799c5
Reviewed-by: Qt CI Patch Build Bot <ci_patchbuild_bot@qt.io>
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>
Reviewed-by: Mahmoud Badri <mahmoud.badri@qt.io>
---
 .../qml2puppet/mockfiles/qt6/EditView3D.qml   |  2 +-
 .../qml2puppet/mockfiles/qt6/SelectionBox.qml | 21 +++++++-----
 .../qml2puppet/editor3d/generalhelper.cpp     | 32 ++++++++++++++++---
 3 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/src/tools/qml2puppet/mockfiles/qt6/EditView3D.qml b/src/tools/qml2puppet/mockfiles/qt6/EditView3D.qml
index 8a4db279628..14f1bb2037a 100644
--- a/src/tools/qml2puppet/mockfiles/qt6/EditView3D.qml
+++ b/src/tools/qml2puppet/mockfiles/qt6/EditView3D.qml
@@ -311,7 +311,7 @@ Item {
                                                      "geometryName": geometryName});
                     selectionBoxes[selectionBoxes.length] = box;
                     box.view3D = Qt.binding(function() {return editView;});
-                    box.visible = Qt.binding(function() {return showSelectionBox;});
+                    box.showBox = Qt.binding(function() {return showSelectionBox;});
                 }
             }
         }
diff --git a/src/tools/qml2puppet/mockfiles/qt6/SelectionBox.qml b/src/tools/qml2puppet/mockfiles/qt6/SelectionBox.qml
index f300fd9502e..fe6cc0af373 100644
--- a/src/tools/qml2puppet/mockfiles/qt6/SelectionBox.qml
+++ b/src/tools/qml2puppet/mockfiles/qt6/SelectionBox.qml
@@ -12,6 +12,7 @@ Node {
     property Node targetNode: null
     property alias model: selectionBoxModel
     property alias geometryName: selectionBoxGeometry.name
+    property bool showBox: true
 
     SelectionBoxGeometry {
         id: selectionBoxGeometry
@@ -31,17 +32,21 @@ Node {
         position: selectionBox.targetNode ? selectionBox.targetNode.position : Qt.vector3d(0, 0, 0)
         pivot: selectionBox.targetNode ? selectionBox.targetNode.pivot : Qt.vector3d(0, 0, 0)
 
-        visible: selectionBox.targetNode && !selectionBoxGeometry.isEmpty
+        visible: selectionBox.targetNode
 
         castsShadows: false
         receivesShadows: false
 
-        materials: [
-            DefaultMaterial {
-                diffuseColor: "#fff600"
-                lighting: DefaultMaterial.NoLighting
-                cullMode: Material.NoCulling
-            }
-        ]
+        DefaultMaterial {
+            id: boxMaterial
+            diffuseColor: "#fff600"
+            lighting: DefaultMaterial.NoLighting
+            cullMode: Material.NoCulling
+        }
+        // We custom set the render node transform for the selectionBox node to correspond
+        // targetNode's parent transform when we calculate the box geometry.
+        // This transform gets reset if visibility of the box changes, so instead we hide the box
+        // by setting null material to it.
+        materials: selectionBox.showBox && !selectionBoxGeometry.isEmpty ? boxMaterial : null
     }
 }
diff --git a/src/tools/qml2puppet/qml2puppet/editor3d/generalhelper.cpp b/src/tools/qml2puppet/qml2puppet/editor3d/generalhelper.cpp
index 57b02f4b168..7e53a766c3e 100644
--- a/src/tools/qml2puppet/qml2puppet/editor3d/generalhelper.cpp
+++ b/src/tools/qml2puppet/qml2puppet/editor3d/generalhelper.cpp
@@ -353,8 +353,32 @@ void GeneralHelper::alignCameras(QQuick3DCamera *camera, const QVariant &nodes)
     }
 
     for (QQuick3DCamera *node : std::as_const(nodeList)) {
-        node->setPosition(camera->position());
-        node->setRotation(camera->rotation());
+        QMatrix4x4 parentTransform;
+        QMatrix4x4 parentRotationTransform;
+        if (node->parentNode()) {
+            QMatrix4x4 rotMat;
+            rotMat.rotate(node->parentNode()->sceneRotation());
+            parentRotationTransform = rotMat.inverted();
+            parentTransform = node->parentNode()->sceneTransform().inverted();
+        }
+
+        QMatrix4x4 localTransform;
+        localTransform.translate(camera->position());
+        localTransform.rotate(camera->rotation());
+
+        QMatrix4x4 finalTransform = parentTransform * localTransform;
+        QVector3D newPos = QVector3D(finalTransform.column(3).x(),
+                                     finalTransform.column(3).y(),
+                                     finalTransform.column(3).z());
+
+        // Rotation must be calculated with sanitized transform that only contains rotation so
+        // that the scaling of ancestor nodes won't distort it
+        QMatrix4x4 finalRotTransform = parentRotationTransform * localTransform;
+        QMatrix3x3 rotationMatrix = finalRotTransform.toGenericMatrix<3, 3>();
+        QQuaternion newRot = QQuaternion::fromRotationMatrix(rotationMatrix).normalized();
+
+        node->setPosition(newPos);
+        node->setRotation(newRot);
     }
 }
 
@@ -376,8 +400,8 @@ QVector3D GeneralHelper::alignView(QQuick3DCamera *camera, const QVariant &nodes
     }
 
     if (cameraNode) {
-        camera->setPosition(cameraNode->position());
-        QVector3D newRotation = cameraNode->eulerRotation();
+        camera->setPosition(cameraNode->scenePosition());
+        QVector3D newRotation = cameraNode->sceneRotation().toEulerAngles();
         newRotation.setZ(0.f);
         camera->setEulerRotation(newRotation);
     }