iostool: Improve deployment speed using delta deploy

The iostool did always deploy the whole bundle, without taking
into account whether anything has actually changed. This meant
that for big bundles anytime the user starts the application
on his device, a full deployment was done.

For a ~1GB bundle this would take around a minute on a recent
Mac and iPhone 12.

This fix uses a new function from the mobiledevice framework
called AMDeviceSecureInstallApplicationBundle.
This function takes a new parameter "ShadowPathKey" which points
to a directory where the last deploy state is captured temporarily.

Before deploying to the device, the function compares
what is to be deployed against the last deploy state and
only deploys the parts that actually changed.

QtCreator provides a temporary folder for this. Due to this,
the initial deployment still does a complete deployment as
no state is available yet. All subsequent deployments
take the captured state into account.

For backwards compatibility, the old deployment method is left intact.

Fixes: QTCREATORBUG-24371
Change-Id: I4df6aa79d41b34c326d78be7952d7eeb23774648
Reviewed-by: Eike Ziller <eike.ziller@qt.io>
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>

src/plugins/ios/iostoolhandler.cpp

@@ -38,6 +38,7 @@
 #include <utils/qtcassert.h>
 #include <utils/qtcprocess.h>
 #include <utils/runextensions.h>
+#include <utils/temporarydirectory.h>
 
 #include <QCoreApplication>
 #include <QDir>
@@ -701,10 +702,14 @@ void IosDeviceToolHandlerPrivate::requestTransferApp(const QString &bundlePath,
 {
     m_bundlePath = bundlePath;
     m_deviceId = deviceId;
+    QString tmpDeltaPath = Utils::TemporaryDirectory::masterDirectoryFilePath().pathAppended("ios").toString();
     QStringList args;
     args << QLatin1String("--id") << deviceId << QLatin1String("--bundle")
          << bundlePath << QLatin1String("--timeout") << QString::number(timeout)
-         << QLatin1String("--install");
+         << QLatin1String("--install")
+         << QLatin1String("--delta-path")
+         << tmpDeltaPath;
+
     start(IosToolHandler::iosDeviceToolPath(), args);
 }
 

src/tools/iostool/CMakeLists.txt

@@ -18,6 +18,7 @@ add_qtc_executable(iostool
     main.cpp
     mobiledevicelib.cpp mobiledevicelib.h
     relayserver.cpp relayserver.h
+    cfutils.h
 )
 
 if (TARGET iostool)

src/tools/iostool/cfutils.h

@@ -0,0 +1,55 @@
+/****************************************************************************
+**
+** Copyright (C) 2022 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <QString>
+
+#include <CoreFoundation/CoreFoundation.h>
+
+namespace Ios {
+
+template<typename CFType>
+struct CFRefDeleter
+{
+    using pointer = CFType;
+    void operator()(CFType ref) { CFRelease(ref); }
+};
+
+inline QString toQStringRelease(CFStringRef str)
+{
+    QString result = QString::fromCFString(str);
+    CFRelease(str);
+    return result;
+}
+
+using CFString_t = std::unique_ptr<CFStringRef, CFRefDeleter<CFStringRef>>;
+using CFUrl_t = std::unique_ptr<CFURLRef, CFRefDeleter<CFURLRef>>;
+using CFPropertyList_t = std::unique_ptr<CFPropertyListRef, CFRefDeleter<CFPropertyListRef>>;
+using CFBundle_t = std::unique_ptr<CFBundleRef, CFRefDeleter<CFBundleRef>>;
+using CFDictionary_t = std::unique_ptr<CFDictionaryRef, CFRefDeleter<CFDictionaryRef>>;
+using CFArray_t = std::unique_ptr<CFArrayRef, CFRefDeleter<CFArrayRef>>;
+
+} // namespace Ios

src/tools/iostool/iosdevicemanager.cpp

@@ -25,19 +25,19 @@
 
 #include "iosdevicemanager.h"
 
+#include "cfutils.h"
 #include "mobiledevicelib.h"
 
-#include <QDebug>
 #include <QDir>
 #include <QFile>
 #include <QHash>
 #include <QLibrary>
+#include <QLoggingCategory>
 #include <QMultiHash>
 #include <QMutex>
 #include <QMutexLocker>
 #include <QProcess>
 #include <QRegularExpression>
-#include <QRegularExpression>
 #include <QSettings>
 #include <QThread>
 #include <QTimer>
@@ -54,6 +54,10 @@ static const bool debugAll = false;
 static const bool verbose = true;
 static const bool noWifi = true;
 
+namespace {
+    Q_LOGGING_CATEGORY(loggingCategory, "qtc.iostool.iosdevicemanager", QtWarningMsg)
+}
+
 // ------- MobileDeviceLib interface --------
 namespace {
 
@@ -277,8 +281,12 @@ public:
     static IosDeviceManagerPrivate *instance();
     explicit IosDeviceManagerPrivate (IosDeviceManager *q);
     bool watchDevices();
-    void requestAppOp(const QString &bundlePath, const QStringList &extraArgs,
+    void requestAppOp(const QString &bundlePath,
-                            Ios::IosDeviceManager::AppOp appOp, const QString &deviceId, int timeout);
+                      const QStringList &extraArgs,
+                      Ios::IosDeviceManager::AppOp appOp,
+                      const QString &deviceId,
+                      int timeout,
+                      const QString &deltaPath);
     void requestDeviceInfo(const QString &deviceId, int timeout);
     QStringList errors();
     void addError(QString errorMsg);
@@ -321,13 +329,17 @@ public:
     QString bundlePath;
     QStringList extraArgs;
     Ios::IosDeviceManager::AppOp appOp;
+    QString deltaPath;
 
-
+    AppOpSession(const QString &deviceId,
-    AppOpSession(const QString &deviceId, const QString &bundlePath,
+                 const QString &bundlePath,
-                        const QStringList &extraArgs, Ios::IosDeviceManager::AppOp appOp);
+                 const QStringList &extraArgs,
+                 Ios::IosDeviceManager::AppOp appOp,
+                 const QString &deltaPath);
 
     void deviceCallbackReturned() override;
     bool installApp();
+    bool installAppNew();
     bool runApp();
     int qmljsDebugPort() const override;
     am_res_t appTransferCallback(CFDictionaryRef dict) override;
@@ -488,9 +500,11 @@ bool IosDeviceManagerPrivate::watchDevices()
 void IosDeviceManagerPrivate::requestAppOp(const QString &bundlePath,
                                            const QStringList &extraArgs,
                                            IosDeviceManager::AppOp appOp,
-                                                 const QString &deviceId, int timeout)
+                                           const QString &deviceId,
+                                           int timeout,
+                                           const QString &deltaPath)
 {
-    AppOpSession *session = new AppOpSession(deviceId, bundlePath, extraArgs, appOp);
+    AppOpSession *session = new AppOpSession(deviceId, bundlePath, extraArgs, appOp, deltaPath);
     session->startDeviceLookup(timeout);
 }
 
@@ -1205,10 +1219,17 @@ bool CommandSession::developerDiskImagePath(QString *path, QString *signaturePat
     return false;
 }
 
-AppOpSession::AppOpSession(const QString &deviceId, const QString &bundlePath,
+AppOpSession::AppOpSession(const QString &deviceId,
-                                       const QStringList &extraArgs, IosDeviceManager::AppOp appOp):
+                           const QString &bundlePath,
-    CommandSession(deviceId), bundlePath(bundlePath), extraArgs(extraArgs), appOp(appOp)
+                           const QStringList &extraArgs,
-{ }
+                           IosDeviceManager::AppOp appOp,
+                           const QString &deltaPath)
+    : CommandSession(deviceId)
+    , bundlePath(bundlePath)
+    , extraArgs(extraArgs)
+    , appOp(appOp)
+    , deltaPath(deltaPath)
+{}
 
 QString AppOpSession::commandName()
 {
@@ -1218,38 +1239,58 @@ QString AppOpSession::commandName()
 bool AppOpSession::installApp()
 {
     bool success = false;
-    if (device != 0) {
+    if (device) {
-        CFURLRef bundleUrl = QUrl::fromLocalFile(bundlePath).toCFURL();
+        if (!installAppNew()) {
+            addError(QString::fromLatin1(
+                "Failed to transfer and install application, trying old way ..."));
+
+            const CFUrl_t bundleUrl(QUrl::fromLocalFile(bundlePath).toCFURL());
+            MobileDeviceLib &mLib = MobileDeviceLib::instance();
+
             CFStringRef key[1] = {CFSTR("PackageType")};
             CFStringRef value[1] = {CFSTR("Developer")};
-        CFDictionaryRef options = CFDictionaryCreate(0, reinterpret_cast<const void**>(&key[0]),
+            const CFDictionary_t options(
-                reinterpret_cast<const void**>(&value[0]), 1,
+                CFDictionaryCreate(0,
+                                   reinterpret_cast<const void **>(&key[0]),
+                                   reinterpret_cast<const void **>(&value[0]),
+                                   1,
                                    &kCFTypeDictionaryKeyCallBacks,
-                                                     &kCFTypeDictionaryValueCallBacks);
+                                   &kCFTypeDictionaryValueCallBacks));
 
-        MobileDeviceLib &mLib = MobileDeviceLib::instance();
             // Transfer bundle with secure API AMDeviceTransferApplication.
-        if (int error = mLib.deviceSecureTransferApplicationPath(0, device, bundleUrl, options,
+            if (int error
-                                                         &appSecureTransferSessionCallback,0)) {
+                = mLib.deviceSecureTransferApplicationPath(0,
-            addError(QString::fromLatin1("TransferAppSession(%1,%2) failed, AMDeviceTransferApplication returned %3 (0x%4)")
+                                                           device,
-                     .arg(bundlePath, deviceId).arg(mobileDeviceErrorString(error)).arg(error));
+                                                           bundleUrl.get(),
+                                                           options.get(),
+                                                           &appSecureTransferSessionCallback,
+                                                           0)) {
+                addError(QString::fromLatin1("TransferAppSession(%1,%2) failed, "
+                                             "AMDeviceTransferApplication returned %3 (0x%4)")
+                             .arg(bundlePath, deviceId)
+                             .arg(mobileDeviceErrorString(error))
+                             .arg(error));
                 success = false;
             } else {
                 // App is transferred. Try installing.
                 if (connectDevice()) {
                     // Secure install app api requires device to be connected.
-                if (am_res_t error = mLib.deviceSecureInstallApplication(0, device, bundleUrl, options,
+                    if (am_res_t error
-                                                                           &appSecureTransferSessionCallback,0)) {
+                        = mLib.deviceSecureInstallApplication(0,
+                                                              device,
+                                                              bundleUrl.get(),
+                                                              options.get(),
+                                                              &appSecureTransferSessionCallback,
+                                                              0)) {
                         const QString errorString = mobileDeviceErrorString(error);
                         if (!errorString.isEmpty()) {
-                        addError(errorString
+                            addError(errorString + QStringLiteral(" (0x")
-                                 + QStringLiteral(" (0x")
+                                     + QString::number(error, 16) + QStringLiteral(")"));
-                                 + QString::number(error, 16)
-                                 + QStringLiteral(")"));
                         } else {
                             addError(QString::fromLatin1("InstallAppSession(%1,%2) failed, "
                                                          "AMDeviceInstallApplication returned 0x%3")
-                                 .arg(bundlePath, deviceId).arg(QString::number(error, 16)));
+                                         .arg(bundlePath, deviceId)
+                                         .arg(QString::number(error, 16)));
                         }
                         success = false;
                     } else {
@@ -1259,29 +1300,82 @@ bool AppOpSession::installApp()
                     disconnectDevice();
                 }
             }
-
+        } else {
-        if (debugAll) {
+            success = true;
-            qDebug() << "AMDeviceSecureTransferApplication finished request with " << (success ? "Success" : "Failure");
         }
 
-        CFRelease(options);
+        qCDebug(loggingCategory) << "AMDeviceSecureTransferApplication finished request with"
-        CFRelease(bundleUrl);
+                                 << (success ? "Success" : "Failure");
 
         progressBase += 100;
     }
 
-
     if (success) {
         sleep(5); // after installation the device needs a bit of quiet....
     }
 
-    if (debugAll) {
+    qCDebug(loggingCategory) << "AMDeviceSecureInstallApplication finished request with"
-        qDebug() << "AMDeviceSecureInstallApplication finished request with " << (success ? "Success" : "Failure");
+                             << (success ? "Success" : "Failure");
+
+    IosDeviceManagerPrivate::instance()->didTransferApp(bundlePath,
+                                                        deviceId,
+                                                        (success ? IosDeviceManager::Success
+                                                                 : IosDeviceManager::Failure));
+    return success;
+}
+
+bool AppOpSession::installAppNew()
+{
+    const CFUrl_t bundleUrl(QUrl::fromLocalFile(bundlePath).toCFURL());
+    MobileDeviceLib &mLib = MobileDeviceLib::instance();
+
+    CFBundle_t bundle(CFBundleCreate(kCFAllocatorDefault, bundleUrl.get()));
+
+    if (!bundle) {
+        addError(QString::fromLatin1("Failed to create bundle"));
+        return false;
     }
 
-    IosDeviceManagerPrivate::instance()->didTransferApp(bundlePath, deviceId,
+    const CFString_t bundleId(CFBundleGetIdentifier(bundle.get()));
-                (success ? IosDeviceManager::Success : IosDeviceManager::Failure));
+    if (!bundleId) {
-    return success;
+        addError(QString::fromLatin1("Failed to retrieve bundle id"));
+        return false;
+    }
+
+    CFUrl_t dpath(QUrl::fromLocalFile(deltaPath).toCFURL());
+
+    CFStringRef keys[] = {
+        CFSTR("CFBundleIdentifier"),
+        CFSTR("CloseOnInvalidate"),
+        CFSTR("InvalidateOnDetach"),
+        CFSTR("IsUserInitiated"),
+        CFSTR("PackageType"),
+        CFSTR("PreferWifi"),
+        CFSTR("ShadowParentKey"),
+    };
+    CFStringRef values[] = {bundleId.get(),
+                            CFSTR("1"),
+                            CFSTR("1"),
+                            CFSTR("1"),
+                            CFSTR("Developer"),
+                            CFSTR("1"),
+                            (CFStringRef)dpath.get()};
+
+    const CFDictionary_t options(CFDictionaryCreate(0,
+                                                 reinterpret_cast<const void **>(&keys[0]),
+                                                 reinterpret_cast<const void **>(&values[0]),
+                                                 7,
+                                                 &kCFTypeDictionaryKeyCallBacks,
+                                                 &kCFTypeDictionaryValueCallBacks));
+
+    if (int error = mLib.deviceSecureInstallApplicationBundle(0,
+                                                              device,
+                                                              bundleUrl.get(),
+                                                              options.get(),
+                                                              &appSecureTransferSessionCallback))
+        return false;
+
+    return true;
 }
 
 void AppOpSession::deviceCallbackReturned()
@@ -1575,9 +1669,14 @@ bool IosDeviceManager::watchDevices() {
     return d->watchDevices();
 }
 
-void IosDeviceManager::requestAppOp(const QString &bundlePath, const QStringList &extraArgs,
+void IosDeviceManager::requestAppOp(const QString &bundlePath,
-                                          AppOp appOp, const QString &deviceId, int timeout) {
+                                    const QStringList &extraArgs,
-    d->requestAppOp(bundlePath, extraArgs, appOp, deviceId, timeout);
+                                    AppOp appOp,
+                                    const QString &deviceId,
+                                    int timeout,
+                                    QString deltaPath)
+{
+    d->requestAppOp(bundlePath, extraArgs, appOp, deviceId, timeout, deltaPath);
 }
 
 void IosDeviceManager::requestDeviceInfo(const QString &deviceId, int timeout)

src/tools/iostool/iosdevicemanager.h

@@ -62,7 +62,7 @@ public:
     static IosDeviceManager *instance();
     bool watchDevices();
     void requestAppOp(const QString &bundlePath, const QStringList &extraArgs, AppOp appOp,
-                      const QString &deviceId, int timeout = 1000);
+                      const QString &deviceId, int timeout = 1000, QString deltaPath = QString());
     void requestDeviceInfo(const QString &deviceId, int timeout = 1000);
     int processGdbServer(ServiceConnRef conn);
     void stopGdbServer(ServiceConnRef conn, int phase);

src/tools/iostool/iostool.cpp

@@ -82,6 +82,12 @@ void IosTool::run(const QStringList &args)
                 printHelp = true;
             }
             bundlePath = args.value(iarg);
+        } else if (arg == QLatin1String("--delta-path")) {
+            if (++iarg == args.size()) {
+                writeMsg(QStringLiteral("missing path after ") + arg);
+                printHelp = true;
+            }
+            m_deltasPath = args.value(iarg);
         } else if (arg == QLatin1String("--install")) {
             appOp = IosDeviceManager::AppOp(appOp | IosDeviceManager::Install);
         } else if (arg == QLatin1String("--run")) {
@@ -163,7 +169,7 @@ void IosTool::run(const QStringList &args)
             break;
         }
         maxProgress = 200;
-        manager->requestAppOp(bundlePath, extraArgs, appOp, deviceId, timeout);
+        manager->requestAppOp(bundlePath, extraArgs, appOp, deviceId, timeout, m_deltasPath);
     }
     if (opLeft == 0)
         doExit(0);

src/tools/iostool/iostool.h

@@ -79,6 +79,7 @@ private:
     Ios::IosDeviceManager::AppOp appOp;
     QFile outFile;
     QString m_qmlPort;
+    QString m_deltasPath;
     QXmlStreamWriter out;
     GdbRelayServer *gdbServer;
     QmlRelayServer *qmlServer;

src/tools/iostool/iostool.qbs

@@ -11,6 +11,7 @@ QtcTool {
     Depends { name: "app_version_header" }
 
     files: [
+        "cfutils.h"
         "Info.plist",
         "gdbrunner.cpp",
         "gdbrunner.h",

src/tools/iostool/mobiledevicelib.cpp

@@ -89,7 +89,15 @@ bool MobileDeviceLib::load()
     m_AMDSetLogLevel = reinterpret_cast<AMDSetLogLevelPtr>(lib.resolve("AMDSetLogLevel"));
     if (m_AMDSetLogLevel == 0)
         addError("MobileDeviceLib does not define AMDSetLogLevel");
-    m_AMDeviceNotificationSubscribe = reinterpret_cast<AMDeviceNotificationSubscribePtr>(lib.resolve("AMDeviceNotificationSubscribe"));
+
+    m_AMDeviceSecureInstallApplicationBundle
+        = reinterpret_cast<AMDeviceSecureInstallApplicationBundlePtr>(
+            lib.resolve("AMDeviceSecureInstallApplicationBundle"));
+    if (m_AMDeviceSecureInstallApplicationBundle == 0)
+        addError("MobileDeviceLib does not define m_AMDeviceSecureInstallApplicationBundle");
+
+    m_AMDeviceNotificationSubscribe = reinterpret_cast<AMDeviceNotificationSubscribePtr>(
+        lib.resolve("AMDeviceNotificationSubscribe"));
     if (m_AMDeviceNotificationSubscribe == 0)
         addError("MobileDeviceLib does not define AMDeviceNotificationSubscribe");
     m_AMDeviceNotificationUnsubscribe = reinterpret_cast<AMDeviceNotificationUnsubscribePtr>(lib.resolve("AMDeviceNotificationUnsubscribe"));
@@ -358,6 +366,21 @@ int MobileDeviceLib::deviceSecureTransferApplicationPath(int zero, AMDeviceRef d
     return returnCode;
 }
 
+int MobileDeviceLib::deviceSecureInstallApplicationBundle(
+    int zero,
+    AMDeviceRef device,
+    CFURLRef url,
+    CFDictionaryRef options,
+    AMDeviceSecureInstallApplicationCallback callback)
+{
+    int returnCode = -1;
+
+    if (m_AMDeviceSecureInstallApplicationBundle) {
+        returnCode = m_AMDeviceSecureInstallApplicationBundle(device, url, options, callback, zero);
+    }
+    return returnCode;
+}
+
 int MobileDeviceLib::deviceSecureInstallApplication(int zero, AMDeviceRef device, CFURLRef url, CFDictionaryRef options, AMDeviceSecureInstallApplicationCallback callback, int arg)
 {
     int returnCode = -1;

src/tools/iostool/mobiledevicelib.h

@@ -101,6 +101,7 @@ typedef am_res_t (MDEV_API *USBMuxConnectByPortPtr)(unsigned int, int, ServiceSo
 // secure Api's
 typedef am_res_t (MDEV_API *AMDeviceSecureStartServicePtr)(AMDeviceRef, CFStringRef, unsigned int *, ServiceConnRef *);
 typedef int (MDEV_API *AMDeviceSecureTransferPathPtr)(int, AMDeviceRef, CFURLRef, CFDictionaryRef, AMDeviceSecureInstallApplicationCallback, int);
+typedef int (MDEV_API *AMDeviceSecureInstallApplicationBundlePtr)(AMDeviceRef, CFURLRef, CFDictionaryRef, AMDeviceSecureInstallApplicationCallback, int zero);
 typedef int (MDEV_API *AMDeviceSecureInstallApplicationPtr)(int, AMDeviceRef, CFURLRef, CFDictionaryRef, AMDeviceSecureInstallApplicationCallback, int);
 typedef int (MDEV_API *AMDServiceConnectionGetSocketPtr)(ServiceConnRef);
 
@@ -158,6 +159,13 @@ public:
     int deviceSecureTransferApplicationPath(int, AMDeviceRef, CFURLRef,
                                             CFDictionaryRef,
                                             AMDeviceSecureInstallApplicationCallback callback, int);
+
+    int deviceSecureInstallApplicationBundle(int zero,
+                                             AMDeviceRef device,
+                                             CFURLRef url,
+                                             CFDictionaryRef options,
+                                             AMDeviceSecureInstallApplicationCallback callback);
+
     int deviceSecureInstallApplication(int zero, AMDeviceRef device, CFURLRef url,
                                        CFDictionaryRef options,
                                        AMDeviceSecureInstallApplicationCallback callback, int arg);
@@ -191,6 +199,7 @@ private:
     AMDeviceMountImagePtr m_AMDeviceMountImage;
     AMDeviceSecureStartServicePtr m_AMDeviceSecureStartService;
     AMDeviceSecureTransferPathPtr m_AMDeviceSecureTransferPath;
+    AMDeviceSecureInstallApplicationBundlePtr m_AMDeviceSecureInstallApplicationBundle;
     AMDeviceSecureInstallApplicationPtr m_AMDeviceSecureInstallApplication;
     AMDServiceConnectionGetSocketPtr m_AMDServiceConnectionGetSocket;
     AMDServiceConnectionSendPtr m_AMDServiceConnectionSend;