GenericDirectUploadService: Reuse TaskTree

Change-Id: Ib450274d3e73059bbc5e8f8e0d586ebb2a7a0dc4
Reviewed-by: Christian Kandeler <christian.kandeler@qt.io>

src/libs/utils/tasktree.h

@@ -62,11 +62,11 @@ class TreeStorage : public TreeStorageBase
 public:
     TreeStorage() : TreeStorageBase(TreeStorage::ctor(), TreeStorage::dtor()) {}
     StorageStruct *operator->() const noexcept { return activeStorage(); }
-
-private:
     StorageStruct *activeStorage() const {
         return static_cast<StorageStruct *>(activeStorageVoid());
     }
+
+private:
     static StorageConstructor ctor() { return [] { return new StorageStruct; }; }
     static StorageDestructor dtor() {
         return [](void *storage) { delete static_cast<StorageStruct *>(storage); };

src/plugins/remotelinux/genericdirectuploadservice.cpp

@@ -16,65 +16,82 @@
 
 #include <QDateTime>
 #include <QDir>
-#include <QFileInfo>
-#include <QHash>
 #include <QList>
-#include <QQueue>
-#include <QString>
 
 using namespace ProjectExplorer;
 using namespace Utils;
+using namespace Utils::Tasking;
 
 namespace RemoteLinux {
 namespace Internal {
 
-enum State { Inactive, PreChecking, Uploading, PostProcessing };
-
 const int MaxConcurrentStatCalls = 10;
 
+struct UploadStorage
+{
+    QList<DeployableFile> filesToUpload;
+    FilesToTransfer filesToTransfer;
+};
+
 class GenericDirectUploadServicePrivate
 {
 public:
-    DeployableFile getFileForProcess(QtcProcess *proc)
+    GenericDirectUploadServicePrivate(GenericDirectUploadService *service) : q(service) {}
-    {
-        const auto it = remoteProcs.find(proc);
-        QTC_ASSERT(it != remoteProcs.end(), return DeployableFile());
-        const DeployableFile file = *it;
-        remoteProcs.erase(it);
-        return file;
-    }
 
+    QDateTime timestampFromStat(const DeployableFile &file, QtcProcess *statProc);
+
+    using FilesToStat = std::function<QList<DeployableFile>(UploadStorage *)>;
+    using StatEndHandler
+          = std::function<void(UploadStorage *, const DeployableFile &, const QDateTime &)>;
+    TaskItem statTask(UploadStorage *storage, const DeployableFile &file,
+                      StatEndHandler statEndHandler);
+    TaskItem statTree(const TreeStorage<UploadStorage> &storage, FilesToStat filesToStat,
+                      StatEndHandler statEndHandler);
+    TaskItem uploadGroup(const TreeStorage<UploadStorage> &storage);
+    TaskItem chmodTask(const DeployableFile &file);
+    TaskItem chmodTree(const TreeStorage<UploadStorage> &storage);
+
+    GenericDirectUploadService *q = nullptr;
     IncrementalDeployment incremental = IncrementalDeployment::NotSupported;
     bool ignoreMissingFiles = false;
-    QHash<QtcProcess *, DeployableFile> remoteProcs;
-    QQueue<DeployableFile> filesToStat;
-    State state = Inactive;
-    QList<DeployableFile> filesToUpload;
-    FileTransfer uploader;
     QList<DeployableFile> deployableFiles;
+    std::unique_ptr<TaskTree> m_taskTree;
 };
 
+QList<DeployableFile> collectFilesToUpload(const DeployableFile &deployable)
+{
+    QList<DeployableFile> collected;
+    FilePath localFile = deployable.localFilePath();
+    if (localFile.isDir()) {
+        const FilePaths files = localFile.dirEntries(QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot);
+        const QString remoteDir = deployable.remoteDirectory() + '/' + localFile.fileName();
+        for (const FilePath &localFilePath : files)
+            collected.append(collectFilesToUpload(DeployableFile(localFilePath, remoteDir)));
+    } else {
+        collected << deployable;
+    }
+    return collected;
+}
+
+static Group packIntoOptionalParallelGroups(const QList<TaskItem> &tasks)
+{
+    QList<TaskItem> groups;
+    int i = 0;
+    while (i < tasks.size()) {
+        const QList<TaskItem> subTasks = tasks.mid(i, MaxConcurrentStatCalls);
+        i += subTasks.size();
+        groups.append(Group { QList<TaskItem> {optional, parallel} + subTasks });
+    }
+    return Group { QList<TaskItem> {optional} + groups };
+}
+
 } // namespace Internal
 
 using namespace Internal;
 
 GenericDirectUploadService::GenericDirectUploadService(QObject *parent)
-    : AbstractRemoteLinuxDeployService(parent), d(new GenericDirectUploadServicePrivate)
+    : AbstractRemoteLinuxDeployService(parent), d(new GenericDirectUploadServicePrivate(this))
 {
-    connect(&d->uploader, &FileTransfer::done, this, [this](const ProcessResultData &result) {
-        QTC_ASSERT(d->state == Uploading, return);
-        if (result.m_error != QProcess::UnknownError || result.m_exitCode != 0) {
-            emit errorMessage(result.m_errorString);
-            setFinished();
-            handleDeploymentDone();
-            return;
-        }
-        d->state = PostProcessing;
-        chmod();
-        queryFiles();
-    });
-    connect(&d->uploader, &FileTransfer::progress,
-            this, &GenericDirectUploadService::progressMessage);
 }
 
 GenericDirectUploadService::~GenericDirectUploadService()
@@ -99,7 +116,6 @@ void GenericDirectUploadService::setIgnoreMissingFiles(bool ignoreMissingFiles)
 
 bool GenericDirectUploadService::isDeploymentNecessary() const
 {
-    QTC_ASSERT(d->filesToUpload.isEmpty(), d->filesToUpload.clear());
     QList<DeployableFile> collected;
     for (int i = 0; i < d->deployableFiles.count(); ++i)
         collected.append(collectFilesToUpload(d->deployableFiles.at(i)));
@@ -109,15 +125,14 @@ bool GenericDirectUploadService::isDeploymentNecessary() const
     return !d->deployableFiles.isEmpty();
 }
 
-void GenericDirectUploadService::doDeploy()
+void GenericDirectUploadService::stopDeployment()
 {
-    QTC_ASSERT(d->state == Inactive, setFinished(); return);
+    d->m_taskTree.reset();
-    d->state = PreChecking;
+    handleDeploymentDone();
-    queryFiles();
 }
 
-QDateTime GenericDirectUploadService::timestampFromStat(const DeployableFile &file,
+QDateTime GenericDirectUploadServicePrivate::timestampFromStat(const DeployableFile &file,
-                                                        QtcProcess *statProc)
+                                                               QtcProcess *statProc)
 {
     bool succeeded = false;
     QString error;
@@ -132,204 +147,208 @@ QDateTime GenericDirectUploadService::timestampFromStat(const DeployableFile &fi
         succeeded = true;
     }
     if (!succeeded) {
-        emit warningMessage(Tr::tr("Failed to retrieve remote timestamp for file \"%1\". "
+        emit q->warningMessage(Tr::tr("Failed to retrieve remote timestamp for file \"%1\". "
-                               "Incremental deployment will not work. Error message was: %2")
+                                      "Incremental deployment will not work. Error message was: %2")
-                            .arg(file.remoteFilePath(), error));
+                               .arg(file.remoteFilePath(), error));
-        return QDateTime();
+        return {};
     }
     const QByteArray output = statProc->readAllStandardOutput().trimmed();
     const QString warningString(Tr::tr("Unexpected stat output for remote file \"%1\": %2")
                                 .arg(file.remoteFilePath()).arg(QString::fromUtf8(output)));
     if (!output.startsWith(file.remoteFilePath().toUtf8())) {
-        emit warningMessage(warningString);
+        emit q->warningMessage(warningString);
-        return QDateTime();
+        return {};
     }
     const QByteArrayList columns = output.mid(file.remoteFilePath().toUtf8().size() + 1).split(' ');
     if (columns.size() < 14) { // Normal Linux stat: 16 columns in total, busybox stat: 15 columns
-        emit warningMessage(warningString);
+        emit q->warningMessage(warningString);
-        return QDateTime();
+        return {};
     }
     bool isNumber;
     const qint64 secsSinceEpoch = columns.at(11).toLongLong(&isNumber);
     if (!isNumber) {
-        emit warningMessage(warningString);
+        emit q->warningMessage(warningString);
-        return QDateTime();
+        return {};
     }
     return QDateTime::fromSecsSinceEpoch(secsSinceEpoch);
 }
 
-void GenericDirectUploadService::checkForStateChangeOnRemoteProcFinished()
+TaskItem GenericDirectUploadServicePrivate::statTask(UploadStorage *storage,
+                                                     const DeployableFile &file,
+                                                     StatEndHandler statEndHandler)
 {
-    if (d->remoteProcs.size() < MaxConcurrentStatCalls && !d->filesToStat.isEmpty())
+    const auto setupHandler = [=](QtcProcess &process) {
-        runStat(d->filesToStat.dequeue());
+        // We'd like to use --format=%Y, but it's not supported by busybox.
-    if (!d->remoteProcs.isEmpty())
+        process.setCommand({q->deviceConfiguration()->filePath("stat"),
-        return;
+                            {"-t", Utils::ProcessArgs::quoteArgUnix(file.remoteFilePath())}});
-    if (d->state == PreChecking) {
+    };
-        uploadFiles();
+    const auto endHandler = [=](const QtcProcess &process) {
-        return;
+        QtcProcess *proc = const_cast<QtcProcess *>(&process);
-    }
+        const QDateTime timestamp = timestampFromStat(file, proc);
-    QTC_ASSERT(d->state == PostProcessing, return);
+        statEndHandler(storage, file, timestamp);
-    emit progressMessage(Tr::tr("All files successfully deployed."));
+    };
-    setFinished();
+    return Process(setupHandler, endHandler, endHandler);
-    handleDeploymentDone();
 }
 
-void GenericDirectUploadService::stopDeployment()
+TaskItem GenericDirectUploadServicePrivate::statTree(const TreeStorage<UploadStorage> &storage,
+                                            FilesToStat filesToStat, StatEndHandler statEndHandler)
 {
-    QTC_ASSERT(d->state != Inactive, return);
+    const auto setupHandler = [=](TaskTree &tree) {
-
+        UploadStorage *storagePtr = storage.activeStorage();
-    setFinished();
+        const QList<DeployableFile> files = filesToStat(storagePtr);
-    handleDeploymentDone();
+        QList<TaskItem> statList;
-}
+        for (const DeployableFile &file : std::as_const(files)) {
-
+            QTC_ASSERT(file.isValid(), continue);
-void GenericDirectUploadService::runStat(const DeployableFile &file)
+            statList.append(statTask(storagePtr, file, statEndHandler));
-{
-    // We'd like to use --format=%Y, but it's not supported by busybox.
-    QtcProcess * const statProc = new QtcProcess(this);
-    statProc->setCommand({deviceConfiguration()->filePath("stat"),
-              {"-t", Utils::ProcessArgs::quoteArgUnix(file.remoteFilePath())}});
-    connect(statProc, &QtcProcess::done, this, [this, statProc, state = d->state] {
-        QTC_ASSERT(d->state == state, return);
-        const DeployableFile file = d->getFileForProcess(statProc);
-        QTC_ASSERT(file.isValid(), return);
-        const QDateTime timestamp = timestampFromStat(file, statProc);
-        statProc->deleteLater();
-        switch (state) {
-        case PreChecking:
-            if (!timestamp.isValid() || hasRemoteFileChanged(file, timestamp))
-                d->filesToUpload.append(file);
-            break;
-        case PostProcessing:
-            if (timestamp.isValid())
-                saveDeploymentTimeStamp(file, timestamp);
-            break;
-        case Inactive:
-        case Uploading:
-            QTC_CHECK(false);
-            break;
         }
-        checkForStateChangeOnRemoteProcFinished();
+        tree.setupRoot(packIntoOptionalParallelGroups(statList));
-    });
+    };
-    d->remoteProcs.insert(statProc, file);
+    return Tree(setupHandler);
-    statProc->start();
 }
 
-QList<DeployableFile> GenericDirectUploadService::collectFilesToUpload(
+TaskItem GenericDirectUploadServicePrivate::uploadGroup(const TreeStorage<UploadStorage> &storage)
-        const DeployableFile &deployable) const
 {
-    QList<DeployableFile> collected;
+    const auto groupSetupHandler = [=] {
-    FilePath localFile = deployable.localFilePath();
+        if (storage->filesToUpload.isEmpty()) {
-    if (localFile.isDir()) {
+            emit q->progressMessage(Tr::tr("No files need to be uploaded."));
-        const FilePaths files = localFile.dirEntries(QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot);
+            return GroupConfig{GroupAction::StopWithDone};
-        const QString remoteDir = deployable.remoteDirectory() + '/' + localFile.fileName();
-        for (const FilePath &localFilePath : files)
-            collected.append(collectFilesToUpload(DeployableFile(localFilePath, remoteDir)));
-    } else {
-        collected << deployable;
-    }
-    return collected;
-}
-
-void GenericDirectUploadService::setFinished()
-{
-    d->state = Inactive;
-    d->filesToStat.clear();
-    for (auto it = d->remoteProcs.begin(); it != d->remoteProcs.end(); ++it) {
-        it.key()->disconnect();
-        it.key()->terminate();
-    }
-    d->remoteProcs.clear();
-    d->uploader.stop();
-    d->filesToUpload.clear();
-}
-
-void GenericDirectUploadService::queryFiles()
-{
-    QTC_ASSERT(d->state == PreChecking || d->state == PostProcessing, return);
-    QTC_ASSERT(d->state == PostProcessing || d->remoteProcs.isEmpty(), return);
-
-    const QList<DeployableFile> &filesToCheck = d->state == PreChecking
-            ? d->deployableFiles : d->filesToUpload;
-    for (const DeployableFile &file : filesToCheck) {
-        if (d->state == PreChecking && (d->incremental != IncrementalDeployment::Enabled
-                                        || hasLocalFileChanged(file))) {
-            d->filesToUpload.append(file);
-            continue;
         }
-        if (d->incremental == IncrementalDeployment::NotSupported)
+        emit q->progressMessage(Tr::tr("%n file(s) need to be uploaded.", "",
-            continue;
+                                       storage->filesToUpload.size()));
-        if (d->remoteProcs.size() >= MaxConcurrentStatCalls)
+        FilesToTransfer files;
-            d->filesToStat << file;
+        for (const DeployableFile &file : std::as_const(storage->filesToUpload)) {
-        else
+            if (!file.localFilePath().exists()) {
-            runStat(file);
+                const QString message = Tr::tr("Local file \"%1\" does not exist.")
-    }
+                                              .arg(file.localFilePath().toUserOutput());
-    checkForStateChangeOnRemoteProcFinished();
+                if (ignoreMissingFiles) {
+                    emit q->warningMessage(message);
+                    continue;
+                }
+                emit q->errorMessage(message);
+                return GroupConfig{GroupAction::StopWithError};
+            }
+            files.append({file.localFilePath(),
+                          q->deviceConfiguration()->filePath(file.remoteFilePath())});
+        }
+        storage->filesToTransfer = files;
+        if (storage->filesToTransfer.isEmpty()) {
+            emit q->progressMessage(Tr::tr("No files need to be uploaded."));
+            return GroupConfig{GroupAction::StopWithDone};
+        }
+        return GroupConfig();
+    };
+
+    const auto setupHandler = [this, storage](FileTransfer &transfer) {
+        transfer.setFilesToTransfer(storage->filesToTransfer);
+        QObject::connect(&transfer, &FileTransfer::progress,
+                         q, &GenericDirectUploadService::progressMessage);
+    };
+    const auto errorHandler = [this](const FileTransfer &transfer) {
+        emit q->errorMessage(transfer.resultData().m_errorString);
+    };
+
+    const Group group {
+        DynamicSetup(groupSetupHandler),
+        Transfer(setupHandler, {}, errorHandler)
+    };
+    return group;
 }
 
-void GenericDirectUploadService::uploadFiles()
+TaskItem GenericDirectUploadServicePrivate::chmodTask(const DeployableFile &file)
 {
-    QTC_ASSERT(d->state == PreChecking, return);
+    const auto setupHandler = [=](QtcProcess &process) {
-    d->state = Uploading;
+        process.setCommand({q->deviceConfiguration()->filePath("chmod"),
-    if (d->filesToUpload.empty()) {
+                {"a+x", Utils::ProcessArgs::quoteArgUnix(file.remoteFilePath())}});
-        emit progressMessage(Tr::tr("No files need to be uploaded."));
+    };
-        setFinished();
+    const auto errorHandler = [=](const QtcProcess &process) {
-        handleDeploymentDone();
+        const QString error = process.errorString();
-        return;
+        if (!error.isEmpty()) {
-    }
+            emit q->warningMessage(Tr::tr("Remote chmod failed for file \"%1\": %2")
-    emit progressMessage(Tr::tr("%n file(s) need to be uploaded.", "", d->filesToUpload.size()));
+                                   .arg(file.remoteFilePath(), error));
-    FilesToTransfer files;
+        } else if (process.exitCode() != 0) {
-    for (const DeployableFile &file : std::as_const(d->filesToUpload)) {
+            emit q->warningMessage(Tr::tr("Remote chmod failed for file \"%1\": %2")
-        if (!file.localFilePath().exists()) {
+                                   .arg(file.remoteFilePath(), process.cleanedStdErr()));
-            const QString message = Tr::tr("Local file \"%1\" does not exist.")
+        }
-                    .arg(file.localFilePath().toUserOutput());
+    };
-            if (d->ignoreMissingFiles) {
+    return Process(setupHandler, {}, errorHandler);
-                emit warningMessage(message);
+}
+
+TaskItem GenericDirectUploadServicePrivate::chmodTree(const TreeStorage<UploadStorage> &storage)
+{
+    const auto setupChmodHandler = [=](TaskTree &tree) {
+        QList<DeployableFile> filesToChmod;
+        for (const DeployableFile &file : std::as_const(storage->filesToUpload)) {
+            if (file.isExecutable())
+                filesToChmod << file;
+        }
+        QList<TaskItem> chmodList;
+        for (const DeployableFile &file : std::as_const(filesToChmod)) {
+            QTC_ASSERT(file.isValid(), continue);
+            chmodList.append(chmodTask(file));
+        }
+        tree.setupRoot(packIntoOptionalParallelGroups(chmodList));
+    };
+    return Tree {setupChmodHandler};
+}
+
+void GenericDirectUploadService::doDeploy()
+{
+    QTC_ASSERT(!d->m_taskTree, return);
+
+    const auto endHandler = [this] {
+        d->m_taskTree.release()->deleteLater();
+        stopDeployment();
+    };
+    const auto doneHandler = [this, endHandler] {
+        emit progressMessage(Tr::tr("All files successfully deployed."));
+        endHandler();
+    };
+
+    const auto preFilesToStat = [this](UploadStorage *storage) {
+        QList<DeployableFile> filesToStat;
+        for (const DeployableFile &file : std::as_const(d->deployableFiles)) {
+            if (d->incremental != IncrementalDeployment::Enabled || hasLocalFileChanged(file)) {
+                storage->filesToUpload.append(file);
                 continue;
-            } else {
-                emit errorMessage(message);
-                setFinished();
-                handleDeploymentDone();
-                return;
             }
+            if (d->incremental == IncrementalDeployment::NotSupported)
+                continue;
+            filesToStat << file;
         }
-        files.append({file.localFilePath(),
+        return filesToStat;
-                      deviceConfiguration()->filePath(file.remoteFilePath())});
+    };
-    }
+    const auto preStatEndHandler = [this](UploadStorage *storage, const DeployableFile &file,
+                                          const QDateTime &timestamp) {
+        if (!timestamp.isValid() || hasRemoteFileChanged(file, timestamp))
+            storage->filesToUpload.append(file);
+    };
 
-    d->uploader.setFilesToTransfer(files);
+    const auto postFilesToStat = [this](UploadStorage *storage) {
-    d->uploader.start();
+        return d->incremental == IncrementalDeployment::NotSupported
-}
+               ? QList<DeployableFile>() : storage->filesToUpload;
+    };
+    const auto postStatEndHandler = [this](UploadStorage *storage, const DeployableFile &file,
+                                           const QDateTime &timestamp) {
+        Q_UNUSED(storage)
+        if (timestamp.isValid())
+            saveDeploymentTimeStamp(file, timestamp);
+    };
 
-void GenericDirectUploadService::chmod()
+    const TreeStorage<UploadStorage> storage;
-{
+    const Group root {
-    QTC_ASSERT(d->state == PostProcessing, return);
+        Storage(storage),
-    if (!Utils::HostOsInfo::isWindowsHost())
+        d->statTree(storage, preFilesToStat, preStatEndHandler),
-        return;
+        d->uploadGroup(storage),
-    for (const DeployableFile &f : std::as_const(d->filesToUpload)) {
+        Group {
-        if (!f.isExecutable())
+            d->chmodTree(storage),
-            continue;
+            d->statTree(storage, postFilesToStat, postStatEndHandler)
-        QtcProcess * const chmodProc = new QtcProcess(this);
+        },
-        chmodProc->setCommand({deviceConfiguration()->filePath("chmod"),
+        OnGroupDone(doneHandler),
-                {"a+x", Utils::ProcessArgs::quoteArgUnix(f.remoteFilePath())}});
+        OnGroupError(endHandler)
-        connect(chmodProc, &QtcProcess::done, this, [this, chmodProc, state = d->state] {
+    };
-            QTC_ASSERT(state == d->state, return);
+
-            const DeployableFile file = d->getFileForProcess(chmodProc);
+    d->m_taskTree.reset(new TaskTree(root));
-            QTC_ASSERT(file.isValid(), return);
+    d->m_taskTree->start();
-            const QString error = chmodProc->errorString();
-            if (!error.isEmpty()) {
-                emit warningMessage(Tr::tr("Remote chmod failed for file \"%1\": %2")
-                                    .arg(file.remoteFilePath(), error));
-            } else if (chmodProc->exitCode() != 0) {
-                emit warningMessage(Tr::tr("Remote chmod failed for file \"%1\": %2")
-                                    .arg(file.remoteFilePath(),
-                                         QString::fromUtf8(chmodProc->readAllStandardError())));
-            }
-            chmodProc->deleteLater();
-            checkForStateChangeOnRemoteProcFinished();
-        });
-        d->remoteProcs.insert(chmodProc, f);
-        chmodProc->start();
-    }
 }
 
 } //namespace RemoteLinux

src/plugins/remotelinux/genericdirectuploadservice.h

@@ -9,13 +9,7 @@
 
 #include <QList>
 
-QT_BEGIN_NAMESPACE
-class QDateTime;
-class QString;
-QT_END_NAMESPACE
-
 namespace ProjectExplorer { class DeployableFile; }
-namespace Utils { class QtcProcess; }
 
 namespace RemoteLinux {
 namespace Internal { class GenericDirectUploadServicePrivate; }
@@ -35,23 +29,11 @@ public:
 
 protected:
     bool isDeploymentNecessary() const override;
-
     void doDeploy() override;
     void stopDeployment() override;
 
 private:
-    void runStat(const ProjectExplorer::DeployableFile &file);
+    friend class Internal::GenericDirectUploadServicePrivate;
-    QDateTime timestampFromStat(const ProjectExplorer::DeployableFile &file,
-                                Utils::QtcProcess *statProc);
-    void checkForStateChangeOnRemoteProcFinished();
-
-    QList<ProjectExplorer::DeployableFile> collectFilesToUpload(
-            const ProjectExplorer::DeployableFile &file) const;
-    void setFinished();
-    void queryFiles();
-    void uploadFiles();
-    void chmod();
-
     Internal::GenericDirectUploadServicePrivate * const d;
 };