RemoteLinux: Add incremental deployment to the TarPackageCreationStep

Patch adds incremental deployment to the tar ball creation. Allows for
only changed files to be deployed in a single tar ball.

Change-Id: I28a0bd1220503125b75a664323b72906b479b821
Reviewed-by: André Hartmann <aha_1980@gmx.de>
Reviewed-by: Christian Kandeler <christian.kandeler@qt.io>

src/plugins/remotelinux/tarpackagecreationstep.cpp

@@ -25,9 +25,13 @@
 
 #include "tarpackagecreationstep.h"
 
+#include <projectexplorer/buildmanager.h>
 #include <projectexplorer/deploymentdata.h>
 #include <projectexplorer/project.h>
 #include <projectexplorer/target.h>
+#include <qtsupport/qtkitinformation.h>
+#include <ssh/sshconnection.h>
+#include <ssh/sshconnectionmanager.h>
 
 #include <QDateTime>
 #include <QDir>
@@ -44,6 +48,7 @@ using namespace ProjectExplorer;
 namespace RemoteLinux {
 namespace {
 const char IgnoreMissingFilesKey[] = "RemoteLinux.TarPackageCreationStep.IgnoreMissingFiles";
+const char IncrementalDeploymentKey[] = "RemoteLinux.TarPackageCreationStep.IncrementalDeployment";
 
 class CreateTarStepWidget : public SimpleBuildStepConfigWidget
 {
@@ -52,13 +57,22 @@ public:
     CreateTarStepWidget(TarPackageCreationStep *step) : SimpleBuildStepConfigWidget(step)
     {
         m_ignoreMissingFilesCheckBox.setText(tr("Ignore missing files"));
+        m_incrementalDeploymentCheckBox.setText(tr("Package modified files only"));
+
         QVBoxLayout *mainLayout = new QVBoxLayout(this);
         mainLayout->setMargin(0);
+        mainLayout->addWidget(&m_incrementalDeploymentCheckBox);
         mainLayout->addWidget(&m_ignoreMissingFilesCheckBox);
+
         m_ignoreMissingFilesCheckBox.setChecked(step->ignoreMissingFiles());
+        m_incrementalDeploymentCheckBox.setChecked(step->isIncrementalDeployment());
+
         connect(&m_ignoreMissingFilesCheckBox, &QAbstractButton::toggled,
                 this, &CreateTarStepWidget::handleIgnoreMissingFilesChanged);
 
+        connect(&m_incrementalDeploymentCheckBox, &QAbstractButton::toggled,
+                this, &CreateTarStepWidget::handleIncrementalDeploymentChanged);
+
         connect(step, &AbstractPackagingStep::packageFilePathChanged,
                 this, &BuildStepConfigWidget::updateSummary);
     }
@@ -82,7 +96,13 @@ private:
         step->setIgnoreMissingFiles(ignoreMissingFiles);
     }
 
+    void handleIncrementalDeploymentChanged(bool incrementalDeployment) {
+        TarPackageCreationStep *step = qobject_cast<TarPackageCreationStep *>(this->step());
+        step->setIncrementalDeployment(incrementalDeployment);
+    }
+
     QCheckBox m_ignoreMissingFilesCheckBox;
+    QCheckBox m_incrementalDeploymentCheckBox;
 };
 
 
@@ -131,21 +151,37 @@ bool TarPackageCreationStep::init(QList<const BuildStep *> &earlierSteps)
 {
     if (!AbstractPackagingStep::init(earlierSteps))
         return false;
+
     m_packagingNeeded = isPackagingNeeded();
-    if (m_packagingNeeded)
-        m_files = target()->deploymentData().allFiles();
+
     return true;
 }
 
 void TarPackageCreationStep::run(QFutureInterface<bool> &fi)
 {
     setPackagingStarted();
+
+    const QList<DeployableFile> &files = target()->deploymentData().allFiles();
+
+    if (m_incrementalDeployment) {
+        m_files.clear();
+        for (const DeployableFile &file : files)
+            addNeededDeploymentFiles(file, target()->kit());
+    } else {
+        m_files = files;
+    }
+
     const bool success = doPackage(fi);
+
     setPackagingFinished(success);
     if (success)
         emit addOutput(tr("Packaging finished successfully."), MessageOutput);
     else
         emit addOutput(tr("Packaging failed."), ErrorMessageOutput);
+
+    connect(BuildManager::instance(), &BuildManager::buildQueueFinished,
+            this, &TarPackageCreationStep::deployFinished);
+
     reportRunResult(fi, success);
 }
 
@@ -159,6 +195,45 @@ bool TarPackageCreationStep::ignoreMissingFiles() const
     return m_ignoreMissingFiles;
 }
 
+void TarPackageCreationStep::setIncrementalDeployment(bool incrementalDeployment)
+{
+    m_incrementalDeployment = incrementalDeployment;
+}
+
+bool TarPackageCreationStep::isIncrementalDeployment() const
+{
+    return m_incrementalDeployment;
+}
+
+void TarPackageCreationStep::addNeededDeploymentFiles(
+        const ProjectExplorer::DeployableFile &deployable,
+        const ProjectExplorer::Kit *kit)
+{
+    const QFileInfo fileInfo = deployable.localFilePath().toFileInfo();
+    if (!fileInfo.isDir()) {
+        if (m_deployTimes.hasChangedSinceLastDeployment(deployable, kit))
+            m_files << deployable;
+        return;
+    }
+
+    const QStringList files = QDir(deployable.localFilePath().toString())
+            .entryList(QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot);
+
+    if (files.isEmpty()) {
+        m_files << deployable;
+        return;
+    }
+
+    for (const QString &fileName : files) {
+        const QString localFilePath = deployable.localFilePath().appendPath(fileName).toString();
+
+        const QString remoteDir = deployable.remoteDirectory() + '/' + fileInfo.fileName();
+
+        // Recurse through the subdirectories
+        addNeededDeploymentFiles(DeployableFile(localFilePath, remoteDir), kit);
+    }
+}
+
 bool TarPackageCreationStep::doPackage(QFutureInterface<bool> &fi)
 {
     emit addOutput(tr("Creating tarball..."), MessageOutput);
@@ -324,6 +399,22 @@ bool TarPackageCreationStep::writeHeader(QFile &tarFile, const QFileInfo &fileIn
     return true;
 }
 
+void TarPackageCreationStep::deployFinished(bool success)
+{
+    disconnect(BuildManager::instance(), &BuildManager::buildQueueFinished,
+               this, &TarPackageCreationStep::deployFinished);
+
+    if (!success)
+        return;
+
+    const Kit *kit = target()->kit();
+
+    // Store files that have been tar'd and successfully deployed
+    const auto files = m_files;
+    for (const DeployableFile &file : files)
+        m_deployTimes.saveDeploymentTimeStamp(file, kit);
+}
+
 QString TarPackageCreationStep::packageFileName() const
 {
     return project()->displayName() + QLatin1String(".tar");
@@ -339,6 +430,7 @@ bool TarPackageCreationStep::fromMap(const QVariantMap &map)
     if (!AbstractPackagingStep::fromMap(map))
         return false;
     setIgnoreMissingFiles(map.value(QLatin1String(IgnoreMissingFilesKey), false).toBool());
+    m_deployTimes.importDeployTimes(map);
     return true;
 }
 
@@ -346,6 +438,7 @@ QVariantMap TarPackageCreationStep::toMap() const
 {
     QVariantMap map = AbstractPackagingStep::toMap();
     map.insert(QLatin1String(IgnoreMissingFilesKey), ignoreMissingFiles());
+    map.unite(m_deployTimes.exportDeployTimes());
     return map;
 }
 

src/plugins/remotelinux/tarpackagecreationstep.h

@@ -26,6 +26,7 @@
 #pragma once
 
 #include "abstractpackagingstep.h"
+#include "deploymenttimeinfo.h"
 #include "remotelinux_export.h"
 
 #include <projectexplorer/deployablefile.h>
@@ -53,7 +54,15 @@ public:
     void setIgnoreMissingFiles(bool ignoreMissingFiles);
     bool ignoreMissingFiles() const;
 
+    void setIncrementalDeployment(bool incrementalDeployment);
+    bool isIncrementalDeployment() const;
+
 private:
+    void deployFinished(bool success);
+
+    void addNeededDeploymentFiles(const ProjectExplorer::DeployableFile &deployable,
+                                  const ProjectExplorer::Kit *kit);
+
     ProjectExplorer::BuildStepConfigWidget *createConfigWidget() override;
     bool fromMap(const QVariantMap &map) override;
     QVariantMap toMap() const override;
@@ -67,6 +76,9 @@ private:
     bool writeHeader(QFile &tarFile, const QFileInfo &fileInfo,
         const QString &remoteFilePath);
 
+    DeploymentTimeInfo m_deployTimes;
+
+    bool m_incrementalDeployment;
     bool m_ignoreMissingFiles;
     bool m_packagingNeeded;
     QList<ProjectExplorer::DeployableFile> m_files;