Docker: implement process interface

Change-Id: I57dd9e060ee35280b663611ebb5ddef20b7d0eb7
Reviewed-by: Jarek Kobus <jaroslaw.kobus@qt.io>

src/libs/utils/CMakeLists.txt

@@ -118,7 +118,7 @@ add_qtc_library(Utils
     processenums.h
     processhandle.cpp processhandle.h
     processinfo.cpp processinfo.h
-    processinterface.h
+    processinterface.cpp processinterface.h
     processreaper.cpp processreaper.h
     processutils.cpp processutils.h
     progressindicator.cpp progressindicator.h

src/libs/utils/processinterface.cpp

@@ -0,0 +1,48 @@
+/****************************************************************************
+**
+** Copyright (C) 2022 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "processinterface.h"
+
+#include "qtcassert.h"
+
+namespace Utils {
+
+/*!
+ * \brief controlSignalToInt
+ * \param controlSignal
+ * \return Converts the ControlSignal enum to the corresponding unix signal
+ */
+int ProcessInterface::controlSignalToInt(ControlSignal controlSignal)
+{
+    switch (controlSignal) {
+    case ControlSignal::Terminate: return 15;
+    case ControlSignal::Kill:      return 9;
+    case ControlSignal::Interrupt: return 2;
+    case ControlSignal::KickOff:   QTC_CHECK(false); return 0;
+    }
+    return 0;
+}
+
+} // Utils

src/libs/utils/processinterface.h

@@ -86,6 +86,8 @@ class QTCREATOR_UTILS_EXPORT ProcessInterface : public QObject
 public:
     ProcessInterface(QObject *parent = nullptr) : QObject(parent) {}
 
+    static int controlSignalToInt(ControlSignal controlSignal);
+
 signals:
     // This should be emitted when being in Starting state only.
     // After emitting this signal the process enters Running state.

src/libs/utils/qtcprocess.cpp

@@ -795,14 +795,8 @@ void QtcProcess::startImpl()
 {
     ProcessInterface *processImpl = nullptr;
     if (d->m_setup.m_commandLine.executable().needsDevice()) {
-        if (s_deviceHooks.processImplHook) { // TODO: replace "if" with an assert for the hook
+        QTC_ASSERT(s_deviceHooks.processImplHook, return);
         processImpl = s_deviceHooks.processImplHook(commandLine().executable());
-        }
-        if (!processImpl) { // TODO: remove this branch when docker is adapted accordingly
-            QTC_ASSERT(s_deviceHooks.startProcessHook, return);
-            s_deviceHooks.startProcessHook(*this);
-            return;
-        }
     } else {
         processImpl = d->createProcessInterface();
     }

src/libs/utils/qtcprocess.h

@@ -219,8 +219,6 @@ class DeviceProcessHooks
 {
 public:
     std::function<ProcessInterface *(const FilePath &)> processImplHook;
-    // TODO: remove this hook
-    std::function<void(QtcProcess &)> startProcessHook;
     std::function<Environment(const FilePath &)> systemEnvironmentForBinary;
 };
 

src/libs/utils/utils.qbs

@@ -226,6 +226,7 @@ Project {
             "processhandle.h",
             "processinfo.cpp",
             "processinfo.h",
+            "processinterface.cpp",
             "processinterface.h",
             "processreaper.cpp",
             "processreaper.h",

src/plugins/docker/dockerdevice.cpp

@@ -58,6 +58,7 @@
 #include <utils/overridecursor.h>
 #include <utils/pathlisteditor.h>
 #include <utils/port.h>
+#include <utils/processinterface.h>
 #include <utils/qtcassert.h>
 #include <utils/qtcprocess.h>
 #include <utils/stringutils.h>
@@ -98,6 +99,8 @@ using namespace Utils;
 namespace Docker {
 namespace Internal {
 
+const QString s_pidMarker = "__qtc$$qtc__";
+
 static Q_LOGGING_CATEGORY(dockerDeviceLog, "qtc.docker.device", QtWarningMsg);
 #define LOG(x) qCDebug(dockerDeviceLog) << this << x << '\n'
 
@@ -222,6 +225,157 @@ public:
     bool m_useFind = true;  // prefer find over ls and hacks, but be able to use ls as fallback
 };
 
+class DockerProcessImpl : public Utils::ProcessInterface
+{
+public:
+    DockerProcessImpl(DockerDevicePrivate *device);
+    virtual ~DockerProcessImpl();
+
+private:
+    void start() override;
+    qint64 write(const QByteArray &data) override;
+    void sendControlSignal(ControlSignal controlSignal) override;
+
+    bool waitForStarted(int msecs) override;
+    bool waitForReadyRead(int msecs) override;
+    bool waitForFinished(int msecs) override;
+
+private:
+    CommandLine fullLocalCommandLine(bool interactive);
+
+private:
+    DockerDevicePrivate *m_devicePrivate = nullptr;
+    // Store the IDevice::ConstPtr in order to extend the lifetime of device for as long
+    // as this object is alive.
+    IDevice::ConstPtr m_device;
+
+    QtcProcess m_process;
+    qint64 m_remotePID = -1;
+    bool m_hasReceivedFirstOutput = false;
+};
+
+CommandLine DockerProcessImpl::fullLocalCommandLine(bool interactive)
+{
+    QStringList args;
+
+    if (!m_setup.m_workingDirectory.isEmpty()) {
+        args.append({"cd", m_setup.m_workingDirectory.path()});
+        args.append("&&");
+    }
+
+    args.append({"echo", s_pidMarker, "&&"});
+
+    const Environment &env = m_setup.m_remoteEnvironment;
+    for (auto it = env.constBegin(); it != env.constEnd(); ++it)
+        args.append(env.key(it) + "='" + env.expandedValueForKey(env.key(it)) + '\'');
+
+    args.append("exec");
+    args.append({m_setup.m_commandLine.executable().path(), m_setup.m_commandLine.arguments()});
+
+    CommandLine shCmd("/bin/sh", {"-c", args.join(" ")});
+    return m_devicePrivate->q->withDockerExecCmd(shCmd, interactive);
+}
+
+DockerProcessImpl::DockerProcessImpl(DockerDevicePrivate *device)
+    : m_devicePrivate(device)
+    , m_device(device->q->sharedFromThis())
+    , m_process(this)
+{
+    connect(&m_process, &QtcProcess::started, this, [this] {
+        qCDebug(dockerDeviceLog) << "Process started:" << m_process.commandLine();
+    });
+
+    connect(&m_process, &QtcProcess::readyReadStandardOutput, this, [this] {
+        if (!m_hasReceivedFirstOutput) {
+            QByteArray output = m_process.readAllStandardOutput();
+            qsizetype idx = output.indexOf('\n');
+            QByteArray firstLine = output.left(idx);
+            QByteArray rest = output.mid(idx+1);
+            qCDebug(dockerDeviceLog) << "Process first line received:" << m_process.commandLine() << firstLine;
+            if (firstLine.startsWith("__qtc")) {
+                bool ok = false;
+                m_remotePID = firstLine.mid(5, firstLine.size() -5 -5).toLongLong(&ok);
+
+                if (ok)
+                    emit started(m_remotePID);
+
+                if (rest.size() > 0)
+                    emit readyRead(rest, {});
+
+                m_hasReceivedFirstOutput = true;
+                return;
+            }
+        }
+        emit readyRead(m_process.readAllStandardOutput(), {});
+    });
+
+    connect(&m_process, &QtcProcess::readyReadStandardError, this, [this] {
+        emit readyRead({}, m_process.readAllStandardError());
+    });
+
+    connect(&m_process, &QtcProcess::done, this, [this] {
+        qCDebug(dockerDeviceLog) << "Process exited:" << m_process.commandLine() << "with code:" << m_process.resultData().m_exitCode;
+        emit done(m_process.resultData());
+    });
+
+}
+
+DockerProcessImpl::~DockerProcessImpl()
+{
+    if (m_process.state() == QProcess::Running)
+        sendControlSignal(ControlSignal::Kill);
+}
+
+void DockerProcessImpl::start()
+{
+    m_process.setProcessImpl(m_setup.m_processImpl);
+    m_process.setProcessMode(m_setup.m_processMode);
+    m_process.setTerminalMode(m_setup.m_terminalMode);
+    m_process.setWriteData(m_setup.m_writeData);
+    m_process.setProcessChannelMode(m_setup.m_processChannelMode);
+    m_process.setExtraData(m_setup.m_extraData);
+    m_process.setStandardInputFile(m_setup.m_standardInputFile);
+    m_process.setAbortOnMetaChars(m_setup.m_abortOnMetaChars);
+    if (m_setup.m_lowPriority)
+        m_process.setLowPriority();
+
+    m_process.setCommand(fullLocalCommandLine(m_setup.m_processMode == ProcessMode::Writer));
+    m_process.start();
+}
+
+qint64 DockerProcessImpl::write(const QByteArray &data)
+{
+    return m_process.writeRaw(data);
+}
+
+void DockerProcessImpl::sendControlSignal(ControlSignal controlSignal)
+{
+    int signal = controlSignalToInt(controlSignal);
+    m_devicePrivate->runInShell(
+        {"kill", {QString("-%1").arg(signal), QString("%2").arg(m_remotePID)}});
+}
+
+bool DockerProcessImpl::waitForStarted(int msecs)
+{
+    Q_UNUSED(msecs)
+    QTC_CHECK(false);
+    return false;
+}
+
+bool DockerProcessImpl::waitForReadyRead(int msecs)
+{
+    Q_UNUSED(msecs)
+    QTC_CHECK(false);
+    return false;
+}
+
+bool DockerProcessImpl::waitForFinished(int msecs)
+{
+    Q_UNUSED(msecs)
+    QTC_CHECK(false);
+    return false;
+}
+
 IDeviceWidget *DockerDevice::createWidget()
 {
     return new DockerDeviceWidget(sharedFromThis());
@@ -331,6 +485,8 @@ void DockerDevicePrivate::stopCurrentContainer()
             m_shell = nullptr;
             return;
         }
+
+        m_shell->terminate();
     }
 
     QtcProcess proc;
@@ -454,6 +610,20 @@ void DockerDevice::setMounts(const QStringList &mounts) const
     d->stopCurrentContainer(); // Force re-start with new mounts.
 }
 
+CommandLine DockerDevice::withDockerExecCmd(const Utils::CommandLine &cmd, bool interactive) const
+{
+    QStringList args;
+
+    args << "exec";
+    if (interactive)
+        args << "-i";
+    args << d->m_container;
+
+    CommandLine dcmd{"docker", args};
+    dcmd.addCommandLineAsArgs(cmd);
+    return dcmd;
+}
+
 const char DockerDeviceDataImageIdKey[] = "DockerDeviceDataImageId";
 const char DockerDeviceDataRepoKey[] = "DockerDeviceDataRepo";
 const char DockerDeviceDataTagKey[] = "DockerDeviceDataTag";
@@ -490,6 +660,11 @@ QtcProcess *DockerDevice::createProcess(QObject *parent) const
     return new DockerDeviceProcess(sharedFromThis(), parent);
 }
 
+ProcessInterface *DockerDevice::createProcessInterface() const
+{
+    return new DockerProcessImpl(d);
+}
+
 bool DockerDevice::canAutoDetectPorts() const
 {
     return true;
@@ -857,8 +1032,8 @@ QByteArray DockerDevice::fileContents(const FilePath &filePath, qint64 limit, qi
     }
 
     QtcProcess proc;
-    proc.setCommand({"dd", args});
-    runProcess(proc);
+    proc.setCommand(withDockerExecCmd({"dd", args}));
+    proc.start();
     proc.waitForFinished();
 
     QByteArray output = proc.readAllStandardOutput();
@@ -894,49 +1069,6 @@ bool DockerDevice::writeFileContents(const FilePath &filePath, const QByteArray
     return proc.exitCode() == 0;
 }
 
-void DockerDevice::runProcess(QtcProcess &process) const
-{
-    updateContainerAccess();
-    if (!DockerApi::isDockerDaemonAvailable(false).value_or(false))
-        return;
-    if (d->m_container.isEmpty()) {
-        LOG("No container set to run " << process.commandLine().toUserOutput());
-        QTC_CHECK(false);
-        process.setResult(ProcessResult::StartFailed);
-        return;
-    }
-
-    const FilePath workingDir = process.workingDirectory();
-    const Environment env = process.environment();
-
-    CommandLine cmd{"docker", {"exec"}};
-    if (!workingDir.isEmpty()) {
-        cmd.addArgs({"-w", mapToDevicePath(workingDir)});
-        if (QTC_GUARD(workingDir.needsDevice())) // warn on local working directory for docker cmd
-            process.setWorkingDirectory(FileUtils::homePath()); // reset working dir for docker exec
-    }
-    if (process.processMode() == ProcessMode::Writer)
-        cmd.addArg("-i");
-    if (env.size() != 0) {
-        process.unsetEnvironment();
-        // FIXME the below would be probably correct if the respective tools would use correct
-        //       environment already, but most are using the host environment which usually makes
-        //       no sense on the device and may degrade performance
-        // const QStringList envList = env.toStringList();
-        // for (const QString &keyValue : envList) {
-        //     cmd.addArg("-e");
-        //     cmd.addArg(keyValue);
-        // }
-    }
-    cmd.addArg(d->m_container);
-    cmd.addCommandLineAsArgs(process.commandLine());
-
-    LOG("Run" << cmd.toUserOutput() << " in " << workingDir.toUserOutput());
-
-    process.setCommand(cmd);
-    process.start();
-}
-
 Environment DockerDevice::systemEnvironment() const
 {
     if (d->m_cachedEnviroment.size() == 0)
@@ -962,12 +1094,11 @@ void DockerDevicePrivate::fetchSystemEnviroment()
     }
 
     QtcProcess proc;
-    proc.setCommand({"env", {}});
-
-    q->runProcess(proc); // FIXME: This only starts.
+    proc.setCommand(q->withDockerExecCmd({"env", {}}));
+    proc.start();
     proc.waitForFinished();
-
     const QString remoteOutput = proc.stdOut();
+
     m_cachedEnviroment = Environment(remoteOutput.split('\n', Qt::SkipEmptyParts), q->osType());
 
     const QString remoteError = proc.stdErr();
@@ -995,6 +1126,14 @@ bool DockerDevicePrivate::runInContainer(const CommandLine &cmd) const
 
 bool DockerDevicePrivate::runInShell(const CommandLine &cmd) const
 {
+    if (QThread::currentThread() != qApp->thread()) {
+        bool result = false;
+        QMetaObject::invokeMethod(const_cast<DockerDevicePrivate*>(this), [this, &cmd, &result] {
+            result = this->runInShell(cmd);
+        }, Qt::BlockingQueuedConnection);
+        return result;
+    }
+
     if (!QTC_GUARD(DockerApi::isDockerDaemonAvailable(false).value_or(false))) {
         LOG("No daemon. Could not run " << cmd.toUserOutput());
         return false;
@@ -1021,11 +1160,16 @@ static QByteArray randomHex()
 
 QByteArray DockerDevicePrivate::outputForRunInShell(const CommandLine &cmd) const
 {
+    QTC_ASSERT(QThread::currentThread() == qApp->thread(), return {});
+
     if (!DockerApi::isDockerDaemonAvailable(false).value_or(false))
         return {};
+
     QTC_ASSERT(m_shell && m_shell->isRunning(), return {});
+
     QMutexLocker l(&m_shellMutex);
     m_shell->readAllStandardOutput(); // clean possible left-overs
+
     const QByteArray oldError = m_shell->readAllStandardError(); // clean possible left-overs
     if (!oldError.isEmpty()) {
         LOG("Unexpected old stderr: " << oldError);
@@ -1034,20 +1178,24 @@ QByteArray DockerDevicePrivate::outputForRunInShell(const CommandLine &cmd) cons
 
     const QByteArray markerWithNewLine("___QC_DOCKER_" + randomHex() + "_OUTPUT_MARKER___\n");
     m_shell->write(cmd.toUserOutput() + "\necho -n \"" + markerWithNewLine + "\"\n");
+
     QByteArray output;
     while (!output.endsWith(markerWithNewLine)) {
         QTC_ASSERT(m_shell->isRunning(), return {});
         m_shell->waitForReadyRead();
         output.append(m_shell->readAllStandardOutput());
     }
+
     LOG("Run command in shell:" << cmd.toUserOutput() << "output size:" << output.size());
     if (QTC_GUARD(output.endsWith(markerWithNewLine)))
         output.chop(markerWithNewLine.size());
+
     const QByteArray currentError = m_shell->readAllStandardError();
     if (!currentError.isEmpty()) {
         LOG("Unexpected current stderr: " << currentError);
         QTC_CHECK(false);
     }
+
     return output;
 }
 

src/plugins/docker/dockerdevice.h

@@ -27,7 +27,6 @@
 
 #include <projectexplorer/devicesupport/idevice.h>
 #include <projectexplorer/devicesupport/idevicefactory.h>
-#include <projectexplorer/devicesupport/sshdeviceprocess.h>
 
 #include <utils/aspects.h>
 
@@ -66,6 +65,8 @@ public:
 
     bool canCreateProcess() const override { return true; }
     Utils::QtcProcess *createProcess(QObject *parent) const override;
+    Utils::ProcessInterface *createProcessInterface() const override;
+
     bool canAutoDetectPorts() const override;
     ProjectExplorer::PortsGatheringMethod::Ptr portsGatheringMethod() const override;
     bool canCreateProcessModel() const override { return false; }
@@ -100,7 +101,6 @@ public:
     QByteArray fileContents(const Utils::FilePath &filePath, qint64 limit, qint64 offset) const override;
     bool writeFileContents(const Utils::FilePath &filePath, const QByteArray &data) const override;
     QDateTime lastModified(const Utils::FilePath &filePath) const override;
-    void runProcess(Utils::QtcProcess &process) const override;
     qint64 fileSize(const Utils::FilePath &filePath) const override;
     QFileDevice::Permissions permissions(const Utils::FilePath &filePath) const override;
     bool setPermissions(const Utils::FilePath &filePath, QFileDevice::Permissions permissions) const override;
@@ -113,6 +113,8 @@ public:
     void updateContainerAccess() const;
     void setMounts(const QStringList &mounts) const;
 
+    Utils::CommandLine withDockerExecCmd(const Utils::CommandLine& cmd, bool interactive = false) const;
+
 protected:
     void fromMap(const QVariantMap &map) final;
     QVariantMap toMap() const final;

src/plugins/docker/dockerdevicewidget.cpp

@@ -25,10 +25,12 @@
 
 #include "dockerdevicewidget.h"
 
+#include <utils/environment.h>
 #include <utils/utilsicons.h>
 #include <utils/hostosinfo.h>
 #include <utils/algorithm.h>
 #include <utils/layoutbuilder.h>
+#include <utils/qtcassert.h>
 
 #include <QCoreApplication>
 #include <QToolButton>

src/plugins/projectexplorer/applicationlauncher.cpp

@@ -399,6 +399,7 @@ void ApplicationLauncherPrivate::start()
         m_stopRequested = false;
 
         CommandLine cmd = m_runnable.command;
+        // FIXME: RunConfiguration::runnable() should give us the correct, on-device path, instead of fixing it up here.
         cmd.setExecutable(m_runnable.device->mapToGlobalPath(cmd.executable()));
         m_process->setCommand(cmd);
         m_process->setWorkingDirectory(m_runnable.workingDirectory);

src/plugins/projectexplorer/devicesupport/devicemanager.cpp

@@ -604,14 +604,6 @@ DeviceManager::DeviceManager(bool isInstance) : d(std::make_unique<DeviceManager
 
     DeviceProcessHooks processHooks;
 
-    // TODO: remove this hook
-    processHooks.startProcessHook = [](QtcProcess &process) {
-        FilePath filePath = process.commandLine().executable();
-        auto device = DeviceManager::deviceForPath(filePath);
-        QTC_ASSERT(device, return);
-        device->runProcess(process);
-    };
-
     processHooks.processImplHook = [](const FilePath &filePath) -> ProcessInterface * {
         auto device = DeviceManager::deviceForPath(filePath);
         QTC_ASSERT(device, return nullptr);

src/plugins/projectexplorer/devicesupport/idevice.cpp

@@ -426,15 +426,8 @@ bool IDevice::setPermissions(const FilePath &filePath, QFile::Permissions) const
 
 ProcessInterface *IDevice::createProcessInterface() const
 {
-// TODO: uncomment below assert when docker device implements this method
-//    QTC_ASSERT(false, return nullptr);
-    return nullptr;
-}
-
-void IDevice::runProcess(QtcProcess &process) const
-{
-    Q_UNUSED(process);
     QTC_CHECK(false);
+    return nullptr;
 }
 
 Environment IDevice::systemEnvironment() const

src/plugins/projectexplorer/devicesupport/idevice.h

@@ -270,7 +270,6 @@ public:
     virtual QFile::Permissions permissions(const Utils::FilePath &filePath) const;
     virtual bool setPermissions(const Utils::FilePath &filePath, QFile::Permissions) const;
     virtual Utils::ProcessInterface *createProcessInterface() const;
-    virtual void runProcess(Utils::QtcProcess &process) const;
     virtual Utils::Environment systemEnvironment() const;
     virtual qint64 fileSize(const Utils::FilePath &filePath) const;
     virtual qint64 bytesAvailable(const Utils::FilePath &filePath) const;

src/plugins/projectexplorer/runcontrol.cpp

@@ -1155,8 +1155,8 @@ void RunControlPrivate::checkState(RunControlState expectedState)
 void RunControlPrivate::setState(RunControlState newState)
 {
     if (!isAllowedTransition(state, newState))
-        qDebug() << "Invalid run control state transition from " << stateName(state)
-                 << " to " << stateName(newState);
+        qDebug() << "Invalid run control state transition from" << stateName(state)
+                 << "to" << stateName(newState);
 
     state = newState;
 

src/plugins/remotelinux/linuxdevice.cpp

@@ -713,17 +713,6 @@ void SshProcessInterfacePrivate::start()
     }
 }
 
-static int controlSignalToInt(ControlSignal controlSignal)
-{
-    switch (controlSignal) {
-    case ControlSignal::Terminate: return 15;
-    case ControlSignal::Kill:      return 9;
-    case ControlSignal::Interrupt: return 2;
-    case ControlSignal::KickOff:   QTC_CHECK(false); return 0;
-    }
-    return 0;
-}
-
 QString SshProcessInterface::pidArgumentForKill() const
 {
     return QString::fromLatin1("-%1 %1").arg(d->m_processId);
@@ -734,7 +723,7 @@ void SshProcessInterfacePrivate::sendControlSignal(ControlSignal controlSignal)
     QTC_ASSERT(controlSignal != ControlSignal::KickOff, return);
     // TODO: In case if m_processId == 0 try sending a signal based on process name.
     const QString args = QString::fromLatin1("-%1 %2")
-            .arg(controlSignalToInt(controlSignal)).arg(q->pidArgumentForKill());
+            .arg(ProcessInterface::controlSignalToInt(controlSignal)).arg(q->pidArgumentForKill());
     CommandLine command = { "kill", args, CommandLine::Raw };
     // Note: This blocking call takes up to 2 ms for local remote.
     m_devicePrivate->runInShell(command);