device: Use multiplex script to allow multithread support

Previously the runInShell and outputForRunInShell methods were
exclusively processed single threaded, meaning all calls
were processed sequentially. With the multiplexed
helper script we can now run multiple processes simultaneously.
( see tst_manual_deviceshell )

Additionally the new script allows us to capture both
stdout and stderr from commands which was not possible previously.

Change-Id: I52f4fb46d872dc274edb9c11872d2f6543741b34
Reviewed-by: David Schulz <david.schulz@qt.io>
Reviewed-by: Jarek Kobus <jaroslaw.kobus@qt.io>
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>

src/libs/utils/deviceshell.cpp

@@ -25,8 +25,9 @@
 
 #include "deviceshell.h"
 
-#include <qtcassert.h>
-#include <qtcprocess.h>
+#include "processinterface.h"
+#include "qtcassert.h"
+#include "qtcprocess.h"
 
 #include <QLoggingCategory>
 #include <QScopeGuard>
@@ -35,6 +36,123 @@ Q_LOGGING_CATEGORY(deviceShellLog, "qtc.utils.deviceshell", QtWarningMsg)
 
 namespace Utils {
 
+namespace {
+
+/*!
+ * The multiplex script waits for input via stdin.
+ *
+ * To start a command, a message is send with the format "<cmd-id> "<base64-encoded-stdin-data>" <commandline>\n"
+ * To stop the script, simply send "exit\n" via stdin
+ *
+ * Once a message is received, two new streams are created that the new process redirects its output to ( $stdoutraw and $stderrraw ).
+ *
+ * These streams are piped through base64 into the two streams stdoutenc and stderrenc.
+ *
+ * Two subshells read from these base64 encoded streams, and prepend the command-id, as well as either "O:" or "E:" depending on whether its the stdout or stderr stream.
+ *
+ * Once the process exits its exit code is send to stdout with the command-id and the type "R".
+ *
+ */
+const QLatin1String r_execScript = QLatin1String(R"(
+#!/bin/sh
+
+readAndMark() {
+    local buffer
+    while read buffer
+    do
+        printf '%s:%s:%s\n' "$1" "$2" "$buffer"
+    done
+}
+
+base64decode()
+{
+    base64 -d 2>/dev/null
+}
+
+base64encode()
+{
+    base64 2>/dev/null
+}
+
+executeAndMark()
+{
+    PID="$1"
+    INDATA="$2"
+    shift
+    shift
+    CMD="$@"
+
+    # Output Streams
+    stdoutenc=$(mktemp -u)
+    stderrenc=$(mktemp -u)
+    mkfifo "$stdoutenc" "$stderrenc"
+
+    # app output streams
+    stdoutraw=$(mktemp -u)
+    stderrraw=$(mktemp -u)
+    mkfifo "$stdoutraw" "$stderrraw"
+
+    # Cleanup
+    trap 'rm -f "$stdoutenc" "$stderrenc" "$stdoutraw" "$stderrraw" ' EXIT
+
+    # Pipe all app output through base64, and then into the output streams
+    cat $stdoutraw | base64encode > "$stdoutenc" &
+    cat $stderrraw | base64encode > "$stderrenc" &
+
+    # Mark the app's output streams
+    readAndMark $PID 'O' < "$stdoutenc" >&1 &
+    readAndMark $PID 'E' < "$stderrenc" >&1 &
+
+    # Start the app ...
+    if [ -z "$INDATA" ]
+    then
+        eval $CMD 1> "$stdoutraw" 2> "$stderrraw"
+    else
+        echo $INDATA | base64decode | eval "$CMD" 1> "$stdoutraw" 2> "$stderrraw"
+    fi
+
+    exitcode=$(echo $? | base64encode)
+
+    wait
+    echo "$PID:R:$exitcode"
+}
+
+execute()
+{
+    PID="$1"
+    INDATA=$(eval echo "$2")
+    shift
+    shift
+    CMD=$@
+    executeAndMark $PID "$INDATA" "$CMD"
+}
+
+cleanup()
+{
+    kill -- -$$
+    exit 1
+}
+
+if ! command -v base64 &> /dev/null
+then
+    echo "base64 command could not be found" >&2
+    exit 1
+fi
+
+trap cleanup 1 2 3 6
+
+echo SCRIPT_INSTALLED >&2
+
+while read -r id inData cmd; do
+    if [ "$id" = "exit" ]; then
+        exit
+    fi
+    execute $id $inData $cmd &
+done
+)");
+
+} // namespace
+
 DeviceShell::DeviceShell()
 {
     m_thread.setObjectName("Shell Thread");
@@ -43,26 +161,14 @@ DeviceShell::DeviceShell()
 
 DeviceShell::~DeviceShell()
 {
+    m_shellProcess->deleteLater();
+
     if (m_thread.isRunning()) {
         m_thread.quit();
         m_thread.wait();
     }
 }
 
-bool DeviceShell::waitForStarted()
-{
-    QTC_ASSERT(m_shellProcess, return false);
-    Q_ASSERT(QThread::currentThread() != &m_thread);
-
-    bool result;
-    QMetaObject::invokeMethod(
-        m_shellProcess,
-        [this] { return m_shellProcess->waitForStarted(); },
-        Qt::BlockingQueuedConnection,
-        &result);
-    return result;
-}
-
 /*!
  * \brief DeviceShell::runInShell
  * \param cmd The command to run
@@ -78,48 +184,8 @@ bool DeviceShell::runInShell(const CommandLine &cmd, const QByteArray &stdInData
     QTC_ASSERT(m_shellProcess, return false);
     Q_ASSERT(QThread::currentThread() != &m_thread);
 
-    bool result = false;
-    QMetaObject::invokeMethod(
-        m_shellProcess,
-        [this, &cmd, &stdInData] { return runInShellImpl(cmd, stdInData); },
-        Qt::BlockingQueuedConnection,
-        &result);
-    return result;
-}
-
-bool DeviceShell::runInShellImpl(const CommandLine &cmd, const QByteArray &stdInData)
-{
-    QTC_ASSERT(QThread::currentThread() == &m_thread, return false);
-
-    QTC_ASSERT(m_shellProcess->isRunning(), return false);
-    QTC_ASSERT(m_shellProcess, return false);
-    QTC_CHECK(m_shellProcess->readAllStandardOutput().isNull()); // clean possible left-overs
-    QTC_CHECK(m_shellProcess->readAllStandardError().isNull());  // clean possible left-overs
-    auto cleanup = qScopeGuard(
-        [this] { m_shellProcess->readAllStandardOutput(); }); // clean on assert
-
-    QString prefix;
-    if (!stdInData.isEmpty())
-        prefix = "echo '" + QString::fromUtf8(stdInData.toBase64()) + "' | base64 -d | ";
-
-    const QString suffix = " > /dev/null 2>&1\necho $?\n";
-    const QString command = prefix + cmd.toUserOutput() + suffix;
-
-    qCDebug(deviceShellLog) << "Running:" << command;
-
-    m_shellProcess->write(command);
-    m_shellProcess->waitForReadyRead();
-
-    const QByteArray output = m_shellProcess->readAllStandardOutput();
-
-    bool ok = false;
-    const int result = output.toInt(&ok);
-
-    qCInfo(deviceShellLog) << "Run command in shell:" << cmd.toUserOutput() << "result: " << output
-                           << " ==>" << result;
-    QTC_ASSERT(ok, return false);
-
-    return result == EXIT_SUCCESS;
+    const RunResult result = run(cmd, stdInData);
+    return result.exitCode == 0;
 }
 
 /*!
@@ -138,59 +204,37 @@ DeviceShell::RunResult DeviceShell::outputForRunInShell(const CommandLine &cmd,
     QTC_ASSERT(m_shellProcess, return {});
     Q_ASSERT(QThread::currentThread() != &m_thread);
 
-    RunResult result;
-    QMetaObject::invokeMethod(
-        m_shellProcess,
-        [this, &cmd, &stdInData] { return outputForRunInShellImpl(cmd, stdInData); },
-        Qt::BlockingQueuedConnection,
-        &result);
-    return result;
+    return run(cmd, stdInData);
 }
 
-DeviceShell::RunResult DeviceShell::outputForRunInShellImpl(const CommandLine &cmd,
-                                                            const QByteArray &stdInData)
+DeviceShell::State DeviceShell::state() const { return m_shellScriptState; }
+
+DeviceShell::RunResult DeviceShell::run(const CommandLine &cmd, const QByteArray &stdInData)
 {
-    QTC_ASSERT(QThread::currentThread() == &m_thread, return {});
+    const RunResult errorResult{-1, {}, {}};
+    QTC_ASSERT(m_shellProcess, return errorResult);
+    QTC_ASSERT(m_shellScriptState == State::Succeeded, return errorResult);
 
-    QTC_ASSERT(m_shellProcess, return {});
-    QTC_CHECK(m_shellProcess->readAllStandardOutput().isNull()); // clean possible left-overs
-    QTC_CHECK(m_shellProcess->readAllStandardError().isNull());  // clean possible left-overs
-    auto cleanup = qScopeGuard(
-        [this] { m_shellProcess->readAllStandardOutput(); }); // clean on assert
+    QMutexLocker lk(&m_commandMutex);
 
-    QString prefix;
-    if (!stdInData.isEmpty())
-        prefix = "echo '" + QString::fromUtf8(stdInData.toBase64()) + "' | base64 -d | ";
+    QWaitCondition waiter;
+    const int id = ++m_currentId;
+    const auto it = m_commandOutput.insert(id, CommandRun{-1, {}, {}, &waiter});
 
-    const QString markerCmd = "echo __qtc$?qtc__ 1>&2\n";
-    const QString suffix = "\n" + markerCmd;
-    const QString command = prefix + cmd.toUserOutput() + suffix;
+    QMetaObject::invokeMethod(m_shellProcess, [this, id, &cmd, &stdInData]() {
+        const QString command = QString("%1 \"%2\" %3\n")
+                                    .arg(id)
+                                    .arg(QString::fromLatin1(stdInData.toBase64()))
+                                    .arg(cmd.toUserOutput());
+        qCDebug(deviceShellLog) << "Running:" << command;
+        m_shellProcess->writeRaw(command.toUtf8());
+    });
 
-    qCDebug(deviceShellLog) << "Running:" << command;
-    m_shellProcess->write(command);
+    waiter.wait(&m_commandMutex);
 
-    RunResult result;
+    const RunResult result = *it;
+    m_commandOutput.erase(it);
 
-    while (true) {
-        m_shellProcess->waitForReadyRead();
-        QByteArray stdErr = m_shellProcess->readAllStandardError();
-        if (stdErr.endsWith("qtc__\n")) {
-            QByteArray marker = stdErr.right(stdErr.length() - stdErr.lastIndexOf("__qtc"));
-            QByteArray exitCodeStr = marker.mid(5, marker.length() - 11);
-            bool ok = false;
-            const int exitCode = exitCodeStr.toInt(&ok);
-
-            result.stdOutput = m_shellProcess->readAllStandardOutput();
-            result.exitCode = ok ? exitCode : -1;
-            break;
-        }
-    }
-
-    //const QByteArray output = m_shellProcess->readAllStandardOutput();
-    qCDebug(deviceShellLog) << "Received output:" << result.stdOutput;
-    qCInfo(deviceShellLog) << "Run command in shell:" << cmd.toUserOutput()
-                           << "output size:" << result.stdOutput.size()
-                           << "exit code:" << result.exitCode;
     return result;
 }
 
@@ -221,7 +265,7 @@ void DeviceShell::setupShellProcess(QtcProcess *shellProcess)
  */
 void DeviceShell::startupFailed(const CommandLine &cmdLine)
 {
-    qCDebug(deviceShellLog) << "Failed to start shell via:" << cmdLine.toUserOutput();
+    qCWarning(deviceShellLog) << "Failed to start shell via:" << cmdLine.toUserOutput();
 }
 
 /*!
@@ -241,7 +285,6 @@ bool DeviceShell::start()
     setupShellProcess(m_shellProcess);
 
     m_shellProcess->setProcessMode(ProcessMode::Writer);
-    m_shellProcess->setWriteData("echo\n");
 
     // Moving the process into its own thread ...
     m_shellProcess->moveToThread(&m_thread);
@@ -252,12 +295,46 @@ bool DeviceShell::start()
         [this] {
             m_shellProcess->start();
 
-            if (!m_shellProcess->waitForStarted() || !m_shellProcess->waitForReadyRead()
-                || m_shellProcess->readAllStandardOutput() != "\n") {
+            if (!m_shellProcess->waitForStarted()) {
                 closeShellProcess();
                 return false;
             }
-            // TODO: Check if necessary tools are available ( e.g. base64, /bin/sh etc. )
+
+            connect(m_shellProcess, &QtcProcess::readyReadStandardOutput, m_shellProcess, [this] {
+                onReadyRead();
+            });
+            connect(m_shellProcess, &QtcProcess::readyReadStandardError, m_shellProcess, [this] {
+                const QByteArray stdErr = m_shellProcess->readAllStandardError();
+
+                if (m_shellScriptState == State::Unknown) {
+                    if (stdErr.contains("SCRIPT_INSTALLED")) {
+                        m_shellScriptState = State::Succeeded;
+                        return;
+                    }
+                    if (stdErr.contains("ERROR_INSTALL_SCRIPT")) {
+                        m_shellScriptState = State::FailedToStart;
+                        qCWarning(deviceShellLog) << "Failed installing device shell script";
+                        return;
+                    }
+                }
+
+                qCWarning(deviceShellLog) << "Received unexpected output on stderr:" << stdErr;
+            });
+
+            connect(m_shellProcess, &QtcProcess::done, m_shellProcess, [this]() {
+                if (m_shellProcess->resultData().m_exitCode != EXIT_SUCCESS
+                    || m_shellProcess->resultData().m_exitStatus != QProcess::NormalExit) {
+                    qCWarning(deviceShellLog) << "Shell exited with error code:"
+                                              << m_shellProcess->resultData().m_exitCode << "("
+                                              << m_shellProcess->exitMessage() << ")";
+                }
+            });
+
+            if (!installShellScript()) {
+                closeShellProcess();
+                return false;
+            }
+
             return true;
         },
         Qt::BlockingQueuedConnection,
@@ -270,16 +347,158 @@ bool DeviceShell::start()
     return result;
 }
 
+bool DeviceShell::installShellScript()
+{
+    const QString installCmd
+        = QString("echo %1 | base64 -d > /tmp/shell.sh 2>/dev/null && "
+                  "chmod +x /tmp/shell.sh && "
+                  "/tmp/shell.sh || echo ERROR_INSTALL_SCRIPT >&2\n")
+              .arg(QString::fromLatin1(
+                  QByteArray(r_execScript.begin(), r_execScript.size()).toBase64()));
+
+    qCDebug(deviceShellLog) << "Install shell script command:" << installCmd;
+    m_shellProcess->write(installCmd);
+
+    while (m_shellScriptState == State::Unknown) {
+        if (!m_shellProcess->waitForReadyRead()) {
+            qCWarning(deviceShellLog) << "Timeout while waiting for device shell script to install";
+            m_shellScriptState = State::FailedToStart;
+            return false;
+        }
+    }
+    return m_shellScriptState == State::Succeeded;
+}
+
 void DeviceShell::closeShellProcess()
 {
     if (m_shellProcess) {
         if (m_shellProcess->isRunning()) {
-            m_shellProcess->write("exit\n");
+            m_shellProcess->write("exit\nexit\n");
             if (!m_shellProcess->waitForFinished(2000))
                 m_shellProcess->terminate();
         }
-        m_shellProcess->deleteLater();
     }
 }
 
+QByteArray::const_iterator next(const QByteArray::const_iterator &bufferEnd,
+                                const QByteArray::const_iterator &itCurrent)
+{
+    for (QByteArray::const_iterator it = itCurrent; it != bufferEnd; ++it) {
+        if (*it == '\n')
+            return it;
+    }
+    return bufferEnd;
+}
+
+QByteArray byteArrayFromRange(QByteArray::const_iterator itStart, QByteArray::const_iterator itEnd)
+{
+    return QByteArray(itStart, std::distance(itStart, itEnd));
+}
+
+QList<std::tuple<int, DeviceShell::ParseType, QByteArray>> parseShellOutput(const QByteArray &data)
+{
+    auto itStart = data.cbegin();
+    const auto itEnd = data.cend();
+
+    QList<std::tuple<int, DeviceShell::ParseType, QByteArray>> result;
+
+    for (auto it = next(itEnd, itStart); it != itEnd; ++it, itStart = it, it = next(itEnd, it)) {
+        const QByteArray lineView = byteArrayFromRange(itStart, it);
+        QTC_ASSERT(lineView.size() > 0, continue);
+
+        const auto pidEnd = lineView.indexOf(':');
+        const auto typeEnd = lineView.indexOf(':', pidEnd + 1);
+
+        QTC_ASSERT(pidEnd != -1 && typeEnd != -1, continue);
+
+        bool ok = false;
+        const QLatin1String sId(lineView.begin(), pidEnd);
+        const quint64 id = QString(sId).toInt(&ok);
+        QTC_ASSERT(ok, continue);
+
+        const QByteArray data = byteArrayFromRange(lineView.begin() + typeEnd + 1, lineView.end());
+        const QByteArray decoded = QByteArray::fromBase64(data);
+
+        DeviceShell::ParseType t;
+        char type = lineView.at(typeEnd - 1);
+        switch (type) {
+        case 'O':
+            t = DeviceShell::ParseType::StdOut;
+            break;
+        case 'E':
+            t = DeviceShell::ParseType::StdErr;
+            break;
+        case 'R':
+            t = DeviceShell::ParseType::ExitCode;
+            break;
+        default:
+            QTC_CHECK(false);
+            continue;
+        }
+
+        result.append(std::make_tuple(id, t, decoded));
+    }
+
+    return result;
+}
+
+/*!
+ * \brief DeviceShell::onReadyRead
+ *
+ * Reads lines coming from the multiplex script.
+ *
+ * The format is: "<command-id>:<type>:base64-encoded-text-or-returnvalue"
+ * The possible <type>'s are:
+ * O for stdout
+ * E for stderr
+ * R for exit code
+ *
+ * Multiple O/E messages may be received for a process. Once
+ * a single "R" is received, the exit code is reported back
+ * and no further messages from that process are expected.
+ */
+void DeviceShell::onReadyRead()
+{
+    m_commandBuffer += m_shellProcess->readAllStandardOutput();
+    const qsizetype lastLineEndIndex = m_commandBuffer.lastIndexOf('\n') + 1;
+
+    if (lastLineEndIndex == 0)
+        return;
+
+    const QByteArray input(m_commandBuffer.cbegin(), lastLineEndIndex);
+
+    const auto result = parseShellOutput(input);
+
+    QMutexLocker lk(&m_commandMutex);
+    for (const auto &line : result) {
+        const auto &[cmdId, type, data] = line;
+
+        const auto itCmd = m_commandOutput.find(cmdId);
+        QTC_ASSERT(itCmd != m_commandOutput.end(), continue);
+
+        switch (type) {
+        case Utils::DeviceShell::ParseType::StdOut:
+            itCmd->stdOut.append(data);
+            break;
+        case Utils::DeviceShell::ParseType::StdErr:
+            itCmd->stdErr.append(data);
+            break;
+        case Utils::DeviceShell::ParseType::ExitCode: {
+            bool ok = false;
+            int exitCode;
+            exitCode = QString::fromUtf8(data.begin(), data.size()).toInt(&ok);
+            QTC_ASSERT(ok, exitCode = -1);
+            itCmd->exitCode = exitCode;
+            itCmd->waiter->wakeOne();
+            break;
+        }
+        }
+    };
+
+    if (lastLineEndIndex == m_commandBuffer.size())
+        m_commandBuffer.clear();
+    else
+        m_commandBuffer = m_commandBuffer.mid(lastLineEndIndex);
+}
+
 } // namespace Utils

src/libs/utils/deviceshell.h

@@ -25,8 +25,11 @@
 
 #include "utils_global.h"
 
+#include <QMap>
+#include <QMutex>
 #include <QProcess>
 #include <QThread>
+#include <QWaitCondition>
 
 #include <memory>
 
@@ -42,10 +45,19 @@ class QTCREATOR_UTILS_EXPORT DeviceShell : public QObject
     Q_OBJECT
 
 public:
+    enum class State { FailedToStart = -1, Unknown = 0, Succeeded = 1 };
+
     struct RunResult
     {
-        int exitCode;
-        QByteArray stdOutput;
+        int exitCode = 0;
+        QByteArray stdOut;
+        QByteArray stdErr;
+    };
+
+    enum class ParseType {
+        StdOut,
+        StdErr,
+        ExitCode,
     };
 
     DeviceShell();
@@ -54,28 +66,43 @@ public:
     bool runInShell(const CommandLine &cmd, const QByteArray &stdInData = {});
     RunResult outputForRunInShell(const CommandLine &cmd, const QByteArray &stdInData = {});
 
-    bool waitForStarted();
+    State state() const;
 
 signals:
     void done();
     void errorOccurred(QProcess::ProcessError error);
 
 protected:
-    bool runInShellImpl(const CommandLine &cmd, const QByteArray &stdInData = {});
-    RunResult outputForRunInShellImpl(const CommandLine &cmd, const QByteArray &stdInData = {});
+    virtual void startupFailed(const CommandLine &cmdLine);
+    RunResult run(const CommandLine &cmd, const QByteArray &stdInData = {});
 
     bool start();
     void close();
 
 private:
     virtual void setupShellProcess(QtcProcess *shellProcess);
-    virtual void startupFailed(const CommandLine &cmdLine);
 
+    bool installShellScript();
     void closeShellProcess();
 
+    void onReadyRead();
+
 private:
+    struct CommandRun : public RunResult
+    {
+        QWaitCondition *waiter;
+    };
+
     QtcProcess *m_shellProcess = nullptr;
     QThread m_thread;
+    int m_currentId{0};
+
+    QMutex m_commandMutex;
+    // QMap is used here to preserve iterators
+    QMap<quint64, CommandRun> m_commandOutput;
+    QByteArray m_commandBuffer;
+
+    State m_shellScriptState = State::Unknown;
 };
 
 } // namespace Utils

src/plugins/docker/dockerdevice.cpp

@@ -135,7 +135,7 @@ public:
     ~DockerDevicePrivate() { stopCurrentContainer(); }
 
     bool runInContainer(const CommandLine &cmd) const;
-    bool runInShell(const CommandLine &cmd) const;
+    bool runInShell(const CommandLine &cmd, const QByteArray &stdInData = {}) const;
     QByteArray outputForRunInShell(const CommandLine &cmd) const;
 
     void updateContainerAccess();
@@ -485,7 +485,8 @@ void DockerDevicePrivate::startContainer()
                                          "or restart Qt Creator."));
     });
 
-    if (!m_shell->waitForStarted()) {
+    if (m_shell->state() != DeviceShell::State::Succeeded) {
+        m_shell.reset();
         DockerApi::recheckDockerDaemon();
         qCWarning(dockerDeviceLog) << "Container shell failed to start";
     }
@@ -979,28 +980,8 @@ bool DockerDevice::writeFileContents(const FilePath &filePath, const QByteArray
     QTC_ASSERT(handlesFile(filePath), return {});
     updateContainerAccess();
 
-// This following would be the generic Unix solution.
-// But it doesn't pass input. FIXME: Why?
-//    QtcProcess proc;
-//    proc.setCommand({"dd", {"of=" + filePath.path()}});
-//    proc.setWriteData(data);
-//    runProcess(proc);
-//    proc.waitForFinished();
-
-    TemporaryFile tempFile("dockertransport-XXXXXX");
-    tempFile.open();
-    tempFile.write(data);
-
-    const QString tempName = tempFile.fileName();
-    tempFile.close();
-
-    CommandLine cmd{"docker", {"cp", tempName, d->m_container + ':' + filePath.path()}};
-
-    QtcProcess proc;
-    proc.setCommand(cmd);
-    proc.runBlocking();
-
-    return proc.exitCode() == 0;
+    QTC_ASSERT(handlesFile(filePath), return {});
+    return d->runInShell({"dd", {"of=" + filePath.path()}}, data);
 }
 
 Environment DockerDevice::systemEnvironment() const
@@ -1058,16 +1039,16 @@ bool DockerDevicePrivate::runInContainer(const CommandLine &cmd) const
     return exitCode == 0;
 }
 
-bool DockerDevicePrivate::runInShell(const CommandLine &cmd) const
+bool DockerDevicePrivate::runInShell(const CommandLine &cmd, const QByteArray& stdInData) const
 {
     QTC_ASSERT(m_shell, return false);
-    return m_shell->runInShell(cmd);
+    return m_shell->runInShell(cmd, stdInData);
 }
 
 QByteArray DockerDevicePrivate::outputForRunInShell(const CommandLine &cmd) const
 {
     QTC_ASSERT(m_shell.get(), return {});
-    return m_shell->outputForRunInShell(cmd).stdOutput;
+    return m_shell->outputForRunInShell(cmd).stdOut;
 }
 
 // Factory

tests/auto/utils/CMakeLists.txt

@@ -9,3 +9,4 @@ add_subdirectory(stringutils)
 add_subdirectory(templateengine)
 add_subdirectory(treemodel)
 add_subdirectory(multicursor)
+add_subdirectory(deviceshell)

tests/auto/utils/deviceshell/CMakeLists.txt

@@ -0,0 +1,8 @@
+file(RELATIVE_PATH RELATIVE_TEST_PATH "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
+file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_LIBEXEC_PATH}")
+
+add_qtc_test(tst_utils_deviceshell
+  DEFINES "TEST_RELATIVE_LIBEXEC_PATH=\"${TEST_RELATIVE_LIBEXEC_PATH}\""
+  DEPENDS Utils
+  SOURCES tst_deviceshell.cpp
+)

tests/auto/utils/deviceshell/deviceshell.qbs

@@ -0,0 +1,12 @@
+Project  {
+    QtcAutotest {
+        name: "DeviceShell autotest"
+
+        Depends { name: "Utils" }
+        Depends { name: "app_version_header" }
+
+        files: [
+            "tst_deviceshell.cpp",
+        ]
+    }
+}

tests/auto/utils/deviceshell/tst_deviceshell.cpp

@@ -0,0 +1,284 @@
+/****************************************************************************
+**
+** Copyright (C) 2022 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include <app/app_version.h>
+
+#include <utils/deviceshell.h>
+#include <utils/environment.h>
+#include <utils/hostosinfo.h>
+#include <utils/launcherinterface.h>
+#include <utils/qtcprocess.h>
+#include <utils/runextensions.h>
+#include <utils/temporarydirectory.h>
+
+#include <QObject>
+#include <QtTest>
+
+using namespace Utils;
+
+class TestShell : public DeviceShell
+{
+public:
+    TestShell(CommandLine cmdLine)
+        : m_cmdLine(std::move(cmdLine))
+    {
+        start();
+    }
+
+private:
+    void setupShellProcess(QtcProcess *shellProcess) override
+    {
+        shellProcess->setCommand(m_cmdLine);
+    }
+
+    CommandLine m_cmdLine;
+};
+
+bool testDocker(const FilePath &executable)
+{
+    QtcProcess p;
+    p.setCommand({executable, {"info"}});
+    p.runBlocking();
+    return p.result() == ProcessResult::FinishedWithSuccess;
+}
+
+class tst_DeviceShell : public QObject
+{
+    Q_OBJECT
+private:
+    QByteArray m_asciiTestData{256, Qt::Uninitialized};
+
+    QList<CommandLine> m_availableShells;
+    bool m_dockerSetupCheckOk{false};
+
+private:
+    QString testString(int length)
+    {
+        QRandomGenerator generator;
+        QString result;
+        for (int i = 0; i < length; ++i)
+            result.append(QChar{generator.bounded('a', 'z')});
+
+        return result;
+    }
+
+private slots:
+    void initTestCase()
+    {
+        TemporaryDirectory::setMasterTemporaryDirectory(
+            QDir::tempPath() + "/" + Core::Constants::IDE_CASED_ID + "-XXXXXX");
+
+        const QString libExecPath(qApp->applicationDirPath() + '/'
+                                  + QLatin1String(TEST_RELATIVE_LIBEXEC_PATH));
+        LauncherInterface::setPathToLauncher(libExecPath);
+
+        std::iota(m_asciiTestData.begin(), m_asciiTestData.end(), 0);
+
+        const FilePath dockerExecutable = Environment::systemEnvironment()
+                                              .searchInPath("docker", {"/usr/local/bin"});
+
+        if (dockerExecutable.exists()) {
+            m_availableShells.append({dockerExecutable, {"run", "-i", "--rm", "alpine"}});
+            if (testDocker(dockerExecutable)) {
+                m_dockerSetupCheckOk = true;
+            } else {
+                // On linux, docker needs some post-install steps: https://docs.docker.com/engine/install/linux-postinstall/
+                // Also check if you can start a simple container from the command line: "docker run -it alpine".
+                qWarning() << "Checking docker failed, tests will be skipped.";
+            }
+        }
+
+        if (!Utils::HostOsInfo::isWindowsHost()) {
+            // Windows by default has bash.exe, which does not work unless a working wsl is installed.
+            // Therefore we only test shells on linux / mac hosts.
+            const auto shells = {"dash", "bash", "sh", "zsh"};
+
+            for (const auto &shell : shells) {
+                const FilePath executable = Environment::systemEnvironment()
+                                                .searchInPath(shell, {"/usr/local/bin"});
+                if (executable.exists())
+                    m_availableShells.append({executable, {}});
+            }
+        }
+
+        if (m_availableShells.isEmpty()) {
+            QSKIP("Skipping deviceshell tests, as no compatible shell could be found");
+        }
+    }
+
+    void cleanupTestCase() { Singleton::deleteAll(); }
+
+    void testArguments_data()
+    {
+        QTest::addColumn<CommandLine>("cmdLine");
+        QTest::addColumn<QString>("testData");
+
+        for (const auto &cmdLine : qAsConst(m_availableShells)) {
+            QTest::newRow((cmdLine.executable().baseName() + " : simple").toUtf8())
+                << cmdLine << "Hallo Welt!";
+            QTest::newRow((cmdLine.executable().baseName() + " : japanese").toUtf8())
+                << cmdLine
+                << QString::fromUtf8(u8"\xe8\xac\x9d\xe3\x81\x8d\xe3\x82\x81\xe9\x80\x80\x31\x30"
+                                     u8"\xe8\x89\xaf\xe3\x81\x9a\xe3"
+                                     u8"\x82\xa4\xe3\x81\xb5\xe3\x81\x8b\xe7\x89\x88\xe8\x84\xb3"
+                                     u8"\xe3\x83\xa9\xe3\x83\xaf\xe6"
+                                     u8"\xad\xa2\xe9\x80\x9a\xe3\x83\xa8\xe3\x83\xb2\xe3\x82\xad");
+            QTest::newRow((cmdLine.executable().baseName() + " : german").toUtf8())
+                << cmdLine
+                << QString::fromUtf8(u8"\x48\x61\x6c\x6c\xc3\xb6\x2c\x20\x77\x69\x65\x20\x67\xc3"
+                                     u8"\xa4\x68\x74\x20\x65\x73\x20"
+                                     u8"\x64\xc3\xbc\x72");
+
+            QTest::newRow((cmdLine.executable().baseName() + " : long").toUtf8())
+                << cmdLine << testString(4096 * 16);
+        }
+    }
+
+    void testArguments()
+    {
+        QFETCH(CommandLine, cmdLine);
+        QFETCH(QString, testData);
+
+        if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) {
+            QSKIP("Docker was found, but does not seem to be set up correctly, skipping.");
+        }
+
+        TestShell shell(cmdLine);
+        QCOMPARE(shell.state(), DeviceShell::State::Succeeded);
+
+        QRandomGenerator generator;
+
+        const DeviceShell::RunResult result = shell.outputForRunInShell({"echo", {testData}});
+        QCOMPARE(result.exitCode, 0);
+        const QString expected = testData + "\n";
+        const QString resultAsUtf8 = QString::fromUtf8(result.stdOut);
+        QCOMPARE(resultAsUtf8.size(), expected.size());
+        QCOMPARE(resultAsUtf8, expected);
+    }
+
+    void testStdin_data()
+    {
+        QTest::addColumn<CommandLine>("cmdLine");
+        QTest::addColumn<QString>("testData");
+
+        for (const auto &cmdLine : qAsConst(m_availableShells)) {
+            QTest::newRow((cmdLine.executable().baseName() + " : simple").toUtf8())
+                << cmdLine << "Hallo Welt!";
+            QTest::newRow((cmdLine.executable().baseName() + " : japanese").toUtf8())
+                << cmdLine
+                << QString::fromUtf8(u8"\xe8\xac\x9d\xe3\x81\x8d\xe3\x82\x81\xe9\x80\x80\x31\x30"
+                                     u8"\xe8\x89\xaf\xe3\x81\x9a\xe3"
+                                     u8"\x82\xa4\xe3\x81\xb5\xe3\x81\x8b\xe7\x89\x88\xe8\x84\xb3"
+                                     u8"\xe3\x83\xa9\xe3\x83\xaf\xe6"
+                                     u8"\xad\xa2\xe9\x80\x9a\xe3\x83\xa8\xe3\x83\xb2\xe3\x82\xad");
+            QTest::newRow((cmdLine.executable().baseName() + " : german").toUtf8())
+                << cmdLine
+                << QString::fromUtf8(u8"\x48\x61\x6c\x6c\xc3\xb6\x2c\x20\x77\x69\x65\x20\x67\xc3"
+                                     u8"\xa4\x68\x74\x20\x65\x73\x20"
+                                     u8"\x64\xc3\xbc\x72");
+
+            QTest::newRow((cmdLine.executable().baseName() + " : long").toUtf8())
+                << cmdLine << testString(4096 * 16);
+        }
+    }
+
+    void testStdin()
+    {
+        QFETCH(CommandLine, cmdLine);
+        QFETCH(QString, testData);
+
+        if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) {
+            QSKIP("Docker was found, but does not seem to be set up correctly, skipping.");
+        }
+
+        TestShell shell(cmdLine);
+        QCOMPARE(shell.state(), DeviceShell::State::Succeeded);
+
+        QRandomGenerator generator;
+
+        const DeviceShell::RunResult result = shell.outputForRunInShell({"cat", {}}, testData.toUtf8());
+        QCOMPARE(result.exitCode, 0);
+        const QString resultAsUtf8 = QString::fromUtf8(result.stdOut);
+        QCOMPARE(resultAsUtf8.size(), testData.size());
+        QCOMPARE(resultAsUtf8, testData);
+    }
+
+    void testAscii_data()
+    {
+        QTest::addColumn<CommandLine>("cmdLine");
+        for (const auto &cmdLine : qAsConst(m_availableShells)) {
+            QTest::newRow(cmdLine.executable().baseName().toUtf8()) << cmdLine;
+        }
+    }
+
+    void testAscii()
+    {
+        QFETCH(CommandLine, cmdLine);
+
+        if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) {
+            QSKIP("Docker was found, but does not seem to be set up correctly, skipping.");
+        }
+
+        TestShell shell(cmdLine);
+        QCOMPARE(shell.state(), DeviceShell::State::Succeeded);
+
+        const DeviceShell::RunResult result = shell.outputForRunInShell({"cat", {}},
+                                                                        m_asciiTestData);
+        QCOMPARE(result.stdOut, m_asciiTestData);
+    }
+
+    void testStdErr_data()
+    {
+        QTest::addColumn<CommandLine>("cmdLine");
+        for (const auto &cmdLine : m_availableShells) {
+            QTest::newRow(cmdLine.executable().baseName().toUtf8()) << cmdLine;
+        }
+    }
+
+    void testStdErr()
+    {
+        QFETCH(CommandLine, cmdLine);
+
+        if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) {
+            QSKIP("Docker was found, but does not seem to be set up correctly, skipping.");
+        }
+
+        TestShell shell(cmdLine);
+        QCOMPARE(shell.state(), DeviceShell::State::Succeeded);
+
+        const DeviceShell::RunResult result = shell.outputForRunInShell({"cat", {}},
+                                                                        m_asciiTestData);
+        QCOMPARE(result.stdOut, m_asciiTestData);
+        QVERIFY(result.stdErr.isEmpty());
+
+        const DeviceShell::RunResult result2 = shell.outputForRunInShell(
+            {"cat", {"/tmp/i-do-not-exist.none"}});
+        QVERIFY(!result2.stdErr.isEmpty());
+    }
+};
+
+QTEST_MAIN(tst_DeviceShell)
+
+#include "tst_deviceshell.moc"

tests/auto/utils/utils.qbs

@@ -14,5 +14,6 @@ Project {
         "templateengine/templateengine.qbs",
         "treemodel/treemodel.qbs",
         "multicursor/multicursor.qbs",
+        "deviceshell/deviceshell.qbs",
     ]
 }

tests/manual/CMakeLists.txt

@@ -17,3 +17,4 @@ add_subdirectory(proparser)
 # add_subdirectory(search)
 add_subdirectory(shootout)
 add_subdirectory(widgets)
+add_subdirectory(deviceshell)

tests/manual/deviceshell/CMakeLists.txt

@@ -0,0 +1,10 @@
+file(RELATIVE_PATH RELATIVE_TEST_PATH "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
+file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_LIBEXEC_PATH}")
+
+add_qtc_test(tst_manual_deviceshell
+  MANUALTEST
+  DEFINES "TEST_RELATIVE_LIBEXEC_PATH=\"${TEST_RELATIVE_LIBEXEC_PATH}\""
+  DEPENDS Utils
+  SOURCES
+    tst_deviceshell.cpp
+)

tests/manual/deviceshell/deviceshell.qbs

@@ -0,0 +1,22 @@
+import qbs.FileInfo
+
+Project  {
+    QtcManualtest {
+        name: "DeviceShell manualtest"
+
+        Depends { name: "Utils" }
+        Depends { name: "app_version_header" }
+
+        files: [
+            "tst_deviceshell.cpp",
+        ]
+        cpp.defines: {
+            var defines = base;
+            var absLibExecPath = FileInfo.joinPaths(qbs.installRoot, qbs.installPrefix,
+                                                    qtc.ide_libexec_path);
+            var relLibExecPath = FileInfo.relativePath(destinationDirectory, absLibExecPath);
+            defines.push('TEST_RELATIVE_LIBEXEC_PATH="' + relLibExecPath + '"');
+            return defines;
+        }
+    }
+}

tests/manual/deviceshell/tst_deviceshell.cpp

@@ -0,0 +1,228 @@
+/****************************************************************************
+**
+** Copyright (C) 2022 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include <app/app_version.h>
+
+#include <utils/deviceshell.h>
+#include <utils/environment.h>
+#include <utils/hostosinfo.h>
+#include <utils/launcherinterface.h>
+#include <utils/qtcprocess.h>
+#include <utils/runextensions.h>
+#include <utils/temporarydirectory.h>
+#include <utils/mapreduce.h>
+
+#include <QObject>
+#include <QtTest>
+
+using namespace Utils;
+
+class TestShell : public DeviceShell
+{
+public:
+    TestShell() { start(); }
+
+    static CommandLine cmdLine() {
+        static CommandLine cmd;
+
+        if (cmd.isEmpty()) {
+            const FilePath dockerExecutable = Environment::systemEnvironment()
+                                                  .searchInPath("docker", {"/usr/local/bin"});
+            const FilePath dashExecutable = Environment::systemEnvironment()
+                                                .searchInPath("dash", {"/usr/local/bin"});
+            const FilePath bashExecutable = Environment::systemEnvironment()
+                                                .searchInPath("bash", {"/usr/local/bin"});
+            const FilePath shExecutable = Environment::systemEnvironment()
+                                                .searchInPath("sh", {"/usr/local/bin"});
+
+            if (dockerExecutable.exists()) {
+                cmd = {dockerExecutable, {"run", "-i", "--rm","alpine"}};
+            } else if (dashExecutable.exists()) {
+                cmd = {dashExecutable, {}};
+            } else if (bashExecutable.exists()) {
+                cmd = {bashExecutable, {}};
+            } else if (shExecutable.exists()) {
+                cmd = {shExecutable, {}};
+            }
+
+            if (cmd.isEmpty()) {
+                return cmd;
+            }
+
+            qDebug() << "Using shell cmd:" << cmd;
+        }
+
+        return cmd;
+    }
+
+private:
+    void setupShellProcess(QtcProcess *shellProcess) override
+    {
+        shellProcess->setCommand(cmdLine());
+    }
+};
+
+class tst_DeviceShell : public QObject
+{
+    Q_OBJECT
+
+    QList<QByteArray> testArrays(const int numArrays)
+    {
+        QRandomGenerator generator;
+        QList<QByteArray> result;
+
+        for (int i = 0; i < numArrays; i++) {
+            QByteArray data;
+            auto numLines = generator.bounded(1, 100);
+            for (int l = 0; l < numLines; l++) {
+                auto numChars = generator.bounded(10, 40);
+                for (int c = 0; c < numChars; c++) {
+                    data += static_cast<char>(generator.bounded('a', 'z'));
+                }
+                data += '\n';
+            }
+            result.append(data);
+        }
+        return result;
+    }
+
+    void test(int maxNumThreads, int numCalls)
+    {
+        TestShell shell;
+        QCOMPARE(shell.state(), DeviceShell::State::Succeeded);
+
+        QThreadPool::globalInstance()->setMaxThreadCount(maxNumThreads);
+
+        QList<QByteArray> testArray = testArrays(numCalls);
+
+        QElapsedTimer t;
+        t.start();
+
+        const QList<QByteArray> result
+            = mapped<QList>(testArray, [&shell](QByteArray data) -> QByteArray {
+                  return shell.outputForRunInShell({"cat", {}}, data).stdOut;
+              }, MapReduceOption::Ordered, QThreadPool::globalInstance());
+
+        QCOMPARE(result, testArray);
+
+        qDebug() << "maxThreads:" << maxNumThreads << ", took:" << t.elapsed() / 1000.0
+                 << "seconds";
+    }
+
+    void testSleep(QList<int> testData, int nThreads)
+    {
+        TestShell shell;
+        QCOMPARE(shell.state(), DeviceShell::State::Succeeded);
+
+        QThreadPool::globalInstance()->setMaxThreadCount(nThreads);
+
+        QElapsedTimer t;
+        t.start();
+
+        const auto result = mapped<QList>(testData, [&shell](const int &time) {
+            shell.runInShell({"sleep", {QString("%1").arg(time)}});
+            return 0;
+        }, MapReduceOption::Unordered, QThreadPool::globalInstance());
+
+        qDebug() << "maxThreads:" << nThreads << ", took:" << t.elapsed() / 1000.0 << "seconds";
+    }
+
+private slots:
+    void initTestCase()
+    {
+        TemporaryDirectory::setMasterTemporaryDirectory(
+            QDir::tempPath() + "/" + Core::Constants::IDE_CASED_ID + "-XXXXXX");
+
+        const QString libExecPath(qApp->applicationDirPath() + '/'
+                                  + QLatin1String(TEST_RELATIVE_LIBEXEC_PATH));
+        LauncherInterface::setPathToLauncher(libExecPath);
+
+        if (TestShell::cmdLine().isEmpty()) {
+            QSKIP("Skipping deviceshell tests, as no compatible shell could be found");
+        }
+    }
+
+    void cleanupTestCase() { Singleton::deleteAll(); }
+
+    void testEncoding_data()
+    {
+        QTest::addColumn<QString>("utf8string");
+
+        QTest::newRow("japanese") << QString::fromUtf8(
+            u8"\xe8\xac\x9d\xe3\x81\x8d\xe3\x82\x81\xe9\x80\x80\x31\x30\xe8\x89\xaf\xe3\x81\x9a\xe3"
+            u8"\x82\xa4\xe3\x81\xb5\xe3\x81\x8b\xe7\x89\x88\xe8\x84\xb3\xe3\x83\xa9\xe3\x83\xaf\xe6"
+            u8"\xad\xa2\xe9\x80\x9a\xe3\x83\xa8\xe3\x83\xb2\xe3\x82\xad\n");
+        QTest::newRow("german") << QString::fromUtf8(
+            u8"\x48\x61\x6c\x6c\xc3\xb6\x2c\x20\x77\x69\x65\x20\x67\xc3\xa4\x68\x74\x20\x65\x73\x20"
+            u8"\x64\xc3\xbc\x72\n");
+    }
+
+    void testEncoding()
+    {
+        QFETCH(QString, utf8string);
+
+        TestShell shell;
+        QCOMPARE(shell.state(), DeviceShell::State::Succeeded);
+
+        const DeviceShell::RunResult r = shell.outputForRunInShell({"cat", {}}, utf8string.toUtf8());
+        const QString output = QString::fromUtf8(r.stdOut);
+        QCOMPARE(output, utf8string);
+    }
+
+    void testThreading_data()
+    {
+        QTest::addColumn<int>("numThreads");
+        QTest::addColumn<int>("numIterations");
+
+        QTest::newRow("multi-threaded") << 10 << 1000;
+        QTest::newRow("single-threaded") << 1 << 1000;
+    }
+
+    void testThreading()
+    {
+        QFETCH(int, numThreads);
+        QFETCH(int, numIterations);
+
+        test(numThreads, numIterations);
+    }
+
+    void testSleepMulti()
+    {
+        QList<int> testData{4, 7, 10, 3, 1, 10, 3, 3, 5, 4};
+        int full = std::accumulate(testData.begin(), testData.end(), 0);
+        qDebug() << "Testing sleep, full time is:" << full << "seconds";
+        QElapsedTimer t;
+        t.start();
+        testSleep(testData, 10);
+        const int multiThreadRunTime = t.restart();
+        testSleep(testData, 1);
+        const int singleThreadRunTime = t.elapsed();
+        QVERIFY(multiThreadRunTime < singleThreadRunTime);
+    }
+};
+
+QTEST_MAIN(tst_DeviceShell)
+
+#include "tst_deviceshell.moc"

tests/manual/manual.qbs

@@ -8,6 +8,7 @@ Project {
     references: [
         "debugger/gui/gui.qbs",
         "debugger/simple/simple.qbs",
+        "deviceshell/deviceshell.qbs",
         "fakevim/fakevim.qbs",
         "pluginview/pluginview.qbs",
         "process/process.qbs",