From 0135c47849bb1962fd379de210a01a918c6e8b4e Mon Sep 17 00:00:00 2001 From: Marcus Tillmanns Date: Fri, 13 May 2022 15:13:35 +0200 Subject: [PATCH] device: Use multiplex script to allow multithread support Previously the runInShell and outputForRunInShell methods were exclusively processed single threaded, meaning all calls were processed sequentially. With the multiplexed helper script we can now run multiple processes simultaneously. ( see tst_manual_deviceshell ) Additionally the new script allows us to capture both stdout and stderr from commands which was not possible previously. Change-Id: I52f4fb46d872dc274edb9c11872d2f6543741b34 Reviewed-by: David Schulz Reviewed-by: Jarek Kobus Reviewed-by: --- src/libs/utils/deviceshell.cpp | 437 +++++++++++++----- src/libs/utils/deviceshell.h | 39 +- src/plugins/docker/dockerdevice.cpp | 35 +- tests/auto/utils/CMakeLists.txt | 1 + tests/auto/utils/deviceshell/CMakeLists.txt | 8 + tests/auto/utils/deviceshell/deviceshell.qbs | 12 + .../utils/deviceshell/tst_deviceshell.cpp | 284 ++++++++++++ tests/auto/utils/utils.qbs | 1 + tests/manual/CMakeLists.txt | 1 + tests/manual/deviceshell/CMakeLists.txt | 10 + tests/manual/deviceshell/deviceshell.qbs | 22 + tests/manual/deviceshell/tst_deviceshell.cpp | 228 +++++++++ tests/manual/manual.qbs | 1 + 13 files changed, 937 insertions(+), 142 deletions(-) create mode 100644 tests/auto/utils/deviceshell/CMakeLists.txt create mode 100644 tests/auto/utils/deviceshell/deviceshell.qbs create mode 100644 tests/auto/utils/deviceshell/tst_deviceshell.cpp create mode 100644 tests/manual/deviceshell/CMakeLists.txt create mode 100644 tests/manual/deviceshell/deviceshell.qbs create mode 100644 tests/manual/deviceshell/tst_deviceshell.cpp diff --git a/src/libs/utils/deviceshell.cpp b/src/libs/utils/deviceshell.cpp index 91ada89a6bd..d46be147096 100644 --- a/src/libs/utils/deviceshell.cpp +++ b/src/libs/utils/deviceshell.cpp @@ -25,8 +25,9 @@ #include "deviceshell.h" -#include -#include +#include "processinterface.h" +#include "qtcassert.h" +#include "qtcprocess.h" #include #include @@ -35,6 +36,123 @@ Q_LOGGING_CATEGORY(deviceShellLog, "qtc.utils.deviceshell", QtWarningMsg) namespace Utils { +namespace { + +/*! + * The multiplex script waits for input via stdin. + * + * To start a command, a message is send with the format " "" \n" + * To stop the script, simply send "exit\n" via stdin + * + * Once a message is received, two new streams are created that the new process redirects its output to ( $stdoutraw and $stderrraw ). + * + * These streams are piped through base64 into the two streams stdoutenc and stderrenc. + * + * Two subshells read from these base64 encoded streams, and prepend the command-id, as well as either "O:" or "E:" depending on whether its the stdout or stderr stream. + * + * Once the process exits its exit code is send to stdout with the command-id and the type "R". + * + */ +const QLatin1String r_execScript = QLatin1String(R"( +#!/bin/sh + +readAndMark() { + local buffer + while read buffer + do + printf '%s:%s:%s\n' "$1" "$2" "$buffer" + done +} + +base64decode() +{ + base64 -d 2>/dev/null +} + +base64encode() +{ + base64 2>/dev/null +} + +executeAndMark() +{ + PID="$1" + INDATA="$2" + shift + shift + CMD="$@" + + # Output Streams + stdoutenc=$(mktemp -u) + stderrenc=$(mktemp -u) + mkfifo "$stdoutenc" "$stderrenc" + + # app output streams + stdoutraw=$(mktemp -u) + stderrraw=$(mktemp -u) + mkfifo "$stdoutraw" "$stderrraw" + + # Cleanup + trap 'rm -f "$stdoutenc" "$stderrenc" "$stdoutraw" "$stderrraw" ' EXIT + + # Pipe all app output through base64, and then into the output streams + cat $stdoutraw | base64encode > "$stdoutenc" & + cat $stderrraw | base64encode > "$stderrenc" & + + # Mark the app's output streams + readAndMark $PID 'O' < "$stdoutenc" >&1 & + readAndMark $PID 'E' < "$stderrenc" >&1 & + + # Start the app ... + if [ -z "$INDATA" ] + then + eval $CMD 1> "$stdoutraw" 2> "$stderrraw" + else + echo $INDATA | base64decode | eval "$CMD" 1> "$stdoutraw" 2> "$stderrraw" + fi + + exitcode=$(echo $? | base64encode) + + wait + echo "$PID:R:$exitcode" +} + +execute() +{ + PID="$1" + INDATA=$(eval echo "$2") + shift + shift + CMD=$@ + executeAndMark $PID "$INDATA" "$CMD" +} + +cleanup() +{ + kill -- -$$ + exit 1 +} + +if ! command -v base64 &> /dev/null +then + echo "base64 command could not be found" >&2 + exit 1 +fi + +trap cleanup 1 2 3 6 + +echo SCRIPT_INSTALLED >&2 + +while read -r id inData cmd; do + if [ "$id" = "exit" ]; then + exit + fi + execute $id $inData $cmd & +done +)"); + +} // namespace + DeviceShell::DeviceShell() { m_thread.setObjectName("Shell Thread"); @@ -43,26 +161,14 @@ DeviceShell::DeviceShell() DeviceShell::~DeviceShell() { + m_shellProcess->deleteLater(); + if (m_thread.isRunning()) { m_thread.quit(); m_thread.wait(); } } -bool DeviceShell::waitForStarted() -{ - QTC_ASSERT(m_shellProcess, return false); - Q_ASSERT(QThread::currentThread() != &m_thread); - - bool result; - QMetaObject::invokeMethod( - m_shellProcess, - [this] { return m_shellProcess->waitForStarted(); }, - Qt::BlockingQueuedConnection, - &result); - return result; -} - /*! * \brief DeviceShell::runInShell * \param cmd The command to run @@ -78,48 +184,8 @@ bool DeviceShell::runInShell(const CommandLine &cmd, const QByteArray &stdInData QTC_ASSERT(m_shellProcess, return false); Q_ASSERT(QThread::currentThread() != &m_thread); - bool result = false; - QMetaObject::invokeMethod( - m_shellProcess, - [this, &cmd, &stdInData] { return runInShellImpl(cmd, stdInData); }, - Qt::BlockingQueuedConnection, - &result); - return result; -} - -bool DeviceShell::runInShellImpl(const CommandLine &cmd, const QByteArray &stdInData) -{ - QTC_ASSERT(QThread::currentThread() == &m_thread, return false); - - QTC_ASSERT(m_shellProcess->isRunning(), return false); - QTC_ASSERT(m_shellProcess, return false); - QTC_CHECK(m_shellProcess->readAllStandardOutput().isNull()); // clean possible left-overs - QTC_CHECK(m_shellProcess->readAllStandardError().isNull()); // clean possible left-overs - auto cleanup = qScopeGuard( - [this] { m_shellProcess->readAllStandardOutput(); }); // clean on assert - - QString prefix; - if (!stdInData.isEmpty()) - prefix = "echo '" + QString::fromUtf8(stdInData.toBase64()) + "' | base64 -d | "; - - const QString suffix = " > /dev/null 2>&1\necho $?\n"; - const QString command = prefix + cmd.toUserOutput() + suffix; - - qCDebug(deviceShellLog) << "Running:" << command; - - m_shellProcess->write(command); - m_shellProcess->waitForReadyRead(); - - const QByteArray output = m_shellProcess->readAllStandardOutput(); - - bool ok = false; - const int result = output.toInt(&ok); - - qCInfo(deviceShellLog) << "Run command in shell:" << cmd.toUserOutput() << "result: " << output - << " ==>" << result; - QTC_ASSERT(ok, return false); - - return result == EXIT_SUCCESS; + const RunResult result = run(cmd, stdInData); + return result.exitCode == 0; } /*! @@ -138,59 +204,37 @@ DeviceShell::RunResult DeviceShell::outputForRunInShell(const CommandLine &cmd, QTC_ASSERT(m_shellProcess, return {}); Q_ASSERT(QThread::currentThread() != &m_thread); - RunResult result; - QMetaObject::invokeMethod( - m_shellProcess, - [this, &cmd, &stdInData] { return outputForRunInShellImpl(cmd, stdInData); }, - Qt::BlockingQueuedConnection, - &result); - return result; + return run(cmd, stdInData); } -DeviceShell::RunResult DeviceShell::outputForRunInShellImpl(const CommandLine &cmd, - const QByteArray &stdInData) +DeviceShell::State DeviceShell::state() const { return m_shellScriptState; } + +DeviceShell::RunResult DeviceShell::run(const CommandLine &cmd, const QByteArray &stdInData) { - QTC_ASSERT(QThread::currentThread() == &m_thread, return {}); + const RunResult errorResult{-1, {}, {}}; + QTC_ASSERT(m_shellProcess, return errorResult); + QTC_ASSERT(m_shellScriptState == State::Succeeded, return errorResult); - QTC_ASSERT(m_shellProcess, return {}); - QTC_CHECK(m_shellProcess->readAllStandardOutput().isNull()); // clean possible left-overs - QTC_CHECK(m_shellProcess->readAllStandardError().isNull()); // clean possible left-overs - auto cleanup = qScopeGuard( - [this] { m_shellProcess->readAllStandardOutput(); }); // clean on assert + QMutexLocker lk(&m_commandMutex); - QString prefix; - if (!stdInData.isEmpty()) - prefix = "echo '" + QString::fromUtf8(stdInData.toBase64()) + "' | base64 -d | "; + QWaitCondition waiter; + const int id = ++m_currentId; + const auto it = m_commandOutput.insert(id, CommandRun{-1, {}, {}, &waiter}); - const QString markerCmd = "echo __qtc$?qtc__ 1>&2\n"; - const QString suffix = "\n" + markerCmd; - const QString command = prefix + cmd.toUserOutput() + suffix; + QMetaObject::invokeMethod(m_shellProcess, [this, id, &cmd, &stdInData]() { + const QString command = QString("%1 \"%2\" %3\n") + .arg(id) + .arg(QString::fromLatin1(stdInData.toBase64())) + .arg(cmd.toUserOutput()); + qCDebug(deviceShellLog) << "Running:" << command; + m_shellProcess->writeRaw(command.toUtf8()); + }); - qCDebug(deviceShellLog) << "Running:" << command; - m_shellProcess->write(command); + waiter.wait(&m_commandMutex); - RunResult result; + const RunResult result = *it; + m_commandOutput.erase(it); - while (true) { - m_shellProcess->waitForReadyRead(); - QByteArray stdErr = m_shellProcess->readAllStandardError(); - if (stdErr.endsWith("qtc__\n")) { - QByteArray marker = stdErr.right(stdErr.length() - stdErr.lastIndexOf("__qtc")); - QByteArray exitCodeStr = marker.mid(5, marker.length() - 11); - bool ok = false; - const int exitCode = exitCodeStr.toInt(&ok); - - result.stdOutput = m_shellProcess->readAllStandardOutput(); - result.exitCode = ok ? exitCode : -1; - break; - } - } - - //const QByteArray output = m_shellProcess->readAllStandardOutput(); - qCDebug(deviceShellLog) << "Received output:" << result.stdOutput; - qCInfo(deviceShellLog) << "Run command in shell:" << cmd.toUserOutput() - << "output size:" << result.stdOutput.size() - << "exit code:" << result.exitCode; return result; } @@ -221,7 +265,7 @@ void DeviceShell::setupShellProcess(QtcProcess *shellProcess) */ void DeviceShell::startupFailed(const CommandLine &cmdLine) { - qCDebug(deviceShellLog) << "Failed to start shell via:" << cmdLine.toUserOutput(); + qCWarning(deviceShellLog) << "Failed to start shell via:" << cmdLine.toUserOutput(); } /*! @@ -241,7 +285,6 @@ bool DeviceShell::start() setupShellProcess(m_shellProcess); m_shellProcess->setProcessMode(ProcessMode::Writer); - m_shellProcess->setWriteData("echo\n"); // Moving the process into its own thread ... m_shellProcess->moveToThread(&m_thread); @@ -252,12 +295,46 @@ bool DeviceShell::start() [this] { m_shellProcess->start(); - if (!m_shellProcess->waitForStarted() || !m_shellProcess->waitForReadyRead() - || m_shellProcess->readAllStandardOutput() != "\n") { + if (!m_shellProcess->waitForStarted()) { closeShellProcess(); return false; } - // TODO: Check if necessary tools are available ( e.g. base64, /bin/sh etc. ) + + connect(m_shellProcess, &QtcProcess::readyReadStandardOutput, m_shellProcess, [this] { + onReadyRead(); + }); + connect(m_shellProcess, &QtcProcess::readyReadStandardError, m_shellProcess, [this] { + const QByteArray stdErr = m_shellProcess->readAllStandardError(); + + if (m_shellScriptState == State::Unknown) { + if (stdErr.contains("SCRIPT_INSTALLED")) { + m_shellScriptState = State::Succeeded; + return; + } + if (stdErr.contains("ERROR_INSTALL_SCRIPT")) { + m_shellScriptState = State::FailedToStart; + qCWarning(deviceShellLog) << "Failed installing device shell script"; + return; + } + } + + qCWarning(deviceShellLog) << "Received unexpected output on stderr:" << stdErr; + }); + + connect(m_shellProcess, &QtcProcess::done, m_shellProcess, [this]() { + if (m_shellProcess->resultData().m_exitCode != EXIT_SUCCESS + || m_shellProcess->resultData().m_exitStatus != QProcess::NormalExit) { + qCWarning(deviceShellLog) << "Shell exited with error code:" + << m_shellProcess->resultData().m_exitCode << "(" + << m_shellProcess->exitMessage() << ")"; + } + }); + + if (!installShellScript()) { + closeShellProcess(); + return false; + } + return true; }, Qt::BlockingQueuedConnection, @@ -270,16 +347,158 @@ bool DeviceShell::start() return result; } +bool DeviceShell::installShellScript() +{ + const QString installCmd + = QString("echo %1 | base64 -d > /tmp/shell.sh 2>/dev/null && " + "chmod +x /tmp/shell.sh && " + "/tmp/shell.sh || echo ERROR_INSTALL_SCRIPT >&2\n") + .arg(QString::fromLatin1( + QByteArray(r_execScript.begin(), r_execScript.size()).toBase64())); + + qCDebug(deviceShellLog) << "Install shell script command:" << installCmd; + m_shellProcess->write(installCmd); + + while (m_shellScriptState == State::Unknown) { + if (!m_shellProcess->waitForReadyRead()) { + qCWarning(deviceShellLog) << "Timeout while waiting for device shell script to install"; + m_shellScriptState = State::FailedToStart; + return false; + } + } + return m_shellScriptState == State::Succeeded; +} + void DeviceShell::closeShellProcess() { if (m_shellProcess) { if (m_shellProcess->isRunning()) { - m_shellProcess->write("exit\n"); + m_shellProcess->write("exit\nexit\n"); if (!m_shellProcess->waitForFinished(2000)) m_shellProcess->terminate(); } - m_shellProcess->deleteLater(); } } +QByteArray::const_iterator next(const QByteArray::const_iterator &bufferEnd, + const QByteArray::const_iterator &itCurrent) +{ + for (QByteArray::const_iterator it = itCurrent; it != bufferEnd; ++it) { + if (*it == '\n') + return it; + } + return bufferEnd; +} + +QByteArray byteArrayFromRange(QByteArray::const_iterator itStart, QByteArray::const_iterator itEnd) +{ + return QByteArray(itStart, std::distance(itStart, itEnd)); +} + +QList> parseShellOutput(const QByteArray &data) +{ + auto itStart = data.cbegin(); + const auto itEnd = data.cend(); + + QList> result; + + for (auto it = next(itEnd, itStart); it != itEnd; ++it, itStart = it, it = next(itEnd, it)) { + const QByteArray lineView = byteArrayFromRange(itStart, it); + QTC_ASSERT(lineView.size() > 0, continue); + + const auto pidEnd = lineView.indexOf(':'); + const auto typeEnd = lineView.indexOf(':', pidEnd + 1); + + QTC_ASSERT(pidEnd != -1 && typeEnd != -1, continue); + + bool ok = false; + const QLatin1String sId(lineView.begin(), pidEnd); + const quint64 id = QString(sId).toInt(&ok); + QTC_ASSERT(ok, continue); + + const QByteArray data = byteArrayFromRange(lineView.begin() + typeEnd + 1, lineView.end()); + const QByteArray decoded = QByteArray::fromBase64(data); + + DeviceShell::ParseType t; + char type = lineView.at(typeEnd - 1); + switch (type) { + case 'O': + t = DeviceShell::ParseType::StdOut; + break; + case 'E': + t = DeviceShell::ParseType::StdErr; + break; + case 'R': + t = DeviceShell::ParseType::ExitCode; + break; + default: + QTC_CHECK(false); + continue; + } + + result.append(std::make_tuple(id, t, decoded)); + } + + return result; +} + +/*! + * \brief DeviceShell::onReadyRead + * + * Reads lines coming from the multiplex script. + * + * The format is: "::base64-encoded-text-or-returnvalue" + * The possible 's are: + * O for stdout + * E for stderr + * R for exit code + * + * Multiple O/E messages may be received for a process. Once + * a single "R" is received, the exit code is reported back + * and no further messages from that process are expected. + */ +void DeviceShell::onReadyRead() +{ + m_commandBuffer += m_shellProcess->readAllStandardOutput(); + const qsizetype lastLineEndIndex = m_commandBuffer.lastIndexOf('\n') + 1; + + if (lastLineEndIndex == 0) + return; + + const QByteArray input(m_commandBuffer.cbegin(), lastLineEndIndex); + + const auto result = parseShellOutput(input); + + QMutexLocker lk(&m_commandMutex); + for (const auto &line : result) { + const auto &[cmdId, type, data] = line; + + const auto itCmd = m_commandOutput.find(cmdId); + QTC_ASSERT(itCmd != m_commandOutput.end(), continue); + + switch (type) { + case Utils::DeviceShell::ParseType::StdOut: + itCmd->stdOut.append(data); + break; + case Utils::DeviceShell::ParseType::StdErr: + itCmd->stdErr.append(data); + break; + case Utils::DeviceShell::ParseType::ExitCode: { + bool ok = false; + int exitCode; + exitCode = QString::fromUtf8(data.begin(), data.size()).toInt(&ok); + QTC_ASSERT(ok, exitCode = -1); + itCmd->exitCode = exitCode; + itCmd->waiter->wakeOne(); + break; + } + } + }; + + if (lastLineEndIndex == m_commandBuffer.size()) + m_commandBuffer.clear(); + else + m_commandBuffer = m_commandBuffer.mid(lastLineEndIndex); +} + } // namespace Utils diff --git a/src/libs/utils/deviceshell.h b/src/libs/utils/deviceshell.h index b800712a0b6..8def78f6d3c 100644 --- a/src/libs/utils/deviceshell.h +++ b/src/libs/utils/deviceshell.h @@ -25,8 +25,11 @@ #include "utils_global.h" +#include +#include #include #include +#include #include @@ -42,10 +45,19 @@ class QTCREATOR_UTILS_EXPORT DeviceShell : public QObject Q_OBJECT public: + enum class State { FailedToStart = -1, Unknown = 0, Succeeded = 1 }; + struct RunResult { - int exitCode; - QByteArray stdOutput; + int exitCode = 0; + QByteArray stdOut; + QByteArray stdErr; + }; + + enum class ParseType { + StdOut, + StdErr, + ExitCode, }; DeviceShell(); @@ -54,28 +66,43 @@ public: bool runInShell(const CommandLine &cmd, const QByteArray &stdInData = {}); RunResult outputForRunInShell(const CommandLine &cmd, const QByteArray &stdInData = {}); - bool waitForStarted(); + State state() const; signals: void done(); void errorOccurred(QProcess::ProcessError error); protected: - bool runInShellImpl(const CommandLine &cmd, const QByteArray &stdInData = {}); - RunResult outputForRunInShellImpl(const CommandLine &cmd, const QByteArray &stdInData = {}); + virtual void startupFailed(const CommandLine &cmdLine); + RunResult run(const CommandLine &cmd, const QByteArray &stdInData = {}); bool start(); void close(); private: virtual void setupShellProcess(QtcProcess *shellProcess); - virtual void startupFailed(const CommandLine &cmdLine); + bool installShellScript(); void closeShellProcess(); + void onReadyRead(); + private: + struct CommandRun : public RunResult + { + QWaitCondition *waiter; + }; + QtcProcess *m_shellProcess = nullptr; QThread m_thread; + int m_currentId{0}; + + QMutex m_commandMutex; + // QMap is used here to preserve iterators + QMap m_commandOutput; + QByteArray m_commandBuffer; + + State m_shellScriptState = State::Unknown; }; } // namespace Utils diff --git a/src/plugins/docker/dockerdevice.cpp b/src/plugins/docker/dockerdevice.cpp index eed68feafc0..285b2f4cf6f 100644 --- a/src/plugins/docker/dockerdevice.cpp +++ b/src/plugins/docker/dockerdevice.cpp @@ -135,7 +135,7 @@ public: ~DockerDevicePrivate() { stopCurrentContainer(); } bool runInContainer(const CommandLine &cmd) const; - bool runInShell(const CommandLine &cmd) const; + bool runInShell(const CommandLine &cmd, const QByteArray &stdInData = {}) const; QByteArray outputForRunInShell(const CommandLine &cmd) const; void updateContainerAccess(); @@ -485,7 +485,8 @@ void DockerDevicePrivate::startContainer() "or restart Qt Creator.")); }); - if (!m_shell->waitForStarted()) { + if (m_shell->state() != DeviceShell::State::Succeeded) { + m_shell.reset(); DockerApi::recheckDockerDaemon(); qCWarning(dockerDeviceLog) << "Container shell failed to start"; } @@ -979,28 +980,8 @@ bool DockerDevice::writeFileContents(const FilePath &filePath, const QByteArray QTC_ASSERT(handlesFile(filePath), return {}); updateContainerAccess(); -// This following would be the generic Unix solution. -// But it doesn't pass input. FIXME: Why? -// QtcProcess proc; -// proc.setCommand({"dd", {"of=" + filePath.path()}}); -// proc.setWriteData(data); -// runProcess(proc); -// proc.waitForFinished(); - - TemporaryFile tempFile("dockertransport-XXXXXX"); - tempFile.open(); - tempFile.write(data); - - const QString tempName = tempFile.fileName(); - tempFile.close(); - - CommandLine cmd{"docker", {"cp", tempName, d->m_container + ':' + filePath.path()}}; - - QtcProcess proc; - proc.setCommand(cmd); - proc.runBlocking(); - - return proc.exitCode() == 0; + QTC_ASSERT(handlesFile(filePath), return {}); + return d->runInShell({"dd", {"of=" + filePath.path()}}, data); } Environment DockerDevice::systemEnvironment() const @@ -1058,16 +1039,16 @@ bool DockerDevicePrivate::runInContainer(const CommandLine &cmd) const return exitCode == 0; } -bool DockerDevicePrivate::runInShell(const CommandLine &cmd) const +bool DockerDevicePrivate::runInShell(const CommandLine &cmd, const QByteArray& stdInData) const { QTC_ASSERT(m_shell, return false); - return m_shell->runInShell(cmd); + return m_shell->runInShell(cmd, stdInData); } QByteArray DockerDevicePrivate::outputForRunInShell(const CommandLine &cmd) const { QTC_ASSERT(m_shell.get(), return {}); - return m_shell->outputForRunInShell(cmd).stdOutput; + return m_shell->outputForRunInShell(cmd).stdOut; } // Factory diff --git a/tests/auto/utils/CMakeLists.txt b/tests/auto/utils/CMakeLists.txt index 5680cde7aaf..60dd1cfbccc 100644 --- a/tests/auto/utils/CMakeLists.txt +++ b/tests/auto/utils/CMakeLists.txt @@ -9,3 +9,4 @@ add_subdirectory(stringutils) add_subdirectory(templateengine) add_subdirectory(treemodel) add_subdirectory(multicursor) +add_subdirectory(deviceshell) diff --git a/tests/auto/utils/deviceshell/CMakeLists.txt b/tests/auto/utils/deviceshell/CMakeLists.txt new file mode 100644 index 00000000000..61492bef322 --- /dev/null +++ b/tests/auto/utils/deviceshell/CMakeLists.txt @@ -0,0 +1,8 @@ +file(RELATIVE_PATH RELATIVE_TEST_PATH "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_LIBEXEC_PATH}") + +add_qtc_test(tst_utils_deviceshell + DEFINES "TEST_RELATIVE_LIBEXEC_PATH=\"${TEST_RELATIVE_LIBEXEC_PATH}\"" + DEPENDS Utils + SOURCES tst_deviceshell.cpp +) diff --git a/tests/auto/utils/deviceshell/deviceshell.qbs b/tests/auto/utils/deviceshell/deviceshell.qbs new file mode 100644 index 00000000000..207848f00d8 --- /dev/null +++ b/tests/auto/utils/deviceshell/deviceshell.qbs @@ -0,0 +1,12 @@ +Project { + QtcAutotest { + name: "DeviceShell autotest" + + Depends { name: "Utils" } + Depends { name: "app_version_header" } + + files: [ + "tst_deviceshell.cpp", + ] + } +} diff --git a/tests/auto/utils/deviceshell/tst_deviceshell.cpp b/tests/auto/utils/deviceshell/tst_deviceshell.cpp new file mode 100644 index 00000000000..40181b7d25e --- /dev/null +++ b/tests/auto/utils/deviceshell/tst_deviceshell.cpp @@ -0,0 +1,284 @@ +/**************************************************************************** +** +** Copyright (C) 2022 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace Utils; + +class TestShell : public DeviceShell +{ +public: + TestShell(CommandLine cmdLine) + : m_cmdLine(std::move(cmdLine)) + { + start(); + } + +private: + void setupShellProcess(QtcProcess *shellProcess) override + { + shellProcess->setCommand(m_cmdLine); + } + + CommandLine m_cmdLine; +}; + +bool testDocker(const FilePath &executable) +{ + QtcProcess p; + p.setCommand({executable, {"info"}}); + p.runBlocking(); + return p.result() == ProcessResult::FinishedWithSuccess; +} + +class tst_DeviceShell : public QObject +{ + Q_OBJECT +private: + QByteArray m_asciiTestData{256, Qt::Uninitialized}; + + QList m_availableShells; + bool m_dockerSetupCheckOk{false}; + +private: + QString testString(int length) + { + QRandomGenerator generator; + QString result; + for (int i = 0; i < length; ++i) + result.append(QChar{generator.bounded('a', 'z')}); + + return result; + } + +private slots: + void initTestCase() + { + TemporaryDirectory::setMasterTemporaryDirectory( + QDir::tempPath() + "/" + Core::Constants::IDE_CASED_ID + "-XXXXXX"); + + const QString libExecPath(qApp->applicationDirPath() + '/' + + QLatin1String(TEST_RELATIVE_LIBEXEC_PATH)); + LauncherInterface::setPathToLauncher(libExecPath); + + std::iota(m_asciiTestData.begin(), m_asciiTestData.end(), 0); + + const FilePath dockerExecutable = Environment::systemEnvironment() + .searchInPath("docker", {"/usr/local/bin"}); + + if (dockerExecutable.exists()) { + m_availableShells.append({dockerExecutable, {"run", "-i", "--rm", "alpine"}}); + if (testDocker(dockerExecutable)) { + m_dockerSetupCheckOk = true; + } else { + // On linux, docker needs some post-install steps: https://docs.docker.com/engine/install/linux-postinstall/ + // Also check if you can start a simple container from the command line: "docker run -it alpine". + qWarning() << "Checking docker failed, tests will be skipped."; + } + } + + if (!Utils::HostOsInfo::isWindowsHost()) { + // Windows by default has bash.exe, which does not work unless a working wsl is installed. + // Therefore we only test shells on linux / mac hosts. + const auto shells = {"dash", "bash", "sh", "zsh"}; + + for (const auto &shell : shells) { + const FilePath executable = Environment::systemEnvironment() + .searchInPath(shell, {"/usr/local/bin"}); + if (executable.exists()) + m_availableShells.append({executable, {}}); + } + } + + if (m_availableShells.isEmpty()) { + QSKIP("Skipping deviceshell tests, as no compatible shell could be found"); + } + } + + void cleanupTestCase() { Singleton::deleteAll(); } + + void testArguments_data() + { + QTest::addColumn("cmdLine"); + QTest::addColumn("testData"); + + for (const auto &cmdLine : qAsConst(m_availableShells)) { + QTest::newRow((cmdLine.executable().baseName() + " : simple").toUtf8()) + << cmdLine << "Hallo Welt!"; + QTest::newRow((cmdLine.executable().baseName() + " : japanese").toUtf8()) + << cmdLine + << QString::fromUtf8(u8"\xe8\xac\x9d\xe3\x81\x8d\xe3\x82\x81\xe9\x80\x80\x31\x30" + u8"\xe8\x89\xaf\xe3\x81\x9a\xe3" + u8"\x82\xa4\xe3\x81\xb5\xe3\x81\x8b\xe7\x89\x88\xe8\x84\xb3" + u8"\xe3\x83\xa9\xe3\x83\xaf\xe6" + u8"\xad\xa2\xe9\x80\x9a\xe3\x83\xa8\xe3\x83\xb2\xe3\x82\xad"); + QTest::newRow((cmdLine.executable().baseName() + " : german").toUtf8()) + << cmdLine + << QString::fromUtf8(u8"\x48\x61\x6c\x6c\xc3\xb6\x2c\x20\x77\x69\x65\x20\x67\xc3" + u8"\xa4\x68\x74\x20\x65\x73\x20" + u8"\x64\xc3\xbc\x72"); + + QTest::newRow((cmdLine.executable().baseName() + " : long").toUtf8()) + << cmdLine << testString(4096 * 16); + } + } + + void testArguments() + { + QFETCH(CommandLine, cmdLine); + QFETCH(QString, testData); + + if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) { + QSKIP("Docker was found, but does not seem to be set up correctly, skipping."); + } + + TestShell shell(cmdLine); + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + QRandomGenerator generator; + + const DeviceShell::RunResult result = shell.outputForRunInShell({"echo", {testData}}); + QCOMPARE(result.exitCode, 0); + const QString expected = testData + "\n"; + const QString resultAsUtf8 = QString::fromUtf8(result.stdOut); + QCOMPARE(resultAsUtf8.size(), expected.size()); + QCOMPARE(resultAsUtf8, expected); + } + + void testStdin_data() + { + QTest::addColumn("cmdLine"); + QTest::addColumn("testData"); + + for (const auto &cmdLine : qAsConst(m_availableShells)) { + QTest::newRow((cmdLine.executable().baseName() + " : simple").toUtf8()) + << cmdLine << "Hallo Welt!"; + QTest::newRow((cmdLine.executable().baseName() + " : japanese").toUtf8()) + << cmdLine + << QString::fromUtf8(u8"\xe8\xac\x9d\xe3\x81\x8d\xe3\x82\x81\xe9\x80\x80\x31\x30" + u8"\xe8\x89\xaf\xe3\x81\x9a\xe3" + u8"\x82\xa4\xe3\x81\xb5\xe3\x81\x8b\xe7\x89\x88\xe8\x84\xb3" + u8"\xe3\x83\xa9\xe3\x83\xaf\xe6" + u8"\xad\xa2\xe9\x80\x9a\xe3\x83\xa8\xe3\x83\xb2\xe3\x82\xad"); + QTest::newRow((cmdLine.executable().baseName() + " : german").toUtf8()) + << cmdLine + << QString::fromUtf8(u8"\x48\x61\x6c\x6c\xc3\xb6\x2c\x20\x77\x69\x65\x20\x67\xc3" + u8"\xa4\x68\x74\x20\x65\x73\x20" + u8"\x64\xc3\xbc\x72"); + + QTest::newRow((cmdLine.executable().baseName() + " : long").toUtf8()) + << cmdLine << testString(4096 * 16); + } + } + + void testStdin() + { + QFETCH(CommandLine, cmdLine); + QFETCH(QString, testData); + + if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) { + QSKIP("Docker was found, but does not seem to be set up correctly, skipping."); + } + + TestShell shell(cmdLine); + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + QRandomGenerator generator; + + const DeviceShell::RunResult result = shell.outputForRunInShell({"cat", {}}, testData.toUtf8()); + QCOMPARE(result.exitCode, 0); + const QString resultAsUtf8 = QString::fromUtf8(result.stdOut); + QCOMPARE(resultAsUtf8.size(), testData.size()); + QCOMPARE(resultAsUtf8, testData); + } + + void testAscii_data() + { + QTest::addColumn("cmdLine"); + for (const auto &cmdLine : qAsConst(m_availableShells)) { + QTest::newRow(cmdLine.executable().baseName().toUtf8()) << cmdLine; + } + } + + void testAscii() + { + QFETCH(CommandLine, cmdLine); + + if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) { + QSKIP("Docker was found, but does not seem to be set up correctly, skipping."); + } + + TestShell shell(cmdLine); + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + const DeviceShell::RunResult result = shell.outputForRunInShell({"cat", {}}, + m_asciiTestData); + QCOMPARE(result.stdOut, m_asciiTestData); + } + + void testStdErr_data() + { + QTest::addColumn("cmdLine"); + for (const auto &cmdLine : m_availableShells) { + QTest::newRow(cmdLine.executable().baseName().toUtf8()) << cmdLine; + } + } + + void testStdErr() + { + QFETCH(CommandLine, cmdLine); + + if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) { + QSKIP("Docker was found, but does not seem to be set up correctly, skipping."); + } + + TestShell shell(cmdLine); + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + const DeviceShell::RunResult result = shell.outputForRunInShell({"cat", {}}, + m_asciiTestData); + QCOMPARE(result.stdOut, m_asciiTestData); + QVERIFY(result.stdErr.isEmpty()); + + const DeviceShell::RunResult result2 = shell.outputForRunInShell( + {"cat", {"/tmp/i-do-not-exist.none"}}); + QVERIFY(!result2.stdErr.isEmpty()); + } +}; + +QTEST_MAIN(tst_DeviceShell) + +#include "tst_deviceshell.moc" diff --git a/tests/auto/utils/utils.qbs b/tests/auto/utils/utils.qbs index 8960168fc74..5237f24a19e 100644 --- a/tests/auto/utils/utils.qbs +++ b/tests/auto/utils/utils.qbs @@ -14,5 +14,6 @@ Project { "templateengine/templateengine.qbs", "treemodel/treemodel.qbs", "multicursor/multicursor.qbs", + "deviceshell/deviceshell.qbs", ] } diff --git a/tests/manual/CMakeLists.txt b/tests/manual/CMakeLists.txt index 48bd9648fac..eecadda01f7 100644 --- a/tests/manual/CMakeLists.txt +++ b/tests/manual/CMakeLists.txt @@ -17,3 +17,4 @@ add_subdirectory(proparser) # add_subdirectory(search) add_subdirectory(shootout) add_subdirectory(widgets) +add_subdirectory(deviceshell) diff --git a/tests/manual/deviceshell/CMakeLists.txt b/tests/manual/deviceshell/CMakeLists.txt new file mode 100644 index 00000000000..39616223488 --- /dev/null +++ b/tests/manual/deviceshell/CMakeLists.txt @@ -0,0 +1,10 @@ +file(RELATIVE_PATH RELATIVE_TEST_PATH "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_LIBEXEC_PATH}") + +add_qtc_test(tst_manual_deviceshell + MANUALTEST + DEFINES "TEST_RELATIVE_LIBEXEC_PATH=\"${TEST_RELATIVE_LIBEXEC_PATH}\"" + DEPENDS Utils + SOURCES + tst_deviceshell.cpp +) diff --git a/tests/manual/deviceshell/deviceshell.qbs b/tests/manual/deviceshell/deviceshell.qbs new file mode 100644 index 00000000000..9a4409793fe --- /dev/null +++ b/tests/manual/deviceshell/deviceshell.qbs @@ -0,0 +1,22 @@ +import qbs.FileInfo + +Project { + QtcManualtest { + name: "DeviceShell manualtest" + + Depends { name: "Utils" } + Depends { name: "app_version_header" } + + files: [ + "tst_deviceshell.cpp", + ] + cpp.defines: { + var defines = base; + var absLibExecPath = FileInfo.joinPaths(qbs.installRoot, qbs.installPrefix, + qtc.ide_libexec_path); + var relLibExecPath = FileInfo.relativePath(destinationDirectory, absLibExecPath); + defines.push('TEST_RELATIVE_LIBEXEC_PATH="' + relLibExecPath + '"'); + return defines; + } + } +} diff --git a/tests/manual/deviceshell/tst_deviceshell.cpp b/tests/manual/deviceshell/tst_deviceshell.cpp new file mode 100644 index 00000000000..f9ab9b4e8a3 --- /dev/null +++ b/tests/manual/deviceshell/tst_deviceshell.cpp @@ -0,0 +1,228 @@ +/**************************************************************************** +** +** Copyright (C) 2022 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace Utils; + +class TestShell : public DeviceShell +{ +public: + TestShell() { start(); } + + static CommandLine cmdLine() { + static CommandLine cmd; + + if (cmd.isEmpty()) { + const FilePath dockerExecutable = Environment::systemEnvironment() + .searchInPath("docker", {"/usr/local/bin"}); + const FilePath dashExecutable = Environment::systemEnvironment() + .searchInPath("dash", {"/usr/local/bin"}); + const FilePath bashExecutable = Environment::systemEnvironment() + .searchInPath("bash", {"/usr/local/bin"}); + const FilePath shExecutable = Environment::systemEnvironment() + .searchInPath("sh", {"/usr/local/bin"}); + + if (dockerExecutable.exists()) { + cmd = {dockerExecutable, {"run", "-i", "--rm","alpine"}}; + } else if (dashExecutable.exists()) { + cmd = {dashExecutable, {}}; + } else if (bashExecutable.exists()) { + cmd = {bashExecutable, {}}; + } else if (shExecutable.exists()) { + cmd = {shExecutable, {}}; + } + + if (cmd.isEmpty()) { + return cmd; + } + + qDebug() << "Using shell cmd:" << cmd; + } + + return cmd; + } + +private: + void setupShellProcess(QtcProcess *shellProcess) override + { + shellProcess->setCommand(cmdLine()); + } +}; + +class tst_DeviceShell : public QObject +{ + Q_OBJECT + + QList testArrays(const int numArrays) + { + QRandomGenerator generator; + QList result; + + for (int i = 0; i < numArrays; i++) { + QByteArray data; + auto numLines = generator.bounded(1, 100); + for (int l = 0; l < numLines; l++) { + auto numChars = generator.bounded(10, 40); + for (int c = 0; c < numChars; c++) { + data += static_cast(generator.bounded('a', 'z')); + } + data += '\n'; + } + result.append(data); + } + return result; + } + + void test(int maxNumThreads, int numCalls) + { + TestShell shell; + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + QThreadPool::globalInstance()->setMaxThreadCount(maxNumThreads); + + QList testArray = testArrays(numCalls); + + QElapsedTimer t; + t.start(); + + const QList result + = mapped(testArray, [&shell](QByteArray data) -> QByteArray { + return shell.outputForRunInShell({"cat", {}}, data).stdOut; + }, MapReduceOption::Ordered, QThreadPool::globalInstance()); + + QCOMPARE(result, testArray); + + qDebug() << "maxThreads:" << maxNumThreads << ", took:" << t.elapsed() / 1000.0 + << "seconds"; + } + + void testSleep(QList testData, int nThreads) + { + TestShell shell; + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + QThreadPool::globalInstance()->setMaxThreadCount(nThreads); + + QElapsedTimer t; + t.start(); + + const auto result = mapped(testData, [&shell](const int &time) { + shell.runInShell({"sleep", {QString("%1").arg(time)}}); + return 0; + }, MapReduceOption::Unordered, QThreadPool::globalInstance()); + + qDebug() << "maxThreads:" << nThreads << ", took:" << t.elapsed() / 1000.0 << "seconds"; + } + +private slots: + void initTestCase() + { + TemporaryDirectory::setMasterTemporaryDirectory( + QDir::tempPath() + "/" + Core::Constants::IDE_CASED_ID + "-XXXXXX"); + + const QString libExecPath(qApp->applicationDirPath() + '/' + + QLatin1String(TEST_RELATIVE_LIBEXEC_PATH)); + LauncherInterface::setPathToLauncher(libExecPath); + + if (TestShell::cmdLine().isEmpty()) { + QSKIP("Skipping deviceshell tests, as no compatible shell could be found"); + } + } + + void cleanupTestCase() { Singleton::deleteAll(); } + + void testEncoding_data() + { + QTest::addColumn("utf8string"); + + QTest::newRow("japanese") << QString::fromUtf8( + u8"\xe8\xac\x9d\xe3\x81\x8d\xe3\x82\x81\xe9\x80\x80\x31\x30\xe8\x89\xaf\xe3\x81\x9a\xe3" + u8"\x82\xa4\xe3\x81\xb5\xe3\x81\x8b\xe7\x89\x88\xe8\x84\xb3\xe3\x83\xa9\xe3\x83\xaf\xe6" + u8"\xad\xa2\xe9\x80\x9a\xe3\x83\xa8\xe3\x83\xb2\xe3\x82\xad\n"); + QTest::newRow("german") << QString::fromUtf8( + u8"\x48\x61\x6c\x6c\xc3\xb6\x2c\x20\x77\x69\x65\x20\x67\xc3\xa4\x68\x74\x20\x65\x73\x20" + u8"\x64\xc3\xbc\x72\n"); + } + + void testEncoding() + { + QFETCH(QString, utf8string); + + TestShell shell; + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + const DeviceShell::RunResult r = shell.outputForRunInShell({"cat", {}}, utf8string.toUtf8()); + const QString output = QString::fromUtf8(r.stdOut); + QCOMPARE(output, utf8string); + } + + void testThreading_data() + { + QTest::addColumn("numThreads"); + QTest::addColumn("numIterations"); + + QTest::newRow("multi-threaded") << 10 << 1000; + QTest::newRow("single-threaded") << 1 << 1000; + } + + void testThreading() + { + QFETCH(int, numThreads); + QFETCH(int, numIterations); + + test(numThreads, numIterations); + } + + void testSleepMulti() + { + QList testData{4, 7, 10, 3, 1, 10, 3, 3, 5, 4}; + int full = std::accumulate(testData.begin(), testData.end(), 0); + qDebug() << "Testing sleep, full time is:" << full << "seconds"; + QElapsedTimer t; + t.start(); + testSleep(testData, 10); + const int multiThreadRunTime = t.restart(); + testSleep(testData, 1); + const int singleThreadRunTime = t.elapsed(); + QVERIFY(multiThreadRunTime < singleThreadRunTime); + } +}; + +QTEST_MAIN(tst_DeviceShell) + +#include "tst_deviceshell.moc" diff --git a/tests/manual/manual.qbs b/tests/manual/manual.qbs index 717d3745d5c..f36312fcd69 100644 --- a/tests/manual/manual.qbs +++ b/tests/manual/manual.qbs @@ -8,6 +8,7 @@ Project { references: [ "debugger/gui/gui.qbs", "debugger/simple/simple.qbs", + "deviceshell/deviceshell.qbs", "fakevim/fakevim.qbs", "pluginview/pluginview.qbs", "process/process.qbs",