diff --git a/src/libs/utils/deviceshell.cpp b/src/libs/utils/deviceshell.cpp index 91ada89a6bd..d46be147096 100644 --- a/src/libs/utils/deviceshell.cpp +++ b/src/libs/utils/deviceshell.cpp @@ -25,8 +25,9 @@ #include "deviceshell.h" -#include -#include +#include "processinterface.h" +#include "qtcassert.h" +#include "qtcprocess.h" #include #include @@ -35,6 +36,123 @@ Q_LOGGING_CATEGORY(deviceShellLog, "qtc.utils.deviceshell", QtWarningMsg) namespace Utils { +namespace { + +/*! + * The multiplex script waits for input via stdin. + * + * To start a command, a message is send with the format " "" \n" + * To stop the script, simply send "exit\n" via stdin + * + * Once a message is received, two new streams are created that the new process redirects its output to ( $stdoutraw and $stderrraw ). + * + * These streams are piped through base64 into the two streams stdoutenc and stderrenc. + * + * Two subshells read from these base64 encoded streams, and prepend the command-id, as well as either "O:" or "E:" depending on whether its the stdout or stderr stream. + * + * Once the process exits its exit code is send to stdout with the command-id and the type "R". + * + */ +const QLatin1String r_execScript = QLatin1String(R"( +#!/bin/sh + +readAndMark() { + local buffer + while read buffer + do + printf '%s:%s:%s\n' "$1" "$2" "$buffer" + done +} + +base64decode() +{ + base64 -d 2>/dev/null +} + +base64encode() +{ + base64 2>/dev/null +} + +executeAndMark() +{ + PID="$1" + INDATA="$2" + shift + shift + CMD="$@" + + # Output Streams + stdoutenc=$(mktemp -u) + stderrenc=$(mktemp -u) + mkfifo "$stdoutenc" "$stderrenc" + + # app output streams + stdoutraw=$(mktemp -u) + stderrraw=$(mktemp -u) + mkfifo "$stdoutraw" "$stderrraw" + + # Cleanup + trap 'rm -f "$stdoutenc" "$stderrenc" "$stdoutraw" "$stderrraw" ' EXIT + + # Pipe all app output through base64, and then into the output streams + cat $stdoutraw | base64encode > "$stdoutenc" & + cat $stderrraw | base64encode > "$stderrenc" & + + # Mark the app's output streams + readAndMark $PID 'O' < "$stdoutenc" >&1 & + readAndMark $PID 'E' < "$stderrenc" >&1 & + + # Start the app ... + if [ -z "$INDATA" ] + then + eval $CMD 1> "$stdoutraw" 2> "$stderrraw" + else + echo $INDATA | base64decode | eval "$CMD" 1> "$stdoutraw" 2> "$stderrraw" + fi + + exitcode=$(echo $? | base64encode) + + wait + echo "$PID:R:$exitcode" +} + +execute() +{ + PID="$1" + INDATA=$(eval echo "$2") + shift + shift + CMD=$@ + executeAndMark $PID "$INDATA" "$CMD" +} + +cleanup() +{ + kill -- -$$ + exit 1 +} + +if ! command -v base64 &> /dev/null +then + echo "base64 command could not be found" >&2 + exit 1 +fi + +trap cleanup 1 2 3 6 + +echo SCRIPT_INSTALLED >&2 + +while read -r id inData cmd; do + if [ "$id" = "exit" ]; then + exit + fi + execute $id $inData $cmd & +done +)"); + +} // namespace + DeviceShell::DeviceShell() { m_thread.setObjectName("Shell Thread"); @@ -43,26 +161,14 @@ DeviceShell::DeviceShell() DeviceShell::~DeviceShell() { + m_shellProcess->deleteLater(); + if (m_thread.isRunning()) { m_thread.quit(); m_thread.wait(); } } -bool DeviceShell::waitForStarted() -{ - QTC_ASSERT(m_shellProcess, return false); - Q_ASSERT(QThread::currentThread() != &m_thread); - - bool result; - QMetaObject::invokeMethod( - m_shellProcess, - [this] { return m_shellProcess->waitForStarted(); }, - Qt::BlockingQueuedConnection, - &result); - return result; -} - /*! * \brief DeviceShell::runInShell * \param cmd The command to run @@ -78,48 +184,8 @@ bool DeviceShell::runInShell(const CommandLine &cmd, const QByteArray &stdInData QTC_ASSERT(m_shellProcess, return false); Q_ASSERT(QThread::currentThread() != &m_thread); - bool result = false; - QMetaObject::invokeMethod( - m_shellProcess, - [this, &cmd, &stdInData] { return runInShellImpl(cmd, stdInData); }, - Qt::BlockingQueuedConnection, - &result); - return result; -} - -bool DeviceShell::runInShellImpl(const CommandLine &cmd, const QByteArray &stdInData) -{ - QTC_ASSERT(QThread::currentThread() == &m_thread, return false); - - QTC_ASSERT(m_shellProcess->isRunning(), return false); - QTC_ASSERT(m_shellProcess, return false); - QTC_CHECK(m_shellProcess->readAllStandardOutput().isNull()); // clean possible left-overs - QTC_CHECK(m_shellProcess->readAllStandardError().isNull()); // clean possible left-overs - auto cleanup = qScopeGuard( - [this] { m_shellProcess->readAllStandardOutput(); }); // clean on assert - - QString prefix; - if (!stdInData.isEmpty()) - prefix = "echo '" + QString::fromUtf8(stdInData.toBase64()) + "' | base64 -d | "; - - const QString suffix = " > /dev/null 2>&1\necho $?\n"; - const QString command = prefix + cmd.toUserOutput() + suffix; - - qCDebug(deviceShellLog) << "Running:" << command; - - m_shellProcess->write(command); - m_shellProcess->waitForReadyRead(); - - const QByteArray output = m_shellProcess->readAllStandardOutput(); - - bool ok = false; - const int result = output.toInt(&ok); - - qCInfo(deviceShellLog) << "Run command in shell:" << cmd.toUserOutput() << "result: " << output - << " ==>" << result; - QTC_ASSERT(ok, return false); - - return result == EXIT_SUCCESS; + const RunResult result = run(cmd, stdInData); + return result.exitCode == 0; } /*! @@ -138,59 +204,37 @@ DeviceShell::RunResult DeviceShell::outputForRunInShell(const CommandLine &cmd, QTC_ASSERT(m_shellProcess, return {}); Q_ASSERT(QThread::currentThread() != &m_thread); - RunResult result; - QMetaObject::invokeMethod( - m_shellProcess, - [this, &cmd, &stdInData] { return outputForRunInShellImpl(cmd, stdInData); }, - Qt::BlockingQueuedConnection, - &result); - return result; + return run(cmd, stdInData); } -DeviceShell::RunResult DeviceShell::outputForRunInShellImpl(const CommandLine &cmd, - const QByteArray &stdInData) +DeviceShell::State DeviceShell::state() const { return m_shellScriptState; } + +DeviceShell::RunResult DeviceShell::run(const CommandLine &cmd, const QByteArray &stdInData) { - QTC_ASSERT(QThread::currentThread() == &m_thread, return {}); + const RunResult errorResult{-1, {}, {}}; + QTC_ASSERT(m_shellProcess, return errorResult); + QTC_ASSERT(m_shellScriptState == State::Succeeded, return errorResult); - QTC_ASSERT(m_shellProcess, return {}); - QTC_CHECK(m_shellProcess->readAllStandardOutput().isNull()); // clean possible left-overs - QTC_CHECK(m_shellProcess->readAllStandardError().isNull()); // clean possible left-overs - auto cleanup = qScopeGuard( - [this] { m_shellProcess->readAllStandardOutput(); }); // clean on assert + QMutexLocker lk(&m_commandMutex); - QString prefix; - if (!stdInData.isEmpty()) - prefix = "echo '" + QString::fromUtf8(stdInData.toBase64()) + "' | base64 -d | "; + QWaitCondition waiter; + const int id = ++m_currentId; + const auto it = m_commandOutput.insert(id, CommandRun{-1, {}, {}, &waiter}); - const QString markerCmd = "echo __qtc$?qtc__ 1>&2\n"; - const QString suffix = "\n" + markerCmd; - const QString command = prefix + cmd.toUserOutput() + suffix; + QMetaObject::invokeMethod(m_shellProcess, [this, id, &cmd, &stdInData]() { + const QString command = QString("%1 \"%2\" %3\n") + .arg(id) + .arg(QString::fromLatin1(stdInData.toBase64())) + .arg(cmd.toUserOutput()); + qCDebug(deviceShellLog) << "Running:" << command; + m_shellProcess->writeRaw(command.toUtf8()); + }); - qCDebug(deviceShellLog) << "Running:" << command; - m_shellProcess->write(command); + waiter.wait(&m_commandMutex); - RunResult result; + const RunResult result = *it; + m_commandOutput.erase(it); - while (true) { - m_shellProcess->waitForReadyRead(); - QByteArray stdErr = m_shellProcess->readAllStandardError(); - if (stdErr.endsWith("qtc__\n")) { - QByteArray marker = stdErr.right(stdErr.length() - stdErr.lastIndexOf("__qtc")); - QByteArray exitCodeStr = marker.mid(5, marker.length() - 11); - bool ok = false; - const int exitCode = exitCodeStr.toInt(&ok); - - result.stdOutput = m_shellProcess->readAllStandardOutput(); - result.exitCode = ok ? exitCode : -1; - break; - } - } - - //const QByteArray output = m_shellProcess->readAllStandardOutput(); - qCDebug(deviceShellLog) << "Received output:" << result.stdOutput; - qCInfo(deviceShellLog) << "Run command in shell:" << cmd.toUserOutput() - << "output size:" << result.stdOutput.size() - << "exit code:" << result.exitCode; return result; } @@ -221,7 +265,7 @@ void DeviceShell::setupShellProcess(QtcProcess *shellProcess) */ void DeviceShell::startupFailed(const CommandLine &cmdLine) { - qCDebug(deviceShellLog) << "Failed to start shell via:" << cmdLine.toUserOutput(); + qCWarning(deviceShellLog) << "Failed to start shell via:" << cmdLine.toUserOutput(); } /*! @@ -241,7 +285,6 @@ bool DeviceShell::start() setupShellProcess(m_shellProcess); m_shellProcess->setProcessMode(ProcessMode::Writer); - m_shellProcess->setWriteData("echo\n"); // Moving the process into its own thread ... m_shellProcess->moveToThread(&m_thread); @@ -252,12 +295,46 @@ bool DeviceShell::start() [this] { m_shellProcess->start(); - if (!m_shellProcess->waitForStarted() || !m_shellProcess->waitForReadyRead() - || m_shellProcess->readAllStandardOutput() != "\n") { + if (!m_shellProcess->waitForStarted()) { closeShellProcess(); return false; } - // TODO: Check if necessary tools are available ( e.g. base64, /bin/sh etc. ) + + connect(m_shellProcess, &QtcProcess::readyReadStandardOutput, m_shellProcess, [this] { + onReadyRead(); + }); + connect(m_shellProcess, &QtcProcess::readyReadStandardError, m_shellProcess, [this] { + const QByteArray stdErr = m_shellProcess->readAllStandardError(); + + if (m_shellScriptState == State::Unknown) { + if (stdErr.contains("SCRIPT_INSTALLED")) { + m_shellScriptState = State::Succeeded; + return; + } + if (stdErr.contains("ERROR_INSTALL_SCRIPT")) { + m_shellScriptState = State::FailedToStart; + qCWarning(deviceShellLog) << "Failed installing device shell script"; + return; + } + } + + qCWarning(deviceShellLog) << "Received unexpected output on stderr:" << stdErr; + }); + + connect(m_shellProcess, &QtcProcess::done, m_shellProcess, [this]() { + if (m_shellProcess->resultData().m_exitCode != EXIT_SUCCESS + || m_shellProcess->resultData().m_exitStatus != QProcess::NormalExit) { + qCWarning(deviceShellLog) << "Shell exited with error code:" + << m_shellProcess->resultData().m_exitCode << "(" + << m_shellProcess->exitMessage() << ")"; + } + }); + + if (!installShellScript()) { + closeShellProcess(); + return false; + } + return true; }, Qt::BlockingQueuedConnection, @@ -270,16 +347,158 @@ bool DeviceShell::start() return result; } +bool DeviceShell::installShellScript() +{ + const QString installCmd + = QString("echo %1 | base64 -d > /tmp/shell.sh 2>/dev/null && " + "chmod +x /tmp/shell.sh && " + "/tmp/shell.sh || echo ERROR_INSTALL_SCRIPT >&2\n") + .arg(QString::fromLatin1( + QByteArray(r_execScript.begin(), r_execScript.size()).toBase64())); + + qCDebug(deviceShellLog) << "Install shell script command:" << installCmd; + m_shellProcess->write(installCmd); + + while (m_shellScriptState == State::Unknown) { + if (!m_shellProcess->waitForReadyRead()) { + qCWarning(deviceShellLog) << "Timeout while waiting for device shell script to install"; + m_shellScriptState = State::FailedToStart; + return false; + } + } + return m_shellScriptState == State::Succeeded; +} + void DeviceShell::closeShellProcess() { if (m_shellProcess) { if (m_shellProcess->isRunning()) { - m_shellProcess->write("exit\n"); + m_shellProcess->write("exit\nexit\n"); if (!m_shellProcess->waitForFinished(2000)) m_shellProcess->terminate(); } - m_shellProcess->deleteLater(); } } +QByteArray::const_iterator next(const QByteArray::const_iterator &bufferEnd, + const QByteArray::const_iterator &itCurrent) +{ + for (QByteArray::const_iterator it = itCurrent; it != bufferEnd; ++it) { + if (*it == '\n') + return it; + } + return bufferEnd; +} + +QByteArray byteArrayFromRange(QByteArray::const_iterator itStart, QByteArray::const_iterator itEnd) +{ + return QByteArray(itStart, std::distance(itStart, itEnd)); +} + +QList> parseShellOutput(const QByteArray &data) +{ + auto itStart = data.cbegin(); + const auto itEnd = data.cend(); + + QList> result; + + for (auto it = next(itEnd, itStart); it != itEnd; ++it, itStart = it, it = next(itEnd, it)) { + const QByteArray lineView = byteArrayFromRange(itStart, it); + QTC_ASSERT(lineView.size() > 0, continue); + + const auto pidEnd = lineView.indexOf(':'); + const auto typeEnd = lineView.indexOf(':', pidEnd + 1); + + QTC_ASSERT(pidEnd != -1 && typeEnd != -1, continue); + + bool ok = false; + const QLatin1String sId(lineView.begin(), pidEnd); + const quint64 id = QString(sId).toInt(&ok); + QTC_ASSERT(ok, continue); + + const QByteArray data = byteArrayFromRange(lineView.begin() + typeEnd + 1, lineView.end()); + const QByteArray decoded = QByteArray::fromBase64(data); + + DeviceShell::ParseType t; + char type = lineView.at(typeEnd - 1); + switch (type) { + case 'O': + t = DeviceShell::ParseType::StdOut; + break; + case 'E': + t = DeviceShell::ParseType::StdErr; + break; + case 'R': + t = DeviceShell::ParseType::ExitCode; + break; + default: + QTC_CHECK(false); + continue; + } + + result.append(std::make_tuple(id, t, decoded)); + } + + return result; +} + +/*! + * \brief DeviceShell::onReadyRead + * + * Reads lines coming from the multiplex script. + * + * The format is: "::base64-encoded-text-or-returnvalue" + * The possible 's are: + * O for stdout + * E for stderr + * R for exit code + * + * Multiple O/E messages may be received for a process. Once + * a single "R" is received, the exit code is reported back + * and no further messages from that process are expected. + */ +void DeviceShell::onReadyRead() +{ + m_commandBuffer += m_shellProcess->readAllStandardOutput(); + const qsizetype lastLineEndIndex = m_commandBuffer.lastIndexOf('\n') + 1; + + if (lastLineEndIndex == 0) + return; + + const QByteArray input(m_commandBuffer.cbegin(), lastLineEndIndex); + + const auto result = parseShellOutput(input); + + QMutexLocker lk(&m_commandMutex); + for (const auto &line : result) { + const auto &[cmdId, type, data] = line; + + const auto itCmd = m_commandOutput.find(cmdId); + QTC_ASSERT(itCmd != m_commandOutput.end(), continue); + + switch (type) { + case Utils::DeviceShell::ParseType::StdOut: + itCmd->stdOut.append(data); + break; + case Utils::DeviceShell::ParseType::StdErr: + itCmd->stdErr.append(data); + break; + case Utils::DeviceShell::ParseType::ExitCode: { + bool ok = false; + int exitCode; + exitCode = QString::fromUtf8(data.begin(), data.size()).toInt(&ok); + QTC_ASSERT(ok, exitCode = -1); + itCmd->exitCode = exitCode; + itCmd->waiter->wakeOne(); + break; + } + } + }; + + if (lastLineEndIndex == m_commandBuffer.size()) + m_commandBuffer.clear(); + else + m_commandBuffer = m_commandBuffer.mid(lastLineEndIndex); +} + } // namespace Utils diff --git a/src/libs/utils/deviceshell.h b/src/libs/utils/deviceshell.h index b800712a0b6..8def78f6d3c 100644 --- a/src/libs/utils/deviceshell.h +++ b/src/libs/utils/deviceshell.h @@ -25,8 +25,11 @@ #include "utils_global.h" +#include +#include #include #include +#include #include @@ -42,10 +45,19 @@ class QTCREATOR_UTILS_EXPORT DeviceShell : public QObject Q_OBJECT public: + enum class State { FailedToStart = -1, Unknown = 0, Succeeded = 1 }; + struct RunResult { - int exitCode; - QByteArray stdOutput; + int exitCode = 0; + QByteArray stdOut; + QByteArray stdErr; + }; + + enum class ParseType { + StdOut, + StdErr, + ExitCode, }; DeviceShell(); @@ -54,28 +66,43 @@ public: bool runInShell(const CommandLine &cmd, const QByteArray &stdInData = {}); RunResult outputForRunInShell(const CommandLine &cmd, const QByteArray &stdInData = {}); - bool waitForStarted(); + State state() const; signals: void done(); void errorOccurred(QProcess::ProcessError error); protected: - bool runInShellImpl(const CommandLine &cmd, const QByteArray &stdInData = {}); - RunResult outputForRunInShellImpl(const CommandLine &cmd, const QByteArray &stdInData = {}); + virtual void startupFailed(const CommandLine &cmdLine); + RunResult run(const CommandLine &cmd, const QByteArray &stdInData = {}); bool start(); void close(); private: virtual void setupShellProcess(QtcProcess *shellProcess); - virtual void startupFailed(const CommandLine &cmdLine); + bool installShellScript(); void closeShellProcess(); + void onReadyRead(); + private: + struct CommandRun : public RunResult + { + QWaitCondition *waiter; + }; + QtcProcess *m_shellProcess = nullptr; QThread m_thread; + int m_currentId{0}; + + QMutex m_commandMutex; + // QMap is used here to preserve iterators + QMap m_commandOutput; + QByteArray m_commandBuffer; + + State m_shellScriptState = State::Unknown; }; } // namespace Utils diff --git a/src/plugins/docker/dockerdevice.cpp b/src/plugins/docker/dockerdevice.cpp index eed68feafc0..285b2f4cf6f 100644 --- a/src/plugins/docker/dockerdevice.cpp +++ b/src/plugins/docker/dockerdevice.cpp @@ -135,7 +135,7 @@ public: ~DockerDevicePrivate() { stopCurrentContainer(); } bool runInContainer(const CommandLine &cmd) const; - bool runInShell(const CommandLine &cmd) const; + bool runInShell(const CommandLine &cmd, const QByteArray &stdInData = {}) const; QByteArray outputForRunInShell(const CommandLine &cmd) const; void updateContainerAccess(); @@ -485,7 +485,8 @@ void DockerDevicePrivate::startContainer() "or restart Qt Creator.")); }); - if (!m_shell->waitForStarted()) { + if (m_shell->state() != DeviceShell::State::Succeeded) { + m_shell.reset(); DockerApi::recheckDockerDaemon(); qCWarning(dockerDeviceLog) << "Container shell failed to start"; } @@ -979,28 +980,8 @@ bool DockerDevice::writeFileContents(const FilePath &filePath, const QByteArray QTC_ASSERT(handlesFile(filePath), return {}); updateContainerAccess(); -// This following would be the generic Unix solution. -// But it doesn't pass input. FIXME: Why? -// QtcProcess proc; -// proc.setCommand({"dd", {"of=" + filePath.path()}}); -// proc.setWriteData(data); -// runProcess(proc); -// proc.waitForFinished(); - - TemporaryFile tempFile("dockertransport-XXXXXX"); - tempFile.open(); - tempFile.write(data); - - const QString tempName = tempFile.fileName(); - tempFile.close(); - - CommandLine cmd{"docker", {"cp", tempName, d->m_container + ':' + filePath.path()}}; - - QtcProcess proc; - proc.setCommand(cmd); - proc.runBlocking(); - - return proc.exitCode() == 0; + QTC_ASSERT(handlesFile(filePath), return {}); + return d->runInShell({"dd", {"of=" + filePath.path()}}, data); } Environment DockerDevice::systemEnvironment() const @@ -1058,16 +1039,16 @@ bool DockerDevicePrivate::runInContainer(const CommandLine &cmd) const return exitCode == 0; } -bool DockerDevicePrivate::runInShell(const CommandLine &cmd) const +bool DockerDevicePrivate::runInShell(const CommandLine &cmd, const QByteArray& stdInData) const { QTC_ASSERT(m_shell, return false); - return m_shell->runInShell(cmd); + return m_shell->runInShell(cmd, stdInData); } QByteArray DockerDevicePrivate::outputForRunInShell(const CommandLine &cmd) const { QTC_ASSERT(m_shell.get(), return {}); - return m_shell->outputForRunInShell(cmd).stdOutput; + return m_shell->outputForRunInShell(cmd).stdOut; } // Factory diff --git a/tests/auto/utils/CMakeLists.txt b/tests/auto/utils/CMakeLists.txt index 5680cde7aaf..60dd1cfbccc 100644 --- a/tests/auto/utils/CMakeLists.txt +++ b/tests/auto/utils/CMakeLists.txt @@ -9,3 +9,4 @@ add_subdirectory(stringutils) add_subdirectory(templateengine) add_subdirectory(treemodel) add_subdirectory(multicursor) +add_subdirectory(deviceshell) diff --git a/tests/auto/utils/deviceshell/CMakeLists.txt b/tests/auto/utils/deviceshell/CMakeLists.txt new file mode 100644 index 00000000000..61492bef322 --- /dev/null +++ b/tests/auto/utils/deviceshell/CMakeLists.txt @@ -0,0 +1,8 @@ +file(RELATIVE_PATH RELATIVE_TEST_PATH "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_LIBEXEC_PATH}") + +add_qtc_test(tst_utils_deviceshell + DEFINES "TEST_RELATIVE_LIBEXEC_PATH=\"${TEST_RELATIVE_LIBEXEC_PATH}\"" + DEPENDS Utils + SOURCES tst_deviceshell.cpp +) diff --git a/tests/auto/utils/deviceshell/deviceshell.qbs b/tests/auto/utils/deviceshell/deviceshell.qbs new file mode 100644 index 00000000000..207848f00d8 --- /dev/null +++ b/tests/auto/utils/deviceshell/deviceshell.qbs @@ -0,0 +1,12 @@ +Project { + QtcAutotest { + name: "DeviceShell autotest" + + Depends { name: "Utils" } + Depends { name: "app_version_header" } + + files: [ + "tst_deviceshell.cpp", + ] + } +} diff --git a/tests/auto/utils/deviceshell/tst_deviceshell.cpp b/tests/auto/utils/deviceshell/tst_deviceshell.cpp new file mode 100644 index 00000000000..40181b7d25e --- /dev/null +++ b/tests/auto/utils/deviceshell/tst_deviceshell.cpp @@ -0,0 +1,284 @@ +/**************************************************************************** +** +** Copyright (C) 2022 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace Utils; + +class TestShell : public DeviceShell +{ +public: + TestShell(CommandLine cmdLine) + : m_cmdLine(std::move(cmdLine)) + { + start(); + } + +private: + void setupShellProcess(QtcProcess *shellProcess) override + { + shellProcess->setCommand(m_cmdLine); + } + + CommandLine m_cmdLine; +}; + +bool testDocker(const FilePath &executable) +{ + QtcProcess p; + p.setCommand({executable, {"info"}}); + p.runBlocking(); + return p.result() == ProcessResult::FinishedWithSuccess; +} + +class tst_DeviceShell : public QObject +{ + Q_OBJECT +private: + QByteArray m_asciiTestData{256, Qt::Uninitialized}; + + QList m_availableShells; + bool m_dockerSetupCheckOk{false}; + +private: + QString testString(int length) + { + QRandomGenerator generator; + QString result; + for (int i = 0; i < length; ++i) + result.append(QChar{generator.bounded('a', 'z')}); + + return result; + } + +private slots: + void initTestCase() + { + TemporaryDirectory::setMasterTemporaryDirectory( + QDir::tempPath() + "/" + Core::Constants::IDE_CASED_ID + "-XXXXXX"); + + const QString libExecPath(qApp->applicationDirPath() + '/' + + QLatin1String(TEST_RELATIVE_LIBEXEC_PATH)); + LauncherInterface::setPathToLauncher(libExecPath); + + std::iota(m_asciiTestData.begin(), m_asciiTestData.end(), 0); + + const FilePath dockerExecutable = Environment::systemEnvironment() + .searchInPath("docker", {"/usr/local/bin"}); + + if (dockerExecutable.exists()) { + m_availableShells.append({dockerExecutable, {"run", "-i", "--rm", "alpine"}}); + if (testDocker(dockerExecutable)) { + m_dockerSetupCheckOk = true; + } else { + // On linux, docker needs some post-install steps: https://docs.docker.com/engine/install/linux-postinstall/ + // Also check if you can start a simple container from the command line: "docker run -it alpine". + qWarning() << "Checking docker failed, tests will be skipped."; + } + } + + if (!Utils::HostOsInfo::isWindowsHost()) { + // Windows by default has bash.exe, which does not work unless a working wsl is installed. + // Therefore we only test shells on linux / mac hosts. + const auto shells = {"dash", "bash", "sh", "zsh"}; + + for (const auto &shell : shells) { + const FilePath executable = Environment::systemEnvironment() + .searchInPath(shell, {"/usr/local/bin"}); + if (executable.exists()) + m_availableShells.append({executable, {}}); + } + } + + if (m_availableShells.isEmpty()) { + QSKIP("Skipping deviceshell tests, as no compatible shell could be found"); + } + } + + void cleanupTestCase() { Singleton::deleteAll(); } + + void testArguments_data() + { + QTest::addColumn("cmdLine"); + QTest::addColumn("testData"); + + for (const auto &cmdLine : qAsConst(m_availableShells)) { + QTest::newRow((cmdLine.executable().baseName() + " : simple").toUtf8()) + << cmdLine << "Hallo Welt!"; + QTest::newRow((cmdLine.executable().baseName() + " : japanese").toUtf8()) + << cmdLine + << QString::fromUtf8(u8"\xe8\xac\x9d\xe3\x81\x8d\xe3\x82\x81\xe9\x80\x80\x31\x30" + u8"\xe8\x89\xaf\xe3\x81\x9a\xe3" + u8"\x82\xa4\xe3\x81\xb5\xe3\x81\x8b\xe7\x89\x88\xe8\x84\xb3" + u8"\xe3\x83\xa9\xe3\x83\xaf\xe6" + u8"\xad\xa2\xe9\x80\x9a\xe3\x83\xa8\xe3\x83\xb2\xe3\x82\xad"); + QTest::newRow((cmdLine.executable().baseName() + " : german").toUtf8()) + << cmdLine + << QString::fromUtf8(u8"\x48\x61\x6c\x6c\xc3\xb6\x2c\x20\x77\x69\x65\x20\x67\xc3" + u8"\xa4\x68\x74\x20\x65\x73\x20" + u8"\x64\xc3\xbc\x72"); + + QTest::newRow((cmdLine.executable().baseName() + " : long").toUtf8()) + << cmdLine << testString(4096 * 16); + } + } + + void testArguments() + { + QFETCH(CommandLine, cmdLine); + QFETCH(QString, testData); + + if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) { + QSKIP("Docker was found, but does not seem to be set up correctly, skipping."); + } + + TestShell shell(cmdLine); + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + QRandomGenerator generator; + + const DeviceShell::RunResult result = shell.outputForRunInShell({"echo", {testData}}); + QCOMPARE(result.exitCode, 0); + const QString expected = testData + "\n"; + const QString resultAsUtf8 = QString::fromUtf8(result.stdOut); + QCOMPARE(resultAsUtf8.size(), expected.size()); + QCOMPARE(resultAsUtf8, expected); + } + + void testStdin_data() + { + QTest::addColumn("cmdLine"); + QTest::addColumn("testData"); + + for (const auto &cmdLine : qAsConst(m_availableShells)) { + QTest::newRow((cmdLine.executable().baseName() + " : simple").toUtf8()) + << cmdLine << "Hallo Welt!"; + QTest::newRow((cmdLine.executable().baseName() + " : japanese").toUtf8()) + << cmdLine + << QString::fromUtf8(u8"\xe8\xac\x9d\xe3\x81\x8d\xe3\x82\x81\xe9\x80\x80\x31\x30" + u8"\xe8\x89\xaf\xe3\x81\x9a\xe3" + u8"\x82\xa4\xe3\x81\xb5\xe3\x81\x8b\xe7\x89\x88\xe8\x84\xb3" + u8"\xe3\x83\xa9\xe3\x83\xaf\xe6" + u8"\xad\xa2\xe9\x80\x9a\xe3\x83\xa8\xe3\x83\xb2\xe3\x82\xad"); + QTest::newRow((cmdLine.executable().baseName() + " : german").toUtf8()) + << cmdLine + << QString::fromUtf8(u8"\x48\x61\x6c\x6c\xc3\xb6\x2c\x20\x77\x69\x65\x20\x67\xc3" + u8"\xa4\x68\x74\x20\x65\x73\x20" + u8"\x64\xc3\xbc\x72"); + + QTest::newRow((cmdLine.executable().baseName() + " : long").toUtf8()) + << cmdLine << testString(4096 * 16); + } + } + + void testStdin() + { + QFETCH(CommandLine, cmdLine); + QFETCH(QString, testData); + + if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) { + QSKIP("Docker was found, but does not seem to be set up correctly, skipping."); + } + + TestShell shell(cmdLine); + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + QRandomGenerator generator; + + const DeviceShell::RunResult result = shell.outputForRunInShell({"cat", {}}, testData.toUtf8()); + QCOMPARE(result.exitCode, 0); + const QString resultAsUtf8 = QString::fromUtf8(result.stdOut); + QCOMPARE(resultAsUtf8.size(), testData.size()); + QCOMPARE(resultAsUtf8, testData); + } + + void testAscii_data() + { + QTest::addColumn("cmdLine"); + for (const auto &cmdLine : qAsConst(m_availableShells)) { + QTest::newRow(cmdLine.executable().baseName().toUtf8()) << cmdLine; + } + } + + void testAscii() + { + QFETCH(CommandLine, cmdLine); + + if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) { + QSKIP("Docker was found, but does not seem to be set up correctly, skipping."); + } + + TestShell shell(cmdLine); + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + const DeviceShell::RunResult result = shell.outputForRunInShell({"cat", {}}, + m_asciiTestData); + QCOMPARE(result.stdOut, m_asciiTestData); + } + + void testStdErr_data() + { + QTest::addColumn("cmdLine"); + for (const auto &cmdLine : m_availableShells) { + QTest::newRow(cmdLine.executable().baseName().toUtf8()) << cmdLine; + } + } + + void testStdErr() + { + QFETCH(CommandLine, cmdLine); + + if (cmdLine.executable().toString().contains("docker") && !m_dockerSetupCheckOk) { + QSKIP("Docker was found, but does not seem to be set up correctly, skipping."); + } + + TestShell shell(cmdLine); + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + const DeviceShell::RunResult result = shell.outputForRunInShell({"cat", {}}, + m_asciiTestData); + QCOMPARE(result.stdOut, m_asciiTestData); + QVERIFY(result.stdErr.isEmpty()); + + const DeviceShell::RunResult result2 = shell.outputForRunInShell( + {"cat", {"/tmp/i-do-not-exist.none"}}); + QVERIFY(!result2.stdErr.isEmpty()); + } +}; + +QTEST_MAIN(tst_DeviceShell) + +#include "tst_deviceshell.moc" diff --git a/tests/auto/utils/utils.qbs b/tests/auto/utils/utils.qbs index 8960168fc74..5237f24a19e 100644 --- a/tests/auto/utils/utils.qbs +++ b/tests/auto/utils/utils.qbs @@ -14,5 +14,6 @@ Project { "templateengine/templateengine.qbs", "treemodel/treemodel.qbs", "multicursor/multicursor.qbs", + "deviceshell/deviceshell.qbs", ] } diff --git a/tests/manual/CMakeLists.txt b/tests/manual/CMakeLists.txt index 48bd9648fac..eecadda01f7 100644 --- a/tests/manual/CMakeLists.txt +++ b/tests/manual/CMakeLists.txt @@ -17,3 +17,4 @@ add_subdirectory(proparser) # add_subdirectory(search) add_subdirectory(shootout) add_subdirectory(widgets) +add_subdirectory(deviceshell) diff --git a/tests/manual/deviceshell/CMakeLists.txt b/tests/manual/deviceshell/CMakeLists.txt new file mode 100644 index 00000000000..39616223488 --- /dev/null +++ b/tests/manual/deviceshell/CMakeLists.txt @@ -0,0 +1,10 @@ +file(RELATIVE_PATH RELATIVE_TEST_PATH "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_LIBEXEC_PATH}") + +add_qtc_test(tst_manual_deviceshell + MANUALTEST + DEFINES "TEST_RELATIVE_LIBEXEC_PATH=\"${TEST_RELATIVE_LIBEXEC_PATH}\"" + DEPENDS Utils + SOURCES + tst_deviceshell.cpp +) diff --git a/tests/manual/deviceshell/deviceshell.qbs b/tests/manual/deviceshell/deviceshell.qbs new file mode 100644 index 00000000000..9a4409793fe --- /dev/null +++ b/tests/manual/deviceshell/deviceshell.qbs @@ -0,0 +1,22 @@ +import qbs.FileInfo + +Project { + QtcManualtest { + name: "DeviceShell manualtest" + + Depends { name: "Utils" } + Depends { name: "app_version_header" } + + files: [ + "tst_deviceshell.cpp", + ] + cpp.defines: { + var defines = base; + var absLibExecPath = FileInfo.joinPaths(qbs.installRoot, qbs.installPrefix, + qtc.ide_libexec_path); + var relLibExecPath = FileInfo.relativePath(destinationDirectory, absLibExecPath); + defines.push('TEST_RELATIVE_LIBEXEC_PATH="' + relLibExecPath + '"'); + return defines; + } + } +} diff --git a/tests/manual/deviceshell/tst_deviceshell.cpp b/tests/manual/deviceshell/tst_deviceshell.cpp new file mode 100644 index 00000000000..f9ab9b4e8a3 --- /dev/null +++ b/tests/manual/deviceshell/tst_deviceshell.cpp @@ -0,0 +1,228 @@ +/**************************************************************************** +** +** Copyright (C) 2022 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace Utils; + +class TestShell : public DeviceShell +{ +public: + TestShell() { start(); } + + static CommandLine cmdLine() { + static CommandLine cmd; + + if (cmd.isEmpty()) { + const FilePath dockerExecutable = Environment::systemEnvironment() + .searchInPath("docker", {"/usr/local/bin"}); + const FilePath dashExecutable = Environment::systemEnvironment() + .searchInPath("dash", {"/usr/local/bin"}); + const FilePath bashExecutable = Environment::systemEnvironment() + .searchInPath("bash", {"/usr/local/bin"}); + const FilePath shExecutable = Environment::systemEnvironment() + .searchInPath("sh", {"/usr/local/bin"}); + + if (dockerExecutable.exists()) { + cmd = {dockerExecutable, {"run", "-i", "--rm","alpine"}}; + } else if (dashExecutable.exists()) { + cmd = {dashExecutable, {}}; + } else if (bashExecutable.exists()) { + cmd = {bashExecutable, {}}; + } else if (shExecutable.exists()) { + cmd = {shExecutable, {}}; + } + + if (cmd.isEmpty()) { + return cmd; + } + + qDebug() << "Using shell cmd:" << cmd; + } + + return cmd; + } + +private: + void setupShellProcess(QtcProcess *shellProcess) override + { + shellProcess->setCommand(cmdLine()); + } +}; + +class tst_DeviceShell : public QObject +{ + Q_OBJECT + + QList testArrays(const int numArrays) + { + QRandomGenerator generator; + QList result; + + for (int i = 0; i < numArrays; i++) { + QByteArray data; + auto numLines = generator.bounded(1, 100); + for (int l = 0; l < numLines; l++) { + auto numChars = generator.bounded(10, 40); + for (int c = 0; c < numChars; c++) { + data += static_cast(generator.bounded('a', 'z')); + } + data += '\n'; + } + result.append(data); + } + return result; + } + + void test(int maxNumThreads, int numCalls) + { + TestShell shell; + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + QThreadPool::globalInstance()->setMaxThreadCount(maxNumThreads); + + QList testArray = testArrays(numCalls); + + QElapsedTimer t; + t.start(); + + const QList result + = mapped(testArray, [&shell](QByteArray data) -> QByteArray { + return shell.outputForRunInShell({"cat", {}}, data).stdOut; + }, MapReduceOption::Ordered, QThreadPool::globalInstance()); + + QCOMPARE(result, testArray); + + qDebug() << "maxThreads:" << maxNumThreads << ", took:" << t.elapsed() / 1000.0 + << "seconds"; + } + + void testSleep(QList testData, int nThreads) + { + TestShell shell; + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + QThreadPool::globalInstance()->setMaxThreadCount(nThreads); + + QElapsedTimer t; + t.start(); + + const auto result = mapped(testData, [&shell](const int &time) { + shell.runInShell({"sleep", {QString("%1").arg(time)}}); + return 0; + }, MapReduceOption::Unordered, QThreadPool::globalInstance()); + + qDebug() << "maxThreads:" << nThreads << ", took:" << t.elapsed() / 1000.0 << "seconds"; + } + +private slots: + void initTestCase() + { + TemporaryDirectory::setMasterTemporaryDirectory( + QDir::tempPath() + "/" + Core::Constants::IDE_CASED_ID + "-XXXXXX"); + + const QString libExecPath(qApp->applicationDirPath() + '/' + + QLatin1String(TEST_RELATIVE_LIBEXEC_PATH)); + LauncherInterface::setPathToLauncher(libExecPath); + + if (TestShell::cmdLine().isEmpty()) { + QSKIP("Skipping deviceshell tests, as no compatible shell could be found"); + } + } + + void cleanupTestCase() { Singleton::deleteAll(); } + + void testEncoding_data() + { + QTest::addColumn("utf8string"); + + QTest::newRow("japanese") << QString::fromUtf8( + u8"\xe8\xac\x9d\xe3\x81\x8d\xe3\x82\x81\xe9\x80\x80\x31\x30\xe8\x89\xaf\xe3\x81\x9a\xe3" + u8"\x82\xa4\xe3\x81\xb5\xe3\x81\x8b\xe7\x89\x88\xe8\x84\xb3\xe3\x83\xa9\xe3\x83\xaf\xe6" + u8"\xad\xa2\xe9\x80\x9a\xe3\x83\xa8\xe3\x83\xb2\xe3\x82\xad\n"); + QTest::newRow("german") << QString::fromUtf8( + u8"\x48\x61\x6c\x6c\xc3\xb6\x2c\x20\x77\x69\x65\x20\x67\xc3\xa4\x68\x74\x20\x65\x73\x20" + u8"\x64\xc3\xbc\x72\n"); + } + + void testEncoding() + { + QFETCH(QString, utf8string); + + TestShell shell; + QCOMPARE(shell.state(), DeviceShell::State::Succeeded); + + const DeviceShell::RunResult r = shell.outputForRunInShell({"cat", {}}, utf8string.toUtf8()); + const QString output = QString::fromUtf8(r.stdOut); + QCOMPARE(output, utf8string); + } + + void testThreading_data() + { + QTest::addColumn("numThreads"); + QTest::addColumn("numIterations"); + + QTest::newRow("multi-threaded") << 10 << 1000; + QTest::newRow("single-threaded") << 1 << 1000; + } + + void testThreading() + { + QFETCH(int, numThreads); + QFETCH(int, numIterations); + + test(numThreads, numIterations); + } + + void testSleepMulti() + { + QList testData{4, 7, 10, 3, 1, 10, 3, 3, 5, 4}; + int full = std::accumulate(testData.begin(), testData.end(), 0); + qDebug() << "Testing sleep, full time is:" << full << "seconds"; + QElapsedTimer t; + t.start(); + testSleep(testData, 10); + const int multiThreadRunTime = t.restart(); + testSleep(testData, 1); + const int singleThreadRunTime = t.elapsed(); + QVERIFY(multiThreadRunTime < singleThreadRunTime); + } +}; + +QTEST_MAIN(tst_DeviceShell) + +#include "tst_deviceshell.moc" diff --git a/tests/manual/manual.qbs b/tests/manual/manual.qbs index 717d3745d5c..f36312fcd69 100644 --- a/tests/manual/manual.qbs +++ b/tests/manual/manual.qbs @@ -8,6 +8,7 @@ Project { references: [ "debugger/gui/gui.qbs", "debugger/simple/simple.qbs", + "deviceshell/deviceshell.qbs", "fakevim/fakevim.qbs", "pluginview/pluginview.qbs", "process/process.qbs",