Deviceshell: Base64 not found fallback

If no base64 is installed on the target, the shell script
cannot work. Previously this would lead to the shell functions
being unavailable.

This change adds a fallback path. In case no base64 is found,
the shell will create a process for each run request instead.

This is much slower than the shell script, but acceptable as
a fallback.

Change-Id: I70591d7e610c4e1c3c258a8e4bef354221d05cb9
Reviewed-by: hjk <hjk@qt.io>

src/libs/utils/deviceshell.cpp

@@ -137,12 +137,6 @@ cleanup()
     exit 1
 }
 
-if [ -z "$(which base64)" ]
-then
-    echo "base64 command could not be found" >&2
-    exit 1
-fi
-
 trap cleanup 1 2 3 6
 
 echo SCRIPT_INSTALLED >&2
@@ -159,7 +153,7 @@ done) > $FINAL_OUT
 
 DeviceShell::DeviceShell()
 {
-    m_thread.setObjectName("Shell Thread");
+    m_thread.setObjectName("DeviceShell");
     m_thread.start();
 }
 
@@ -213,8 +207,26 @@ DeviceShell::RunResult DeviceShell::outputForRunInShell(const CommandLine &cmd,
 
 DeviceShell::State DeviceShell::state() const { return m_shellScriptState; }
 
+QStringList DeviceShell::missingFeatures() const { return m_missingFeatures; }
+
 DeviceShell::RunResult DeviceShell::run(const CommandLine &cmd, const QByteArray &stdInData)
 {
+    if (m_shellScriptState == State::NoScript) {
+        // Fallback ...
+        QtcProcess proc;
+        proc.setCommand(createFallbackCommand(cmd));
+        proc.setWriteData(stdInData);
+
+        proc.start();
+        proc.waitForFinished();
+
+        return RunResult{
+            proc.exitCode(),
+            proc.readAllStandardOutput(),
+            proc.readAllStandardError()
+        };
+    }
+
     const RunResult errorResult{-1, {}, {}};
     QTC_ASSERT(m_shellProcess, return errorResult);
     QTC_ASSERT(m_shellScriptState == State::Succeeded, return errorResult);
@@ -262,6 +274,18 @@ void DeviceShell::setupShellProcess(QtcProcess *shellProcess)
     shellProcess->setCommand(CommandLine{"bash"});
 }
 
+/*!
+* \brief DeviceShell::createFallbackCommand
+* \param cmd The command to run
+* \return The command to run in case the shell script is not available
+*
+* Creates a command to run in case the shell script is not available
+*/
+CommandLine DeviceShell::createFallbackCommand(const CommandLine &cmd)
+{
+    return cmd;
+}
+
 /*!
  * \brief DeviceShell::startupFailed
  *
@@ -297,6 +321,7 @@ bool DeviceShell::start()
     QMetaObject::invokeMethod(
         m_shellProcess,
         [this] {
+            qCDebug(deviceShellLog) << "Starting shell process:" << m_shellProcess->commandLine().toUserOutput();
             m_shellProcess->start();
 
             if (!m_shellProcess->waitForStarted()) {
@@ -304,26 +329,18 @@ bool DeviceShell::start()
                 return false;
             }
 
+            if (!installShellScript()) {
+                if (m_shellScriptState == State::FailedToStart)
+                    closeShellProcess();
+            } else {
                 connect(m_shellProcess, &QtcProcess::readyReadStandardOutput, m_shellProcess, [this] {
                     onReadyRead();
                 });
                 connect(m_shellProcess, &QtcProcess::readyReadStandardError, m_shellProcess, [this] {
                     const QByteArray stdErr = m_shellProcess->readAllStandardError();
-
-                if (m_shellScriptState == State::Unknown) {
-                    if (stdErr.contains("SCRIPT_INSTALLED")) {
-                        m_shellScriptState = State::Succeeded;
-                        return;
-                    }
-                    if (stdErr.contains("ERROR_INSTALL_SCRIPT")) {
-                        m_shellScriptState = State::FailedToStart;
-                        qCWarning(deviceShellLog) << "Failed installing device shell script";
-                        return;
-                    }
-                }
-
                     qCWarning(deviceShellLog) << "Received unexpected output on stderr:" << stdErr;
                 });
+            }
 
             connect(m_shellProcess, &QtcProcess::done, m_shellProcess, [this] {
                 if (m_shellProcess->resultData().m_exitCode != EXIT_SUCCESS
@@ -334,11 +351,6 @@ bool DeviceShell::start()
                 }
             });
 
-            if (!installShellScript()) {
-                closeShellProcess();
-                return false;
-            }
-
             return true;
         },
         Qt::BlockingQueuedConnection,
@@ -351,23 +363,61 @@ bool DeviceShell::start()
     return result;
 }
 
+bool DeviceShell::checkCommand(const QByteArray &command)
+{
+    const QByteArray checkBase64Cmd = "(which base64 || echo '<missing>')\n";
+
+    m_shellProcess->writeRaw(checkBase64Cmd);
+    if (!m_shellProcess->waitForReadyRead()) {
+        qCWarning(deviceShellLog) << "Timeout while trying to check for" << command;
+        return false;
+    }
+    QByteArray out = m_shellProcess->readAllStandardOutput();
+    if (out.contains("<missing>")) {
+        m_shellScriptState = State::NoScript;
+        qCWarning(deviceShellLog) << "Command" << command << "was not found";
+        m_missingFeatures.append(QString::fromUtf8(command));
+        return false;
+    }
+
+    return true;
+}
+
 bool DeviceShell::installShellScript()
 {
-    const QByteArray runScriptCmd = "scriptData=$(echo "
+    if (!checkCommand("base64")) {
-            + QByteArray(r_execScript.begin(), r_execScript.size()).toBase64()
+        m_shellScriptState = State::NoScript;
-            + " | base64 -d 2>/dev/null ) && /bin/sh -c \"$scriptData\" || echo ERROR_INSTALL_SCRIPT >&2\n";
+        return false;
+    }
 
-    qCDebug(deviceShellLog) << "Install shell script command:" << runScriptCmd;
+    const static QByteArray shellScriptBase64
-    m_shellProcess->writeRaw(runScriptCmd);
+        = QByteArray(r_execScript.begin(), r_execScript.size()).toBase64();
+    const QByteArray scriptCmd = "(scriptData=$(echo " + shellScriptBase64
+                                 + " | base64 -d 2>/dev/null ) && /bin/sh -c \"$scriptData\") || "
+                                   "echo ERROR_INSTALL_SCRIPT >&2\n";
+
+    qCDebug(deviceShellLog) << "Installing shell script:" << scriptCmd;
+    m_shellProcess->writeRaw(scriptCmd);
 
     while (m_shellScriptState == State::Unknown) {
-        if (!m_shellProcess->waitForReadyRead()) {
+        if (!m_shellProcess->waitForReadyRead(5000)) {
-            qCWarning(deviceShellLog) << "Timeout while waiting for device shell script to install";
+            qCWarning(deviceShellLog) << "Timeout while waiting for shell script installation";
-            m_shellScriptState = State::FailedToStart;
+            return false;
+        }
+
+        QByteArray out = m_shellProcess->readAllStandardError();
+        if (out.contains("SCRIPT_INSTALLED")) {
+            m_shellScriptState = State::Succeeded;
+            return true;
+        }
+        if (out.contains("ERROR_INSTALL_SCRIPT")) {
+            m_shellScriptState = State::NoScript;
+            qCWarning(deviceShellLog) << "Failed installing device shell script";
             return false;
         }
     }
-    return m_shellScriptState == State::Succeeded;
+
+    return true;
 }
 
 void DeviceShell::closeShellProcess()

src/libs/utils/deviceshell.h

@@ -24,7 +24,7 @@ class QTCREATOR_UTILS_EXPORT DeviceShell : public QObject
     Q_OBJECT
 
 public:
-    enum class State { FailedToStart = -1, Unknown = 0, Succeeded = 1 };
+    enum class State { FailedToStart = -1, Unknown = 0, Succeeded = 1, NoScript = 2 };
 
     struct RunResult
     {
@@ -49,6 +49,8 @@ public:
 
     State state() const;
 
+    QStringList missingFeatures() const;
+
 signals:
     void done(const ProcessResultData &resultData);
 
@@ -60,12 +62,15 @@ protected:
 
 private:
     virtual void setupShellProcess(QtcProcess *shellProcess);
+    virtual CommandLine createFallbackCommand(const CommandLine &cmdLine);
 
     bool installShellScript();
     void closeShellProcess();
 
     void onReadyRead();
 
+    bool checkCommand(const QByteArray &command);
+
 private:
     struct CommandRun : public RunResult
     {
@@ -82,6 +87,7 @@ private:
     QByteArray m_commandBuffer;
 
     State m_shellScriptState = State::Unknown;
+    QStringList m_missingFeatures;
 };
 
 } // namespace Utils

src/plugins/docker/dockerdevice.cpp

@@ -85,9 +85,10 @@ static Q_LOGGING_CATEGORY(dockerDeviceLog, "qtc.docker.device", QtWarningMsg);
 class ContainerShell : public Utils::DeviceShell
 {
 public:
-    ContainerShell(DockerSettings *settings, const QString &containerId)
+    ContainerShell(DockerSettings *settings, const QString &containerId, const FilePath &devicePath)
         : m_settings(settings)
         , m_containerId(containerId)
+        , m_devicePath(devicePath)
     {
     }
 
@@ -97,9 +98,17 @@ private:
         shellProcess->setCommand({m_settings->dockerBinaryPath.filePath(), {"container", "start", "-i", "-a", m_containerId}});
     }
 
+    CommandLine createFallbackCommand(const CommandLine &cmdLine)
+    {
+        CommandLine result = cmdLine;
+        result.setExecutable(cmdLine.executable().onDevice(m_devicePath));
+        return result;
+    }
+
 private:
     DockerSettings *m_settings;
     QString m_containerId;
+    FilePath m_devicePath;
 };
 
 class DockerDevicePrivate : public QObject
@@ -118,6 +127,7 @@ public:
 
     void updateContainerAccess();
 
+    bool createContainer();
     void startContainer();
     void stopCurrentContainer();
     void fetchSystemEnviroment();
@@ -199,11 +209,12 @@ DockerProcessImpl::DockerProcessImpl(DockerDevicePrivate *device)
             QByteArray output = m_process.readAllStandardOutput();
             qsizetype idx = output.indexOf('\n');
             QByteArray firstLine = output.left(idx);
-            QByteArray rest = output.mid(idx+1);
+            QByteArray rest = output.mid(idx + 1);
-            qCDebug(dockerDeviceLog) << "Process first line received:" << m_process.commandLine() << firstLine;
+            qCDebug(dockerDeviceLog)
+                << "Process first line received:" << m_process.commandLine() << firstLine;
             if (firstLine.startsWith("__qtc")) {
                 bool ok = false;
-                m_remotePID = firstLine.mid(5, firstLine.size() -5 -5).toLongLong(&ok);
+                m_remotePID = firstLine.mid(5, firstLine.size() - 5 - 5).toLongLong(&ok);
 
                 if (ok)
                     emit started(m_remotePID);
@@ -223,10 +234,10 @@ DockerProcessImpl::DockerProcessImpl(DockerDevicePrivate *device)
     });
 
     connect(&m_process, &QtcProcess::done, this, [this] {
-        qCDebug(dockerDeviceLog) << "Process exited:" << m_process.commandLine() << "with code:" << m_process.resultData().m_exitCode;
+        qCDebug(dockerDeviceLog) << "Process exited:" << m_process.commandLine()
+                                 << "with code:" << m_process.resultData().m_exitCode;
         emit done(m_process.resultData());
     });
-
 }
 
 DockerProcessImpl::~DockerProcessImpl()
@@ -391,19 +402,23 @@ static QString getLocalIPv4Address()
     return QString();
 }
 
-void DockerDevicePrivate::startContainer()
+bool DockerDevicePrivate::createContainer()
 {
     if (!m_settings)
-        return;
+        return false;
 
     const QString display = HostOsInfo::isLinuxHost() ? QString(":0")
                                                       : QString(getLocalIPv4Address() + ":0.0");
-    CommandLine dockerCreate{m_settings->dockerBinaryPath.filePath(), {"create",
+    CommandLine dockerCreate{m_settings->dockerBinaryPath.filePath(),
+                             {"create",
                               "-i",
                               "--rm",
-                                        "-e", QString("DISPLAY=%1").arg(display),
+                              "-e",
-                                        "-e", "XAUTHORITY=/.Xauthority",
+                              QString("DISPLAY=%1").arg(display),
-                                        "--net", "host"}};
+                              "-e",
+                              "XAUTHORITY=/.Xauthority",
+                              "--net",
+                              "host"}};
 
 #ifdef Q_OS_UNIX
     // no getuid() and getgid() on Windows.
@@ -423,7 +438,7 @@ void DockerDevicePrivate::startContainer()
 
     dockerCreate.addArgs({"--entrypoint", "/bin/sh", m_data.repoAndTag()});
 
-    LOG("RUNNING: " << dockerCreate.toUserOutput());
+    qCDebug(dockerDeviceLog) << "RUNNING: " << dockerCreate.toUserOutput();
     QtcProcess createProcess;
     createProcess.setCommand(dockerCreate);
     createProcess.runBlocking();
@@ -432,16 +447,29 @@ void DockerDevicePrivate::startContainer()
         qCWarning(dockerDeviceLog) << "Failed creating docker container:";
         qCWarning(dockerDeviceLog) << "Exit Code:" << createProcess.exitCode();
         qCWarning(dockerDeviceLog) << createProcess.allOutput();
-        return;
+        return false;
     }
 
     m_container = createProcess.cleanedStdOut().trimmed();
     if (m_container.isEmpty())
-        return;
+        return false;
-    LOG("Container via process: " << m_container);
 
-    m_shell = std::make_unique<ContainerShell>(m_settings, m_container);
+    LOG("ContainerId: " << m_container);
-    connect(m_shell.get(), &DeviceShell::done, this, [this] (const ProcessResultData &resultData) {
+    return true;
+}
+
+void DockerDevicePrivate::startContainer()
+{
+    if (!createContainer())
+        return;
+
+    m_shell = std::make_unique<ContainerShell>(m_settings,
+                                               m_container,
+                                               FilePath::fromString(
+                                                   QString("device://%1/")
+                                                       .arg(this->q->id().toString())));
+
+    connect(m_shell.get(), &DeviceShell::done, this, [this](const ProcessResultData &resultData) {
         if (resultData.m_error != QProcess::UnknownError)
             return;
 
@@ -977,7 +1005,7 @@ void DockerDevice::aboutToBeRemoved() const
 
 void DockerDevicePrivate::fetchSystemEnviroment()
 {
-    if (m_shell) {
+    if (m_shell && m_shell->state() == DeviceShell::State::Succeeded) {
         const QByteArray output = outputForRunInShell({"env", {}});
         const QString out = QString::fromUtf8(output.data(), output.size());
         m_cachedEnviroment = Environment(out.split('\n', Qt::SkipEmptyParts), q->osType());

src/plugins/remotelinux/linuxdevice.cpp

@@ -751,8 +751,9 @@ class ShellThreadHandler : public QObject
     class LinuxDeviceShell : public DeviceShell
     {
     public:
-        LinuxDeviceShell(const CommandLine &cmdLine)
+        LinuxDeviceShell(const CommandLine &cmdLine, const FilePath &devicePath)
             : m_cmdLine(cmdLine)
+            , m_devicePath(devicePath)
         {
         }
 
@@ -763,8 +764,16 @@ class ShellThreadHandler : public QObject
             shellProcess->setCommand(m_cmdLine);
         }
 
+        CommandLine createFallbackCommand(const CommandLine &cmdLine) override
+        {
+            CommandLine result = cmdLine;
+            result.setExecutable(cmdLine.executable().onDevice(m_devicePath));
+            return result;
+        }
+
     private:
         const CommandLine m_cmdLine;
+        const FilePath m_devicePath;
     };
 
 public:
@@ -792,7 +801,7 @@ public:
                     << m_displaylessSshParameters.host());
         cmd.addArg("/bin/sh");
 
-        m_shell.reset(new LinuxDeviceShell(cmd));
+        m_shell.reset(new LinuxDeviceShell(cmd, FilePath::fromString(QString("ssh://%1/").arg(parameters.userAtHost()))));
         connect(m_shell.get(), &DeviceShell::done, this, [this] { m_shell.reset(); });
         return m_shell->start();
     }
@@ -1052,6 +1061,7 @@ Environment LinuxDevice::systemEnvironment() const
 LinuxDevicePrivate::LinuxDevicePrivate(LinuxDevice *parent)
     : q(parent)
 {
+    m_shellThread.setObjectName("LinuxDeviceShell");
     m_handler = new ShellThreadHandler();
     m_handler->moveToThread(&m_shellThread);
     QObject::connect(&m_shellThread, &QThread::finished, m_handler, &QObject::deleteLater);
@@ -1091,11 +1101,7 @@ bool LinuxDevicePrivate::runInShell(const CommandLine &cmd, const QByteArray &da
     DEBUG(cmd.toUserOutput());
     QTC_ASSERT(setupShell(), return false);
 
-    bool ret = false;
-    QMetaObject::invokeMethod(m_handler, [this, &cmd, &data] {
     return m_handler->runInShell(cmd, data);
-    }, Qt::BlockingQueuedConnection, &ret);
-    return ret;
 }
 
 QByteArray LinuxDevicePrivate::outputForRunInShell(const CommandLine &cmd)
@@ -1104,20 +1110,20 @@ QByteArray LinuxDevicePrivate::outputForRunInShell(const CommandLine &cmd)
     DEBUG(cmd);
     QTC_ASSERT(setupShell(), return {});
 
-    QByteArray ret;
-    QMetaObject::invokeMethod(m_handler, [this, &cmd] {
     return m_handler->outputForRunInShell(cmd);
-    }, Qt::BlockingQueuedConnection, &ret);
-    return ret;
 }
 
 void LinuxDevicePrivate::attachToSharedConnection(SshConnectionHandle *connectionHandle,
                                                   const SshParameters &sshParameters)
 {
     QString socketFilePath;
+
+    Qt::ConnectionType connectionType = QThread::currentThread() == m_handler->thread() ? Qt::DirectConnection : Qt::BlockingQueuedConnection;
+
     QMetaObject::invokeMethod(m_handler, [this, connectionHandle, sshParameters] {
         return m_handler->attachToSharedConnection(connectionHandle, sshParameters);
-    }, Qt::BlockingQueuedConnection, &socketFilePath);
+    }, connectionType, &socketFilePath);
+
     if (!socketFilePath.isEmpty())
         emit connectionHandle->connected(socketFilePath);
 }

tests/auto/utils/qtcprocess/tst_qtcprocess.cpp

@@ -94,6 +94,8 @@ class tst_QtcProcess : public QObject
 private slots:
     void initTestCase();
 
+    void multiRead();
+
     void splitArgs_data();
     void splitArgs();
     void prepareArgs_data();
@@ -211,6 +213,31 @@ Q_DECLARE_METATYPE(ProcessArgs::SplitError)
 Q_DECLARE_METATYPE(Utils::OsType)
 Q_DECLARE_METATYPE(Utils::ProcessResult)
 
+void tst_QtcProcess::multiRead()
+{
+    QByteArray buffer;
+    QtcProcess process;
+
+    process.setCommand({"/bin/sh", {}});
+    process.setProcessChannelMode(QProcess::SeparateChannels);
+    process.setProcessMode(Utils::ProcessMode::Writer);
+
+    process.start();
+    QVERIFY(process.waitForStarted());
+
+    process.writeRaw("echo hi\n");
+
+    QVERIFY(process.waitForReadyRead(1000));
+    buffer = process.readAllStandardOutput();
+    QCOMPARE(buffer, QByteArray("hi\n"));
+
+    process.writeRaw("echo you\n");
+
+    QVERIFY(process.waitForReadyRead(1000));
+    buffer = process.readAllStandardOutput();
+    QCOMPARE(buffer, QByteArray("you\n"));
+}
+
 void tst_QtcProcess::splitArgs_data()
 {
     QTest::addColumn<QString>("in");