Utils: Merge {synchronous,qtc}process.{h,cpp} file pairs

Mechanical to prepare merging the actual classes.
Adapting #includes.

Change-Id: I77a2c28129287778bc870c30cb890cd26bc2e62b
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

src/libs/utils/qtcprocess.cpp

@@ -24,17 +24,22 @@
 ****************************************************************************/
 
 #include "qtcprocess.h"
-#include "stringutils.h"
 
-#include <utils/qtcassert.h>
-#include <utils/hostosinfo.h>
+#include "stringutils.h"
+#include "executeondestruction.h"
+#include "hostosinfo.h"
+#include "qtcassert.h"
+#include "qtcprocess.h"
 
 #include <QCoreApplication>
-#include <QDir>
 #include <QDebug>
+#include <QDir>
+#include <QLoggingCategory>
 #include <QRegularExpression>
 #include <QStack>
+#include <QTextCodec>
 #include <QThread>
+#include <QTimer>
 
 #ifdef QT_GUI_LIB
 // qmlpuppet does not use that.
@@ -42,6 +47,10 @@
 #include <QMessageBox>
 #endif
 
+#include <algorithm>
+#include <limits.h>
+#include <memory>
+
 #ifdef Q_OS_WIN
 #define CALLBACK WINAPI
 #include <qt_windows.h>
@@ -1845,4 +1854,575 @@ QString QtcProcess::locateBinary(const QString &binary)
     return locateBinary(QString::fromLocal8Bit(path), binary);
 }
 
+
+/*!
+    \class Utils::SynchronousProcess
+
+    \brief The SynchronousProcess class runs a synchronous process in its own
+    event loop that blocks only user input events. Thus, it allows for the GUI to
+    repaint and append output to log windows.
+
+    The stdOut(), stdErr() signals are emitted unbuffered as the process
+    writes them.
+
+    The stdOutBuffered(), stdErrBuffered() signals are emitted with complete
+    lines based on the '\\n' marker if they are enabled using
+    stdOutBufferedSignalsEnabled()/setStdErrBufferedSignalsEnabled().
+    They would typically be used for log windows.
+
+    There is a timeout handling that takes effect after the last data have been
+    read from stdout/stdin (as opposed to waitForFinished(), which measures time
+    since it was invoked). It is thus also suitable for slow processes that
+    continuously output data (like version system operations).
+
+    The property timeOutMessageBoxEnabled influences whether a message box is
+    shown asking the user if they want to kill the process on timeout (default: false).
+
+    There are also static utility functions for dealing with fully synchronous
+    processes, like reading the output with correct timeout handling.
+
+    Caution: This class should NOT be used if there is a chance that the process
+    triggers opening dialog boxes (for example, by file watchers triggering),
+    as this will cause event loop problems.
+*/
+
+enum { debug = 0 };
+enum { syncDebug = 0 };
+
+enum { defaultMaxHangTimerCount = 10 };
+
+static Q_LOGGING_CATEGORY(processLog, "qtc.utils.synchronousprocess", QtWarningMsg);
+
+// ----------- SynchronousProcessResponse
+void SynchronousProcessResponse::clear()
+{
+    result = StartFailed;
+    exitCode = -1;
+    rawStdOut.clear();
+    rawStdErr.clear();
+}
+
+QString SynchronousProcessResponse::exitMessage(const QString &binary, int timeoutS) const
+{
+    switch (result) {
+    case Finished:
+        return SynchronousProcess::tr("The command \"%1\" finished successfully.").arg(QDir::toNativeSeparators(binary));
+    case FinishedError:
+        return SynchronousProcess::tr("The command \"%1\" terminated with exit code %2.").arg(QDir::toNativeSeparators(binary)).arg(exitCode);
+    case TerminatedAbnormally:
+        return SynchronousProcess::tr("The command \"%1\" terminated abnormally.").arg(QDir::toNativeSeparators(binary));
+    case StartFailed:
+        return SynchronousProcess::tr("The command \"%1\" could not be started.").arg(QDir::toNativeSeparators(binary));
+    case Hang:
+        return SynchronousProcess::tr("The command \"%1\" did not respond within the timeout limit (%2 s).")
+                .arg(QDir::toNativeSeparators(binary)).arg(timeoutS);
+    }
+    return QString();
+}
+
+QByteArray SynchronousProcessResponse::allRawOutput() const
+{
+    if (!rawStdOut.isEmpty() && !rawStdErr.isEmpty()) {
+        QByteArray result = rawStdOut;
+        if (!result.endsWith('\n'))
+            result += '\n';
+        result += rawStdErr;
+        return result;
+    }
+    return !rawStdOut.isEmpty() ? rawStdOut : rawStdErr;
+}
+
+QString SynchronousProcessResponse::allOutput() const
+{
+    const QString out = stdOut();
+    const QString err = stdErr();
+
+    if (!out.isEmpty() && !err.isEmpty()) {
+        QString result = out;
+        if (!result.endsWith('\n'))
+            result += '\n';
+        result += err;
+        return result;
+    }
+    return !out.isEmpty() ? out : err;
+}
+
+QString SynchronousProcessResponse::stdOut() const
+{
+    return QtcProcess::normalizeNewlines(codec->toUnicode(rawStdOut));
+}
+
+QString SynchronousProcessResponse::stdErr() const
+{
+    return QtcProcess::normalizeNewlines(codec->toUnicode(rawStdErr));
+}
+
+QTCREATOR_UTILS_EXPORT QDebug operator<<(QDebug str, const SynchronousProcessResponse& r)
+{
+    QDebug nsp = str.nospace();
+    nsp << "SynchronousProcessResponse: result=" << r.result << " ex=" << r.exitCode << '\n'
+        << r.rawStdOut.size() << " bytes stdout, stderr=" << r.rawStdErr << '\n';
+    return str;
+}
+
+SynchronousProcessResponse::Result defaultExitCodeInterpreter(int code)
+{
+    return code ? SynchronousProcessResponse::FinishedError
+                : SynchronousProcessResponse::Finished;
+}
+
+// Data for one channel buffer (stderr/stdout)
+class ChannelBuffer : public QObject
+{
+    Q_OBJECT
+
+public:
+    void clearForRun();
+
+    QString linesRead();
+    void append(const QByteArray &text, bool emitSignals);
+
+    QByteArray rawData;
+    QString incompleteLineBuffer; // lines not yet signaled
+    QTextCodec *codec = nullptr; // Not owner
+    std::unique_ptr<QTextCodec::ConverterState> codecState;
+    int rawDataPos = 0;
+    bool bufferedSignalsEnabled = false;
+    bool firstBuffer = true;
+
+signals:
+    void outputBuffered(const QString &text, bool firstTime);
+};
+
+void ChannelBuffer::clearForRun()
+{
+    firstBuffer = true;
+    rawDataPos = 0;
+    rawData.clear();
+    codecState.reset(new QTextCodec::ConverterState);
+    incompleteLineBuffer.clear();
+}
+
+/* Check for complete lines read from the device and return them, moving the
+ * buffer position. */
+QString ChannelBuffer::linesRead()
+{
+    // Convert and append the new input to the buffer of incomplete lines
+    const char *start = rawData.constData() + rawDataPos;
+    const int len = rawData.size() - rawDataPos;
+    incompleteLineBuffer.append(codec->toUnicode(start, len, codecState.get()));
+    rawDataPos = rawData.size();
+
+    // Any completed lines in the incompleteLineBuffer?
+    const int lastLineIndex = qMax(incompleteLineBuffer.lastIndexOf('\n'),
+                                   incompleteLineBuffer.lastIndexOf('\r'));
+    if (lastLineIndex == -1)
+        return QString();
+
+    // Get completed lines and remove them from the incompleteLinesBuffer:
+    const QString lines = QtcProcess::normalizeNewlines(incompleteLineBuffer.left(lastLineIndex + 1));
+    incompleteLineBuffer = incompleteLineBuffer.mid(lastLineIndex + 1);
+
+    return lines;
+}
+
+void ChannelBuffer::append(const QByteArray &text, bool emitSignals)
+{
+    if (text.isEmpty())
+        return;
+    rawData += text;
+    if (!emitSignals)
+        return;
+
+    // Buffered. Emit complete lines?
+    if (bufferedSignalsEnabled) {
+        const QString lines = linesRead();
+        if (!lines.isEmpty()) {
+            emit outputBuffered(lines, firstBuffer);
+            firstBuffer = false;
+        }
+    }
+}
+
+// ----------- SynchronousProcessPrivate
+class SynchronousProcessPrivate {
+public:
+    void clearForRun();
+
+    QTextCodec *m_codec = QTextCodec::codecForLocale();
+    QtcProcess m_process;
+    QTimer m_timer;
+    QEventLoop m_eventLoop;
+    SynchronousProcessResponse m_result;
+    FilePath m_binary;
+    ChannelBuffer m_stdOut;
+    ChannelBuffer m_stdErr;
+    ExitCodeInterpreter m_exitCodeInterpreter = defaultExitCodeInterpreter;
+
+    int m_hangTimerCount = 0;
+    int m_maxHangTimerCount = defaultMaxHangTimerCount;
+    bool m_startFailure = false;
+    bool m_timeOutMessageBoxEnabled = false;
+    bool m_waitingForUser = false;
+};
+
+void SynchronousProcessPrivate::clearForRun()
+{
+    m_hangTimerCount = 0;
+    m_stdOut.clearForRun();
+    m_stdOut.codec = m_codec;
+    m_stdErr.clearForRun();
+    m_stdErr.codec = m_codec;
+    m_result.clear();
+    m_result.codec = m_codec;
+    m_startFailure = false;
+    m_binary = {};
+}
+
+// ----------- SynchronousProcess
+SynchronousProcess::SynchronousProcess() :
+    d(new SynchronousProcessPrivate)
+{
+    d->m_timer.setInterval(1000);
+    connect(&d->m_timer, &QTimer::timeout, this, &SynchronousProcess::slotTimeout);
+    connect(&d->m_process, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished),
+            this, &SynchronousProcess::finished);
+    connect(&d->m_process, &QProcess::errorOccurred, this, &SynchronousProcess::error);
+    connect(&d->m_process, &QProcess::readyReadStandardOutput,
+            this, [this]() {
+                d->m_hangTimerCount = 0;
+                processStdOut(true);
+            });
+    connect(&d->m_process, &QProcess::readyReadStandardError,
+            this, [this]() {
+                d->m_hangTimerCount = 0;
+                processStdErr(true);
+            });
+    connect(&d->m_stdOut, &ChannelBuffer::outputBuffered, this, &SynchronousProcess::stdOutBuffered);
+    connect(&d->m_stdErr, &ChannelBuffer::outputBuffered, this, &SynchronousProcess::stdErrBuffered);
+}
+
+SynchronousProcess::~SynchronousProcess()
+{
+    disconnect(&d->m_timer, nullptr, this, nullptr);
+    disconnect(&d->m_process, nullptr, this, nullptr);
+    delete d;
+}
+
+void SynchronousProcess::setTimeoutS(int timeoutS)
+{
+    if (timeoutS > 0)
+        d->m_maxHangTimerCount = qMax(2, timeoutS);
+    else
+        d->m_maxHangTimerCount = INT_MAX / 1000;
+}
+
+int SynchronousProcess::timeoutS() const
+{
+    return d->m_maxHangTimerCount == (INT_MAX / 1000) ? -1 : d->m_maxHangTimerCount;
+}
+
+void SynchronousProcess::setCodec(QTextCodec *c)
+{
+    QTC_ASSERT(c, return);
+    d->m_codec = c;
+}
+
+QTextCodec *SynchronousProcess::codec() const
+{
+    return d->m_codec;
+}
+
+bool SynchronousProcess::stdOutBufferedSignalsEnabled() const
+{
+    return d->m_stdOut.bufferedSignalsEnabled;
+}
+
+void SynchronousProcess::setStdOutBufferedSignalsEnabled(bool v)
+{
+    d->m_stdOut.bufferedSignalsEnabled = v;
+}
+
+bool SynchronousProcess::stdErrBufferedSignalsEnabled() const
+{
+    return d->m_stdErr.bufferedSignalsEnabled;
+}
+
+void SynchronousProcess::setStdErrBufferedSignalsEnabled(bool v)
+{
+    d->m_stdErr.bufferedSignalsEnabled = v;
+}
+
+Environment SynchronousProcess::environment() const
+{
+    return d->m_process.environment();
+}
+
+bool SynchronousProcess::timeOutMessageBoxEnabled() const
+{
+    return d->m_timeOutMessageBoxEnabled;
+}
+
+void SynchronousProcess::setTimeOutMessageBoxEnabled(bool v)
+{
+    d->m_timeOutMessageBoxEnabled = v;
+}
+
+void SynchronousProcess::setEnvironment(const Environment &e)
+{
+    d->m_process.setEnvironment(Environment(e));
+}
+
+void SynchronousProcess::setDisableUnixTerminal()
+{
+    d->m_process.setDisableUnixTerminal();
+}
+
+void SynchronousProcess::setExitCodeInterpreter(const ExitCodeInterpreter &interpreter)
+{
+    QTC_ASSERT(interpreter, return);
+    d->m_exitCodeInterpreter = interpreter;
+}
+
+ExitCodeInterpreter SynchronousProcess::exitCodeInterpreter() const
+{
+    return d->m_exitCodeInterpreter;
+}
+
+void SynchronousProcess::setWorkingDirectory(const QString &workingDirectory)
+{
+    d->m_process.setWorkingDirectory(workingDirectory);
+}
+
+QString SynchronousProcess::workingDirectory() const
+{
+    return d->m_process.workingDirectory();
+}
+
+QProcess::ProcessChannelMode SynchronousProcess::processChannelMode () const
+{
+    return d->m_process.processChannelMode();
+}
+
+void SynchronousProcess::setProcessChannelMode(QProcess::ProcessChannelMode m)
+{
+    d->m_process.setProcessChannelMode(m);
+}
+
+#ifdef QT_GUI_LIB
+static bool isGuiThread()
+{
+    return QThread::currentThread() == QCoreApplication::instance()->thread();
+}
+#endif
+
+SynchronousProcessResponse SynchronousProcess::run(const CommandLine &cmd,
+                                                   const QByteArray &writeData)
+{
+    // FIXME: Implement properly
+    if (cmd.executable().needsDevice()) {
+        QtcProcess proc;
+        proc.setEnvironment(Environment(d->m_process.environment()));
+        proc.setWorkingDirectory(d->m_process.workingDirectory());
+        proc.setCommand(cmd);
+
+        // writeData ?
+        proc.start();
+
+        proc.waitForFinished();
+
+        SynchronousProcessResponse res;
+        res.result = SynchronousProcessResponse::Finished;
+        res.exitCode = proc.exitCode();
+        res.rawStdOut = proc.readAllStandardOutput();
+        res.rawStdErr = proc.readAllStandardError();
+        return res;
+    };
+
+    qCDebug(processLog).noquote() << "Starting:" << cmd.toUserOutput();
+    ExecuteOnDestruction logResult([this] {
+        qCDebug(processLog) << d->m_result;
+    });
+
+    d->clearForRun();
+
+    d->m_binary = cmd.executable();
+    // using QProcess::start() and passing program, args and OpenMode results in a different
+    // quoting of arguments than using QProcess::setArguments() beforehand and calling start()
+    // only with the OpenMode
+    d->m_process.setCommand(cmd);
+    if (!writeData.isEmpty()) {
+        connect(&d->m_process, &QProcess::started, this, [this, writeData] {
+            d->m_process.write(writeData);
+            d->m_process.closeWriteChannel();
+        });
+    }
+    d->m_process.setOpenMode(writeData.isEmpty() ? QIODevice::ReadOnly : QIODevice::ReadWrite);
+    d->m_process.start();
+
+    // On Windows, start failure is triggered immediately if the
+    // executable cannot be found in the path. Do not start the
+    // event loop in that case.
+    if (!d->m_startFailure) {
+        d->m_timer.start();
+#ifdef QT_GUI_LIB
+        if (isGuiThread())
+            QApplication::setOverrideCursor(Qt::WaitCursor);
+#endif
+        d->m_eventLoop.exec(QEventLoop::ExcludeUserInputEvents);
+        processStdOut(false);
+        processStdErr(false);
+
+        d->m_result.rawStdOut = d->m_stdOut.rawData;
+        d->m_result.rawStdErr = d->m_stdErr.rawData;
+
+        d->m_timer.stop();
+#ifdef QT_GUI_LIB
+        if (isGuiThread())
+            QApplication::restoreOverrideCursor();
+#endif
+    }
+
+    return  d->m_result;
+}
+
+SynchronousProcessResponse SynchronousProcess::runBlocking(const CommandLine &cmd)
+{
+    // FIXME: Implement properly
+    if (cmd.executable().needsDevice()) {
+        QtcProcess proc;
+        proc.setEnvironment(Environment(d->m_process.environment()));
+        proc.setWorkingDirectory(d->m_process.workingDirectory());
+        proc.setCommand(cmd);
+
+        // writeData ?
+        proc.start();
+
+        proc.waitForFinished();
+
+        SynchronousProcessResponse res;
+        res.result = SynchronousProcessResponse::Finished;
+        res.exitCode = proc.exitCode();
+        res.rawStdOut = proc.readAllStandardOutput();
+        res.rawStdErr = proc.readAllStandardError();
+        return res;
+    };
+
+    qCDebug(processLog).noquote() << "Starting blocking:" << cmd.toUserOutput();
+    ExecuteOnDestruction logResult([this] {
+        qCDebug(processLog) << d->m_result;
+    });
+
+    d->clearForRun();
+
+    d->m_binary = cmd.executable();
+    d->m_process.setOpenMode(QIODevice::ReadOnly);
+    d->m_process.setCommand(cmd);
+    d->m_process.start();
+    if (!d->m_process.waitForStarted(d->m_maxHangTimerCount * 1000)) {
+        d->m_result.result = SynchronousProcessResponse::StartFailed;
+        return d->m_result;
+    }
+    d->m_process.closeWriteChannel();
+    if (!d->m_process.waitForFinished(d->m_maxHangTimerCount * 1000)) {
+        d->m_result.result = SynchronousProcessResponse::Hang;
+        d->m_process.terminate();
+        if (!d->m_process.waitForFinished(1000)) {
+            d->m_process.kill();
+            d->m_process.waitForFinished(1000);
+        }
+    }
+
+    if (d->m_process.state() != QProcess::NotRunning)
+        return d->m_result;
+
+    d->m_result.exitCode = d->m_process.exitCode();
+    if (d->m_result.result == SynchronousProcessResponse::StartFailed) {
+        if (d->m_process.exitStatus() != QProcess::NormalExit)
+            d->m_result.result = SynchronousProcessResponse::TerminatedAbnormally;
+        else
+            d->m_result.result = (exitCodeInterpreter())(d->m_result.exitCode);
+    }
+    processStdOut(false);
+    processStdErr(false);
+
+    d->m_result.rawStdOut = d->m_stdOut.rawData;
+    d->m_result.rawStdErr = d->m_stdErr.rawData;
+
+    return d->m_result;
+}
+
+bool SynchronousProcess::terminate()
+{
+    return d->m_process.stopProcess();
+}
+
+void SynchronousProcess::slotTimeout()
+{
+    if (!d->m_waitingForUser && (++d->m_hangTimerCount > d->m_maxHangTimerCount)) {
+        if (debug)
+            qDebug() << Q_FUNC_INFO << "HANG detected, killing";
+        d->m_waitingForUser = true;
+        const bool terminate = !d->m_timeOutMessageBoxEnabled || askToKill(d->m_binary.toString());
+        d->m_waitingForUser = false;
+        if (terminate) {
+            d->m_process.stopProcess();
+            d->m_result.result = SynchronousProcessResponse::Hang;
+        } else {
+            d->m_hangTimerCount = 0;
+        }
+    } else {
+        if (debug)
+            qDebug() << Q_FUNC_INFO << d->m_hangTimerCount;
+    }
+}
+
+void SynchronousProcess::finished(int exitCode, QProcess::ExitStatus e)
+{
+    if (debug)
+        qDebug() << Q_FUNC_INFO << exitCode << e;
+    d->m_hangTimerCount = 0;
+
+    switch (e) {
+    case QProcess::NormalExit:
+        d->m_result.result = d->m_exitCodeInterpreter(exitCode);
+        d->m_result.exitCode = exitCode;
+        break;
+    case QProcess::CrashExit:
+        // Was hang detected before and killed?
+        if (d->m_result.result != SynchronousProcessResponse::Hang)
+            d->m_result.result = SynchronousProcessResponse::TerminatedAbnormally;
+        d->m_result.exitCode = -1;
+        break;
+    }
+    d->m_eventLoop.quit();
+}
+
+void SynchronousProcess::error(QProcess::ProcessError e)
+{
+    d->m_hangTimerCount = 0;
+    if (debug)
+        qDebug() << Q_FUNC_INFO << e;
+    // Was hang detected before and killed?
+    if (d->m_result.result != SynchronousProcessResponse::Hang)
+        d->m_result.result = SynchronousProcessResponse::StartFailed;
+    d->m_startFailure = true;
+    d->m_eventLoop.quit();
+}
+
+void SynchronousProcess::processStdOut(bool emitSignals)
+{
+    // Handle binary data
+    d->m_stdOut.append(d->m_process.readAllStandardOutput(), emitSignals);
+}
+
+void SynchronousProcess::processStdErr(bool emitSignals)
+{
+    // Handle binary data
+    d->m_stdErr.append(d->m_process.readAllStandardError(), emitSignals);
+}
+
 } // namespace Utils
+
+#include "qtcprocess.moc"
+