Started Console process handling for CDB.

Introduced "Suspend" mode for the process stub and corresponding
mode enumeration in console process (Windows).

src/libs/utils/consoleprocess.cpp

@@ -0,0 +1,86 @@
+/**************************************************************************
+**
+** This file is part of Qt Creator
+**
+** Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+**
+** Contact:  Qt Software Information (qt-info@nokia.com)
+**
+** Commercial Usage
+**
+** Licensees holding valid Qt Commercial licenses may use this file in
+** accordance with the Qt Commercial License Agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Nokia.
+**
+** GNU Lesser General Public License Usage
+**
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** If you are unsure which license is appropriate for your use, please
+** contact the sales department at qt-sales@nokia.com.
+**
+**************************************************************************/
+
+#include "consoleprocess.h"
+
+namespace Core {
+namespace Utils {
+
+QString ConsoleProcess::modeOption(Mode m)
+{
+    switch (m) {
+        case Debug:
+        return QLatin1String("debug");
+        case Suspend:
+        return QLatin1String("suspend");
+        case Run:
+        break;
+    }
+    return QLatin1String("run");
+}
+
+QString ConsoleProcess::msgCommChannelFailed(const QString &error)
+{
+    return tr("Cannot set up communication channel: %1").arg(error);
+}
+
+QString ConsoleProcess::msgPromptToClose()
+{
+    //! Showed in a terminal which might have
+    //! a different character set on Windows.
+    return tr("Press <RETURN> to close this window...");
+}
+
+QString ConsoleProcess::msgCannotCreateTempFile(const QString &why)
+{
+    return tr("Cannot create temporary file: %1").arg(why);
+}
+
+QString ConsoleProcess::msgCannotCreateTempDir(const QString & dir, const QString &why)
+{
+    return tr("Cannot create temporary directory '%1': %2").arg(dir, why);
+}
+
+QString ConsoleProcess::msgUnexpectedOutput()
+{
+    return tr("Unexpected output from helper program.");
+}
+
+QString ConsoleProcess::msgCannotChangeToWorkDir(const QString & dir, const QString &why)
+{
+    return tr("Cannot change to working directory '%1': %2").arg(dir, why);
+}
+
+QString ConsoleProcess::msgCannotExecute(const QString & p, const QString &why)
+{
+    return tr("Cannot execute '%1': %2").arg(p, why);
+}
+
+}
+}

src/libs/utils/consoleprocess.h

@@ -59,14 +59,15 @@ class QWORKBENCH_UTILS_EXPORT ConsoleProcess : public QObject, public AbstractPr
     Q_OBJECT
 
 public:
+    enum Mode { Run, Debug, Suspend };
     ConsoleProcess(QObject *parent = 0);
     ~ConsoleProcess();
 
     bool start(const QString &program, const QStringList &args);
     void stop();
 
-    void setDebug(bool on) { m_debug = on; }
+    void setMode(Mode m) { m_mode = m; }
-    bool isDebug() const { return m_debug; }
+    Mode mode() const { return m_mode; }
 
     bool isRunning() const; // This reflects the state of the console+stub
     qint64 applicationPID() const { return m_appPid; }
@@ -99,6 +100,15 @@ private slots:
 #endif
 
 private:
+    static QString modeOption(Mode m);
+    static QString msgCommChannelFailed(const QString &error);
+    static QString msgPromptToClose();
+    static QString msgCannotCreateTempFile(const QString &why);
+    static QString msgCannotCreateTempDir(const QString & dir, const QString &why);
+    static QString msgUnexpectedOutput();
+    static QString msgCannotChangeToWorkDir(const QString & dir, const QString &why);
+    static QString msgCannotExecute(const QString & p, const QString &why);
+
     QString stubServerListen();
     void stubServerShutdown();
 #ifdef Q_OS_WIN
@@ -106,7 +116,7 @@ private:
     void cleanupInferior();
 #endif
 
-    bool m_debug;
+    Mode m_mode;
     qint64 m_appPid;
     int m_appCode;
     QString m_executable;

src/libs/utils/consoleprocess_unix.cpp

@@ -44,14 +44,13 @@
 
 using namespace Core::Utils;
 
-ConsoleProcess::ConsoleProcess(QObject *parent)
+ConsoleProcess::ConsoleProcess(QObject *parent)  :
-    : QObject(parent)
+    QObject(parent),
+    m_mode(Run),
+    m_appPid(0),
+    m_stubSocket(0),
+    m_settings(0)
 {
-    m_debug = false;
-    m_appPid = 0;
-    m_stubSocket = 0;
-    m_settings = 0;
-
     connect(&m_stubServer, SIGNAL(newConnection()), SLOT(stubConnectionAvailable()));
 
     m_process.setProcessChannelMode(QProcess::ForwardedChannels);
@@ -69,9 +68,9 @@ bool ConsoleProcess::start(const QString &program, const QStringList &args)
     if (isRunning())
         return false;
 
-    QString err = stubServerListen();
+    const QString err = stubServerListen();
     if (!err.isEmpty()) {
-        emit processError(tr("Cannot set up communication channel: %1").arg(err));
+        emit processError(msgCommChannelFailed(err));
         return false;
     }
 
@@ -79,7 +78,7 @@ bool ConsoleProcess::start(const QString &program, const QStringList &args)
         m_tempFile = new QTemporaryFile();
         if (!m_tempFile->open()) {
             stubServerShutdown();
-            emit processError(tr("Cannot create temporary file: %1").arg(m_tempFile->errorString()));
+            emit processError(msgCannotCreateTempFile(m_tempFile->errorString()));
             delete m_tempFile;
             m_tempFile = 0;
             return false;
@@ -94,13 +93,13 @@ bool ConsoleProcess::start(const QString &program, const QStringList &args)
     QStringList xtermArgs = terminalEmulator(m_settings).split(QLatin1Char(' ')); // FIXME: quoting
     xtermArgs
 #ifdef Q_OS_MAC
-              << (QCoreApplication::applicationDirPath() + "/../Resources/qtcreator_process_stub")
+              << (QCoreApplication::applicationDirPath() + QLatin1String("/../Resources/qtcreator_process_stub"))
 #else
-              << (QCoreApplication::applicationDirPath() + "/qtcreator_process_stub")
+              << (QCoreApplication::applicationDirPath() + QLatin1String("/qtcreator_process_stub"))
 #endif
-              << (m_debug ? "debug" : "exec")
+              << modeOption(m_mode)
               << m_stubServer.fullServerName()
-              << tr("Press <RETURN> to close this window...")
+              << msgPromptToClose()
               << workingDirectory()
               << (m_tempFile ? m_tempFile->fileName() : 0)
               << program << args;
@@ -145,7 +144,7 @@ QString ConsoleProcess::stubServerListen()
         {
             QTemporaryFile tf;
             if (!tf.open())
-                return tr("Cannot create temporary file: %1").arg(tf.errorString());
+                return msgCannotCreateTempFile(tf.errorString());
             stubFifoDir = QFile::encodeName(tf.fileName());
         }
         // By now the temp file was deleted again
@@ -153,9 +152,9 @@ QString ConsoleProcess::stubServerListen()
         if (!::mkdir(m_stubServerDir.constData(), 0700))
             break;
         if (errno != EEXIST)
-            return tr("Cannot create temporary directory '%1': %2").arg(stubFifoDir, strerror(errno));
+            return msgCannotCreateTempDir(stubFifoDir, QString::fromLocal8Bit(strerror(errno)));
     }
-    QString stubServer  = stubFifoDir + "/stub-socket";
+    const QString stubServer  = stubFifoDir + "/stub-socket";
     if (!m_stubServer.listen(stubServer)) {
         ::rmdir(m_stubServerDir.constData());
         return tr("Cannot create socket '%1': %2").arg(stubServer, m_stubServer.errorString());
@@ -190,11 +189,9 @@ void ConsoleProcess::readStubOutput()
         QByteArray out = m_stubSocket->readLine();
         out.chop(1); // \n
         if (out.startsWith("err:chdir ")) {
-            emit processError(tr("Cannot change to working directory '%1': %2")
+            emit processError(msgCannotChangeToWorkDir(workingDirectory(), errorMsg(out.mid(10).toInt())));
-                              .arg(workingDirectory(), errorMsg(out.mid(10).toInt())));
         } else if (out.startsWith("err:exec ")) {
-            emit processError(tr("Cannot execute '%1': %2")
+            emit processError(msgCannotExecute(m_executable, errorMsg(out.mid(9).toInt())));
-                              .arg(m_executable, errorMsg(out.mid(9).toInt())));
         } else if (out.startsWith("pid ")) {
             // Will not need it any more
             delete m_tempFile;
@@ -213,7 +210,7 @@ void ConsoleProcess::readStubOutput()
             m_appPid = 0;
             emit processStopped();
         } else {
-            emit processError(tr("Unexpected output from helper program."));
+            emit processError(msgUnexpectedOutput());
             m_process.terminate();
             break;
         }
@@ -250,7 +247,7 @@ QString ConsoleProcess::defaultTerminalEmulator()
 
 QString ConsoleProcess::terminalEmulator(const QSettings *settings)
 {
-    QString dflt = defaultTerminalEmulator() + QLatin1String(" -e");
+    const QString dflt = defaultTerminalEmulator() + QLatin1String(" -e");
     if (!settings)
         return dflt;
     return settings->value(QLatin1String("General/TerminalEmulator"), dflt).toString();

src/libs/utils/consoleprocess_win.cpp

@@ -42,18 +42,17 @@
 
 using namespace Core::Utils;
 
-ConsoleProcess::ConsoleProcess(QObject *parent)
+ConsoleProcess::ConsoleProcess(QObject *parent) :
-    : QObject(parent)
+    QObject(parent),
+    m_mode(Run),
+    m_appPid(0),
+    m_pid(0),
+    m_hInferior(NULL),
+    m_tempFile(0),
+    m_stubSocket(0),
+    processFinishedNotifier(0),
+    inferiorFinishedNotifier(0)
 {
-    m_debug = false;
-    m_appPid = 0;
-    m_pid = 0;
-    m_hInferior = NULL;
-    m_tempFile = 0;
-    m_stubSocket = 0;
-    processFinishedNotifier = 0;
-    inferiorFinishedNotifier = 0;
-
     connect(&m_stubServer, SIGNAL(newConnection()), SLOT(stubConnectionAvailable()));
 }
 
@@ -67,9 +66,9 @@ bool ConsoleProcess::start(const QString &program, const QStringList &args)
     if (isRunning())
         return false;
 
-    QString err = stubServerListen();
+    const QString err = stubServerListen();
     if (!err.isEmpty()) {
-        emit processError(tr("Cannot set up communication channel: %1").arg(err));
+        emit processError(msgCommChannelFailed(err));
         return false;
     }
 
@@ -77,7 +76,7 @@ bool ConsoleProcess::start(const QString &program, const QStringList &args)
         m_tempFile = new QTemporaryFile();
         if (!m_tempFile->open()) {
             stubServerShutdown();
-            emit processError(tr("Cannot create temporary file: %1").arg(m_tempFile->errorString()));
+            emit processError(msgCannotCreateTempFile(m_tempFile->errorString()));
             delete m_tempFile;
             m_tempFile = 0;
             return false;
@@ -102,15 +101,15 @@ bool ConsoleProcess::start(const QString &program, const QStringList &args)
         workDir.append('\\');
 
     QStringList stubArgs;
-    stubArgs << (m_debug ? "debug" : "exec")
+    stubArgs << modeOption(m_mode)
              << m_stubServer.fullServerName()
              << workDir
              << (m_tempFile ? m_tempFile->fileName() : 0)
              << createWinCommandline(program, args)
-             << tr("Press <RETURN> to close this window...");
+             << msgPromptToClose();
 
-    QString cmdLine = createWinCommandline(
+    const QString cmdLine = createWinCommandline(
-            QCoreApplication::applicationDirPath() + "/qtcreator_process_stub.exe", stubArgs);
+            QCoreApplication::applicationDirPath() + QLatin1String("/qtcreator_process_stub.exe"), stubArgs);
 
     bool success = CreateProcessW(0, (WCHAR*)cmdLine.utf16(),
                                   0, 0, FALSE, CREATE_NEW_CONSOLE,
@@ -180,13 +179,11 @@ void ConsoleProcess::readStubOutput()
         QByteArray out = m_stubSocket->readLine();
         out.chop(2); // \r\n
         if (out.startsWith("err:chdir ")) {
-            emit processError(tr("Cannot change to working directory '%1': %2")
+            emit processError(msgCannotChangeToWorkDir(workingDirectory(), winErrorMessage(out.mid(10).toInt())));
-                              .arg(workingDirectory(), winErrorMessage(out.mid(10).toInt())));
         } else if (out.startsWith("err:exec ")) {
-            emit processError(tr("Cannot execute '%1': %2")
+            emit processError(msgCannotExecute(m_executable, winErrorMessage(out.mid(9).toInt())));
-                              .arg(m_executable, winErrorMessage(out.mid(9).toInt())));
         } else if (out.startsWith("pid ")) {
-            // Will not need it any more
+            // Wil not need it any more
             delete m_tempFile;
             m_tempFile = 0;
 
@@ -204,7 +201,7 @@ void ConsoleProcess::readStubOutput()
             connect(inferiorFinishedNotifier, SIGNAL(activated(HANDLE)), SLOT(inferiorExited()));
             emit processStarted();
         } else {
-            emit processError(tr("Unexpected output from helper program."));
+            emit processError(msgUnexpectedOutput());
             TerminateProcess(m_pid->hProcess, (unsigned)-1);
             break;
         }

src/libs/utils/process_stub_win.c

@@ -39,6 +39,8 @@
 static FILE *qtcFd;
 static wchar_t *sleepMsg;
 
+enum RunMode { Run, Debug, Suspend };
+
 /* Print some "press enter" message, wait for that, exit. */
 static void doExit(int code)
 {
@@ -112,6 +114,7 @@ int main()
     STARTUPINFOW si;
     PROCESS_INFORMATION pi;
     DEBUG_EVENT dbev;
+    enum RunMode mode = Run;
 
     argv = CommandLineToArgvW(GetCommandLine(), &argc);
 
@@ -158,8 +161,20 @@ int main()
     si.cb = sizeof(si);
 
     creationFlags = CREATE_UNICODE_ENVIRONMENT;
-    if (!wcscmp(argv[ArgAction], L"debug"))
+    if (!wcscmp(argv[ArgAction], L"debug")) {
+        mode = Debug;
+    } else if (!wcscmp(argv[ArgAction], L"suspend")) {
+        mode = Suspend;
+    }
+
+    switch (mode) {
+    case Debug:
         creationFlags |= DEBUG_ONLY_THIS_PROCESS;
+        break;
+    case Suspend:
+        creationFlags |= CREATE_SUSPENDED;
+        break;
+    }
     if (!CreateProcessW(0, argv[ArgCmdLine], 0, 0, FALSE, creationFlags, env, 0, &si, &pi)) {
         /* Only expected error: no such file or direcotry, i.e. executable not found */
         sendMsg("err:exec %d\n", GetLastError());
@@ -172,7 +187,7 @@ int main()
        So instead we start a debugged process, eat all the initial
        debug events, suspend the process and detach from it. If gdb
        tries to attach *now*, everything goes smoothly. Yay. */
-    if (creationFlags & DEBUG_ONLY_THIS_PROCESS) {
+    if (mode == Debug) {
         do {
             if (!WaitForDebugEvent (&dbev, INFINITE))
                 systemError("Cannot fetch debug event, error %d\n");

src/libs/utils/utils.pro

@@ -26,7 +26,8 @@ SOURCES += \
     savedaction.cpp \
     submiteditorwidget.cpp \
     synchronousprocess.cpp \
-    submitfieldwidget.cpp
+    submitfieldwidget.cpp \
+    consoleprocess.cpp
 
 win32 {
     SOURCES += abstractprocess_win.cpp \

src/plugins/debugger/cdb/cdbdebugengine.cpp

@@ -392,7 +392,7 @@ CdbDebugEngine::CdbDebugEngine(DebuggerManager *parent, const QSharedPointer<Cdb
     IDebuggerEngine(parent),
     m_d(new CdbDebugEnginePrivate(parent, options, this))
 {
-    // m_d->m_consoleStubProc.setDebug(true);
+    m_d->m_consoleStubProc.setMode(Core::Utils::ConsoleProcess::Suspend);
     connect(&m_d->m_consoleStubProc, SIGNAL(processError(QString)), this, SLOT(slotConsoleStubError(QString)));
     connect(&m_d->m_consoleStubProc, SIGNAL(processStarted()), this, SLOT(slotConsoleStubStarted()));
     connect(&m_d->m_consoleStubProc, SIGNAL(wrapperStopped()), this, SLOT(slotConsoleStubTerminated()));
@@ -469,10 +469,12 @@ bool CdbDebugEngine::startDebugger()
     m_d->m_debuggerManager->showStatusMessage("Starting Debugger", -1);
     QString errorMessage;
     bool rc = false;
+    bool needWatchTimer = false;
     m_d->clearForRun();
     switch (mode) {
     case AttachExternal:
         rc = startAttachDebugger(m_d->m_debuggerManager->m_attachedPID, &errorMessage);
+        needWatchTimer = true;
         break;
     case StartInternal:
     case StartExternal:
@@ -484,7 +486,9 @@ bool CdbDebugEngine::startDebugger()
             rc = m_d->m_consoleStubProc.start(m_d->m_debuggerManager->m_executable, m_d->m_debuggerManager->m_processArgs);
             if (!rc)
                 errorMessage = tr("The console stub process was unable to start '%1'.").arg(m_d->m_debuggerManager->m_executable);
+            // continues in slotConsoleStubStarted()...
         } else {
+            needWatchTimer = true;
             rc = startDebuggerWithExecutable(mode, &errorMessage);
         }
         break;
@@ -494,7 +498,8 @@ bool CdbDebugEngine::startDebugger()
     }
     if (rc) {
         m_d->m_debuggerManager->showStatusMessage(tr("Debugger Running"), -1);
-        startWatchTimer();
+        if (needWatchTimer)
+            startWatchTimer();
     } else {
         qWarning("%s\n", qPrintable(errorMessage));
     }
@@ -503,12 +508,12 @@ bool CdbDebugEngine::startDebugger()
 
 bool CdbDebugEngine::startAttachDebugger(qint64 pid, QString *errorMessage)
 {
-    // Need to aatrach invasively, otherwise, no notification signals
+    // Need to attrach invasively, otherwise, no notification signals
     // for for CreateProcess/ExitProcess occur.
-    const HRESULT hr = m_d->m_cif.debugClient->AttachProcess(NULL, pid,
+    const ULONG flags = DEBUG_ATTACH_INVASIVE_RESUME_PROCESS;
-                                                          DEBUG_ATTACH_INVASIVE_RESUME_PROCESS);
+    const HRESULT hr = m_d->m_cif.debugClient->AttachProcess(NULL, pid, flags);
     if (debugCDB)
-        qDebug() << "Attaching to " << pid << " returns " << hr;
+        qDebug() << "Attaching to " << pid << " returns " << hr << executionStatusString(m_d->m_cif.debugControl);
     if (FAILED(hr)) {
         *errorMessage = tr("AttachProcess failed for pid %1: %2").arg(pid).arg(msgDebugEngineComResult(hr));
         return false;
@@ -560,7 +565,6 @@ bool CdbDebugEngine::startDebuggerWithExecutable(DebuggerStartMode sm, QString *
         m_d->m_mode = sm;
     }
     m_d->m_debuggerManagerAccess->notifyInferiorRunning();
-
     return true;
 }
 
@@ -1318,7 +1322,9 @@ void CdbDebugEngine::slotConsoleStubStarted()
     QString errorMessage;
     if (startAttachDebugger(appPid, &errorMessage)) {
         m_d->m_debuggerManager->m_attachedPID = appPid;
+        startWatchTimer();
         m_d->m_debuggerManagerAccess->notifyInferiorPidChanged(appPid);
+        m_d->m_debuggerManagerAccess->notifyInferiorRunning();
     } else {
         QMessageBox::critical(m_d->m_debuggerManager->mainWindow(), tr("Debugger Error"), errorMessage);
     }
@@ -1343,7 +1349,8 @@ void CdbDebugEnginePrivate::notifyCrashed()
 void CdbDebugEnginePrivate::handleDebugEvent()
 {
     if (debugCDB)
-        qDebug() << Q_FUNC_INFO << m_hDebuggeeProcess;
+        qDebug() << Q_FUNC_INFO << '\n'  << m_hDebuggeeProcess << m_breakEventMode
+                << executionStatusString(m_cif.debugControl);
 
     // restore mode and do special handling
     const HandleBreakEventMode mode = m_breakEventMode;

src/plugins/debugger/cdb/cdbdebugeventcallback.cpp

@@ -34,7 +34,7 @@
 #include "breakhandler.h"
 #include "cdbstacktracecontext.h"
 
-enum { cppExceptionCode = 0xe06d7363 };
+enum { cppExceptionCode = 0xe06d7363, startupCompleteTrap = 0x406d1388 };
 
 #include <QtCore/QDebug>
 #include <QtCore/QTextStream>
@@ -245,6 +245,9 @@ void formatException(const EXCEPTION_RECORD64 *e, QTextStream &str)
     case cppExceptionCode:
         str << "C++ exception";
         break;
+    case startupCompleteTrap:
+        str << "Startup complete";
+        break;
     case EXCEPTION_ACCESS_VIOLATION: {
             const bool writeOperation = e->ExceptionInformation[0];
             str << (writeOperation ? "write" : "read")
@@ -341,7 +344,7 @@ static bool isFatalException(LONG code)
     switch (code) {
     case EXCEPTION_BREAKPOINT:
     case EXCEPTION_SINGLE_STEP:
-    case 0x406d1388: // Mysterious exception at start of application
+    case startupCompleteTrap: // Mysterious exception at start of application
         return false;
     default:
         break;
@@ -362,7 +365,7 @@ STDMETHODIMP CdbDebugEventCallback::Exception(
     }
     const bool fatal = isFatalException(Exception->ExceptionCode);
     if (debugCDB)
-        qDebug() << Q_FUNC_INFO << '\n' << fatal << msg;
+        qDebug() << Q_FUNC_INFO << "\nex=" << Exception->ExceptionCode << " fatal=" << fatal << msg;
     m_pEngine->m_d->m_debuggerManagerAccess->showApplicationOutput(msg);
     if (fatal)
         m_pEngine->m_d->notifyCrashed();

src/plugins/debugger/cdb/cdbdumperhelper.cpp

@@ -352,7 +352,7 @@ bool CdbDumperHelper::ensureInitialized(QString *errorMessage)
                 return false;
             case CallLoadNoQtApp:
                 m_access->showDebuggerOutput(m_messagePrefix, QCoreApplication::translate("CdbDumperHelper", "The debuggee does not appear to be Qt application."));
-                disable();
+                m_state = Disabled; // No message here
                 return true;
             }
         break;
@@ -367,7 +367,7 @@ bool CdbDumperHelper::ensureInitialized(QString *errorMessage)
         m_access->showDebuggerOutput(m_messagePrefix, m_helper.toString());
         m_state = Initialized;
     } else {
-        disable();
+        m_state = Disabled; // No message here
         *errorMessage = QCoreApplication::translate("CdbDumperHelper", "The custom dumper library could not be initialized: %1").arg(*errorMessage);
         m_access->showDebuggerOutput(m_messagePrefix, *errorMessage);
         m_access->showQtDumperLibraryWarning(*errorMessage);

src/plugins/debugger/gdbengine.cpp

@@ -117,7 +117,7 @@ GdbEngine::GdbEngine(DebuggerManager *parent)
 {
     q = parent;
     qq = parent->engineInterface();
-    m_stubProc.setDebug(true);
+    m_stubProc.setMode(Core::Utils::ConsoleProcess::Debug);
     initializeVariables();
     initializeConnections();
 }