Debugger: Add wow64 support to the cdbengine.

Change-Id: I1819c42438609553ce277e01b7c8a2d4fffbfb1b
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@digia.com>

src/plugins/debugger/cdb/cdbengine.cpp

@@ -356,6 +356,7 @@ CdbEngine::CdbEngine(const DebuggerStartParameters &sp, const OptionsPtr &option
     m_notifyEngineShutdownOnTermination(false),
     m_hasDebuggee(false),
     m_cdbIs64Bit(false),
+    m_wow64State(wow64Uninitialized),
     m_elapsedLogTime(0),
     m_sourceStepInto(false),
     m_watchPointX(0),
@@ -387,6 +388,7 @@ void CdbEngine::init()
     m_watchPointX = m_watchPointY = 0;
     m_ignoreCdbOutput = false;
     m_watchInameToName.clear();
+    m_wow64State = wow64Uninitialized;
 
     m_outputBuffer.clear();
     m_builtinCommandQueue.clear();
@@ -642,6 +644,8 @@ bool CdbEngine::launchCDB(const DebuggerStartParameters &sp, QString *errorMessa
 #else
             false;
 #endif
+    if (!m_cdbIs64Bit)
+        m_wow64State = noWow64Stack;
     const QFileInfo extensionFi(CdbEngine::extensionLibraryName(m_cdbIs64Bit));
     if (!extensionFi.isFile()) {
         *errorMessage = QString::fromLatin1("Internal error: The extension %1 cannot be found.").
@@ -2220,6 +2224,10 @@ void CdbEngine::processStop(const GdbMi &stopReason, bool conditionalBreakPointT
                 case ParseStackStepOut: // Hit on a frame with no source while step into.
                     executeStepOut();
                     return;
+                case ParseStackWow64:
+                    postBuiltinCommand("!wow64exts.info", 0, &CdbEngine::handleCheckWow64,
+                                       0, qVariantFromValue(stack));
+                    return;
                 }
             } else {
                 showMessage(QString::fromLatin1(stopReason["stackerror"].data()), LogError);
@@ -2247,6 +2255,52 @@ void CdbEngine::processStop(const GdbMi &stopReason, bool conditionalBreakPointT
         showStoppedByExceptionMessageBox(exceptionBoxMessage);
 }
 
+void CdbEngine::handleCheckWow64(const CdbBuiltinCommandPtr &cmd)
+{
+    // Using the stack command from the wow64exts cdb extension to
+    // check if there is a 32bit subsystem in this debuggee.
+    if (cmd->reply.first().startsWith("Could not get the address of the 32bit PEB")) {
+        m_wow64State = noWow64Stack;
+        if (cmd->cookie.canConvert<GdbMi>())
+            parseStackTrace(qvariant_cast<GdbMi>(cmd->cookie), false);
+        return;
+    }
+    postBuiltinCommand("k", 0, &CdbEngine::ensureUsing32BitStackInWow64, 0, cmd->cookie);
+}
+
+void CdbEngine::ensureUsing32BitStackInWow64(const CdbEngine::CdbBuiltinCommandPtr &cmd)
+{
+    // Parsing the header of the stack output to check which bitness
+    // the cdb is currently using.
+    foreach (const QByteArray &line, cmd->reply) {
+        if (!line.startsWith("Child"))
+            continue;
+        if (line.startsWith("ChildEBP")) {
+            m_wow64State = wow64Stack32Bit;
+            return;
+        } else if (line.startsWith("Child-SP")) {
+            m_wow64State = wow64Stack64Bit;
+            postBuiltinCommand("!wow64exts.sw", 0, &CdbEngine::handleSwitchWow64Stack);
+            return;
+        }
+    }
+    m_wow64State = noWow64Stack;
+    if (cmd->cookie.canConvert<GdbMi>())
+        parseStackTrace(qvariant_cast<GdbMi>(cmd->cookie), false);
+}
+
+void CdbEngine::handleSwitchWow64Stack(const CdbEngine::CdbBuiltinCommandPtr &cmd)
+{
+    if (cmd->reply.first() == "Switched to 32bit mode")
+        m_wow64State = wow64Stack32Bit;
+    else if (cmd->reply.first() == "Switched to 64bit mode")
+        m_wow64State = wow64Stack64Bit;
+    else
+        m_wow64State = noWow64Stack;
+    // reload threads and the stack after switching the mode
+    postCommandSequence(CommandListThreads | CommandListStack);
+}
+
 void CdbEngine::handleSessionAccessible(unsigned long cdbExState)
 {
     const DebuggerState s = state();
@@ -2866,6 +2920,10 @@ unsigned CdbEngine::parseStackTrace(const GdbMi &data, bool sourceStepInto)
     StackFrames frames = parseFrames(data, &incomplete);
     const int count = frames.size();
     for (int i = 0; i < count; i++) {
+        if (m_wow64State == wow64Uninitialized) {
+            showMessage(QString::fromLatin1("Checking for wow64 subsystem..."), LogMisc);
+            return ParseStackWow64;
+        }
         const bool hasFile = !frames.at(i).file.isEmpty();
         // jmp-frame hit by step into, do another 't' and abort sequence.
         if (!hasFile && i == 0 && sourceStepInto) {
@@ -2910,7 +2968,10 @@ void CdbEngine::handleStackTrace(const CdbExtensionCommandPtr &command)
     if (command->success) {
         GdbMi data;
         data.fromString(command->reply);
-        parseStackTrace(data, false);
+        if (parseStackTrace(data, false) == ParseStackWow64) {
+            postBuiltinCommand("!wow64exts.info", 0, &CdbEngine::handleCheckWow64,
+                               0, qVariantFromValue(data));
+        }
         postCommandSequence(command->commandSequence);
     } else {
         showMessage(QString::fromLocal8Bit(command->errorMessage), LogError);
@@ -3145,3 +3206,5 @@ void CdbEngine::handleCustomSpecialStop(const QVariant &v)
 
 } // namespace Internal
 } // namespace Debugger
+
+Q_DECLARE_METATYPE(Debugger::Internal::GdbMi)

src/plugins/debugger/cdb/cdbengine.h

@@ -177,7 +177,8 @@ private:
     enum ParseStackResultFlags // Flags returned by parseStackTrace
     {
         ParseStackStepInto = 1, // Need to execute a step, hit on a call frame in "Step into"
-        ParseStackStepOut = 2 // Need to step out, hit on a frame without debug information
+        ParseStackStepOut = 2, // Need to step out, hit on a frame without debug information
+        ParseStackWow64 = 3 // Hit on a frame with 32bit emulation, switch debugger to 32 bit mode
     };
 
 
@@ -219,6 +220,9 @@ private:
     void handleExpression(const CdbExtensionCommandPtr &);
     void handleResolveSymbol(const CdbBuiltinCommandPtr &command);
     void handleResolveSymbol(const QList<quint64> &addresses, const QVariant &cookie);
+    void handleCheckWow64(const CdbBuiltinCommandPtr &cmd);
+    void ensureUsing32BitStackInWow64(const CdbBuiltinCommandPtr &cmd);
+    void handleSwitchWow64Stack(const CdbBuiltinCommandPtr &cmd);
     void jumpToAddress(quint64 address);
 
     // Extension commands
@@ -263,6 +267,12 @@ private:
     bool m_notifyEngineShutdownOnTermination;
     bool m_hasDebuggee;
     bool m_cdbIs64Bit;
+    enum Wow64State {
+        wow64Uninitialized,
+        noWow64Stack,
+        wow64Stack32Bit,
+        wow64Stack64Bit
+    } m_wow64State;
     QTime m_logTime;
     mutable int m_elapsedLogTime;
     QByteArray m_extensionMessageBuffer;