Utils: print assert backtrace on windows

Change-Id: Ifd9753b22641547c16fb6843dc7a3cf5f5b7c86e
Reviewed-by: Christian Stenger <christian.stenger@qt.io>
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>
Reviewed-by: David Schulz <david.schulz@qt.io>
Reviewed-by: Alessandro Portale <alessandro.portale@qt.io>

src/libs/utils/CMakeLists.txt

@@ -217,6 +217,14 @@ extend_qtc_library(Utils
     mimeutils.cpp
 )
 
+if (MSVC)
+    find_library(DbgHelpLib dbghelp)
+endif()
+
+extend_qtc_library(Utils CONDITION MSVC
+  DEPENDS ${DbgHelpLib}
+)
+
 extend_qtc_library(Utils CONDITION WIN32
   SOURCES
     touchbar/touchbar.cpp

src/libs/utils/qtcassert.cpp

@@ -5,11 +5,20 @@
 
 #include <QByteArray>
 #include <QDebug>
+#include <QMutex>
 
 #if defined(Q_OS_UNIX)
 #include <stdio.h>
 #include <signal.h>
 #include <execinfo.h>
+#elif defined(_MSC_VER)
+#ifdef QTCREATOR_PCH_H
+#define CALLBACK WINAPI
+#define OUT
+#define IN
+#endif
+#include <Windows.h>
+#include <DbgHelp.h>
 #endif
 
 namespace Utils {
@@ -26,6 +35,74 @@ void dumpBacktrace(int maxdepth)
     for (int i = 0; i < size; ++i)
         qDebug() << "0x" + QByteArray::number(quintptr(bt[i]), 16) << lines[i];
     free(lines);
+#elif defined(_MSC_VER)
+    DWORD machineType;
+#if defined(Q_OS_WIN64)
+    machineType = IMAGE_FILE_MACHINE_AMD64;
+#else
+    return;
+#endif
+    static QMutex mutex;
+    mutex.lock();
+    HANDLE process = GetCurrentProcess();
+    HANDLE thread = GetCurrentThread();
+    CONTEXT ctx;
+    RtlCaptureContext(&ctx);
+    STACKFRAME64 frame;
+    memset(&frame, 0, sizeof(STACKFRAME64));
+#if defined(Q_OS_WIN64)
+    frame.AddrPC.Offset = ctx.Rip;
+    frame.AddrPC.Mode = AddrModeFlat;
+    frame.AddrStack.Offset = ctx.Rsp;
+    frame.AddrStack.Mode = AddrModeFlat;
+    frame.AddrFrame.Offset = ctx.Rbp;
+    frame.AddrFrame.Mode = AddrModeFlat;
+#endif
+    int depth = 0;
+
+    static bool symbolsInitialized = false;
+    if (!symbolsInitialized) {
+        SymInitialize(process, NULL, TRUE);
+        SymSetOptions(SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS);
+        symbolsInitialized = true;
+    }
+
+    while (StackWalk64(machineType,
+                       process,
+                       thread,
+                       &frame,
+                       &ctx,
+                       NULL,
+                       &SymFunctionTableAccess64,
+                       &SymGetModuleBase64,
+                       NULL)) {
+        char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];
+        PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer;
+        pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+        pSymbol->MaxNameLen = MAX_SYM_NAME;
+
+        DWORD64 displacement = 0;
+        if (!SymFromAddr(process, frame.AddrPC.Offset, &displacement, pSymbol))
+            break;
+
+        DWORD symDisplacement = 0;
+        IMAGEHLP_LINE64 lineInfo;
+        SymSetOptions(SYMOPT_LOAD_LINES);
+
+        QString out = QString("%1: 0x%2 at %3")
+                          .arg(depth)
+                          .arg(QString::number(pSymbol->Address, 16))
+                          .arg(QString::fromLatin1(&pSymbol->Name[0], pSymbol->NameLen));
+        if (SymGetLineFromAddr64(process, frame.AddrPC.Offset, &symDisplacement, &lineInfo)) {
+            out.append(QString(" at %3:%4")
+                           .arg(QString::fromLatin1(lineInfo.FileName),
+                                QString::number(lineInfo.LineNumber)));
+        }
+        qDebug() << out;
+        if (++depth == maxdepth)
+            break;
+    }
+    mutex.unlock();
 #endif
 }
 

src/libs/utils/utils.qbs

@@ -17,6 +17,8 @@ Project {
                 libs.push("user32", "iphlpapi", "ws2_32", "shell32", "ole32");
                 if (qbs.toolchainType === "mingw")
                     libs.push("uuid");
+                else if (qbs.toolchainType === "msvc")
+                    libs.push("dbghelp");
             } else if (qbs.targetOS.contains("unix")) {
                 if (!qbs.targetOS.contains("macos"))
                     libs.push("X11");

src/tools/processlauncher/CMakeLists.txt

@@ -22,3 +22,11 @@ add_qtc_executable(qtcreator_processlauncher
     ${UTILSDIR}/singleton.cpp
     ${UTILSDIR}/singleton.h
     )
+
+if (MSVC)
+    find_library(DbgHelpLib dbghelp)
+endif()
+
+extend_qtc_executable(qtcreator_processlauncher CONDITION MSVC
+  DEPENDS ${DbgHelpLib}
+)

src/tools/processlauncher/processlauncher.qbs

@@ -11,7 +11,9 @@ QtcTool {
 
     Properties {
         condition: qbs.targetOS.contains("windows")
-        cpp.dynamicLibraries: "user32"
+        cpp.dynamicLibraries: {
+            return qbs.toolchainType === "msvc" ? ["user32", "dbghelp"] : ["user32"];
+        }
     }
 
     files: [

src/tools/sdktool/CMakeLists.txt

@@ -90,6 +90,13 @@ extend_qtc_library(sdktoolLib CONDITION APPLE
     ${FWFoundation}
 )
 
+if (MSVC)
+    find_library(DbgHelpLib dbghelp)
+endif()
+
+extend_qtc_library(sdktoolLib CONDITION MSVC
+  DEPENDS ${DbgHelpLib}
+)
 extend_qtc_library(sdktoolLib CONDITION WIN32
   PUBLIC_DEPENDS
     user32 iphlpapi ws2_32 shell32

src/tools/sdktool/sdktoollib.qbs

@@ -24,8 +24,12 @@ QtcLibrary {
         return defines;
     }
     cpp.dynamicLibraries: {
+        var libs = [];
         if (qbs.targetOS.contains("windows"))
-            return ["user32", "shell32"]
+            libs.push("user32", "shell32");
+        if (qbs.toolchainType === "msvc")
+            libs.push("dbghelp");
+        return libs;
     }
     Properties {
         condition: qbs.targetOS.contains("macos")