Debugger: Add an option to control heap debugging

Added a checkbox (temporarily) in CDB settings page that toggles
the heap debugging on Windows.
Although it is "technically" a CDB option, it woks with LLDB on Windows
as well.
The default behavior is to keep the heap debugging off.
The checkbox can still be overridden by the environment variable.

Fixes: QTCREATORBUG-32102
Change-Id: I580cd0d66da38776f3a51632ede6b5f60d4d4cfd
Reviewed-by: hjk <hjk@qt.io>
Reviewed-by: David Schulz <david.schulz@qt.io>

src/plugins/debugger/cdb/cdbengine.cpp

@@ -406,6 +406,10 @@ void CdbEngine::setupEngine()
     if (!oldCdbExtensionPath.isEmpty())
         inferiorEnvironment.appendOrSet(cdbExtensionPathVariableC, oldCdbExtensionPath);
 
+    if (!inferiorEnvironment.hasKey(Debugger::Constants::NO_DEBUG_HEAP)) {
+        const QString value = s.enableHeapDebugging() ? "0" : "1";
+        inferiorEnvironment.set(Debugger::Constants::NO_DEBUG_HEAP, value);
+    }
     m_process.setEnvironment(inferiorEnvironment);
     if (!sp.inferior.workingDirectory.isEmpty())
         m_process.setWorkingDirectory(sp.inferior.workingDirectory);

src/plugins/debugger/cdb/cdboptionspage.cpp

@@ -171,6 +171,7 @@ CdbOptionsPageWidget::CdbOptionsPageWidget()
 
     m_breakEventWidget->setBreakEvents(settings().cdbBreakEvents());
 
+    // clang-format off
     Column {
         Row {
             Group {
@@ -188,7 +189,8 @@ CdbOptionsPageWidget::CdbOptionsPageWidget()
                     s.ignoreFirstChanceAccessViolation,
                     s.cdbBreakOnCrtDbgReport,
                     s.cdbBreakPointCorrection,
-                    s.cdbUsePythonDumper
+                    s.cdbUsePythonDumper,
+                    s.enableHeapDebugging
                 }
             }
         },
@@ -209,6 +211,7 @@ CdbOptionsPageWidget::CdbOptionsPageWidget()
         st
 
     }.attachTo(this);
+    // clang-format on
 }
 
 void CdbOptionsPageWidget::apply()

src/plugins/debugger/debuggeractions.cpp

@@ -190,6 +190,14 @@ DebuggerSettings::DebuggerSettings() :
     ignoreFirstChanceAccessViolation.setSettingsKey(cdbSettingsGroup, "IgnoreFirstChanceAccessViolation");
     ignoreFirstChanceAccessViolation.setLabelText(Tr::tr("Ignore first chance access violations"));
 
+    enableHeapDebugging.setSettingsKey(cdbSettingsGroup, "EnableHeapDebugging");
+    enableHeapDebugging.setLabelText(Tr::tr("Enable heap debugging"));
+    enableHeapDebugging.setToolTip(
+        "<p>"
+        + Tr::tr("Allocate memory using the debug heap rather than the normal heap. The debug heap "
+                 "enables additional checks to help diagnose heap related bugs. However it comes "
+                 "at a performance cost when allocating memory in the debugged process"));
+
     //
     // Locals & Watchers
     sortStructMembers.setSettingsKey(debugModeGroup, "SortStructMembers");
@@ -255,6 +263,7 @@ DebuggerSettings::DebuggerSettings() :
     page5.registerAspect(&firstChanceExceptionTaskEntry);
     page5.registerAspect(&secondChanceExceptionTaskEntry);
     page5.registerAspect(&ignoreFirstChanceAccessViolation);
+    page5.registerAspect(&enableHeapDebugging);
 
     // Page 6
     page6.registerAspect(&cdbSymbolPaths);

src/plugins/debugger/debuggeractions.h

@@ -89,6 +89,7 @@ public:
     Utils::BoolAspect firstChanceExceptionTaskEntry;
     Utils::BoolAspect secondChanceExceptionTaskEntry;
     Utils::BoolAspect ignoreFirstChanceAccessViolation;
+    Utils::BoolAspect enableHeapDebugging;
 
     // Page 6: CDB Paths
     Utils::StringListAspect cdbSymbolPaths;

src/plugins/debugger/debuggerinternalconstants.h

@@ -53,6 +53,8 @@ const char DISASSEMBLER_SOURCE_FILE[]         = "DisassemblerSourceFile";
 
 const char CRT_DEBUG_REPORT[]                 = "CrtDbgReport";
 
+const char NO_DEBUG_HEAP[]                    = "_NO_DEBUG_HEAP";
+
 } // namespace Constants
 
 enum ModelRoles

src/plugins/debugger/lldb/lldbengine.cpp

@@ -264,22 +264,31 @@ void LldbEngine::handleLldbStarted()
     for (const FilePath &path : rp.solibSearchPath)
         executeDebuggerCommand("settings append target.exec-search-paths " + path.toString());
 
+    const ProcessRunData &inferior = rp.inferior;
+    const FilePath &executable = inferior.command.executable();
     DebuggerCommand cmd2("setupInferior");
-    cmd2.arg("executable", rp.inferior.command.executable().path());
+    cmd2.arg("executable", executable.path());
     cmd2.arg("breakonmain", rp.breakOnMain);
     cmd2.arg("useterminal", usesTerminal());
     cmd2.arg("startmode", rp.startMode);
     cmd2.arg("nativemixed", isNativeMixedActive());
-    cmd2.arg("workingdirectory", rp.inferior.workingDirectory.path());
-    QStringList environment = rp.inferior.environment.toStringList();
+    cmd2.arg("workingdirectory", inferior.workingDirectory.path());
+    Environment environment = inferior.environment;
     // Prevent lldb from automatically setting OS_ACTIVITY_DT_MODE to mirror
     // NSLog to stderr, as that will also mirror os_log, which we pick up in
     // AppleUnifiedLogger::preventsStderrLogging(), and end up disabling Qt's
     // default stderr logger. We prefer Qt's own stderr logging if we can.
-    environment << "IDE_DISABLED_OS_ACTIVITY_DT_MODE=1";
-    cmd2.arg("environment", environment);
-    cmd2.arg("processargs", toHex(ProcessArgs::splitArgs(rp.inferior.command.arguments(),
-                                                         HostOsInfo::hostOs()).join(QChar(0))));
+    environment.set("IDE_DISABLED_OS_ACTIVITY_DT_MODE", "1");
+    if (executable.osType() == Utils::OsTypeWindows
+        && !environment.hasKey(Debugger::Constants::NO_DEBUG_HEAP)) {
+        const QString value = settings().enableHeapDebugging() ? "0" : "1";
+        environment.set(Debugger::Constants::NO_DEBUG_HEAP, value);
+    }
+    cmd2.arg("environment", environment.toStringList());
+    cmd2.arg(
+        "processargs",
+        toHex(ProcessArgs::splitArgs(inferior.command.arguments(), HostOsInfo::hostOs())
+                  .join(QChar(0))));
     cmd2.arg("platform", rp.platform);
     cmd2.arg("symbolfile", rp.symbolFile.path());