Debugger: Use LD_PRELOAD to load debugging helper.

on UNIX. Either set the LD_PRELOAD environment variable
using a gdb command or have the TermGdbAdapter set the
variable for the debuggee. For the remote adapter,
switch on toolchain. dlopen() is a fallback for
platforms where it is not supported and attaching
to running processes. Fixes a crash with gdb 7.0
(and spurious gdb 6.8 crashes with dlopen()).
Reviewed-by: hjk <qtc-committer@nokia.com>

src/plugins/debugger/gdb/abstractgdbadapter.h

@@ -48,6 +48,11 @@ class AbstractGdbAdapter : public QObject
     Q_OBJECT
 
 public:
+    enum  DumperHandling { DumperNotAvailable,
+                           DumperLoadedByAdapter,
+                           DumperLoadedByGdbPreload,
+                           DumperLoadedByGdb };
+
     AbstractGdbAdapter(GdbEngine *engine, QObject *parent = 0);
     virtual ~AbstractGdbAdapter();
 
@@ -61,7 +66,7 @@ public:
     virtual void shutdown();
     virtual const char *inferiorShutdownCommand() const;
 
-    virtual bool dumpersAvailable() const = 0;
+    virtual DumperHandling dumperHandling() const = 0;
 
     static QString msgGdbStopFailed(const QString &why);
     static QString msgInferiorStopFailed(const QString &why);

src/plugins/debugger/gdb/attachgdbadapter.h

@@ -48,7 +48,7 @@ class AttachGdbAdapter : public AbstractGdbAdapter
 public:
     AttachGdbAdapter(GdbEngine *engine, QObject *parent = 0);
 
-    bool dumpersAvailable() const { return true; }
+    virtual DumperHandling dumperHandling() const { return DumperLoadedByGdb; }
 
     void startAdapter();
     void startInferior();

src/plugins/debugger/gdb/coregdbadapter.h

@@ -52,7 +52,7 @@ class CoreGdbAdapter : public AbstractGdbAdapter
 public:
     CoreGdbAdapter(GdbEngine *engine, QObject *parent = 0);
 
-    bool dumpersAvailable() const { return false; }
+    virtual DumperHandling dumperHandling() const { return DumperNotAvailable; }
 
     void startAdapter();
     void startInferior();

src/plugins/debugger/gdb/gdbengine.cpp

@@ -1197,7 +1197,8 @@ void GdbEngine::handleStopResponse(const GdbMi &data)
         }
     }
 
-    bool initHelpers = (m_debuggingHelperState == DebuggingHelperUninitialized);
+    bool initHelpers = m_debuggingHelperState == DebuggingHelperUninitialized
+                       || m_debuggingHelperState == DebuggingHelperLoadTried;
     // Don't load helpers on stops triggered by signals unless it's
     // an intentional trap.
     if (initHelpers && reason == "signal-received"
@@ -1529,7 +1530,7 @@ AbstractGdbAdapter *GdbEngine::createAdapter(const DebuggerStartParametersPtr &s
     case AttachCore:
         return new CoreGdbAdapter(this);
     case StartRemote:
-        return new RemoteGdbAdapter(this);
+        return new RemoteGdbAdapter(this, sp->toolChainType);
     case AttachExternal:
         return new AttachGdbAdapter(this);
     default:
@@ -1556,7 +1557,7 @@ void GdbEngine::startDebugger(const DebuggerStartParametersPtr &sp)
     m_gdbAdapter = createAdapter(sp);
     connectAdapter();
 
-    if (startModeAllowsDumpers())
+    if (m_gdbAdapter->dumperHandling() != AbstractGdbAdapter::DumperNotAvailable)
         connectDebuggingHelperActions();
 
     m_gdbAdapter->startAdapter();
@@ -2769,7 +2770,7 @@ bool GdbEngine::hasDebuggingHelperForType(const QString &type) const
     if (!theDebuggerBoolSetting(UseDebuggingHelpers))
         return false;
 
-    if (!startModeAllowsDumpers()) {
+    if (m_gdbAdapter->dumperHandling() == AbstractGdbAdapter::DumperNotAvailable) {
         // "call" is not possible in gdb when looking at core files
         return type == __("QString") || type.endsWith(__("::QString"))
             || type == __("QStringList") || type.endsWith(__("::QStringList"));
@@ -2811,7 +2812,7 @@ void GdbEngine::runDirectDebuggingHelper(const WatchData &data, bool dumpChildre
 
 void GdbEngine::runDebuggingHelper(const WatchData &data0, bool dumpChildren)
 {
-    if (!startModeAllowsDumpers()) {
+    if (m_debuggingHelperState != DebuggingHelperAvailable) {
         runDirectDebuggingHelper(data0, dumpChildren);
         return;
     }
@@ -3847,14 +3848,50 @@ void GdbEngine::assignValueInDebugger(const QString &expression, const QString &
     postCommand(_("-var-assign assign ") + value, Discardable, CB(handleVarAssign));
 }
 
+QString GdbEngine::qtDumperLibraryName() const
+{
+    return m_manager->qtDumperLibraryName();
+}
+
+bool GdbEngine::checkDebuggingHelpers()
+{
+    if (!manager()->qtDumperLibraryEnabled())
+        return false;
+    const QString lib = qtDumperLibraryName();
+    //qDebug() << "DUMPERLIB:" << lib;
+    const QFileInfo fi(lib);
+    if (!fi.exists()) {
+        const QStringList &locations = manager()->qtDumperLibraryLocations();
+        const QString loc = locations.join(QLatin1String(", "));
+        const QString msg = tr("The debugging helper library was not found at %1.").arg(loc);
+        debugMessage(msg);
+        manager()->showQtDumperLibraryWarning(msg);
+        return false;
+    }
+    return true;
+}
+
+void GdbEngine::setDebuggingHelperState(DebuggingHelperState s)
+{
+    m_debuggingHelperState = s;
+}
+
 void GdbEngine::tryLoadDebuggingHelpers()
 {
     if (isSynchroneous())
         return;
-
+    switch (m_debuggingHelperState) {
-    if (m_debuggingHelperState != DebuggingHelperUninitialized)
+    case DebuggingHelperUninitialized:
+        break;
+    case DebuggingHelperLoadTried:
+        tryQueryDebuggingHelpers();
         return;
-    if (!startModeAllowsDumpers()) {
+    case DebuggingHelperAvailable:
+    case DebuggingHelperUnavailable:
+        return;
+    }
+
+    if (m_gdbAdapter->dumperHandling() == AbstractGdbAdapter::DumperNotAvailable) {
         // Load at least gdb macro based dumpers.
         QFile file(_(":/gdb/gdbmacros.txt"));
         file.open(QIODevice::ReadOnly);
@@ -3868,22 +3905,11 @@ void GdbEngine::tryLoadDebuggingHelpers()
 
     PENDING_DEBUG("TRY LOAD CUSTOM DUMPERS");
     m_debuggingHelperState = DebuggingHelperUnavailable;
-    if (!manager()->qtDumperLibraryEnabled())
+    if (!checkDebuggingHelpers())
         return;
-    const QString lib = manager()->qtDumperLibraryName();
-    const QStringList &locations = manager()->qtDumperLibraryLocations();
-    //qDebug() << "DUMPERLIB:" << lib;
-    // @TODO: same in CDB engine...
-    const QFileInfo fi(lib);
-    if (!fi.exists()) {
-        const QString loc = locations.join(QLatin1String(", "));
-        const QString msg = tr("The debugging helper library was not found at %1.").arg(loc);
-        debugMessage(msg);
-        manager()->showQtDumperLibraryWarning(msg);
-        return;
-    }
 
     m_debuggingHelperState = DebuggingHelperLoadTried;
+    const QString lib = manager()->qtDumperLibraryName();
 #if defined(Q_OS_WIN)
     if (m_dumperInjectionLoad) {
         /// Launch asynchronous remote thread to load.
@@ -3929,29 +3955,20 @@ void GdbEngine::tryLoadDebuggingHelpers()
 
 void GdbEngine::tryQueryDebuggingHelpers()
 {
-#if !X
     // retrieve list of dumpable classes
     postCommand(_("call (void*)qDumpObjectData440(1,%1+1,0,0,0,0,0,0)"), EmbedToken);
     postCommand(_("p (char*)&qDumpOutBuffer"), CB(handleQueryDebuggingHelper));
-#else
-    m_debuggingHelperState = DebuggingHelperUnavailable;
-#endif
 }
 
 void GdbEngine::recheckDebuggingHelperAvailability()
 {
-    if (startModeAllowsDumpers()) {
+    if (m_gdbAdapter->dumperHandling() != AbstractGdbAdapter::DumperNotAvailable) {
         // retreive list of dumpable classes
         postCommand(_("call (void*)qDumpObjectData440(1,%1+1,0,0,0,0,0,0)"), EmbedToken);
         postCommand(_("p (char*)&qDumpOutBuffer"), CB(handleQueryDebuggingHelper));
     }
 }
 
-bool GdbEngine::startModeAllowsDumpers() const
-{
-    return m_gdbAdapter->dumpersAvailable();
-}
-
 void GdbEngine::watchPoint(const QPoint &pnt)
 {
     //qDebug() << "WATCH " << pnt;
@@ -4309,7 +4326,12 @@ bool GdbEngine::startGdb(const QStringList &args, const QString &gdb, const QStr
               ).arg(scriptFileName));
         }
     }
-
+    if (m_gdbAdapter->dumperHandling() == AbstractGdbAdapter::DumperLoadedByGdbPreload
+        && checkDebuggingHelpers()) {        
+        const QString cmd = QLatin1String("set environment LD_PRELOAD ") + manager()->qtDumperLibraryName();
+        postCommand(cmd);
+        m_debuggingHelperState = DebuggingHelperLoadTried;
+    }
     return true;
 }
 

src/plugins/debugger/gdb/gdbengine.h

@@ -449,8 +449,9 @@ private: ////////// View & Data Stuff //////////
     QMap<QString, QString> m_varToType;
 
 private: ////////// Dumper Management //////////
-
+    QString qtDumperLibraryName() const;
-    bool startModeAllowsDumpers() const;
+    bool checkDebuggingHelpers();
+    void setDebuggingHelperState(DebuggingHelperState);
     void tryLoadDebuggingHelpers();
     void tryQueryDebuggingHelpers();
     Q_SLOT void recheckDebuggingHelperAvailability();

src/plugins/debugger/gdb/plaingdbadapter.cpp

@@ -58,6 +58,15 @@ PlainGdbAdapter::PlainGdbAdapter(GdbEngine *engine, QObject *parent)
         engine, SLOT(readDebugeeOutput(QByteArray)));
 }
 
+AbstractGdbAdapter::DumperHandling PlainGdbAdapter::dumperHandling() const
+{
+#ifdef Q_OS_WIN
+    return DumperLoadedByGdb;
+#else
+    return DumperLoadedByGdbPreload;
+#endif
+}
+
 void PlainGdbAdapter::startAdapter()
 {
     QTC_ASSERT(state() == EngineStarting, qDebug() << state());

src/plugins/debugger/gdb/plaingdbadapter.h

@@ -50,7 +50,7 @@ class PlainGdbAdapter : public AbstractGdbAdapter
 public:
     PlainGdbAdapter(GdbEngine *engine, QObject *parent = 0);
 
-    bool dumpersAvailable() const { return true; }
+    virtual DumperHandling dumperHandling() const;
 
     void startAdapter();
     void startInferior();

src/plugins/debugger/gdb/remotegdbadapter.cpp

@@ -34,6 +34,7 @@
 
 #include <utils/qtcassert.h>
 #include <utils/fancymainwindow.h>
+#include <projectexplorer/toolchain.h>
 
 #include <QtCore/QFileInfo>
 #include <QtGui/QMessageBox>
@@ -51,8 +52,9 @@ namespace Internal {
 //
 ///////////////////////////////////////////////////////////////////////
 
-RemoteGdbAdapter::RemoteGdbAdapter(GdbEngine *engine, QObject *parent)
+RemoteGdbAdapter::RemoteGdbAdapter(GdbEngine *engine, int toolChainType, QObject *parent) :
-    : AbstractGdbAdapter(engine, parent)
+    AbstractGdbAdapter(engine, parent),
+    m_toolChainType(toolChainType)
 {
     connect(&m_uploadProc, SIGNAL(error(QProcess::ProcessError)),
         this, SLOT(uploadProcError(QProcess::ProcessError)));
@@ -62,6 +64,23 @@ RemoteGdbAdapter::RemoteGdbAdapter(GdbEngine *engine, QObject *parent)
         this, SLOT(readUploadStandardError()));
 }
 
+AbstractGdbAdapter::DumperHandling RemoteGdbAdapter::dumperHandling() const
+{
+    switch (m_toolChainType) {
+    case ProjectExplorer::ToolChain::MinGW:
+    case ProjectExplorer::ToolChain::MSVC:
+    case ProjectExplorer::ToolChain::WINCE:
+    case ProjectExplorer::ToolChain::WINSCW:
+    case ProjectExplorer::ToolChain::GCCE:
+    case ProjectExplorer::ToolChain::RVCT_ARMV5:
+    case ProjectExplorer::ToolChain::RVCT_ARMV6:
+        return DumperLoadedByGdb;
+    default:
+        break;
+    }
+    return DumperLoadedByGdbPreload;
+}
+
 void RemoteGdbAdapter::startAdapter()
 {
     QTC_ASSERT(state() == EngineStarting, qDebug() << state());

src/plugins/debugger/gdb/remotegdbadapter.h

@@ -46,9 +46,9 @@ class RemoteGdbAdapter : public AbstractGdbAdapter
     Q_OBJECT
 
 public:
-    RemoteGdbAdapter(GdbEngine *engine, QObject *parent = 0);
+    RemoteGdbAdapter(GdbEngine *engine, int toolChainType, QObject *parent = 0);
 
-    bool dumpersAvailable() const { return true; }
+    virtual DumperHandling dumperHandling() const;
 
     void startAdapter();
     void startInferior();
@@ -67,6 +67,8 @@ private:
     void handleFileExecAndSymbols(const GdbResponse &response);
     void handleTargetRemote(const GdbResponse &response);
 
+    const int m_toolChainType;
+
     QProcess m_uploadProc;
 };
 

src/plugins/debugger/gdb/termgdbadapter.cpp

@@ -69,6 +69,15 @@ TermGdbAdapter::~TermGdbAdapter()
     m_stubProc.disconnect(); // Avoid spurious state transitions from late exiting stub
 }
 
+AbstractGdbAdapter::DumperHandling TermGdbAdapter::dumperHandling() const
+{
+#ifdef Q_OS_WIN    
+    return DumperLoadedByGdb;
+#else
+    return DumperLoadedByAdapter; // Handles loading itself via LD_PRELOAD
+#endif
+}
+
 void TermGdbAdapter::startAdapter()
 {
     QTC_ASSERT(state() == EngineStarting, qDebug() << state());
@@ -82,7 +91,14 @@ void TermGdbAdapter::startAdapter()
 //    m_stubProc.blockSignals(false);
 
     m_stubProc.setWorkingDirectory(startParameters().workingDir);
-    m_stubProc.setEnvironment(startParameters().environment);
+    // Set environment + dumper preload.
+    QStringList environment = startParameters().environment;
+    if (dumperHandling() == DumperLoadedByGdbPreload
+        && m_engine->checkDebuggingHelpers()) {
+        environment.push_back(QLatin1String("LD_PRELOAD=") + m_engine->qtDumperLibraryName());        
+        m_engine->setDebuggingHelperState(DebuggingHelperLoadTried);
+    }
+    m_stubProc.setEnvironment(environment);
     // FIXME: Starting the stub implies starting the inferior. This is
     // fairly unclean as far as the state machine and error reporting go.
     if (!m_stubProc.start(startParameters().executable,

src/plugins/debugger/gdb/termgdbadapter.h

@@ -51,7 +51,7 @@ public:
     TermGdbAdapter(GdbEngine *engine, QObject *parent = 0);
     ~TermGdbAdapter();
 
-    bool dumpersAvailable() const { return true; }
+    virtual DumperHandling dumperHandling() const;
 
     void startAdapter();
     void startInferior();

src/plugins/debugger/gdb/trkgdbadapter.h

@@ -164,7 +164,8 @@ public:
         QIODevice::OpenMode mode = QIODevice::ReadWrite);
     void write(const QByteArray &data);
     bool isTrkAdapter() const { return true; }
-    bool dumpersAvailable() const { return false; }
+
+    virtual DumperHandling dumperHandling() const { return DumperNotAvailable; }
 
 private:
     void startAdapter();