debugger: rework dumping arrays of plain data

Simple array data can be fetched in one go.
The "normal" way gdb asks the server for contents is one item
at a time, at ~3ms per round trip. Now we get 2000 in 200ms.

This also introduces a re-usable readRawMemory function and
removes a few checkAccess calls which will be triggered anyways.

Change-Id: Ic07a3d6593fd2ea45f7a8058509118fe22a845bb
Reviewed-by: hjk <qthjk@ovi.com>

share/qtcreator/dumper/bridge.py

@@ -1,4 +1,6 @@
 
+import binascii
+
 cdbLoaded = False
 lldbLoaded = False
 gdbLoaded = False
@@ -287,6 +289,29 @@ try:
         removeTempFile(filename, file)
         return lines
 
+    def selectedInferior():
+        try:
+            # Does not exist in 7.3.
+            return gdb.selected_inferior()
+        except:
+            pass
+        # gdb.Inferior is new in gdb 7.2
+        return gdb.inferiors()[0]
+
+    def readRawMemory(base, size):
+        try:
+            inferior = selectedInferior()
+            return binascii.hexlify(inferior.read_memory(base, size))
+        except:
+            pass
+        s = ""
+        t = lookupType("unsigned char").pointer()
+        base = base.cast(t)
+        for i in xrange(size):
+            s += "%02x" % int(base.dereference())
+            base += 1
+        return s
+
     #######################################################################
     #
     # Types

share/qtcreator/dumper/dumper.py

@@ -42,11 +42,6 @@ def removeTempFile(name, file):
     except:
         pass
 
-try:
-    import binascii
-except:
-    pass
-
 verbosity = 0
 verbosity = 1
 
@@ -69,8 +64,14 @@ Hex4EncodedLittleEndianWithoutQuotes, \
 Hex2EncodedLocal8Bit, \
 JulianDate, \
 MillisecondsSinceMidnight, \
-JulianDateAndMillisecondsSinceMidnight \
+JulianDateAndMillisecondsSinceMidnight, \
-    = range(17)
+Hex2EncodedInt1, \
+Hex2EncodedInt2, \
+Hex2EncodedInt4, \
+Hex2EncodedInt8, \
+Hex2EncodedFloat4, \
+Hex2EncodedFloat8 \
+    = range(23)
 
 # Display modes
 StopDisplay, \
@@ -479,6 +480,26 @@ def isSimpleType(typeobj):
         or code == FloatCode \
         or code == EnumCode
 
+def simpleEncoding(typeobj):
+    code = typeobj.code
+    if code == BoolCode or code == CharCode:
+        return Hex2EncodedInt1
+    if code == IntCode:
+        if typeobj.sizeof == 1:
+            return Hex2EncodedInt1
+        if typeobj.sizeof == 2:
+            return Hex2EncodedInt2
+        if typeobj.sizeof == 4:
+            return Hex2EncodedInt4
+        if typeobj.sizeof == 8:
+            return Hex2EncodedInt8
+    if code == FloatCode:
+        if typeobj.sizeof == 4:
+            return Hex2EncodedFloat4
+        if typeobj.sizeof == 8:
+            return Hex2EncodedFloat8
+    return None
+
 def warn(message):
     if True or verbosity > 0:
         print "XXX: %s\n" % message.encode("latin1")
@@ -668,80 +689,23 @@ def findFirstZero(p, maximum):
         p = p + 1
     return maximum + 1
 
-def extractCharArray(p, maxsize):
+def encodeCArray(p, innerType, suffix):
-    p = p.cast(lookupType("unsigned char").pointer())
+    t = lookupType(innerType)
-    s = ""
+    p = p.cast(t.pointer())
-    i = 0
+    limit = findFirstZero(p, qqStringCutOff)
-    while i < maxsize:
+    s = readRawMemory(p, limit * t.sizeof)
-        c = int(p.dereference())
+    if limit > qqStringCutOff:
-        if c == 0:
+        s += suffix
-            return s
-        s += "%c" % c
-        p += 1
-        i += 1
-    if p.dereference() != 0:
-        s += "..."
     return s
 
-def extractByteArray(value):
+def encodeCharArray(p):
-    d_ptr = value['d'].dereference()
+    return encodeCArray(p, "unsigned char", "2e2e2e")
-    data = d_ptr['data']
-    size = d_ptr['size']
-    alloc = d_ptr['alloc']
-    check(0 <= size and size <= alloc and alloc <= 100*1000*1000)
-    checkRef(d_ptr["ref"])
-    if size > 0:
-        checkAccess(data, 4)
-        checkAccess(data + size) == 0
-    return extractCharArray(data, min(qqStringCutOff, size))
 
-def encodeCharArray(p, maxsize = None, limit = None):
+def encodeChar2Array(p):
-    if maxsize is None:
+    return encodeCArray(p, "unsigned short", "2e002e002e00")
-        maxsize = qqStringCutOff
-    t = lookupType("unsigned char").pointer()
-    p = p.cast(t)
-    if limit is None:
-        limit = findFirstZero(p, maxsize)
-    s = ""
-    try:
-        # gdb.Inferior is new in gdb 7.2
-        inferior = gdb.inferiors()[0]
-        s = binascii.hexlify(inferior.read_memory(p, limit))
-    except:
-        for i in xrange(limit):
-            s += "%02x" % int(p.dereference())
-            p += 1
-    if limit > maxsize:
-        s += "2e2e2e"
-    return s
 
-def encodeChar2Array(p, maxsize = None):
+def encodeChar4Array(p):
-    if maxsize == None:
+    return encodeCArray(p, "unsigned int", "2e0000002e0000002e000000")
-        maxsize = qqStringCutOff
-    t = lookupType("unsigned short").pointer()
-    p = p.cast(t)
-    limit = findFirstZero(p, maxsize)
-    s = ""
-    for i in xrange(limit):
-        s += "%04x" % int(p.dereference())
-        p += 1
-    if i == maxsize:
-        s += "2e002e002e00"
-    return s
-
-def encodeChar4Array(p, maxsize = None):
-    if maxsize == None:
-        maxsize = qqStringCutOff
-    t = lookupType("unsigned int").pointer()
-    p = p.cast(t)
-    limit = findFirstZero(p, maxsize)
-    s = ""
-    for i in xrange(limit):
-        s += "%08x" % int(p.dereference())
-        p += 1
-    if i > maxsize:
-        s += "2e0000002e0000002e000000"
-    return s
 
 def qByteArrayData(value):
     private = value['d']
@@ -758,13 +722,15 @@ def qByteArrayData(value):
 
 def encodeByteArray(value):
     data, size, alloc = qByteArrayData(value)
+    if alloc != 0:
         check(0 <= size and size <= alloc and alloc <= 100*1000*1000)
-    if size > 0:
+    limit = min(size, qqStringCutOff)
-        checkAccess(data, 4)
+    s = readRawMemory(data, limit)
-        checkAccess(data + size) == 0
+    if limit < size:
-    return encodeCharArray(data, limit = size)
+        s += "2e2e2e"
+    return s
 
-def qQStringData(value):
+def qStringData(value):
     private = value['d']
     checkRef(private['ref'])
     try:
@@ -778,26 +744,11 @@ def qQStringData(value):
         return private['data'], int(private['size']), int(private['alloc'])
 
 def encodeString(value):
-    data, size, alloc = qQStringData(value)
+    data, size, alloc = qStringData(value)
-
     if alloc != 0:
         check(0 <= size and size <= alloc and alloc <= 100*1000*1000)
-    if size > 0:
-        checkAccess(data, 4)
-        checkAccess(data + size) == 0
-    s = ""
     limit = min(size, qqStringCutOff)
-    try:
+    s = readRawMemory(data, 2 * limit)
-        # gdb.Inferior is new in gdb 7.2
-        inferior = gdb.inferiors()[0]
-        s = binascii.hexlify(inferior.read_memory(data, 2 * limit))
-    except:
-        p = data
-        for i in xrange(limit):
-            val = int(p.dereference())
-            s += "%02x" % (val % 256)
-            s += "%02x" % (val / 256)
-            p += 1
     if limit < size:
         s += "2e002e002e00"
     return s
@@ -1082,7 +1033,7 @@ class Dumper:
             for item in listOfLocals([]):
                 self.expandedINames.add(item.iname)
                 self.expandedINames.discard("")
-                warn("EXPANDED: %s" % self.expandedINames)
+                #warn("EXPANDED: %s" % self.expandedINames)
 
         # Take care of the return value of the last function call.
         if len(resultVarName) > 0:
@@ -1353,21 +1304,25 @@ class Dumper:
             self.putName(name)
             self.putItem(value)
 
-    def tryPutArrayContents(self, type, base, n):
+    def tryPutArrayContents(self, typeobj, base, n):
-        if isSimpleType(type):
+        if not isSimpleType(typeobj):
-            self.put('{value="')
-            self.put('"},{value="'.join([str((base + i).dereference())
-                for i in xrange(n)]))
-            self.put('"}');
-            return True
             return False
+        size = n * typeobj.sizeof;
+        self.put('childtype="%s",' % typeobj)
+        self.put('addrbase="0x%x",' % long(base))
+        self.put('addrstep="0x%x",' % long(typeobj.sizeof))
+        self.put('arrayencoding="%s",' % simpleEncoding(typeobj))
+        self.put('arraydata="')
+        self.put(readRawMemory(base, size))
+        self.put('",')
+        return True
 
     def putArrayData(self, type, base, n,
             childNumChild = None, maxNumChild = 10000):
         base = base.cast(type.pointer())
+        if not self.tryPutArrayContents(type, base, n):
             with Children(self, n, type, childNumChild, maxNumChild,
                     base, type.sizeof):
-            if not self.tryPutArrayContents(type, base, n):
                 for i in self.childRange():
                     self.putSubItem(i, (base + i).dereference())
 
@@ -1507,9 +1462,9 @@ class Dumper:
             if self.currentIName in self.expandedINames:
                 p = value.cast(targetType.pointer())
                 ts = targetType.sizeof
+                if not self.tryPutArrayContents(targetType, p, type.sizeof/ts):
                     with Children(self, childType=targetType,
                             addrBase=p, addrStep=ts):
-                    if not self.tryPutArrayContents(targetType, p, type.sizeof/ts):
                         self.putFields(value)
             return
 
@@ -1557,7 +1512,7 @@ class Dumper:
                 # Use Latin1 as default for char *.
                 self.putAddress(value.address)
                 self.putType(typeName)
-                self.putValue(encodeCharArray(value, 100), Hex2EncodedLatin1)
+                self.putValue(encodeCharArray(value), Hex2EncodedLatin1)
                 self.putNumChild(0)
                 return
 
@@ -1578,7 +1533,7 @@ class Dumper:
                 # Explicitly requested Latin1 formatting.
                 self.putAddress(value.address)
                 self.putType(typeName)
-                self.putValue(encodeCharArray(value, 100), Hex2EncodedLatin1)
+                self.putValue(encodeCharArray(value), Hex2EncodedLatin1)
                 self.putNumChild(0)
                 return
 
@@ -1586,7 +1541,7 @@ class Dumper:
                 # Explicitly requested UTF-8 formatting.
                 self.putAddress(value.address)
                 self.putType(typeName)
-                self.putValue(encodeCharArray(value, 100), Hex2EncodedUtf8)
+                self.putValue(encodeCharArray(value), Hex2EncodedUtf8)
                 self.putNumChild(0)
                 return
 
@@ -1594,7 +1549,7 @@ class Dumper:
                 # Explicitly requested local 8 bit formatting.
                 self.putAddress(value.address)
                 self.putType(typeName)
-                self.putValue(encodeCharArray(value, 100), Hex2EncodedLocal8Bit)
+                self.putValue(encodeCharArray(value), Hex2EncodedLocal8Bit)
                 self.putNumChild(0)
                 return
 
@@ -1602,7 +1557,7 @@ class Dumper:
                 # Explicitly requested UTF-16 formatting.
                 self.putAddress(value.address)
                 self.putType(typeName)
-                self.putValue(encodeChar2Array(value), Hex4EncodedBigEndian)
+                self.putValue(encodeChar2Array(value), Hex4EncodedLittleEndian)
                 self.putNumChild(0)
                 return
 
@@ -1610,7 +1565,7 @@ class Dumper:
                 # Explicitly requested UCS-4 formatting.
                 self.putAddress(value.address)
                 self.putType(typeName)
-                self.putValue(encodeChar4Array(value), Hex8EncodedBigEndian)
+                self.putValue(encodeChar4Array(value), Hex8EncodedLittleEndian)
                 self.putNumChild(0)
                 return
 
@@ -1860,7 +1815,8 @@ def threadnames(arg):
     out = '['
     oldthread = gdb.selected_thread()
     try:
-        for thread in gdb.inferiors()[0].threads():
+        inferior = selectedInferior()
+        for thread in inferior.threads():
             maximalStackDepth = int(arg)
             thread.switch()
             e = gdb.selected_frame ()

share/qtcreator/dumper/qttypes.py

@@ -459,9 +459,6 @@ def qdump__QList(d, value):
     check(begin >= 0 and end >= 0 and end <= 1000 * 1000 * 1000)
     size = end - begin
     check(size >= 0)
-    #if n > 0:
-    #    checkAccess(&list.front())
-    #    checkAccess(&list.back())
     checkRef(d_ptr["ref"])
 
     # Additional checks on pointer arrays.
@@ -1650,7 +1647,6 @@ def qdump__QVector(d, value):
     d.putItemCount(size)
     d.putNumChild(size)
     if d.isExpanded():
-        d.putField("size", size)
         d.putArrayData(innerType, p, size)
 
 
@@ -2569,8 +2565,10 @@ if False:
     def qdump__Function(d, value):
         min = value["min"]
         max = value["max"]
-        var = extractByteArray(value["var"])
+        data, size, alloc = qByteArrayData(value["var"])
-        f = extractByteArray(value["f"])
+        var = extractCString(data)
+        data, size, alloc = qByteArrayData(value["f"])
+        f = extractCString(data)
         d.putValue("%s, %s=%f..%f" % (f, var, min, max))
         d.putNumChild(0)
         d.putField("typeformats", "Normal,Displayed");

src/plugins/debugger/watchutils.cpp

@@ -592,7 +592,7 @@ QString decodeData(const QByteArray &ba, int encoding)
         case Hex8EncodedBigEndian: { // 10, %08x encoded 32 bit data
             const QChar doubleQuote(QLatin1Char('"'));
             QByteArray decodedBa = QByteArray::fromHex(ba);
-            for (int i = 0; i < decodedBa.size(); i += 4) {
+            for (int i = 0; i < decodedBa.size() - 3; i += 4) {
                 char c = decodedBa.at(i);
                 decodedBa[i] = decodedBa.at(i + 3);
                 decodedBa[i + 3] = c;
@@ -643,6 +643,54 @@ QString decodeData(const QByteArray &ba, int encoding)
     return QCoreApplication::translate("Debugger", "<Encoding error>");
 }
 
+template <class T>
+void decodeArrayHelper(QList<WatchData> *list, const WatchData &tmplate,
+    const QByteArray &rawData)
+{
+    const QByteArray ba = QByteArray::fromHex(rawData);
+    const T *p = (const T *) ba.data();
+    WatchData data;
+    const QByteArray exp = "*(" + gdbQuoteTypes(tmplate.type) + "*)0x";
+    for (int i = 0, n = ba.size() / sizeof(T); i < n; ++i) {
+        data = tmplate;
+        data.sortId = i;
+        data.iname += QByteArray::number(i);
+        data.name = QString::fromLatin1("[%1]").arg(i);
+        data.value = QString::number(p[i]);
+        data.address += i * sizeof(T);
+        data.exp = exp + QByteArray::number(data.address, 16);
+        data.setAllUnneeded();
+        list->append(data);
+    }
+}
+
+void decodeArray(QList<WatchData> *list, const WatchData &tmplate,
+    const QByteArray &rawData, int encoding)
+{
+    switch (encoding) {
+        case Hex2EncodedInt1:
+            decodeArrayHelper<uchar>(list, tmplate, rawData);
+            break;
+        case Hex2EncodedInt2:
+            decodeArrayHelper<ushort>(list, tmplate, rawData);
+            break;
+        case Hex2EncodedInt4:
+            decodeArrayHelper<uint>(list, tmplate, rawData);
+            break;
+        case Hex2EncodedInt8:
+            decodeArrayHelper<quint64>(list, tmplate, rawData);
+            break;
+        case Hex2EncodedFloat4:
+            decodeArrayHelper<float>(list, tmplate, rawData);
+            break;
+        case Hex2EncodedFloat8:
+            decodeArrayHelper<double>(list, tmplate, rawData);
+            break;
+        default:
+            qDebug() << "ENCODING ERROR: " << encoding;
+    }
+}
+
 // Editor tooltip support
 bool isCppEditor(Core::IEditor *editor)
 {
@@ -870,6 +918,13 @@ void parseWatchData(const QSet<QByteArray> &expandedINames,
     setWatchDataChildCount(childtemplate, item.findChild("childnumchild"));
     //qDebug() << "CHILD TEMPLATE:" << childtemplate.toString();
 
+    mi = item.findChild("arraydata");
+    if (mi.isValid()) {
+        int encoding = item.findChild("arrayencoding").data().toInt();
+        childtemplate.iname = data.iname + '.';
+        childtemplate.address = addressBase;
+        decodeArray(list, childtemplate, mi.data(), encoding);
+    } else {
         for (int i = 0, n = children.children().size(); i != n; ++i) {
             const GdbMi &child = children.children().at(i);
             WatchData data1 = childtemplate;
@@ -906,6 +961,7 @@ void parseWatchData(const QSet<QByteArray> &expandedINames,
             }
             parseWatchData(expandedINames, data1, child, list);
         }
+    }
 }
 
 

src/plugins/debugger/watchutils.h

@@ -72,7 +72,13 @@ enum DebuggerEncoding
     Hex2EncodedLocal8BitWithQuotes         = 13,
     JulianDate                             = 14,
     MillisecondsSinceMidnight              = 15,
-    JulianDateAndMillisecondsSinceMidnight = 16
+    JulianDateAndMillisecondsSinceMidnight = 16,
+    Hex2EncodedInt1                        = 17,
+    Hex2EncodedInt2                        = 18,
+    Hex2EncodedInt4                        = 19,
+    Hex2EncodedInt8                        = 20,
+    Hex2EncodedFloat4                      = 21,
+    Hex2EncodedFloat8                      = 22
 };
 
 bool isEditorDebuggable(Core::IEditor *editor);
@@ -103,6 +109,9 @@ QString cppExpressionAt(TextEditor::ITextEditor *editor, int pos,
 QString cppFunctionAt(const QString &fileName, int line);
 // Decode string data as returned by the dumper helpers.
 QString decodeData(const QByteArray &baIn, int encoding);
+// Decode string data as returned by the dumper helpers.
+void decodeArray(WatchData *list, const WatchData &tmplate,
+    const QByteArray &rawData, int encoding);
 
 // Get variables that are not initialized at a certain line
 // of a function from the code model. Shadowed variables will

tests/manual/debugger/simple/simple_test_app.cpp

@@ -2714,7 +2714,7 @@ namespace stdmap {
 
     void testStdMapUIntFloatIterator()
     {
-        typedef std::map<uint, float> Map;
+        typedef std::map<int, float> Map;
         Map map;
         map[11] = 11.0;
         map[22] = 22.0;
@@ -2732,11 +2732,11 @@ namespace stdmap {
 
         BREAK_HERE;
         // Expand map.
-        // Check map <6 items> std::map<unsigned int, float>.
+        // Check map <6 items> std::map<int, float>.
         // Check map.11 11 float.
-        // Check it1.first 11 unsigned int.
+        // Check it1.first 11 int.
         // Check it1.second 11 float.
-        // Check it1.first 55 unsigned int.
+        // Check it1.first 55 int.
         // Check it1.second 55 float.
         // Continue.
         dummyStatement(&map, &it1, &it2, &it3, &it4, &it5, &it6);