Help index filter optimizations

Move retrieval away from prepare search, and do some caching. Also split
the keyword search into individual chunks per help database that are
executed on the UI thread individually.

Change-Id: I0b8fe4abfc3cba46620985752d3d90638e10512f
Reviewed-by: Eike Ziller <eike.ziller@theqtcompany.com>

src/plugins/coreplugin/helpmanager.cpp

@@ -111,7 +111,7 @@ HelpManager::~HelpManager()
     delete d;
 }
 
-QObject *HelpManager::instance()
+HelpManager *HelpManager::instance()
 {
     Q_ASSERT(m_instance);
     return m_instance;
@@ -248,51 +248,6 @@ QMap<QString, QUrl> HelpManager::linksForIdentifier(const QString &id)
     return d->m_helpEngine->linksForIdentifier(id);
 }
 
-// This should go into Qt 4.8 once we start using it for Qt Creator
-QStringList HelpManager::findKeywords(const QString &key, Qt::CaseSensitivity caseSensitivity,
-                                      int maxHits)
-{
-    QTC_ASSERT(!d->m_needsSetup, return QStringList());
-
-    const QLatin1String sqlite("QSQLITE");
-    const QLatin1String name("HelpManager::findKeywords");
-
-    QSet<QString> keywords;
-    QSet<QString> keywordsToSort;
-
-    DbCleaner cleaner(name);
-    QSqlDatabase db = QSqlDatabase::addDatabase(sqlite, name);
-    if (db.driver() && db.driver()->lastError().type() == QSqlError::NoError) {
-        QHelpEngineCore core(collectionFilePath());
-        core.setAutoSaveFilter(false);
-        core.setCurrentFilter(tr("Unfiltered"));
-        core.setupData();
-        const QStringList &registeredDocs = core.registeredDocumentations();
-        foreach (const QString &nameSpace, registeredDocs) {
-            db.setDatabaseName(core.documentationFileName(nameSpace));
-            if (db.open()) {
-                QSqlQuery query = QSqlQuery(db);
-                query.setForwardOnly(true);
-                query.exec(QString::fromLatin1("SELECT DISTINCT Name FROM IndexTable WHERE Name LIKE "
-                    "'%%1%' LIMIT %2").arg(key, QString::number(maxHits)));
-                while (query.next()) {
-                    const QString &keyValue = query.value(0).toString();
-                    if (!keyValue.isEmpty()) {
-                        if (keyValue.startsWith(key, caseSensitivity))
-                            keywordsToSort.insert(keyValue);
-                        else
-                            keywords.insert(keyValue);
-                    }
-                }
-            }
-        }
-    }
-
-    QStringList keywordsSorted = keywordsToSort.toList();
-    Utils::sort(keywordsSorted);
-    return keywordsSorted + keywords.toList();
-}
-
 QUrl HelpManager::findFile(const QUrl &url)
 {
     QTC_ASSERT(!d->m_needsSetup, return QUrl());

src/plugins/coreplugin/helpmanager.h

@@ -63,7 +63,7 @@ public:
 
     typedef QHash<QString, QStringList> Filters;
 
-    static QObject *instance();
+    static HelpManager *instance();
     static QString collectionFilePath();
 
     static void registerDocumentation(const QStringList &fileNames);
@@ -74,9 +74,6 @@ public:
 
     static QMap<QString, QUrl> linksForKeyword(const QString &key);
     static QMap<QString, QUrl> linksForIdentifier(const QString &id);
-    static QStringList findKeywords(const QString &key,
-                             Qt::CaseSensitivity caseSensitivity = Qt::CaseInsensitive,
-                             int maxHits = INT_MAX);
 
     static QUrl findFile(const QUrl &url);
     static QByteArray fileData(const QUrl &url);

src/plugins/help/help.pro

@@ -1,4 +1,4 @@
-QT += help network printsupport
+QT += help network printsupport sql
 !isEmpty(QT.webkitwidgets.name): QT += webkitwidgets webkit
 else: DEFINES += QT_NO_WEBKIT
 

src/plugins/help/helpindexfilter.cpp

@@ -37,8 +37,14 @@
 #include <extensionsystem/pluginmanager.h>
 #include <coreplugin/icore.h>
 #include <coreplugin/helpmanager.h>
+#include <utils/algorithm.h>
 
 #include <QIcon>
+#include <QSqlDatabase>
+#include <QSqlDriver>
+#include <QSqlError>
+#include <QSqlQuery>
+
 
 using namespace Core;
 using namespace Help;
@@ -47,6 +53,7 @@ using namespace Help::Internal;
 Q_DECLARE_METATYPE(ILocatorFilter*)
 
 HelpIndexFilter::HelpIndexFilter()
+    : m_needsUpdate(true)
 {
     setId("HelpIndexFilter");
     setDisplayName(tr("Help Index"));
@@ -54,6 +61,12 @@ HelpIndexFilter::HelpIndexFilter()
     setShortcutString(QString(QLatin1Char('?')));
 
     m_icon = QIcon(QLatin1String(":/help/images/bookmark.png"));
+    connect(HelpManager::instance(), &HelpManager::setupFinished,
+            this, &HelpIndexFilter::invalidateCache);
+    connect(HelpManager::instance(), &HelpManager::documentationChanged,
+            this, &HelpIndexFilter::invalidateCache);
+    connect(HelpManager::instance(), &HelpManager::collectionFileChanged,
+            this, &HelpIndexFilter::invalidateCache);
 }
 
 HelpIndexFilter::~HelpIndexFilter()
@@ -62,21 +75,62 @@ HelpIndexFilter::~HelpIndexFilter()
 
 void HelpIndexFilter::prepareSearch(const QString &entry)
 {
-    if (entry.length() < 2)
+    Q_UNUSED(entry)
-        m_keywords = Core::HelpManager::findKeywords(entry, caseSensitivity(entry), 200);
+    QStringList namespaces = HelpManager::registeredNamespaces();
-    else
+    m_helpDatabases = Utils::transform(namespaces, [](const QString &ns) {
-        m_keywords = Core::HelpManager::findKeywords(entry, caseSensitivity(entry));
+        return HelpManager::fileFromNamespace(ns);
+    });
 }
 
 QList<LocatorFilterEntry> HelpIndexFilter::matchesFor(QFutureInterface<LocatorFilterEntry> &future, const QString &entry)
 {
-    Q_UNUSED(entry) // search is already done in the GUI thread in prepareSearch
+    m_mutex.lock(); // guard m_needsUpdate
+    bool forceUpdate = m_needsUpdate;
+    m_mutex.unlock();
+
+    if (forceUpdate || m_searchTermCache.size() < 2 || m_searchTermCache.isEmpty()
+            || !entry.contains(m_searchTermCache)) {
+        int limit = entry.size() < 2 ? 200 : INT_MAX;
+        QSet<QString> results;
+        foreach (const QString &filePath, m_helpDatabases) {
+            QSet<QString> result;
+            QMetaObject::invokeMethod(this, "searchMatches", Qt::BlockingQueuedConnection,
+                                      Q_RETURN_ARG(QSet<QString>, result),
+                                      Q_ARG(QString, filePath),
+                                      Q_ARG(QString, entry),
+                                      Q_ARG(int, limit));
+            results.unite(result);
+        }
+        m_mutex.lock(); // guard m_needsUpdate
+        m_needsUpdate = false;
+        m_mutex.unlock();
+        m_keywordCache = results;
+        m_searchTermCache = entry;
+    }
+
+    Qt::CaseSensitivity cs = caseSensitivity(entry);
     QList<LocatorFilterEntry> entries;
-    foreach (const QString &keyword, m_keywords) {
+    QStringList keywords;
+    QStringList unsortedKeywords;
+    keywords.reserve(m_keywordCache.size());
+    unsortedKeywords.reserve(m_keywordCache.size());
+    QSet<QString> allresults;
+    foreach (const QString &keyword, m_keywordCache) {
         if (future.isCanceled())
             break;
-        entries.append(LocatorFilterEntry(this, keyword, QVariant(), m_icon));
+        if (keyword.startsWith(entry, cs)) {
+            keywords.append(keyword);
+            allresults.insert(keyword);
+        } else if (keyword.contains(entry, cs)) {
+            unsortedKeywords.append(keyword);
+            allresults.insert(keyword);
         }
+    }
+    Utils::sort(keywords);
+    keywords << unsortedKeywords;
+    m_keywordCache = allresults;
+    foreach (const QString &keyword, keywords)
+        entries.append(LocatorFilterEntry(this, keyword, QVariant(), m_icon));
 
     return entries;
 }
@@ -95,5 +149,42 @@ void HelpIndexFilter::accept(LocatorFilterEntry selection) const
 void HelpIndexFilter::refresh(QFutureInterface<void> &future)
 {
     Q_UNUSED(future)
-    // Nothing to refresh
+    invalidateCache();
+}
+
+QSet<QString> HelpIndexFilter::searchMatches(const QString &databaseFilePath,
+                                           const QString &term, int limit)
+{
+    static const QLatin1String sqlite("QSQLITE");
+    static const QLatin1String name("HelpManager::findKeywords");
+
+    QSet<QString> keywords;
+
+    { // make sure db is destroyed before removeDatabase call
+        QSqlDatabase db = QSqlDatabase::addDatabase(sqlite, name);
+        if (db.driver() && db.driver()->lastError().type() == QSqlError::NoError) {
+            db.setDatabaseName(databaseFilePath);
+            if (db.open()) {
+                QSqlQuery query = QSqlQuery(db);
+                query.setForwardOnly(true);
+                query.exec(QString::fromLatin1("SELECT DISTINCT Name FROM IndexTable WHERE Name LIKE "
+                    "'%%1%' LIMIT %2").arg(term, QString::number(limit)));
+                while (query.next()) {
+                    const QString &keyValue = query.value(0).toString();
+                    if (!keyValue.isEmpty())
+                        keywords.insert(keyValue);
+                }
+                db.close();
+            }
+        }
+    }
+    QSqlDatabase::removeDatabase(name);
+    return keywords;
+}
+
+void HelpIndexFilter::invalidateCache()
+{
+    m_mutex.lock();
+    m_needsUpdate = true;
+    m_mutex.unlock();
 }

src/plugins/help/helpindexfilter.h

@@ -34,6 +34,7 @@
 #include <coreplugin/locator/ilocatorfilter.h>
 
 #include <QIcon>
+#include <QMutex>
 
 namespace Help {
 namespace Internal {
@@ -48,17 +49,26 @@ public:
 
     // ILocatorFilter
     void prepareSearch(const QString &entry);
-    QList<Core::LocatorFilterEntry> matchesFor(QFutureInterface<Core::LocatorFilterEntry> &future, const QString &entry);
+    QList<Core::LocatorFilterEntry> matchesFor(QFutureInterface<Core::LocatorFilterEntry> &future,
+                                               const QString &entry);
     void accept(Core::LocatorFilterEntry selection) const;
     void refresh(QFutureInterface<void> &future);
 
+    Q_INVOKABLE QSet<QString> searchMatches(const QString &databaseFilePath,
+                                          const QString &term, int limit);
 signals:
     void linkActivated(const QUrl &link) const;
     void linksActivated(const QMap<QString, QUrl> &links, const QString &key) const;
 
 private:
+    void invalidateCache();
+
+    QStringList m_helpDatabases;
+    QSet<QString> m_keywordCache;
+    QString m_searchTermCache;
+    bool m_needsUpdate;
+    QMutex m_mutex;
     QIcon m_icon;
-    QStringList m_keywords;
 };
 
 } // namespace Internal