Mimetypes v3: Fix issues with providers overriding mime types

from other providers. Providers are sorted by "most specific first". If
some provider already contains some mime type, and didn't find e.g. a
match, another provider down the line may not add a match for that mime
type.

An example are the text/x-csrc and text/x-c++src mime types. We override
the versions in freedesktop.org.xml in the CppEditor plugin.

The freedesktop version of text/x-csrc has a magic for "#include" at the
beginning of the file, which we removed. A file foo.wxyz that starts
with "#include" should not open in our C++ editor, but without this
patch it does.

If you remove some extension/globpattern from the text/x-c++src mime
type in our mime type settings (Environment > MIME Types), for example
"*.cc", files with that extension should no longer open in our C++
editor. Without this patch they do.

Change-Id: I88049dce7ec2c8e57612f88464c6ce611336132d
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>
Reviewed-by: Jarek Kobus <jaroslaw.kobus@qt.io>

src/libs/utils/mimetypes2/mimedatabase.cpp

@@ -244,8 +244,9 @@ MimeGlobMatchResult MimeDatabasePrivate::findByFileName(const QString &fileName)
 {
     MimeGlobMatchResult result;
     const QString fileNameExcludingPath = QFileInfo(fileName).fileName();
+    QList<QString> checkedMimeTypes;
     for (const auto &provider : providers())
-        provider->addFileNameMatches(fileNameExcludingPath, result);
+        provider->addFileNameMatches(fileNameExcludingPath, result, checkedMimeTypes);
     return result;
 }
 
@@ -325,13 +326,15 @@ QStringList MimeDatabasePrivate::parents(const QString &mimeName)
 {
     Q_ASSERT(!mutex.tryLock());
     QStringList result;
-    for (const auto &provider : providers())
+    for (const auto &provider : providers()) {
+        if (provider->hasMimeTypeForName(mimeName)) {
             provider->addParents(mimeName, result);
-    if (result.isEmpty()) {
+            break;
+        }
+    }
     const QString parent = fallbackParent(mimeName);
     if (!parent.isEmpty())
         result.append(parent);
-    }
     return result;
 }
 
@@ -339,9 +342,13 @@ QStringList MimeDatabasePrivate::listAliases(const QString &mimeName)
 {
     QMutexLocker locker(&mutex);
     QStringList result;
-    for (const auto &provider : providers())
+    for (const auto &provider : providers()) {
+        if (provider->hasMimeTypeForName(mimeName)) {
             provider->addAliases(mimeName, result);
             return result;
+        }
+    }
+    return result;
 }
 
 bool MimeDatabasePrivate::mimeInherits(const QString &mime, const QString &parent)
@@ -378,8 +385,9 @@ MimeType MimeDatabasePrivate::findByData(const QByteArray &data, int *accuracyPt
 
     *accuracyPtr = 0;
     MimeType candidate;
+    QList<QString> checkedMimeTypes;
     for (const auto &provider : providers())
-        provider->findByMagic(data, accuracyPtr, candidate);
+        provider->findByMagic(data, accuracyPtr, candidate, checkedMimeTypes);
 
     if (candidate.isValid())
         return candidate;

src/libs/utils/mimetypes2/mimeglobpattern.cpp

@@ -255,13 +255,16 @@ void MimeAllGlobPatterns::removeMimeType(const QString &mimeType)
 }
 
 void MimeGlobPatternList::match(MimeGlobMatchResult &result,
-                                 const QString &fileName) const
+                                const QString &fileName,
+                                const QList<QString> &ignoreMimeTypes) const
 {
 
     MimeGlobPatternList::const_iterator it = this->constBegin();
     const MimeGlobPatternList::const_iterator endIt = this->constEnd();
     for (; it != endIt; ++it) {
         const MimeGlobPattern &glob = *it;
+        if (ignoreMimeTypes.contains(glob.mimeType()))
+            continue;
         if (glob.matchFileName(fileName)) {
             const QString pattern = glob.pattern();
             const int suffixLen = isSimplePattern(pattern) ? pattern.length() - 2 : 0;
@@ -270,10 +273,12 @@ void MimeGlobPatternList::match(MimeGlobMatchResult &result,
     }
 }
 
-void MimeAllGlobPatterns::matchingGlobs(const QString &fileName, MimeGlobMatchResult &result) const
+void MimeAllGlobPatterns::matchingGlobs(const QString &fileName,
+                                        MimeGlobMatchResult &result,
+                                        const QList<QString> &ignoreMimeTypes) const
 {
     // First try the high weight matches (>50), if any.
-    m_highWeightGlobs.match(result, fileName);
+    m_highWeightGlobs.match(result, fileName, ignoreMimeTypes);
 
     // Now use the "fast patterns" dict, for simple *.foo patterns with weight 50
     // (which is most of them, so this optimization is definitely worth it)
@@ -285,14 +290,16 @@ void MimeAllGlobPatterns::matchingGlobs(const QString &fileName, MimeGlobMatchRe
 
         const QStringList matchingMimeTypes = m_fastPatterns.value(simpleExtension);
         const QString simplePattern = QLatin1String("*.") + simpleExtension;
-        for (const QString &mime : matchingMimeTypes)
+        for (const QString &mime : matchingMimeTypes) {
+            if (!ignoreMimeTypes.contains(mime))
                 result.addMatch(mime, 50, simplePattern, simpleExtension.size());
+        }
         // Can't return yet; *.tar.bz2 has to win over *.bz2, so we need the low-weight mimetypes anyway,
         // at least those with weight 50.
     }
 
     // Finally, try the low weight matches (<=50)
-    m_lowWeightGlobs.match(result, fileName);
+    m_lowWeightGlobs.match(result, fileName, ignoreMimeTypes);
 }
 
 void MimeAllGlobPatterns::clear()

src/libs/utils/mimetypes2/mimeglobpattern_p.h

@@ -140,7 +140,9 @@ public:
         erase(std::remove_if(begin(), end(), isMimeTypeEqual), end());
     }
 
-    void match(MimeGlobMatchResult &result, const QString &fileName) const;
+    void match(MimeGlobMatchResult &result,
+               const QString &fileName,
+               const QList<QString> &ignoreMimeTypes) const;
 };
 
 /*!
@@ -157,7 +159,9 @@ public:
 
     void addGlob(const MimeGlobPattern &glob);
     void removeMimeType(const QString &mimeType);
-    void matchingGlobs(const QString &fileName, MimeGlobMatchResult &result) const;
+    void matchingGlobs(const QString &fileName,
+                       MimeGlobMatchResult &result,
+                       const QList<QString> &ignoreMimeTypes) const;
     void clear();
 
     PatternsMap m_fastPatterns; // example: "doc" -> "application/msword", "text/plain"

src/libs/utils/mimetypes2/mimeprovider.cpp

@@ -239,29 +239,62 @@ MimeType MimeBinaryProvider::mimeTypeForName(const QString &name)
     return mimeTypeForNameUnchecked(name);
 }
 
-void MimeBinaryProvider::addFileNameMatches(const QString &fileName, MimeGlobMatchResult &result)
+void MimeBinaryProvider::addFileNameMatches(const QString &fileName,
+                                            MimeGlobMatchResult &result,
+                                            QList<QString> &checkedMimeTypes)
 {
+    // TODO checkedMimeTypes
     if (fileName.isEmpty())
         return;
     Q_ASSERT(m_cacheFile);
     const QString lowerFileName = fileName.toLower();
     // Check literals (e.g. "Makefile")
-    matchGlobList(result, m_cacheFile, m_cacheFile->getUint32(PosLiteralListOffset), fileName);
+    matchGlobList(result,
+                  m_cacheFile,
+                  m_cacheFile->getUint32(PosLiteralListOffset),
+                  fileName,
+                  checkedMimeTypes);
     // Check the very common *.txt cases with the suffix tree
     if (result.m_matchingMimeTypes.isEmpty()) {
         const int reverseSuffixTreeOffset = m_cacheFile->getUint32(PosReverseSuffixTreeOffset);
         const int numRoots = m_cacheFile->getUint32(reverseSuffixTreeOffset);
         const int firstRootOffset = m_cacheFile->getUint32(reverseSuffixTreeOffset + 4);
-        matchSuffixTree(result, m_cacheFile, numRoots, firstRootOffset, lowerFileName, lowerFileName.length() - 1, false);
+        matchSuffixTree(result,
+                        m_cacheFile,
+                        numRoots,
+                        firstRootOffset,
+                        lowerFileName,
+                        lowerFileName.length() - 1,
+                        false,
+                        checkedMimeTypes);
         if (result.m_matchingMimeTypes.isEmpty())
-            matchSuffixTree(result, m_cacheFile, numRoots, firstRootOffset, fileName, fileName.length() - 1, true);
+            matchSuffixTree(result,
+                            m_cacheFile,
+                            numRoots,
+                            firstRootOffset,
+                            fileName,
+                            fileName.length() - 1,
+                            true,
+                            checkedMimeTypes);
     }
     // Check complex globs (e.g. "callgrind.out[0-9]*" or "README*")
     if (result.m_matchingMimeTypes.isEmpty())
-        matchGlobList(result, m_cacheFile, m_cacheFile->getUint32(PosGlobListOffset), fileName);
+        matchGlobList(result,
+                      m_cacheFile,
+                      m_cacheFile->getUint32(PosGlobListOffset),
+                      fileName,
+                      checkedMimeTypes);
+    // add all mime types from this provider to checkedMimeTypes, so they
+    // don't get checked again by another provider (if this provider overrides
+    // a mime type from another provider)
+    addAllMimeTypeNames(checkedMimeTypes);
 }
 
-void MimeBinaryProvider::matchGlobList(MimeGlobMatchResult &result, CacheFile *cacheFile, int off, const QString &fileName)
+void MimeBinaryProvider::matchGlobList(MimeGlobMatchResult &result,
+                                       CacheFile *cacheFile,
+                                       int off,
+                                       const QString &fileName,
+                                       const QList<QString> &ignoreMimeTypes)
 {
     const int numGlobs = cacheFile->getUint32(off);
     //qDebug() << "Loading" << numGlobs << "globs from" << cacheFile->file.fileName() << "at offset" << cacheFile->globListOffset;
@@ -275,6 +308,8 @@ void MimeBinaryProvider::matchGlobList(MimeGlobMatchResult &result, CacheFile *c
         const QString pattern = QLatin1String(cacheFile->getCharStar(globOffset));
 
         const char *mimeType = cacheFile->getCharStar(mimeTypeOffset);
+        if (ignoreMimeTypes.contains(QLatin1String(mimeType)))
+            continue;
         //qDebug() << pattern << mimeType << weight << caseSensitive;
         MimeGlobPattern glob(pattern, QString() /*unused*/, weight, qtCaseSensitive);
 
@@ -283,7 +318,14 @@ void MimeBinaryProvider::matchGlobList(MimeGlobMatchResult &result, CacheFile *c
     }
 }
 
-bool MimeBinaryProvider::matchSuffixTree(MimeGlobMatchResult &result, MimeBinaryProvider::CacheFile *cacheFile, int numEntries, int firstOffset, const QString &fileName, int charPos, bool caseSensitiveCheck)
+bool MimeBinaryProvider::matchSuffixTree(MimeGlobMatchResult &result,
+                                         MimeBinaryProvider::CacheFile *cacheFile,
+                                         int numEntries,
+                                         int firstOffset,
+                                         const QString &fileName,
+                                         int charPos,
+                                         bool caseSensitiveCheck,
+                                         const QList<QString> &ignoreMimeTypes)
 {
     QChar fileChar = fileName[charPos];
     int min = 0;
@@ -302,7 +344,14 @@ bool MimeBinaryProvider::matchSuffixTree(MimeGlobMatchResult &result, MimeBinary
             int childrenOffset = cacheFile->getUint32(off + 8);
             bool success = false;
             if (charPos > 0)
-                success = matchSuffixTree(result, cacheFile, numChildren, childrenOffset, fileName, charPos, caseSensitiveCheck);
+                success = matchSuffixTree(result,
+                                          cacheFile,
+                                          numChildren,
+                                          childrenOffset,
+                                          fileName,
+                                          charPos,
+                                          caseSensitiveCheck,
+                                          ignoreMimeTypes);
             if (!success) {
                 for (int i = 0; i < numChildren; ++i) {
                     const int childOff = childrenOffset + 12 * i;
@@ -311,6 +360,8 @@ bool MimeBinaryProvider::matchSuffixTree(MimeGlobMatchResult &result, MimeBinary
                         break;
                     const int mimeTypeOffset = cacheFile->getUint32(childOff + 4);
                     const char *mimeType = cacheFile->getCharStar(mimeTypeOffset);
+                    if (ignoreMimeTypes.contains(QLatin1String(mimeType)))
+                        continue;
                     const int flagsAndWeight = cacheFile->getUint32(childOff + 8);
                     const int weight = flagsAndWeight & 0xff;
                     const bool caseSensitive = flagsAndWeight & 0x100;
@@ -355,7 +406,10 @@ bool MimeBinaryProvider::matchMagicRule(MimeBinaryProvider::CacheFile *cacheFile
     return false;
 }
 
-void MimeBinaryProvider::findByMagic(const QByteArray &data, int *accuracyPtr, MimeType &candidate)
+void MimeBinaryProvider::findByMagic(const QByteArray &data,
+                                     int *accuracyPtr,
+                                     MimeType &candidate,
+                                     QList<QString> &checkedMimeTypes)
 {
     const int magicListOffset = m_cacheFile->getUint32(PosMagicListOffset);
     const int numMatches = m_cacheFile->getUint32(magicListOffset);
@@ -369,6 +423,8 @@ void MimeBinaryProvider::findByMagic(const QByteArray &data, int *accuracyPtr, M
         if (matchMagicRule(m_cacheFile, numMatchlets, firstMatchletOffset, data)) {
             const int mimeTypeOffset = m_cacheFile->getUint32(off + 4);
             const char *mimeType = m_cacheFile->getCharStar(mimeTypeOffset);
+            if (checkedMimeTypes.contains(QLatin1String(mimeType)))
+                continue;
             *accuracyPtr = m_cacheFile->getUint32(off);
             // Return the first match. We have no rules for conflicting magic data...
             // (mime.cache itself is sorted, but what about local overrides with a lower prio?)
@@ -376,6 +432,10 @@ void MimeBinaryProvider::findByMagic(const QByteArray &data, int *accuracyPtr, M
             return;
         }
     }
+    // add all mime types from this provider to checkedMimeTypes, so they
+    // don't get checked again by another provider (if this provider overrides
+    // a mime type from another provider)
+    addAllMimeTypeNames(checkedMimeTypes);
 }
 
 void MimeBinaryProvider::addParents(const QString &mime, QStringList &result)
@@ -713,16 +773,27 @@ MimeType MimeXMLProvider::mimeTypeForName(const QString &name)
     return m_nameMimeTypeMap.value(name);
 }
 
-void MimeXMLProvider::addFileNameMatches(const QString &fileName, MimeGlobMatchResult &result)
+void MimeXMLProvider::addFileNameMatches(const QString &fileName,
+                                         MimeGlobMatchResult &result,
+                                         QList<QString> &checkedMimeTypes)
 {
-    m_mimeTypeGlobs.matchingGlobs(fileName, result);
+    m_mimeTypeGlobs.matchingGlobs(fileName, result, checkedMimeTypes);
+    // add all mime types from this provider to checkedMimeTypes, so they
+    // don't get checked again by another provider (if this provider overrides
+    // a mime type from another provider)
+    addAllMimeTypeNames(checkedMimeTypes);
 }
 
-void MimeXMLProvider::findByMagic(const QByteArray &data, int *accuracyPtr, MimeType &candidate)
+void MimeXMLProvider::findByMagic(const QByteArray &data,
+                                  int *accuracyPtr,
+                                  MimeType &candidate,
+                                  QList<QString> &checkedMimeTypes)
 {
     QString candidateName;
     bool foundOne = false;
     for (const MimeMagicRuleMatcher &matcher : qAsConst(m_magicMatchers)) {
+        if (checkedMimeTypes.contains(matcher.mimetype()))
+            continue;
         if (matcher.matches(data)) {
             const int priority = matcher.priority();
             if (priority > *accuracyPtr) {
@@ -734,6 +805,10 @@ void MimeXMLProvider::findByMagic(const QByteArray &data, int *accuracyPtr, Mime
     }
     if (foundOne)
         candidate = mimeTypeForName(candidateName);
+    // add all mime types from this provider to checkedMimeTypes, so they
+    // don't get checked again by another provider (if this provider overrides
+    // a mime type from another provider)
+    addAllMimeTypeNames(checkedMimeTypes);
 }
 
 void MimeXMLProvider::ensureLoaded()
@@ -884,4 +959,44 @@ MimeXMLProvider::MimeXMLProvider(MimeDatabasePrivate *db,
     }
 }
 
+bool MimeBinaryProvider::hasMimeTypeForName(const QString &name)
+{
+    loadMimeTypeList();
+    return m_mimetypeNames.contains(name);
+}
+
+void MimeBinaryProvider::addAllMimeTypeNames(QList<QString> &result)
+{
+    // similar to addAllMimeTypes
+    loadMimeTypeList();
+    if (result.isEmpty()) { // fast path
+        result = QList(m_mimetypeNames.cbegin(), m_mimetypeNames.cend());
+    } else {
+        for (const QString &name : qAsConst(m_mimetypeNames))
+            if (!result.contains(name))
+                result.append(name);
+    }
+}
+
+bool MimeXMLProvider::hasMimeTypeForName(const QString &name)
+{
+    return m_nameMimeTypeMap.contains(name);
+}
+
+void MimeXMLProvider::addAllMimeTypeNames(QList<QString> &result)
+{
+    // similar to addAllMimeTypes
+    if (result.isEmpty()) { // fast path
+        result = m_nameMimeTypeMap.keys();
+    } else {
+        for (auto it = m_nameMimeTypeMap.constBegin(), end = m_nameMimeTypeMap.constEnd();
+             it != end;
+             ++it) {
+            const QString newMime = it.key();
+            if (!result.contains(newMime))
+                result.append(newMime);
+        }
+    }
+}
+
 } // namespace Utils

src/libs/utils/mimetypes2/mimeprovider_p.h

@@ -70,11 +70,18 @@ public:
     virtual bool isValid() = 0;
     virtual bool isInternalDatabase() const = 0;
     virtual MimeType mimeTypeForName(const QString &name) = 0;
-    virtual void addFileNameMatches(const QString &fileName, MimeGlobMatchResult &result) = 0;
+    virtual void addFileNameMatches(const QString &fileName,
+                                    MimeGlobMatchResult &result,
+                                    QList<QString> &checkedMimeTypes)
+        = 0;
     virtual void addParents(const QString &mime, QStringList &result) = 0;
     virtual QString resolveAlias(const QString &name) = 0;
     virtual void addAliases(const QString &name, QStringList &result) = 0;
-    virtual void findByMagic(const QByteArray &data, int *accuracyPtr, MimeType &candidate) = 0;
+    virtual void findByMagic(const QByteArray &data,
+                             int *accuracyPtr,
+                             MimeType &candidate,
+                             QList<QString> &checkedMimeTypes)
+        = 0;
     virtual void addAllMimeTypes(QList<MimeType> &result) = 0;
     virtual bool loadMimeTypePrivate(MimeTypePrivate &) { return false; }
     virtual void loadIcon(MimeTypePrivate &) {}
@@ -83,6 +90,10 @@ public:
 
     QString directory() const { return m_directory; }
 
+    // added for Qt Creator
+    virtual bool hasMimeTypeForName(const QString &name) = 0;
+    virtual void addAllMimeTypeNames(QList<QString> &result) = 0;
+
     MimeDatabasePrivate *m_db;
     QString m_directory;
 };
@@ -99,22 +110,42 @@ public:
     bool isValid() override;
     bool isInternalDatabase() const override;
     MimeType mimeTypeForName(const QString &name) override;
-    void addFileNameMatches(const QString &fileName, MimeGlobMatchResult &result) override;
+    void addFileNameMatches(const QString &fileName,
+                            MimeGlobMatchResult &result,
+                            QList<QString> &checkedMimeTypes) override;
     void addParents(const QString &mime, QStringList &result) override;
     QString resolveAlias(const QString &name) override;
     void addAliases(const QString &name, QStringList &result) override;
-    void findByMagic(const QByteArray &data, int *accuracyPtr, MimeType &candidate) override;
+    void findByMagic(const QByteArray &data,
+                     int *accuracyPtr,
+                     MimeType &candidate,
+                     QList<QString> &checkedMimeTypes) override;
     void addAllMimeTypes(QList<MimeType> &result) override;
     bool loadMimeTypePrivate(MimeTypePrivate &) override;
     void loadIcon(MimeTypePrivate &) override;
     void loadGenericIcon(MimeTypePrivate &) override;
     void ensureLoaded() override;
 
+    // added for Qt Creator
+    bool hasMimeTypeForName(const QString &name) override;
+    void addAllMimeTypeNames(QList<QString> &result) override;
+
 private:
     struct CacheFile;
 
-    void matchGlobList(MimeGlobMatchResult &result, CacheFile *cacheFile, int offset, const QString &fileName);
+    void matchGlobList(MimeGlobMatchResult &result,
-    bool matchSuffixTree(MimeGlobMatchResult &result, CacheFile *cacheFile, int numEntries, int firstOffset, const QString &fileName, int charPos, bool caseSensitiveCheck);
+                       CacheFile *cacheFile,
+                       int offset,
+                       const QString &fileName,
+                       const QList<QString> &ignoreMimeTypes);
+    bool matchSuffixTree(MimeGlobMatchResult &result,
+                         CacheFile *cacheFile,
+                         int numEntries,
+                         int firstOffset,
+                         const QString &fileName,
+                         int charPos,
+                         bool caseSensitiveCheck,
+                         const QList<QString> &ignoreMimeTypes);
     bool matchMagicRule(CacheFile *cacheFile, int numMatchlets, int firstOffset, const QByteArray &data);
     QLatin1String iconForMime(CacheFile *cacheFile, int posListOffset, const QByteArray &inputMime);
     void loadMimeTypeList();
@@ -153,11 +184,16 @@ public:
     bool isValid() override;
     bool isInternalDatabase() const override;
     MimeType mimeTypeForName(const QString &name) override;
-    void addFileNameMatches(const QString &fileName, MimeGlobMatchResult &result) override;
+    void addFileNameMatches(const QString &fileName,
+                            MimeGlobMatchResult &result,
+                            QList<QString> &checkedMimeTypes) override;
     void addParents(const QString &mime, QStringList &result) override;
     QString resolveAlias(const QString &name) override;
     void addAliases(const QString &name, QStringList &result) override;
-    void findByMagic(const QByteArray &data, int *accuracyPtr, MimeType &candidate) override;
+    void findByMagic(const QByteArray &data,
+                     int *accuracyPtr,
+                     MimeType &candidate,
+                     QList<QString> &checkedMimeTypes) override;
     void addAllMimeTypes(QList<MimeType> &result) override;
     void ensureLoaded() override;
 
@@ -170,6 +206,10 @@ public:
     void addAlias(const QString &alias, const QString &name);
     void addMagicMatcher(const MimeMagicRuleMatcher &matcher);
 
+    // added for Qt Creator
+    bool hasMimeTypeForName(const QString &name) override;
+    void addAllMimeTypeNames(QList<QString> &result) override;
+
 private:
     void load(const QString &fileName);
     void load(const char *data, qsizetype len);