diff --git a/src/plugins/clangtools/documentclangtoolrunner.cpp b/src/plugins/clangtools/documentclangtoolrunner.cpp
index 8dab531dcb6..5640411e7cf 100644
--- a/src/plugins/clangtools/documentclangtoolrunner.cpp
+++ b/src/plugins/clangtools/documentclangtoolrunner.cpp
@@ -83,9 +83,7 @@ static void removeClangToolRefactorMarkers(TextEditor::TextEditorWidget *editor)
 {
     if (!editor)
         return;
-    editor->setRefactorMarkers(
-        TextEditor::RefactorMarker::filterOutType(editor->refactorMarkers(),
-                                                  Constants::CLANG_TOOL_FIXIT_AVAILABLE_MARKER_ID));
+    editor->clearRefactorMarkers(Constants::CLANG_TOOL_FIXIT_AVAILABLE_MARKER_ID);
 }
 
 void DocumentClangToolRunner::scheduleRun()
@@ -291,7 +289,7 @@ void DocumentClangToolRunner::onDone(const AnalyzeOutputData &output)
 
     for (auto editor : TextEditor::BaseTextEditor::textEditorsForDocument(doc)) {
         if (TextEditor::TextEditorWidget *widget = editor->editorWidget()) {
-            widget->setRefactorMarkers(markers + widget->refactorMarkers());
+            widget->setRefactorMarkers(markers, Constants::CLANG_TOOL_FIXIT_AVAILABLE_MARKER_ID);
             if (!m_editorsWithMarkers.contains(widget))
                 m_editorsWithMarkers << widget;
         }
diff --git a/src/plugins/cppeditor/cppeditorwidget.cpp b/src/plugins/cppeditor/cppeditorwidget.cpp
index edc65f76526..a6daf681db5 100644
--- a/src/plugins/cppeditor/cppeditorwidget.cpp
+++ b/src/plugins/cppeditor/cppeditorwidget.cpp
@@ -590,8 +590,7 @@ void CppEditorWidget::onCodeWarningsUpdated(unsigned revision,
 
     setExtraSelections(TextEditorWidget::CodeWarningsSelection,
                        unselectLeadingWhitespace(selections));
-    setRefactorMarkers(refactorMarkers + RefactorMarker::filterOutType(
-            this->refactorMarkers(), Constants::CPP_CLANG_FIXIT_AVAILABLE_MARKER_ID));
+    setRefactorMarkers(refactorMarkers, Constants::CPP_CLANG_FIXIT_AVAILABLE_MARKER_ID);
 }
 
 void CppEditorWidget::onIfdefedOutBlocksUpdated(unsigned revision,
diff --git a/src/plugins/cppeditor/cppfunctiondecldeflink.cpp b/src/plugins/cppeditor/cppfunctiondecldeflink.cpp
index b43e87758e2..aa17721ea37 100644
--- a/src/plugins/cppeditor/cppfunctiondecldeflink.cpp
+++ b/src/plugins/cppeditor/cppfunctiondecldeflink.cpp
@@ -279,8 +279,7 @@ void FunctionDeclDefLink::hideMarker(CppEditorWidget *editor)
 {
     if (!hasMarker)
         return;
-    editor->setRefactorMarkers(RefactorMarker::filterOutType(
-        editor->refactorMarkers(), Constants::CPP_FUNCTION_DECL_DEF_LINK_MARKER_ID));
+    editor->clearRefactorMarkers(Constants::CPP_FUNCTION_DECL_DEF_LINK_MARKER_ID);
     hasMarker = false;
 }
 
@@ -289,8 +288,7 @@ void FunctionDeclDefLink::showMarker(CppEditorWidget *editor)
     if (hasMarker)
         return;
 
-    QList<RefactorMarker> markers = RefactorMarker::filterOutType(
-        editor->refactorMarkers(), Constants::CPP_FUNCTION_DECL_DEF_LINK_MARKER_ID);
+    RefactorMarkers markers;
     RefactorMarker marker;
 
     // show the marker at the end of the linked area, with a special case
@@ -321,7 +319,7 @@ void FunctionDeclDefLink::showMarker(CppEditorWidget *editor)
             cppEditor->applyDeclDefLinkChanges(true);
     };
     markers += marker;
-    editor->setRefactorMarkers(markers);
+    editor->setRefactorMarkers(markers, Constants::CPP_FUNCTION_DECL_DEF_LINK_MARKER_ID);
 
     hasMarker = true;
 }
diff --git a/src/plugins/languageclient/client.cpp b/src/plugins/languageclient/client.cpp
index bb5a4594dc5..219a4547368 100644
--- a/src/plugins/languageclient/client.cpp
+++ b/src/plugins/languageclient/client.cpp
@@ -202,8 +202,7 @@ public:
             for (Core::IEditor *editor : editors) {
                 if (auto textEditor = qobject_cast<BaseTextEditor *>(editor)) {
                     TextEditorWidget *widget = textEditor->editorWidget();
-                    widget->setRefactorMarkers(
-                        RefactorMarker::filterOutType(widget->refactorMarkers(), m_id));
+                    widget->clearRefactorMarkers(m_id);
                     widget->removeHoverHandler(&m_hoverHandler);
                 }
             }
@@ -910,8 +909,7 @@ void Client::deactivateDocument(TextEditor::TextDocument *document)
             TextEditor::TextEditorWidget *widget = textEditor->editorWidget();
             widget->removeHoverHandler(&d->m_hoverHandler);
             widget->setExtraSelections(TextEditor::TextEditorWidget::CodeSemanticsSelection, {});
-            widget->setRefactorMarkers(
-                TextEditor::RefactorMarker::filterOutType(widget->refactorMarkers(), id()));
+            widget->clearRefactorMarkers(id());
             updateEditorToolBar(editor);
         }
     }
@@ -1158,7 +1156,7 @@ void Client::documentContentsChanged(TextEditor::TextDocument *document,
         TextEditorWidget *widget = editor->editorWidget();
         QTC_ASSERT(widget, continue);
         delete d->m_documentHighlightsTimer.take(widget);
-        widget->setRefactorMarkers(RefactorMarker::filterOutType(widget->refactorMarkers(), id()));
+        widget->clearRefactorMarkers(id());
     }
     d->m_documentUpdateTimer.start();
 }
diff --git a/src/plugins/languageclient/languageclientutils.cpp b/src/plugins/languageclient/languageclientutils.cpp
index c421b2a9376..06ed8dcae65 100644
--- a/src/plugins/languageclient/languageclientutils.cpp
+++ b/src/plugins/languageclient/languageclientutils.cpp
@@ -200,7 +200,7 @@ void updateCodeActionRefactoringMarker(Client *client,
     const RefactorMarkers markers = markersAtBlock.values();
     for (BaseTextEditor *editor : editors) {
         if (TextEditorWidget *editorWidget = editor->editorWidget())
-            editorWidget->setRefactorMarkers(markers + editorWidget->refactorMarkers());
+            editorWidget->setRefactorMarkers(markers, client->id());
     }
 }
 
diff --git a/src/plugins/qmljseditor/qmljseditor.cpp b/src/plugins/qmljseditor/qmljseditor.cpp
index ccc94a5ea42..1bd8ef89ff8 100644
--- a/src/plugins/qmljseditor/qmljseditor.cpp
+++ b/src/plugins/qmljseditor/qmljseditor.cpp
@@ -311,8 +311,7 @@ void QmlJSEditorWidget::updateContextPane()
 
         if (m_contextPane->isAvailable(this, info.document, newNode) &&
             !m_contextPane->widget()->isVisible()) {
-            QList<RefactorMarker> markers
-                = RefactorMarker::filterOutType(refactorMarkers(), QT_QUICK_TOOLBAR_MARKER_ID);
+            RefactorMarkers markers;
             if (UiObjectMember *m = newNode->uiObjectMemberCast()) {
                 const int start = qualifiedTypeNameId(m)->identifierToken.begin();
                 for (UiQualifiedId *q = qualifiedTypeNameId(m); q; q = q->next) {
@@ -333,10 +332,9 @@ void QmlJSEditorWidget::updateContextPane()
                     }
                 }
             }
-            setRefactorMarkers(markers);
+            setRefactorMarkers(markers, QT_QUICK_TOOLBAR_MARKER_ID);
         } else if (oldNode != newNode) {
-            setRefactorMarkers(
-                RefactorMarker::filterOutType(refactorMarkers(), QT_QUICK_TOOLBAR_MARKER_ID));
+            clearRefactorMarkers(QT_QUICK_TOOLBAR_MARKER_ID);
         }
         m_oldCursorPosition = position();
 
@@ -877,8 +875,7 @@ void QmlJSEditorWidget::showContextPane()
                              &scopeChain,
                              newNode, false, true);
         m_oldCursorPosition = position();
-        setRefactorMarkers(
-            RefactorMarker::filterOutType(refactorMarkers(), QT_QUICK_TOOLBAR_MARKER_ID));
+        clearRefactorMarkers(QT_QUICK_TOOLBAR_MARKER_ID);
     }
 }
 
diff --git a/src/plugins/texteditor/texteditor.cpp b/src/plugins/texteditor/texteditor.cpp
index e7f428f83b6..4592980540e 100644
--- a/src/plugins/texteditor/texteditor.cpp
+++ b/src/plugins/texteditor/texteditor.cpp
@@ -8572,6 +8572,38 @@ void TextEditorWidget::setRefactorMarkers(const RefactorMarkers &markers)
         emit requestBlockUpdate(marker.cursor.block());
 }
 
+void TextEditorWidget::setRefactorMarkers(const RefactorMarkers &newMarkers, const Utils::Id &type)
+{
+    RefactorMarkers markers = d->m_refactorOverlay->markers();
+    auto first = std::partition(markers.begin(),
+                                markers.end(),
+                                [type](const RefactorMarker &marker) {
+                                    return marker.type == type;
+                                });
+
+    for (auto it = markers.begin(); it != first; ++it)
+        emit requestBlockUpdate(it->cursor.block());
+    markers.erase(markers.begin(), first);
+    markers.append(newMarkers);
+    d->m_refactorOverlay->setMarkers(markers);
+    for (const RefactorMarker &marker : newMarkers)
+        emit requestBlockUpdate(marker.cursor.block());
+}
+
+void TextEditorWidget::clearRefactorMarkers(const Utils::Id &type)
+{
+    RefactorMarkers markers = d->m_refactorOverlay->markers();
+    for (auto it = markers.begin(); it != markers.end();) {
+        if (it->type == type) {
+            emit requestBlockUpdate(it->cursor.block());
+            it = markers.erase(it);
+        } else {
+            ++it;
+        }
+    }
+    d->m_refactorOverlay->setMarkers(markers);
+}
+
 bool TextEditorWidget::inFindScope(const QTextCursor &cursor) const
 {
     return d->m_find->inScope(cursor);
diff --git a/src/plugins/texteditor/texteditor.h b/src/plugins/texteditor/texteditor.h
index e7cb5b34347..c05425a5b07 100644
--- a/src/plugins/texteditor/texteditor.h
+++ b/src/plugins/texteditor/texteditor.h
@@ -310,6 +310,8 @@ public:
 
     RefactorMarkers refactorMarkers() const;
     void setRefactorMarkers(const RefactorMarkers &markers);
+    void setRefactorMarkers(const RefactorMarkers &markers, const Utils::Id &type);
+    void clearRefactorMarkers(const Utils::Id &type);
 
     enum Side { Left, Right };
     QAction *insertExtraToolBarWidget(Side side, QWidget *widget);