Introduce codemodelbackend process and library

This is a partial result of wip/clang-oop. More will follow.

This allows us to invoke the completion out of the Qt Creator process
and thus safes us as against libclang crashes.

At this point only the completion use case is supported.

Some notes on the individual components:

 src/libs/codemodelbackendipc
  * library encapsulating the inter process communication handling
  * used by the backend application and in a follow-up change by the
    creator integration

 src/libs/3rdparty/sqlite
  * version 3.8.10.2
  * dependency of codemodelbackendipc, will be used to storage indexing
    data, among others

 src/tools/codemodelbackend
  * the backend application

 tests/unit:
  * unit tests

Change-Id: I91a48e27467581a22fb760a18d8eb926008fea60
Reviewed-by: Alessandro Portale <alessandro.portale@theqtcompany.com>
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@theqtcompany.com>
Reviewed-by: Marco Bubke <marco.bubke@theqtcompany.com>
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@theqtcompany.com>

src/plugins/clangcodemodel/clang_installation.pri

@@ -1,101 +0,0 @@
-isEmpty(LLVM_INSTALL_DIR):LLVM_INSTALL_DIR=$$(LLVM_INSTALL_DIR)
-LLVM_INSTALL_DIR = $$clean_path($$LLVM_INSTALL_DIR)
-
-DEFINES += CLANG_COMPLETION
-DEFINES += CLANG_HIGHLIGHTING
-#DEFINES += CLANG_INDEXING
-
-defineReplace(findLLVMConfig) {
-    LLVM_CONFIG_VARIANTS = \
-        llvm-config llvm-config-3.2 llvm-config-3.3 llvm-config-3.4 \
-        llvm-config-3.5 llvm-config-3.6 llvm-config-4.0 llvm-config-4.1
-
-    # Prefer llvm-config* from LLVM_INSTALL_DIR
-    !isEmpty(LLVM_INSTALL_DIR) {
-        for(variant, LLVM_CONFIG_VARIANTS) {
-            variant=$$LLVM_INSTALL_DIR/bin/$$variant
-            exists($$variant) {
-                return($$variant)
-            }
-        }
-    }
-
-    # Find llvm-config* in PATH
-    ENV_PATH = $$(PATH)
-    ENV_PATH = $$split(ENV_PATH, $$QMAKE_DIRLIST_SEP)
-    for(variant, LLVM_CONFIG_VARIANTS) {
-        for(path, ENV_PATH) {
-            subvariant = $$path/$$variant
-            exists($$subvariant) {
-                return($$subvariant)
-            }
-        }
-    }
-
-    # Fallback
-    return(llvm-config)
-}
-
-defineReplace(findClangOnWindows) {
-    FILE_EXTS = a dll
-    win32-msvc*: FILE_EXTS = lib dll
-    for (suffix, $$list(lib bin)) {
-        for (libname, $$list(clang libclang)) {
-            for (ext, FILE_EXTS) {
-                exists("$${LLVM_INSTALL_DIR}/$${suffix}/$${libname}.$${ext}") {
-                    return($${LLVM_INSTALL_DIR}/$${suffix}/)
-                }
-            }
-        }
-    }
-    error("Cannot find clang shared library at $${LLVM_INSTALL_DIR}")
-}
-
-win32 {
-    LLVM_INCLUDEPATH = "$$LLVM_INSTALL_DIR/include"
-    CLANG_LIB_PATH = $$findClangOnWindows()
-    CLANG_LIB = clang
-    !exists("$${CLANG_LIB_PATH}/clang.*"):CLANG_LIB = libclang
-
-    LLVM_LIBS = -L"$${CLANG_LIB_PATH}" -l$${CLANG_LIB}
-    LLVM_LIBS += -ladvapi32 -lshell32
-    LLVM_VERSION = 3.4
-}
-
-unix {
-    LLVM_CONFIG = $$findLLVMConfig()
-
-    LLVM_VERSION = $$system($$LLVM_CONFIG --version 2>/dev/null)
-    LLVM_VERSION = $$replace(LLVM_VERSION, ^(\\d+\\.\\d+\\.\\d+).*$, \\1)
-    message("... version $$LLVM_VERSION")
-
-    LLVM_INCLUDEPATH = $$system($$LLVM_CONFIG --includedir 2>/dev/null)
-    isEmpty(LLVM_INCLUDEPATH):LLVM_INCLUDEPATH=$$LLVM_INSTALL_DIR/include
-    LLVM_LIBDIR = $$system($$LLVM_CONFIG --libdir 2>/dev/null)
-    isEmpty(LLVM_LIBDIR):LLVM_LIBDIR=$$LLVM_INSTALL_DIR/lib
-
-    exists ($${LLVM_LIBDIR}/libclang.*) {
-        #message("LLVM was build with autotools")
-        CLANG_LIB = clang
-    } else {
-        exists ($${LLVM_LIBDIR}/liblibclang.*) {
-            #message("LLVM was build with CMake")
-            CLANG_LIB = libclang
-        } else {
-            exists ($${LLVM_INSTALL_DIR}/lib/libclang.*) {
-                #message("libclang placed separately from LLVM")
-                CLANG_LIB = clang
-                LLVM_LIBDIR = $${LLVM_INSTALL_DIR}/lib
-                LLVM_INCLUDEPATH=$${LLVM_INSTALL_DIR}/include
-            } else {
-                error("Cannot find Clang shared library!")
-            }
-        }
-    }
-
-    LLVM_LIBS = -L$${LLVM_LIBDIR}
-    LLVM_LIBS += -l$${CLANG_LIB}
-}
-
-DEFINES += CLANG_VERSION=\\\"$${LLVM_VERSION}\\\"
-DEFINES += "\"CLANG_RESOURCE_DIR=\\\"$${LLVM_LIBDIR}/clang/$${LLVM_VERSION}/include\\\"\""

src/plugins/clangcodemodel/clangcodemodel.pro

@@ -1,5 +1,5 @@
 include(../../qtcreatorplugin.pri)
-include(clang_installation.pri)
+include(../../shared/clang/clang_installation.pri)
 
 message("Building ClangCodeModel plugin with Clang from $$LLVM_INSTALL_DIR")
 message("  INCLUDEPATH += $$LLVM_INCLUDEPATH")

src/plugins/clangcodemodel/clangcodemodel.qbs

@@ -1,6 +1,6 @@
 import qbs
 import qbs.File
-import qbs.Process
+import QtcClangInstallation as Clang
 import QtcProcessOutputReader
 
 QtcPlugin {
@@ -13,56 +13,22 @@ QtcPlugin {
     Depends { name: "TextEditor" }
     Depends { name: "Utils" }
 
-    property string llvmInstallDirFromEnv: qbs.getEnv("LLVM_INSTALL_DIR")
-
     property bool clangCompletion: true
     property bool clangHighlighting: true
     property bool clangIndexing: false
 
-    property string llvmConfig: {
-        var llvmConfigVariants = [
-            "llvm-config", "llvm-config-3.2", "llvm-config-3.3", "llvm-config-3.4",
-            "llvm-config-3.5", "llvm-config-3.6", "llvm-config-4.0", "llvm-config-4.1"
-        ];
+    property string llvmConfig: Clang.llvmConfig(qbs)
+    property string llvmIncludeDir: Clang.includeDir(llvmConfig, QtcProcessOutputReader)
+    property string llvmLibDir: Clang.libDir(llvmConfig, QtcProcessOutputReader)
+    property string llvmLibs: Clang.libraries(qbs.targetOS)
+    property string llvmVersion: Clang.version(llvmConfig, QtcProcessOutputReader)
 
-        // Prefer llvm-config* from LLVM_INSTALL_DIR
-        if (llvmInstallDirFromEnv) {
-            for (var i = 0; i < llvmConfigVariants.length; ++i) {
-                var variant = llvmInstallDirFromEnv + "/bin/" + llvmConfigVariants[i];
-                if (File.exists(variant))
-                    return variant;
-            }
-        }
-
-        // Find llvm-config* in PATH
-        var pathListString = qbs.getEnv("PATH");
-        var separator = qbs.hostOS.contains("windows") ? ";" : ":";
-        var pathList = pathListString.split(separator);
-        for (var i = 0; i < llvmConfigVariants.length; ++i) {
-            for (var j = 0; j < pathList.length; ++j) {
-                var variant = pathList[j] + "/" + llvmConfigVariants[i];
-                if (File.exists(variant))
-                    return variant;
-            }
-        }
-
-        return undefined;
-    }
     condition: llvmConfig
 
-    property string llvmIncludeDir: QtcProcessOutputReader.readOutput(llvmConfig, ["--includedir"])
-    property string llvmLibDir: QtcProcessOutputReader.readOutput(llvmConfig, ["--libdir"])
-    property string llvmVersion: QtcProcessOutputReader.readOutput(llvmConfig, ["--version"])
-        .replace(/(\d+\.\d+\.\d+).*/, "$1")
-
     cpp.includePaths: base.concat(llvmIncludeDir)
     cpp.libraryPaths: base.concat(llvmLibDir)
     cpp.rpaths: cpp.libraryPaths
-
-    property string llvmLib: "clang"
-    property stringList additionalLibraries: qbs.targetOS.contains("windows")
-                                             ? ["advapi32", "shell32"] : []
-    cpp.dynamicLibraries: base.concat(llvmLib).concat(additionalLibraries)
+    cpp.dynamicLibraries: base.concat(llvmLibs)
 
     cpp.defines: {
         var defines = base;

src/plugins/clangcodemodel/clangcodemodelunittestfiles.pri

@@ -0,0 +1,5 @@
+INCLUDEPATH += $$PWD
+
+SOURCES += $$PWD/completionchunkstotextconverter.cpp
+
+HEADERS += $$PWD/completionchunkstotextconverter.h

src/plugins/clangcodemodel/completionchunkstotextconverter.cpp

@@ -0,0 +1,90 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://www.qt.io/licensing.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+#include "completionchunkstotextconverter.h"
+
+namespace ClangCodeModel {
+namespace Internal {
+
+void CompletionChunksToTextConverter::parseChunks(const QVector<CodeModelBackEnd::CodeCompletionChunk> &codeCompletionChunks)
+{
+    m_text.clear();
+
+    for (const auto &codeCompletionChunk : codeCompletionChunks)
+        parse(codeCompletionChunk);
+}
+
+const QString &CompletionChunksToTextConverter::text() const
+{
+    return m_text;
+}
+
+QString CompletionChunksToTextConverter::convert(const QVector<CodeModelBackEnd::CodeCompletionChunk> &codeCompletionChunks)
+{
+    CompletionChunksToTextConverter converter;
+
+    converter.parseChunks(codeCompletionChunks);
+
+    return converter.text();
+}
+
+void CompletionChunksToTextConverter::parse(const CodeModelBackEnd::CodeCompletionChunk &codeCompletionChunk)
+{
+    using CodeModelBackEnd::CodeCompletionChunk;
+
+    switch (codeCompletionChunk.kind()) {
+        case CodeCompletionChunk::ResultType: parseResultType(codeCompletionChunk.text()); break;
+        case CodeCompletionChunk::Optional: parseOptional(codeCompletionChunk); break;
+        default: parseText(codeCompletionChunk.text()); break;
+    }
+}
+
+void CompletionChunksToTextConverter::parseResultType(const Utf8String &resultTypeText)
+{
+    m_text += resultTypeText.toString() + QChar(QChar::Space);
+}
+
+void CompletionChunksToTextConverter::parseText(const Utf8String &text)
+{
+    m_text += text.toString();
+}
+
+void CompletionChunksToTextConverter::parseOptional(const CodeModelBackEnd::CodeCompletionChunk &optionalCodeCompletionChunk)
+{
+    m_text += QStringLiteral("<i>");
+
+    m_text += convert(optionalCodeCompletionChunk.optionalChunks());
+
+    m_text += QStringLiteral("</i>");
+}
+
+} // namespace Internal
+} // namespace ClangCodeModel
+

src/plugins/clangcodemodel/completionchunkstotextconverter.h

@@ -0,0 +1,65 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt.io/licensing
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://www.qt.io/licensing.  For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+#ifndef CLANGCODEMODEL_INTERNAL_COMPLETIONCHUNKSTOTEXTCONVERTER_H
+#define CLANGCODEMODEL_INTERNAL_COMPLETIONCHUNKSTOTEXTCONVERTER_H
+
+#include <codemodelbackendipc/codecompletionchunk.h>
+
+#include <QString>
+
+#include <sqlite/utf8string.h>
+
+namespace ClangCodeModel {
+namespace Internal {
+
+class CompletionChunksToTextConverter
+{
+public:
+    void parseChunks(const QVector<CodeModelBackEnd::CodeCompletionChunk> &codeCompletionChunks);
+
+    const QString &text() const;
+
+    static QString convert(const QVector<CodeModelBackEnd::CodeCompletionChunk> &codeCompletionChunks);
+
+private:
+    void parse(const CodeModelBackEnd::CodeCompletionChunk & codeCompletionChunk);
+    void parseResultType(const Utf8String &text);
+    void parseText(const Utf8String &text);
+    void parseOptional(const CodeModelBackEnd::CodeCompletionChunk & optionalCodeCompletionChunk);
+
+private:
+    QString m_text;
+};
+
+} // namespace Internal
+} // namespace ClangCodeModel
+
+#endif // CLANGCODEMODEL_INTERNAL_COMPLETIONCHUNKSTOTEXTCONVERTER_H