Python: add pyside uic extra compiler

To be able to autocomplete code from .ui forms we need to feed the
python language server the compiled form. The uic extra compiler
generates a temporary ui_name.py file for uncompiled or unsaved ui
files. These files are inside a folder that gets appended to the python
path environment variable for the python language server.

Change-Id: I9f48d2012162f33986639315189c41e0a7e0dad2
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

src/plugins/languageclient/languageclientinterface.cpp

@@ -99,7 +99,9 @@ void BaseClientInterface::parseCurrentMessage()
     m_currentMessage = BaseMessage();
 }
 
-StdIOClientInterface::StdIOClientInterface() {}
+StdIOClientInterface::StdIOClientInterface()
+    : m_env(Utils::Environment::systemEnvironment())
+{}
 
 StdIOClientInterface::~StdIOClientInterface()
 {
@@ -124,6 +126,7 @@ void StdIOClientInterface::startImpl()
     connect(m_process, &QtcProcess::started, this, &StdIOClientInterface::started);
     m_process->setCommand(m_cmd);
     m_process->setWorkingDirectory(m_workingDirectory);
+    m_process->setEnvironment(m_env);
     m_process->start();
 }
 
@@ -137,6 +140,11 @@ void StdIOClientInterface::setWorkingDirectory(const FilePath &workingDirectory)
     m_workingDirectory = workingDirectory;
 }
 
+void StdIOClientInterface::setEnvironment(const Utils::Environment &environment)
+{
+    m_env = environment;
+}
+
 void StdIOClientInterface::sendData(const QByteArray &data)
 {
     if (!m_process || m_process->state() != QProcess::Running) {

src/plugins/languageclient/languageclientinterface.h

@@ -29,6 +29,7 @@
 
 #include <languageserverprotocol/jsonrpcmessages.h>
 
+#include <utils/environment.h>
 #include <utils/qtcprocess.h>
 
 #include <QBuffer>
@@ -84,12 +85,14 @@ public:
     // These functions only have an effect if they are called before start
     void setCommandLine(const Utils::CommandLine &cmd);
     void setWorkingDirectory(const Utils::FilePath &workingDirectory);
+    void setEnvironment(const Utils::Environment &environment);
 
 protected:
     void sendData(const QByteArray &data) final;
     Utils::CommandLine m_cmd;
     Utils::FilePath m_workingDirectory;
     Utils::QtcProcess *m_process = nullptr;
+    Utils::Environment m_env;
 
 private:
     void readError();

src/plugins/python/CMakeLists.txt

@@ -5,6 +5,7 @@ add_qtc_plugin(Python
     pipsupport.cpp pipsupport.h
     pyside.cpp pyside.h
     pysidebuildconfiguration.cpp pysidebuildconfiguration.h
+    pysideuicextracompiler.cpp pysideuicextracompiler.h
     python.qrc
     pythonconstants.h
     pythoneditor.cpp pythoneditor.h

src/plugins/python/pysideuicextracompiler.cpp

@@ -0,0 +1,72 @@
+/****************************************************************************
+**
+** Copyright (C) 2022 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "pysideuicextracompiler.h"
+
+#include <utils/qtcprocess.h>
+
+using namespace ProjectExplorer;
+
+namespace Python {
+namespace Internal {
+
+PySideUicExtraCompiler::PySideUicExtraCompiler(const Utils::FilePath &pySideUic,
+                                               const Project *project,
+                                               const Utils::FilePath &source,
+                                               const Utils::FilePaths &targets,
+                                               QObject *parent)
+    : ProcessExtraCompiler(project, source, targets, parent)
+    , m_pySideUic(pySideUic)
+{
+}
+
+Utils::FilePath PySideUicExtraCompiler::pySideUicPath() const
+{
+    return m_pySideUic;
+}
+
+Utils::FilePath PySideUicExtraCompiler::command() const
+{
+    return m_pySideUic;
+}
+
+FileNameToContentsHash PySideUicExtraCompiler::handleProcessFinished(
+    Utils::QtcProcess *process)
+{
+    FileNameToContentsHash result;
+    if (process->exitStatus() != QProcess::NormalExit && process->exitCode() != 0)
+        return result;
+
+    const Utils::FilePaths targetList = targets();
+    if (targetList.size() != 1)
+        return result;
+    // As far as I can discover in the UIC sources, it writes out local 8-bit encoding. The
+    // conversion below is to normalize both the encoding, and the line terminators.
+    result[targetList.first()] = QString::fromLocal8Bit(process->readAllStandardOutput()).toUtf8();
+    return result;
+}
+
+} // namespace Internal
+} // namespace Python

src/plugins/python/pysideuicextracompiler.h

@@ -0,0 +1,53 @@
+/****************************************************************************
+**
+** Copyright (C) 2022 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <projectexplorer/extracompiler.h>
+
+namespace Python {
+namespace Internal {
+
+class PySideUicExtraCompiler : public ProjectExplorer::ProcessExtraCompiler
+{
+public:
+    PySideUicExtraCompiler(const Utils::FilePath &pySideUic,
+                           const ProjectExplorer::Project *project,
+                           const Utils::FilePath &source,
+                           const Utils::FilePaths &targets,
+                           QObject *parent = nullptr);
+
+    Utils::FilePath pySideUicPath() const;
+
+private:
+    Utils::FilePath command() const override;
+    ProjectExplorer::FileNameToContentsHash handleProcessFinished(
+        Utils::QtcProcess *process) override;
+
+    Utils::FilePath m_pySideUic;
+};
+
+} // namespace Internal
+} // namespace Python

src/plugins/python/python.qbs

@@ -23,6 +23,8 @@ QtcPlugin {
             "pyside.h",
             "pysidebuildconfiguration.cpp",
             "pysidebuildconfiguration.h",
+            "pysideuicextracompiler.cpp",
+            "pysideuicextracompiler.h",
             "python.qrc",
             "pythonconstants.h",
             "pythoneditor.cpp",

src/plugins/python/pythonlanguageclient.cpp

@@ -26,18 +26,24 @@
 #include "pythonlanguageclient.h"
 
 #include "pipsupport.h"
+#include "pysideuicextracompiler.h"
 #include "pythonconstants.h"
 #include "pythonplugin.h"
 #include "pythonproject.h"
+#include "pythonrunconfiguration.h"
 #include "pythonsettings.h"
 #include "pythonutils.h"
 
 #include <coreplugin/editormanager/editormanager.h>
 #include <coreplugin/icore.h>
 #include <coreplugin/progressmanager/progressmanager.h>
+#include <languageclient/languageclientinterface.h>
 #include <languageclient/languageclientmanager.h>
+#include <languageserverprotocol/textsynchronization.h>
 #include <languageserverprotocol/workspace.h>
+#include <projectexplorer/extracompiler.h>
 #include <projectexplorer/session.h>
+#include <projectexplorer/target.h>
 #include <texteditor/textdocument.h>
 #include <texteditor/texteditor.h>
 #include <utils/infobar.h>
@@ -57,6 +63,7 @@
 #include <QTimer>
 
 using namespace LanguageClient;
+using namespace LanguageServerProtocol;
 using namespace ProjectExplorer;
 using namespace Utils;
 
@@ -67,8 +74,9 @@ static constexpr char startPylsInfoBarId[] = "Python::StartPyls";
 static constexpr char installPylsInfoBarId[] = "Python::InstallPyls";
 static constexpr char enablePylsInfoBarId[] = "Python::EnablePyls";
 
-struct PythonLanguageServerState
+class PythonLanguageServerState
 {
+public:
     enum {
         CanNotBeInstalled,
         CanBeInstalled,
@@ -454,35 +462,119 @@ void PyLSSettings::setInterpreter(const QString &interpreterId)
     m_executable = interpreter.command;
 }
 
-class PyLSClient : public Client
+class PyLSInterface : public StdIOClientInterface
 {
 public:
-    using Client::Client;
-    void openDocument(TextEditor::TextDocument *document) override
+    PyLSInterface()
+        : m_extraPythonPath("QtCreator-pyls-XXXXXX")
     {
-        using namespace LanguageServerProtocol;
-        if (reachable()) {
-            const FilePath documentPath = document->filePath();
-            if (isSupportedDocument(document) && !pythonProjectForFile(documentPath)) {
-                const FilePath workspacePath = documentPath.parentDir();
-                if (!extraWorkspaceDirs.contains(workspacePath)) {
-                    WorkspaceFoldersChangeEvent event;
-                    event.setAdded({WorkSpaceFolder(DocumentUri::fromFilePath(workspacePath),
-                                                    workspacePath.fileName())});
-                    DidChangeWorkspaceFoldersParams params;
-                    params.setEvent(event);
-                    DidChangeWorkspaceFoldersNotification change(params);
-                    sendMessage(change);
-                    extraWorkspaceDirs.append(workspacePath);
-                }
+        Environment env = Environment::systemEnvironment();
+        env.appendOrSet("PYTHONPATH",
+                        m_extraPythonPath.path().toString(),
+                        OsSpecificAspects::pathListSeparator(env.osType()));
+        setEnvironment(env);
+    }
+    TemporaryDirectory m_extraPythonPath;
+};
+
+BaseClientInterface *PyLSSettings::createInterfaceWithProject(
+    ProjectExplorer::Project *project) const
+{
+    auto interface = new PyLSInterface;
+    interface->setCommandLine(command());
+    if (project)
+        interface->setWorkingDirectory(project->projectDirectory());
+    return interface;
+}
+
+PyLSClient::PyLSClient(BaseClientInterface *interface)
+    : Client(interface)
+    , m_extraCompilerOutputDir(static_cast<PyLSInterface *>(interface)->m_extraPythonPath.path())
+{
+}
+
+void PyLSClient::openDocument(TextEditor::TextDocument *document)
+{
+    using namespace LanguageServerProtocol;
+    if (reachable()) {
+        const FilePath documentPath = document->filePath();
+        if (PythonProject *project = pythonProjectForFile(documentPath)) {
+            if (Target *target = project->activeTarget()) {
+                if (auto rc = qobject_cast<PythonRunConfiguration *>(target->activeRunConfiguration()))
+                    updateExtraCompilers(project, rc->extraCompilers());
+            }
+        } else if (isSupportedDocument(document)) {
+            const FilePath workspacePath = documentPath.parentDir();
+            if (!m_extraWorkspaceDirs.contains(workspacePath)) {
+                WorkspaceFoldersChangeEvent event;
+                event.setAdded({WorkSpaceFolder(DocumentUri::fromFilePath(workspacePath),
+                                                workspacePath.fileName())});
+                DidChangeWorkspaceFoldersParams params;
+                params.setEvent(event);
+                DidChangeWorkspaceFoldersNotification change(params);
+                sendMessage(change);
+                m_extraWorkspaceDirs.append(workspacePath);
             }
         }
-        Client::openDocument(document);
     }
+    Client::openDocument(document);
+}
 
-private:
-    FilePaths extraWorkspaceDirs;
-};
+void PyLSClient::projectClosed(ProjectExplorer::Project *project)
+{
+    for (ProjectExplorer::ExtraCompiler *compiler : m_extraCompilers.value(project))
+        closeExtraCompiler(compiler);
+    Client::projectClosed(project);
+}
+
+void PyLSClient::updateExtraCompilers(ProjectExplorer::Project *project,
+                                      const QList<PySideUicExtraCompiler *> &extraCompilers)
+{
+    auto oldCompilers = m_extraCompilers.take(project);
+    for (PySideUicExtraCompiler *extraCompiler : extraCompilers) {
+        QTC_ASSERT(extraCompiler->targets().size() == 1 , continue);
+        int index = oldCompilers.indexOf(extraCompiler);
+        if (index < 0) {
+            m_extraCompilers[project] << extraCompiler;
+            connect(extraCompiler,
+                    &ExtraCompiler::contentsChanged,
+                    this,
+                    [this, extraCompiler](const FilePath &file) {
+                        updateExtraCompilerContents(extraCompiler, file);
+                    });
+            if (extraCompiler->isDirty())
+                static_cast<ExtraCompiler *>(extraCompiler)->run();
+        } else {
+            m_extraCompilers[project] << oldCompilers.takeAt(index);
+        }
+    }
+    for (ProjectExplorer::ExtraCompiler *compiler : oldCompilers)
+        closeExtraCompiler(compiler);
+}
+
+void PyLSClient::updateExtraCompilerContents(ExtraCompiler *compiler, const FilePath &file)
+{
+    const QString text = QString::fromUtf8(compiler->content(file));
+    const FilePath target = m_extraCompilerOutputDir.pathAppended(file.fileName());
+
+    target.writeFileContents(compiler->content(file));
+}
+
+void PyLSClient::closeExtraCompiler(ProjectExplorer::ExtraCompiler *compiler)
+{
+    const FilePath file = compiler->targets().first();
+    m_extraCompilerOutputDir.pathAppended(file.fileName()).removeFile();
+    compiler->disconnect(this);
+}
+
+PyLSClient *PyLSClient::clientForPython(const FilePath &python)
+{
+    if (auto setting = PyLSConfigureAssistant::languageServerForPython(python)) {
+        if (auto client = LanguageClientManager::clientsForSetting(setting).value(0))
+            return qobject_cast<PyLSClient *>(client);
+    }
+    return nullptr;
+}
 
 Client *PyLSSettings::createClient(BaseClientInterface *interface) const
 {

src/plugins/python/pythonlanguageclient.h

@@ -26,18 +26,46 @@
 #pragma once
 
 #include <utils/fileutils.h>
+#include <utils/temporarydirectory.h>
 
 #include <languageclient/client.h>
 #include <languageclient/languageclientsettings.h>
 
 namespace Core { class IDocument; }
-namespace LanguageClient { class Client; }
+namespace ProjectExplorer { class ExtraCompiler; }
 namespace TextEditor { class TextDocument; }
 
 namespace Python {
 namespace Internal {
 
-struct PythonLanguageServerState;
+class PySideUicExtraCompiler;
+class PythonLanguageServerState;
+
+class PyLSClient : public LanguageClient::Client
+{
+    Q_OBJECT
+public:
+    explicit PyLSClient(LanguageClient::BaseClientInterface *interface);
+
+    void openDocument(TextEditor::TextDocument *document) override;
+    void projectClosed(ProjectExplorer::Project *project) override;
+
+    void updateExtraCompilers(ProjectExplorer::Project *project,
+                              const QList<PySideUicExtraCompiler *> &extraCompilers);
+
+    static PyLSClient *clientForPython(const Utils::FilePath &python);
+
+private:
+    void updateExtraCompilerContents(ProjectExplorer::ExtraCompiler *compiler,
+                                     const Utils::FilePath &file);
+    void closeExtraDoc(const Utils::FilePath &file);
+    void closeExtraCompiler(ProjectExplorer::ExtraCompiler *compiler);
+
+    Utils::FilePaths m_extraWorkspaceDirs;
+    Utils::FilePath m_extraCompilerOutputDir;
+
+    QHash<ProjectExplorer::Project *, QList<ProjectExplorer::ExtraCompiler *>> m_extraCompilers;
+};
 
 class PyLSSettings : public LanguageClient::StdIOSettings
 {
@@ -56,6 +84,9 @@ public:
     LanguageClient::Client *createClient(LanguageClient::BaseClientInterface *interface) const final;
 
 private:
+    LanguageClient::BaseClientInterface *createInterfaceWithProject(
+        ProjectExplorer::Project *project) const override;
+
     static QJsonObject defaultConfiguration();
 
     QString m_interpreterId;

src/plugins/python/pythonproject.cpp

@@ -242,7 +242,6 @@ static FileType getFileType(const FilePath &f)
 void PythonBuildSystem::triggerParsing()
 {
     ParseGuard guard = guardParsingRun();
-
     parse();
 
     const QDir baseDir(projectDirectory().toString());

src/plugins/python/pythonrunconfiguration.cpp

@@ -27,6 +27,7 @@
 
 #include "pyside.h"
 #include "pysidebuildconfiguration.h"
+#include "pysideuicextracompiler.h"
 #include "pythonconstants.h"
 #include "pythonlanguageclient.h"
 #include "pythonproject.h"
@@ -137,13 +138,6 @@ private:
 
 ////////////////////////////////////////////////////////////////
 
-class PythonRunConfiguration : public RunConfiguration
-{
-public:
-    PythonRunConfiguration(Target *target, Id id);
-    void currentInterpreterChanged();
-};
-
 PythonRunConfiguration::PythonRunConfiguration(Target *target, Id id)
     : RunConfiguration(target, id)
 {
@@ -202,6 +196,13 @@ PythonRunConfiguration::PythonRunConfiguration(Target *target, Id id)
     });
 
     connect(target, &Target::buildSystemUpdated, this, &RunConfiguration::update);
+    connect(target, &Target::buildSystemUpdated, this, &PythonRunConfiguration::updateExtraCompilers);
+    currentInterpreterChanged();
+}
+
+PythonRunConfiguration::~PythonRunConfiguration()
+{
+    qDeleteAll(m_extraCompilers);
 }
 
 void PythonRunConfiguration::currentInterpreterChanged()
@@ -210,6 +211,7 @@ void PythonRunConfiguration::currentInterpreterChanged()
     BuildStepList *buildSteps = target()->activeBuildConfiguration()->buildSteps();
 
     Utils::FilePath pySideProjectPath;
+    m_pySideUicPath.clear();
     const PipPackage pySide6Package("PySide6");
     const PipPackageInfo info = pySide6Package.info(python);
 
@@ -217,10 +219,18 @@ void PythonRunConfiguration::currentInterpreterChanged()
         if (file.fileName() == HostOsInfo::withExecutableSuffix("pyside6-project")) {
             pySideProjectPath = info.location.resolvePath(file);
             pySideProjectPath = pySideProjectPath.cleanPath();
-            break;
+            if (!m_pySideUicPath.isEmpty())
+                break;
+        } else if (file.fileName() == HostOsInfo::withExecutableSuffix("pyside6-uic")) {
+            m_pySideUicPath = info.location.resolvePath(file);
+            m_pySideUicPath = m_pySideUicPath.cleanPath();
+            if (!pySideProjectPath.isEmpty())
+                break;
         }
     }
 
+    updateExtraCompilers();
+
     if (auto pySideBuildStep = buildSteps->firstOfType<PySideBuildStep>())
         pySideBuildStep->updatePySideProjectPath(pySideProjectPath);
 
@@ -234,6 +244,48 @@ void PythonRunConfiguration::currentInterpreterChanged()
     }
 }
 
+QList<PySideUicExtraCompiler *> PythonRunConfiguration::extraCompilers() const
+{
+    return m_extraCompilers;
+}
+
+void PythonRunConfiguration::updateExtraCompilers()
+{
+    QList<PySideUicExtraCompiler *> oldCompilers = m_extraCompilers;
+    m_extraCompilers.clear();
+
+    if (m_pySideUicPath.isExecutableFile()) {
+        auto uiMatcher = [](const ProjectExplorer::Node *node) {
+            if (const ProjectExplorer::FileNode *fileNode = node->asFileNode())
+                return fileNode->fileType() == ProjectExplorer::FileType::Form;
+            return false;
+        };
+        const FilePaths uiFiles = project()->files(uiMatcher);
+        for (const FilePath &uiFile : uiFiles) {
+            Utils::FilePath generated = uiFile.parentDir();
+            generated = generated.pathAppended("/ui_" + uiFile.baseName() + ".py");
+            int index = Utils::indexOf(oldCompilers, [&](PySideUicExtraCompiler *oldCompiler) {
+                return oldCompiler->pySideUicPath() == m_pySideUicPath
+                       && oldCompiler->project() == project() && oldCompiler->source() == uiFile
+                       && oldCompiler->targets() == Utils::FilePaths{generated};
+            });
+            if (index < 0) {
+                m_extraCompilers << new PySideUicExtraCompiler(m_pySideUicPath,
+                                                               project(),
+                                                               uiFile,
+                                                               {generated},
+                                                               this);
+            } else {
+                m_extraCompilers << oldCompilers.takeAt(index);
+            }
+        }
+    }
+    const FilePath python = aspect<InterpreterAspect>()->currentInterpreter().command;
+    if (auto client = PyLSClient::clientForPython(python))
+        client->updateExtraCompilers(project(), m_extraCompilers);
+    qDeleteAll(oldCompilers);
+}
+
 PythonRunConfigurationFactory::PythonRunConfigurationFactory()
 {
     registerRunConfiguration<PythonRunConfiguration>(Constants::C_PYTHONRUNCONFIGURATION_ID);

src/plugins/python/pythonrunconfiguration.h

@@ -31,6 +31,24 @@
 namespace Python {
 namespace Internal {
 
+class PySideUicExtraCompiler;
+
+class PythonRunConfiguration : public ProjectExplorer::RunConfiguration
+{
+    Q_OBJECT
+public:
+    PythonRunConfiguration(ProjectExplorer::Target *target, Utils::Id id);
+    ~PythonRunConfiguration() override;
+    void currentInterpreterChanged();
+    QList<PySideUicExtraCompiler *> extraCompilers() const;
+
+private:
+    void updateExtraCompilers();
+    Utils::FilePath m_pySideUicPath;
+
+    QList<PySideUicExtraCompiler *> m_extraCompilers;
+};
+
 class PythonRunConfigurationFactory : public ProjectExplorer::RunConfigurationFactory
 {
 public: