TaskTree: Implement AssetDownloader example

This is an equivalent of the downloader used for
the car-configurator example.

Change-Id: Ia0ab13ea7d4557d375b44aa2ba06be5a81651251
Reviewed-by: hjk <hjk@qt.io>
This commit is contained in:
Jarek Kobus
2024-05-27 20:02:04 +02:00
parent 0214cac51d
commit bcba7184f0
7 changed files with 707 additions and 0 deletions

View File

@@ -13,6 +13,7 @@ Project {
"shootout/shootout.qbs", "shootout/shootout.qbs",
"spinner/spinner.qbs", "spinner/spinner.qbs",
"subdirfilecontainer/subdirfilecontainer.qbs", "subdirfilecontainer/subdirfilecontainer.qbs",
"tasking/assetdownloader/assetdownloader.qbs",
"tasking/dataexchange/dataexchange.qbs", "tasking/dataexchange/dataexchange.qbs",
"tasking/demo/demo.qbs", "tasking/demo/demo.qbs",
"tasking/imagescaling/imagescaling.qbs", "tasking/imagescaling/imagescaling.qbs",

View File

@@ -1,3 +1,4 @@
add_subdirectory(assetdownloader)
add_subdirectory(dataexchange) add_subdirectory(dataexchange)
add_subdirectory(demo) add_subdirectory(demo)
add_subdirectory(imagescaling) add_subdirectory(imagescaling)

View File

@@ -0,0 +1,8 @@
add_qtc_test(tst_tasking_assetdownloader
MANUALTEST
DEPENDS Tasking Qt::Concurrent Qt::Network Qt::Widgets Qt::GuiPrivate Qt::CorePrivate
SOURCES
assetdownloader.cpp
assetdownloader.h
main.cpp
)

View File

@@ -0,0 +1,532 @@
// Copyright (C) 2024 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "assetdownloader.h"
#include <tasking/concurrentcall.h>
#include <tasking/networkquery.h>
#include <tasking/tasktreerunner.h>
#if QT_VERSION >= QT_VERSION_CHECK(6, 6, 0)
#include <QtCore/private/qzipreader_p.h>
#else
#include <QtGui/private/qzipreader_p.h>
#endif
#include <QtCore/QDir>
#include <QtCore/QFile>
#include <QtCore/QJsonArray>
#include <QtCore/QJsonDocument>
#include <QtCore/QJsonObject>
#include <QtCore/QStandardPaths>
#include <QtCore/QTemporaryDir>
#include <QtCore/QTemporaryFile>
using namespace Tasking;
struct DownloadableAssets
{
QUrl remoteUrl;
QList<QUrl> allAssets;
QList<QUrl> assetsToDownload;
};
class AssetDownloaderPrivate
{
public:
AssetDownloaderPrivate(AssetDownloader *q) : m_q(q) {}
AssetDownloader *m_q = nullptr;
TaskTreeRunner m_taskTreeRunner;
QString m_lastProgressText;
QNetworkAccessManager *m_manager = nullptr;
QString m_jsonFileName;
QString m_zipFileName;
QDir m_preferredLocalDownloadDir =
QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation);
QUrl m_offlineAssetsFilePath;
QUrl m_downloadBase;
void setProgress(int progressValue, int progressMaximum, const QString &progressText)
{
m_lastProgressText = progressText;
emit m_q->progressChanged(progressValue, progressMaximum, progressText);
}
void updateProgress(int progressValue, int progressMaximum)
{
setProgress(progressValue, progressMaximum, m_lastProgressText);
}
void clearProgress(const QString &progressText)
{
setProgress(0, 0, progressText);
}
void setupDownload(NetworkQuery *query, const QString &progressText)
{
query->setNetworkAccessManager(m_manager);
clearProgress(progressText);
QObject::connect(query, &NetworkQuery::started, query, [this, query] {
QNetworkReply *reply = query->reply();
QObject::connect(reply, &QNetworkReply::downloadProgress,
query, [this](qint64 bytesReceived, qint64 totalBytes) {
updateProgress((totalBytes > 0) ? 100.0 * bytesReceived / totalBytes : 0, 100);
});
});
}
};
static bool isWritableDir(const QDir &dir)
{
if (dir.exists()) {
QTemporaryFile file(dir.filePath(QString::fromLatin1("tmp")));
return file.open();
}
return false;
}
static bool sameFileContent(const QFileInfo &first, const QFileInfo &second)
{
if (first.exists() ^ second.exists())
return false;
if (first.size() != second.size())
return false;
QFile firstFile(first.absoluteFilePath());
QFile secondFile(second.absoluteFilePath());
if (firstFile.open(QFile::ReadOnly) && secondFile.open(QFile::ReadOnly)) {
char char1;
char char2;
int readBytes1 = 0;
int readBytes2 = 0;
while (!firstFile.atEnd()) {
readBytes1 = firstFile.read(&char1, 1);
readBytes2 = secondFile.read(&char2, 1);
if (readBytes1 != readBytes2 || readBytes1 != 1)
return false;
if (char1 != char2)
return false;
}
return true;
}
return false;
}
static bool createDirectory(const QDir &dir)
{
if (dir.exists())
return true;
if (!createDirectory(dir.absoluteFilePath(QString::fromUtf8(".."))))
return false;
return dir.mkpath(QString::fromUtf8("."));
}
static bool canBeALocalBaseDir(const QDir &dir)
{
if (dir.exists())
return !dir.isEmpty() || isWritableDir(dir);
return createDirectory(dir) && isWritableDir(dir);
}
static QString pathFromUrl(const QUrl &url)
{
return url.isLocalFile() ? url.toLocalFile() : url.toString();
}
static QList<QUrl> filterDownloadableAssets(const QList<QUrl> &assetFiles, const QDir &expectedDir)
{
QList<QUrl> downloadList;
std::copy_if(assetFiles.begin(), assetFiles.end(), std::back_inserter(downloadList),
[&](const QUrl &assetPath) {
return !QFileInfo::exists(expectedDir.absoluteFilePath(assetPath.toString()));
});
return downloadList;
}
static bool allAssetsPresent(const QList<QUrl> &assetFiles, const QDir &expectedDir)
{
return std::all_of(assetFiles.begin(), assetFiles.end(), [&](const QUrl &assetPath) {
return QFileInfo::exists(expectedDir.absoluteFilePath(assetPath.toString()));
});
}
AssetDownloader::AssetDownloader(QObject *parent)
: QObject(parent)
, d(new AssetDownloaderPrivate(this))
{}
AssetDownloader::~AssetDownloader() = default;
void AssetDownloader::setNetworkAccessManager(QNetworkAccessManager *manager)
{
d->m_manager = manager;
}
QUrl AssetDownloader::downloadBase() const
{
return d->m_downloadBase;
}
void AssetDownloader::setDownloadBase(const QUrl &downloadBase)
{
if (d->m_downloadBase != downloadBase) {
d->m_downloadBase = downloadBase;
emit downloadBaseChanged(d->m_downloadBase);
}
}
QUrl AssetDownloader::preferredLocalDownloadDir() const
{
return QUrl::fromLocalFile(d->m_preferredLocalDownloadDir.absolutePath());
}
void AssetDownloader::setPreferredLocalDownloadDir(const QUrl &localDir)
{
if (!localDir.isLocalFile())
qWarning() << "preferredLocalDownloadDir Should be a local directory";
const QString path = pathFromUrl(localDir);
if (d->m_preferredLocalDownloadDir != path) {
d->m_preferredLocalDownloadDir.setPath(path);
emit preferredLocalDownloadDirChanged(preferredLocalDownloadDir());
}
}
QUrl AssetDownloader::offlineAssetsFilePath() const
{
return d->m_offlineAssetsFilePath;
}
void AssetDownloader::setOfflineAssetsFilePath(const QUrl &offlineAssetsFilePath)
{
if (d->m_offlineAssetsFilePath != offlineAssetsFilePath) {
d->m_offlineAssetsFilePath = offlineAssetsFilePath;
emit offlineAssetsFilePathChanged(d->m_offlineAssetsFilePath);
}
}
QString AssetDownloader::jsonFileName() const
{
return d->m_jsonFileName;
}
void AssetDownloader::setJsonFileName(const QString &jsonFileName)
{
if (d->m_jsonFileName != jsonFileName) {
d->m_jsonFileName = jsonFileName;
emit jsonFileNameChanged(d->m_jsonFileName);
}
}
QString AssetDownloader::zipFileName() const
{
return d->m_zipFileName;
}
void AssetDownloader::setZipFileName(const QString &zipFileName)
{
if (d->m_zipFileName != zipFileName) {
d->m_zipFileName = zipFileName;
emit zipFileNameChanged(d->m_zipFileName);
}
}
static QDir baseLocalDir(const QDir &preferredLocalDir)
{
if (canBeALocalBaseDir(preferredLocalDir))
return preferredLocalDir;
qWarning().noquote() << "AssetDownloader: Cannot set \"" << preferredLocalDir
<< "\" as a local download directory!";
return QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation);
}
static void precheckLocalFile(const QUrl &url)
{
if (url.isEmpty())
return;
QFile file(pathFromUrl(url));
if (!file.open(QIODevice::ReadOnly))
qWarning() << "Cannot open local file" << url;
}
static void readAssetsFileContent(QPromise<DownloadableAssets> &promise, const QByteArray &content)
{
const QJsonObject json = QJsonDocument::fromJson(content).object();
const QJsonArray assetsArray = json[u"assets"].toArray();
DownloadableAssets result;
result.remoteUrl = json[u"url"].toString();
for (const QJsonValue &asset : assetsArray) {
if (promise.isCanceled())
return;
result.allAssets.append(asset.toString());
}
result.assetsToDownload = result.allAssets;
if (result.allAssets.isEmpty() || result.remoteUrl.isEmpty())
promise.future().cancel();
else
promise.addResult(result);
}
static void unzip(QPromise<void> &promise, const QByteArray &content, const QDir &directory,
const QString &fileName)
{
const QString zipFilePath = directory.absoluteFilePath(fileName);
QFile zipFile(zipFilePath);
if (!zipFile.open(QIODevice::WriteOnly)) {
promise.future().cancel();
return;
}
zipFile.write(content);
zipFile.close();
if (promise.isCanceled())
return;
QZipReader reader(zipFilePath);
const bool extracted = reader.extractAll(directory.absolutePath());
reader.close();
if (extracted)
QFile::remove(zipFilePath);
else
promise.future().cancel();
}
static void writeAsset(QPromise<void> &promise, const QByteArray &content, const QString &filePath)
{
const QFileInfo fileInfo(filePath);
QFile file(fileInfo.absoluteFilePath());
if (!createDirectory(fileInfo.dir()) || !file.open(QFile::WriteOnly)) {
promise.future().cancel();
return;
}
if (promise.isCanceled())
return;
file.write(content);
file.close();
}
static void copyAndCheck(QPromise<void> &promise, const QString &sourcePath, const QString &destPath)
{
QFile sourceFile(sourcePath);
QFile destFile(destPath);
const QFileInfo sourceFileInfo(sourceFile.fileName());
const QFileInfo destFileInfo(destFile.fileName());
if (destFile.exists() && !destFile.remove()) {
promise.future().cancel();
return;
}
if (!createDirectory(destFileInfo.absolutePath())) {
promise.future().cancel();
return;
}
if (promise.isCanceled())
return;
if (!sourceFile.copy(destFile.fileName()) && !sameFileContent(sourceFileInfo, destFileInfo))
promise.future().cancel();
}
void AssetDownloader::start()
{
if (d->m_taskTreeRunner.isRunning())
return;
struct InternalStorage
{
QTemporaryDir temporaryDir;
QDir tempDir;
QDir baseLocalDir;
QByteArray jsonContent;
DownloadableAssets assets;
QByteArray zipContent;
int doneCount = 0;
};
const Storage<InternalStorage> storage;
const auto onSetup = [this, storage] {
if (!storage->temporaryDir.isValid()) {
qWarning() << "Cannot create a temporary directory.";
return SetupResult::StopWithError;
}
storage->tempDir = storage->temporaryDir.path();
storage->baseLocalDir = baseLocalDir(d->m_preferredLocalDownloadDir);
precheckLocalFile(d->m_offlineAssetsFilePath);
return SetupResult::Continue;
};
const auto onJsonDownloadSetup = [this](NetworkQuery &query) {
query.setRequest(QNetworkRequest(d->m_downloadBase.resolved(d->m_jsonFileName)));
d->setupDownload(&query, tr("Downloading JSON file..."));
};
const auto onJsonDownloadDone = [this, storage](const NetworkQuery &query, DoneWith result) {
if (result == DoneWith::Success) {
storage->jsonContent = query.reply()->readAll();
return DoneResult::Success;
}
qWarning() << "Cannot download" << d->m_downloadBase.resolved(d->m_jsonFileName)
<< query.reply()->errorString();
if (d->m_offlineAssetsFilePath.isEmpty()) {
qWarning() << "Also there is no local file as a replacement";
return DoneResult::Error;
}
QFile file(pathFromUrl(d->m_offlineAssetsFilePath));
if (!file.open(QIODevice::ReadOnly)) {
qWarning() << "Also failed to open" << d->m_offlineAssetsFilePath;
return DoneResult::Error;
}
storage->jsonContent = file.readAll();
return DoneResult::Success;
};
const auto onReadAssetsFileSetup = [storage](ConcurrentCall<DownloadableAssets> &async) {
async.setConcurrentCallData(readAssetsFileContent, storage->jsonContent);
};
const auto onReadAssetsFileDone = [storage](const ConcurrentCall<DownloadableAssets> &async) {
storage->assets = async.result();
};
const auto onSkipIfAllAssetsPresent = [storage] {
return allAssetsPresent(storage->assets.allAssets, storage->baseLocalDir)
? SetupResult::StopWithSuccess : SetupResult::Continue;
};
const auto onZipDownloadSetup = [this, storage](NetworkQuery &query) {
if (d->m_zipFileName.isEmpty())
return SetupResult::StopWithSuccess;
query.setRequest(QNetworkRequest(d->m_downloadBase.resolved(d->m_zipFileName)));
d->setupDownload(&query, tr("Downloading zip file..."));
return SetupResult::Continue;
};
const auto onZipDownloadDone = [storage](const NetworkQuery &query, DoneWith result) {
if (result == DoneWith::Success)
storage->zipContent = query.reply()->readAll();
return DoneResult::Success; // Ignore zip download failure
};
const auto onUnzipSetup = [this, storage](ConcurrentCall<void> &async) {
if (storage->zipContent.isEmpty())
return SetupResult::StopWithSuccess;
async.setConcurrentCallData(unzip, storage->zipContent, storage->tempDir, d->m_zipFileName);
d->clearProgress(tr("Unzipping..."));
return SetupResult::Continue;
};
const auto onUnzipDone = [storage](DoneWith result) {
if (result == DoneWith::Success) {
// Avoid downloading assets that are present in unzipped tree
InternalStorage &storageData = *storage;
storageData.assets.assetsToDownload =
filterDownloadableAssets(storageData.assets.allAssets, storageData.tempDir);
} else {
qWarning() << "ZipFile failed";
}
return DoneResult::Success; // Ignore unzip failure
};
const LoopUntil downloadIterator([storage](int iteration) {
return iteration < storage->assets.assetsToDownload.count();
});
const Storage<QByteArray> assetStorage;
const auto onAssetsDownloadGroupSetup = [this, storage] {
d->setProgress(0, storage->assets.assetsToDownload.size(), tr("Downloading assets..."));
};
const auto onAssetDownloadSetup = [this, storage, downloadIterator](NetworkQuery &query) {
query.setNetworkAccessManager(d->m_manager);
query.setRequest(QNetworkRequest(storage->assets.remoteUrl.resolved(
storage->assets.assetsToDownload.at(downloadIterator.iteration()))));
};
const auto onAssetDownloadDone = [assetStorage](const NetworkQuery &query, DoneWith result) {
if (result == DoneWith::Success)
*assetStorage = query.reply()->readAll();
};
const auto onAssetWriteSetup = [storage, downloadIterator, assetStorage](
ConcurrentCall<void> &async) {
const QString filePath = storage->tempDir.absoluteFilePath(
storage->assets.assetsToDownload.at(downloadIterator.iteration()).toString());
async.setConcurrentCallData(writeAsset, *assetStorage, filePath);
};
const auto onAssetWriteDone = [this, storage](DoneWith result) {
if (result != DoneWith::Success) {
qWarning() << "Asset write failed";
return;
}
InternalStorage &storageData = *storage;
++storageData.doneCount;
d->updateProgress(storageData.doneCount, storageData.assets.assetsToDownload.size());
};
const LoopUntil copyIterator([storage](int iteration) {
return iteration < storage->assets.allAssets.count();
});
const auto onAssetsCopyGroupSetup = [this, storage] {
storage->doneCount = 0;
d->setProgress(0, storage->assets.allAssets.size(), tr("Copying assets..."));
};
const auto onAssetCopySetup = [storage, copyIterator](ConcurrentCall<void> &async) {
const QString fileName = storage->assets.allAssets.at(copyIterator.iteration()).toString();
const QString sourcePath = storage->tempDir.absoluteFilePath(fileName);
const QString destPath = storage->baseLocalDir.absoluteFilePath(fileName);
async.setConcurrentCallData(copyAndCheck, sourcePath, destPath);
};
const auto onAssetCopyDone = [this, storage] (DoneWith result) {
if (result != DoneWith::Success) {
qWarning() << "Asset copy failed";
return;
}
InternalStorage &storageData = *storage;
++storageData.doneCount;
d->updateProgress(storageData.doneCount, storageData.assets.allAssets.size());
};
const Group root {
storage,
onGroupSetup(onSetup),
NetworkQueryTask(onJsonDownloadSetup, onJsonDownloadDone),
ConcurrentCallTask<DownloadableAssets>(onReadAssetsFileSetup, onReadAssetsFileDone, CallDoneIf::Success),
Group {
onGroupSetup(onSkipIfAllAssetsPresent),
NetworkQueryTask(onZipDownloadSetup, onZipDownloadDone),
ConcurrentCallTask<void>(onUnzipSetup, onUnzipDone),
Group {
parallelIdealThreadCountLimit,
downloadIterator,
onGroupSetup(onAssetsDownloadGroupSetup),
Group {
assetStorage,
NetworkQueryTask(onAssetDownloadSetup, onAssetDownloadDone),
ConcurrentCallTask<void>(onAssetWriteSetup, onAssetWriteDone)
}
},
Group {
parallelIdealThreadCountLimit,
copyIterator,
onGroupSetup(onAssetsCopyGroupSetup),
ConcurrentCallTask<void>(onAssetCopySetup, onAssetCopyDone)
}
}
};
d->m_taskTreeRunner.start(root,[this](TaskTree *) { emit started(); },
[this](DoneWith result) { emit finished(result == DoneWith::Success); });
}

View File

@@ -0,0 +1,102 @@
// Copyright (C) 2024 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#ifndef ASSETDOWNLOADER_H
#define ASSETDOWNLOADER_H
//
// W A R N I N G
// -------------
//
// This file is not part of the Qt API. It exists purely as an
// implementation detail. This header file may change from version to
// version without notice, or even be removed.
//
// We mean it.
//
#include <QtCore/QObject>
#include <QtCore/QUrl>
#include <memory>
QT_BEGIN_NAMESPACE
class QNetworkAccessManager;
QT_END_NAMESPACE
class AssetDownloaderPrivate;
class AssetDownloader : public QObject
{
Q_OBJECT
Q_PROPERTY(
QUrl downloadBase
READ downloadBase
WRITE setDownloadBase
NOTIFY downloadBaseChanged)
Q_PROPERTY(
QUrl preferredLocalDownloadDir
READ preferredLocalDownloadDir
WRITE setPreferredLocalDownloadDir
NOTIFY preferredLocalDownloadDirChanged)
Q_PROPERTY(
QUrl offlineAssetsFilePath
READ offlineAssetsFilePath
WRITE setOfflineAssetsFilePath
NOTIFY offlineAssetsFilePathChanged)
Q_PROPERTY(
QString jsonFileName
READ jsonFileName
WRITE setJsonFileName
NOTIFY jsonFileNameChanged)
Q_PROPERTY(
QString zipFileName
READ zipFileName
WRITE setZipFileName
NOTIFY zipFileNameChanged)
public:
AssetDownloader(QObject *parent = nullptr);
~AssetDownloader();
void setNetworkAccessManager(QNetworkAccessManager *manager);
QUrl downloadBase() const;
void setDownloadBase(const QUrl &downloadBase);
QUrl preferredLocalDownloadDir() const;
void setPreferredLocalDownloadDir(const QUrl &localDir);
QUrl offlineAssetsFilePath() const;
void setOfflineAssetsFilePath(const QUrl &offlineAssetsFilePath);
QString jsonFileName() const;
void setJsonFileName(const QString &jsonFileName);
QString zipFileName() const;
void setZipFileName(const QString &zipFileName);
public Q_SLOTS:
void start();
Q_SIGNALS:
void started();
void finished(bool success);
void progressChanged(int progressValue, int progressMaximum, const QString &progressText);
void downloadBaseChanged(const QUrl &);
void preferredLocalDownloadDirChanged(const QUrl &url);
void offlineAssetsFilePathChanged(const QUrl &);
void jsonFileNameChanged(const QString &);
void zipFileNameChanged(const QString &);
private:
std::unique_ptr<AssetDownloaderPrivate> d;
};
#endif // ASSETDOWNLOADER_H

View File

@@ -0,0 +1,13 @@
QtcManualTest {
name: "Tasking assetdownloader"
type: ["application"]
Depends { name: "Qt"; submodules: ["concurrent", "network", "widgets", "core-private", "gui-private"] }
Depends { name: "Tasking" }
files: [
"assetdownloader.cpp",
"assetdownloader.h",
"main.cpp",
]
}

View File

@@ -0,0 +1,50 @@
// Copyright (C) 2024 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "assetdownloader.h"
#include <QApplication>
#include <QMessageBox>
#include <QNetworkAccessManager>
#include <QProgressDialog>
int main(int argc, char *argv[])
{
QApplication app(argc, argv);
app.setOrganizationName("QtProject");
app.setApplicationName("Asset Downloader");
QProgressDialog progress;
progress.setAutoClose(false);
progress.setRange(0, 0);
QObject::connect(&progress, &QProgressDialog::canceled, &app, &QApplication::quit);
QNetworkAccessManager manager;
AssetDownloader downloader;
downloader.setNetworkAccessManager(&manager);
downloader.setJsonFileName("car-configurator-assets-v1.json");
downloader.setZipFileName("car-configurator-assets-v1.zip");
downloader.setDownloadBase(QUrl("https://download.qt.io/learning/examples/"));
QObject::connect(&downloader, &AssetDownloader::started,
&progress, &QProgressDialog::show);
QObject::connect(&downloader, &AssetDownloader::progressChanged, &progress,
[&](int progressValue, int progressMaximum, const QString &progressText) {
progress.setLabelText(progressText);
progress.setMaximum(progressMaximum);
progress.setValue(progressValue);
});
QObject::connect(&downloader, &AssetDownloader::finished, &progress, [&](bool success) {
progress.reset();
progress.hide();
if (success)
QMessageBox::information(nullptr, "Asset Downloader", "Download Finished Successfully.");
else
QMessageBox::warning(nullptr, "Asset Downloader", "Download Finished with an Error.");
});
downloader.start();
return app.exec();
}