From ae94dc614a444e643c3f2645f137c18bde1fe763 Mon Sep 17 00:00:00 2001 From: Aleix Pol Date: Tue, 15 Jun 2021 19:06:33 +0200 Subject: [PATCH 1/3] flatpak: Improve startup time by leveraging libappstream caching Especially on the pinephone it is noticeable, this should help starting faster by reusing the xml parsing instead of doing it from scratch on every run. --- .../FlatpakBackend/FlatpakBackend.cpp | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/libdiscover/backends/FlatpakBackend/FlatpakBackend.cpp b/libdiscover/backends/FlatpakBackend/FlatpakBackend.cpp index 620924844..4a6663660 100644 --- a/libdiscover/backends/FlatpakBackend/FlatpakBackend.cpp +++ b/libdiscover/backends/FlatpakBackend/FlatpakBackend.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -672,9 +673,8 @@ void FlatpakBackend::integrateRemote(FlatpakInstallation *flatpakInstallation, F const QString appstreamDirPath = source.appstreamDir(); const QString appstreamIconsPath = source.appstreamDir() + QLatin1String("/icons/"); - const QString appDirFileName = appstreamDirPath + QLatin1String("/appstream.xml.gz"); - if (!QFile::exists(appDirFileName)) { - qWarning() << "No" << appDirFileName << "appstream metadata found for" << source.name(); + if (!QFile::exists(appstreamDirPath)) { + qWarning() << "No" << appstreamDirPath << "appstream metadata found for" << source.name(); metadataRefreshed(); return; } @@ -703,16 +703,20 @@ void FlatpakBackend::integrateRemote(FlatpakInstallation *flatpakInstallation, F fw->deleteLater(); }); acquireFetching(true); - fw->setFuture(QtConcurrent::run(&m_threadPool, [appDirFileName]() -> QList { - AppStream::Metadata metadata; - metadata.setFormatStyle(AppStream::Metadata::FormatStyleCollection); - AppStream::Metadata::MetadataError error = metadata.parseFile(appDirFileName, AppStream::Metadata::FormatKindXml); - if (error != AppStream::Metadata::MetadataErrorNoError) { - qWarning() << "Failed to parse appstream metadata: " << error; + AppStream::Pool *pool = new AppStream::Pool(this); + pool->clearMetadataLocations(); + pool->addMetadataLocation(appstreamDirPath); + pool->setCacheFlags(AppStream::Pool::CacheFlagUseUser); + + const QString subdir = flatpak_installation_get_id(flatpakInstallation) + QLatin1Char('/') + sourceName; + pool->setCacheLocation(QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + "/flatpak-appstream/" + subdir); + QDir().mkpath(pool->cacheLocation()); + fw->setFuture(QtConcurrent::run(&m_threadPool, [pool, appstreamDirPath]() -> QList { + if (!pool->load()) { + qWarning() << "Could not open the AppStream metadata pool" << pool->lastError(); return {}; } - - return metadata.components(); + return pool->components(); })); } -- GitLab From 20a1cdaa93dbe8247e8c37ea925326fc64ea8efc Mon Sep 17 00:00:00 2001 From: Aleix Pol Date: Mon, 5 Jul 2021 23:36:14 +0200 Subject: [PATCH 2/3] flatpak: lazy load flatpak resources Only create them as they are needed instead of right at startup. --- .../FlatpakBackend/FlatpakBackend.cpp | 586 ++++++++++-------- .../backends/FlatpakBackend/FlatpakBackend.h | 23 +- .../FlatpakBackend/FlatpakResource.cpp | 15 +- .../backends/FlatpakBackend/FlatpakResource.h | 2 +- 4 files changed, 341 insertions(+), 285 deletions(-) diff --git a/libdiscover/backends/FlatpakBackend/FlatpakBackend.cpp b/libdiscover/backends/FlatpakBackend/FlatpakBackend.cpp index 4a6663660..389841733 100644 --- a/libdiscover/backends/FlatpakBackend/FlatpakBackend.cpp +++ b/libdiscover/backends/FlatpakBackend/FlatpakBackend.cpp @@ -50,6 +50,80 @@ DISCOVER_BACKEND_PLUGIN(FlatpakBackend) +class FlatpakSource +{ +public: + FlatpakSource(FlatpakBackend *backend, FlatpakInstallation *installation, FlatpakRemote *remote) + : m_remote(remote) + , m_installation(installation) + , m_backend(backend) + { + g_object_ref(m_remote); + g_object_ref(m_installation); + } + + ~FlatpakSource() + { + g_object_unref(m_remote); + g_object_unref(m_installation); + } + + bool isEnabled() const + { + return !flatpak_remote_get_disabled(m_remote); + } + + QString appstreamDir() const + { + g_autoptr(GFile) appstreamDir = flatpak_remote_get_appstream_dir(m_remote, nullptr); + if (!appstreamDir) { + qWarning() << "No appstream dir for" << flatpak_remote_get_name(m_remote); + return {}; + } + g_autofree char *path_str = g_file_get_path(appstreamDir); + return QString::fromUtf8(path_str); + } + + QString name() const + { + return QString::fromUtf8(flatpak_remote_get_name(m_remote)); + } + + FlatpakInstallation *installation() const + { + return m_installation; + } + + void addResource(FlatpakResource *resource) + { + // Update app with all possible information we have + if (!m_backend->parseMetadataFromAppBundle(resource)) { + qWarning() << "Failed to parse metadata from app bundle for" << resource->name(); + } + + m_backend->updateAppState(resource); + + m_resources.insert(resource->uniqueId(), resource); + if (!resource->extends().isEmpty()) { + m_backend->m_extends.append(resource->extends()); + m_backend->m_extends.removeDuplicates(); + } + + QObject::connect(resource, &FlatpakResource::sizeChanged, m_backend, [this, resource] { + if (!m_backend->isFetching()) + Q_EMIT m_backend->resourcesChanged(resource, {"size", "sizeDescription"}); + }); + } + + AppStream::Pool *m_pool = nullptr; + QHash m_resources; + +private: + FlatpakRemote *const m_remote; + FlatpakInstallation *const m_installation; + FlatpakBackend *const m_backend; +}; + QDebug operator<<(QDebug debug, const FlatpakResource::Id &id) { QDebugStateSaver saver(debug); @@ -97,8 +171,8 @@ FlatpakBackend::FlatpakBackend(QObject *parent) } connect(m_reviews.data(), &OdrsReviewsBackend::ratingsReady, this, [this] { - m_reviews->emitRatingFetched(this, kTransform>(m_resources, [](AbstractResource *r) { - return r; + m_reviews->emitRatingFetched(this, kAppend>(m_flatpakSources, [](const auto &source) { + return kTransform>(source->m_resources.values()); })); }); @@ -234,16 +308,90 @@ FlatpakInstalledRef *FlatpakBackend::getInstalledRefForApp(FlatpakResource *reso return ref; } -FlatpakResource *FlatpakBackend::getAppForInstalledRef(FlatpakInstallation *flatpakInstallation, FlatpakInstalledRef *ref) const +QString refToBundleId(FlatpakRef *ref) +{ + return QString(flatpak_ref_get_kind(ref) == FLATPAK_REF_KIND_APP ? "app/" : "runtime/") + flatpak_ref_get_name(ref) + '/' + flatpak_ref_get_arch(ref) + '/' + + flatpak_ref_get_branch(ref); +} + +FlatpakResource *FlatpakBackend::getAppForInstalledRef(FlatpakInstallation *installation, FlatpakInstalledRef *ref) const { - auto r = m_resources.value(idForInstalledRef(flatpakInstallation, ref, {})); - if (!r) - r = m_resources.value(idForInstalledRef(flatpakInstallation, ref, QStringLiteral(".desktop"))); + auto id = idForInstalledRef(installation, ref, {}); + for (const auto &source : m_flatpakSources) { + auto ret = source->m_resources.value(id); + if (ret) { + return ret; + } + } + auto id2 = idForInstalledRef(installation, ref, QStringLiteral(".desktop")); + for (const auto &source : m_flatpakSources) { + auto ret = source->m_resources.value(id2); + if (ret) { + return ret; + } + } + + const QLatin1String name(flatpak_ref_get_name(FLATPAK_REF(ref))); + + const QString origin = QString::fromUtf8(flatpak_installed_ref_get_origin(ref)); + auto source = findSource(installation, origin); + const QString pathExports = FlatpakResource::installationPath(installation) + QLatin1String("/exports/"); + const QString pathApps = pathExports + QLatin1String("share/applications/"); + AppStream::Component cid; + if (source && source->m_pool) { + QList comps = source->m_pool->componentsById(name); + if (comps.isEmpty()) { + comps = source->m_pool->componentsById(name + ".desktop"); + } + + if (comps.isEmpty()) { + const QString bundleId = refToBundleId(FLATPAK_REF(ref)); + comps = kFilter>(comps, [&bundleId](const AppStream::Component &comp) -> bool { + return comp.bundle(AppStream::Bundle::Kind::KindFlatpak).id() == bundleId; + }); + } + + if (comps.count() >= 1) { + Q_ASSERT(comps.count() == 1); + cid = comps.constFirst(); + } + } + + if (!cid.isValid()) { + AppStream::Metadata metadata; + const QString fnDesktop = pathApps + name + QLatin1String(".desktop"); + AppStream::Metadata::MetadataError error = metadata.parseFile(fnDesktop, AppStream::Metadata::FormatKindDesktopEntry); + if (error != AppStream::Metadata::MetadataErrorNoError) { + if (QFile::exists(fnDesktop)) + qDebug() << "Failed to parse appstream metadata:" << error << fnDesktop; + + cid.setId(QString::fromLatin1(flatpak_ref_get_name(FLATPAK_REF(ref)))); +#if FLATPAK_CHECK_VERSION(1, 1, 2) + cid.setName(QString::fromUtf8(flatpak_installed_ref_get_appdata_name(ref))); +#endif + } else + cid = metadata.component(); + } + + FlatpakResource *resource = resourceForComponent(cid, source); + resource->setIconPath(pathExports); + resource->setState(AbstractResource::Installed); + resource->updateFromRef(FLATPAK_REF(ref)); + source->addResource(resource); - // if (!r) { // qDebug() << "no" << flatpak_ref_get_name(FLATPAK_REF(ref)); - // } - return r; + return resource; +} + +QSharedPointer FlatpakBackend::findSource(FlatpakInstallation *installation, const QString &origin) const +{ + for (const auto &source : m_flatpakSources) { + if (source->installation() == installation && source->name() == origin) { + return source; + } + } + + Q_UNREACHABLE(); } FlatpakResource *FlatpakBackend::getRuntimeForApp(FlatpakResource *resource) const @@ -256,11 +404,13 @@ FlatpakResource *FlatpakBackend::getRuntimeForApp(FlatpakResource *resource) con return runtime; } - for (auto it = m_resources.constBegin(), itEnd = m_resources.constEnd(); it != itEnd; ++it) { - const auto &id = it.key(); - if (id.type == FlatpakResource::Runtime && id.id == runtimeInfo.at(0) && id.branch == runtimeInfo.at(2)) { - runtime = *it; - break; + for (const auto &source : m_flatpakSources) { + for (auto it = source->m_resources.constBegin(), itEnd = source->m_resources.constEnd(); it != itEnd; ++it) { + const auto &id = it.key(); + if (id.type == FlatpakResource::Runtime && id.id == runtimeInfo.at(0) && id.branch == runtimeInfo.at(2)) { + runtime = *it; + break; + } } } @@ -390,7 +540,11 @@ void FlatpakBackend::addAppFromFlatpakBundle(const QUrl &url, ResultsStream *str resource->setState(FlatpakResource::None); resource->setType(FlatpakResource::DesktopApp); - addResource(resource); + if (!m_localSource) { + m_localSource.reset(new FlatpakSource(this, preferredInstallation(), nullptr)); + m_flatpakSources += m_localSource; + } + m_localSource->addResource(resource); stream->resourcesFound({resource}); } @@ -462,10 +616,12 @@ void FlatpakBackend::addAppFromFlatpakRef(const QUrl &url, ResultsStream *stream resource->updateFromRef(ref); QUrl runtimeUrl = QUrl(settings.value(QStringLiteral("Flatpak Ref/RuntimeRepo")).toString()); + auto refSource = QSharedPointer::create(this, preferredInstallation(), nullptr); + m_flatpakSources += refSource; if (!runtimeUrl.isEmpty()) { // We need to fetch metadata to find information about required runtime auto fw = new QFutureWatcher(this); - connect(fw, &QFutureWatcher::finished, this, [this, resource, fw, runtimeUrl, stream]() { + connect(fw, &QFutureWatcher::finished, this, [this, resource, fw, runtimeUrl, stream, refSource]() { fw->deleteLater(); const auto metadata = fw->result(); // Even when we failed to fetch information about runtime we still want to show the application @@ -477,20 +633,23 @@ void FlatpakBackend::addAppFromFlatpakRef(const QUrl &url, ResultsStream *stream auto runtime = getRuntimeForApp(resource); if (!runtime || (runtime && !runtime->isInstalled())) { auto repoStream = new ResultsStream(QLatin1String("FlatpakStream-searchrepo-") + runtimeUrl.toString()); - connect(repoStream, &ResultsStream::resourcesFound, this, [this, resource, stream](const QVector &resources) { - for (auto res : resources) { - installApplication(res); - } - addResource(resource); - stream->resourcesFound({resource}); - stream->finish(); - }); + connect(repoStream, + &ResultsStream::resourcesFound, + this, + [this, resource, stream, refSource](const QVector &resources) { + for (auto res : resources) { + installApplication(res); + } + refSource->addResource(resource); + stream->resourcesFound({resource}); + stream->finish(); + }); auto fetchRemoteResource = new FlatpakFetchRemoteResourceJob(runtimeUrl, repoStream, this); fetchRemoteResource->start(); return; } else { - addResource(resource); + refSource->addResource(resource); } } stream->resourcesFound({resource}); @@ -498,7 +657,7 @@ void FlatpakBackend::addAppFromFlatpakRef(const QUrl &url, ResultsStream *stream }); fw->setFuture(QtConcurrent::run(&m_threadPool, &FlatpakRunnables::fetchMetadata, resource, m_cancellable)); } else { - addResource(resource); + refSource->addResource(resource); stream->resourcesFound({resource}); stream->finish(); } @@ -558,60 +717,6 @@ void FlatpakBackend::addSourceFromFlatpakRepo(const QUrl &url, ResultsStream *st stream->resourcesFound({resource}); } -void FlatpakBackend::addResource(FlatpakResource *resource) -{ - // Update app with all possible information we have - if (!parseMetadataFromAppBundle(resource)) { - qWarning() << "Failed to parse metadata from app bundle for" << resource->name(); - } - - updateAppState(resource); - - m_resources.insert(resource->uniqueId(), resource); - if (!resource->extends().isEmpty()) { - m_extends.append(resource->extends()); - m_extends.removeDuplicates(); - } - - connect(resource, &FlatpakResource::sizeChanged, this, [this, resource] { - if (!isFetching()) - Q_EMIT resourcesChanged(resource, {"size", "sizeDescription"}); - }); -} - -class FlatpakSource -{ -public: - FlatpakSource(FlatpakRemote *remote) - : m_remote(remote) - { - } - - bool isEnabled() const - { - return !flatpak_remote_get_disabled(m_remote); - } - - QString appstreamDir() const - { - g_autoptr(GFile) appstreamDir = flatpak_remote_get_appstream_dir(m_remote, nullptr); - if (!appstreamDir) { - qWarning() << "No appstream dir for" << flatpak_remote_get_name(m_remote); - return {}; - } - g_autofree char *path_str = g_file_get_path(appstreamDir); - return QString::fromUtf8(path_str); - } - - QString name() const - { - return QString::fromUtf8(flatpak_remote_get_name(m_remote)); - } - -private: - FlatpakRemote *m_remote; -}; - void FlatpakBackend::loadAppsFromAppstreamData() { for (auto installation : qAsConst(m_installations)) { @@ -641,7 +746,7 @@ bool FlatpakBackend::loadAppsFromAppstreamData(FlatpakInstallation *flatpakInsta g_autoptr(GFile) fileTimestamp = flatpak_remote_get_appstream_timestamp(remote, flatpak_get_default_arch()); g_autofree char *path_str = g_file_get_path(fileTimestamp); - QFileInfo fileInfo = QFileInfo(QString::fromUtf8(path_str)); + QFileInfo fileInfo(QFile::encodeName(path_str)); // Refresh appstream metadata in case they have never been refreshed or the cache is older than 6 hours if (!fileInfo.exists() || fileInfo.lastModified().toUTC().secsTo(QDateTime::currentDateTimeUtc()) > 21600) { refreshAppstreamMetadata(flatpakInstallation, remote); @@ -656,7 +761,6 @@ void FlatpakBackend::metadataRefreshed() { m_refreshAppstreamMetadataJobs--; if (m_refreshAppstreamMetadataJobs == 0) { - loadInstalledApps(); checkForUpdates(); } } @@ -665,45 +769,34 @@ void FlatpakBackend::integrateRemote(FlatpakInstallation *flatpakInstallation, F { Q_ASSERT(m_refreshAppstreamMetadataJobs != 0); - FlatpakSource source(remote); - if (!source.isEnabled() || flatpak_remote_get_noenumerate(remote)) { + QSharedPointer source(new FlatpakSource(this, flatpakInstallation, remote)); + if (!source->isEnabled() || flatpak_remote_get_noenumerate(remote)) { metadataRefreshed(); return; } - const QString appstreamDirPath = source.appstreamDir(); - const QString appstreamIconsPath = source.appstreamDir() + QLatin1String("/icons/"); + const QString appstreamDirPath = source->appstreamDir(); if (!QFile::exists(appstreamDirPath)) { - qWarning() << "No" << appstreamDirPath << "appstream metadata found for" << source.name(); + qWarning() << "No" << appstreamDirPath << "appstream metadata found for" << source->name(); metadataRefreshed(); return; } - auto fw = new QFutureWatcher>(this); - const auto sourceName = source.name(); - connect(fw, &QFutureWatcher>::finished, this, [this, fw, flatpakInstallation, appstreamIconsPath, sourceName]() { - const auto components = fw->result(); - QVector resources; - for (const AppStream::Component &appstreamComponent : components) { - FlatpakResource *resource = new FlatpakResource(appstreamComponent, flatpakInstallation, this); - resource->setIconPath(appstreamIconsPath); - resource->setOrigin(sourceName); - if (resource->resourceType() == FlatpakResource::Runtime) { - resources.prepend(resource); - } else { - resources.append(resource); - } - } - for (auto resource : qAsConst(resources)) { - addResource(resource); + AppStream::Pool *pool = new AppStream::Pool(this); + source->m_pool = pool; + auto fw = new QFutureWatcher(this); + const auto sourceName = source->name(); + connect(fw, &QFutureWatcher::finished, this, [this, fw, pool, source]() { + if (fw->result()) { + m_flatpakSources += source; + } else { + qWarning() << "Could not open the AppStream metadata pool" << pool->lastError(); } - metadataRefreshed(); acquireFetching(false); fw->deleteLater(); }); acquireFetching(true); - AppStream::Pool *pool = new AppStream::Pool(this); pool->clearMetadataLocations(); pool->addMetadataLocation(appstreamDirPath); pool->setCacheFlags(AppStream::Pool::CacheFlagUseUser); @@ -711,86 +804,7 @@ void FlatpakBackend::integrateRemote(FlatpakInstallation *flatpakInstallation, F const QString subdir = flatpak_installation_get_id(flatpakInstallation) + QLatin1Char('/') + sourceName; pool->setCacheLocation(QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + "/flatpak-appstream/" + subdir); QDir().mkpath(pool->cacheLocation()); - fw->setFuture(QtConcurrent::run(&m_threadPool, [pool, appstreamDirPath]() -> QList { - if (!pool->load()) { - qWarning() << "Could not open the AppStream metadata pool" << pool->lastError(); - return {}; - } - return pool->components(); - })); -} - -void FlatpakBackend::loadInstalledApps() -{ - for (auto installation : qAsConst(m_installations)) { - // Load installed applications and update existing resources with info from installed application - if (!loadInstalledApps(installation)) { - qWarning() << "Failed to load installed packages from installation" << installation; - } - } -} - -bool FlatpakBackend::loadInstalledApps(FlatpakInstallation *flatpakInstallation) -{ - Q_ASSERT(flatpakInstallation); - - g_autoptr(GError) localError = nullptr; - g_autoptr(GPtrArray) refs = flatpak_installation_list_installed_refs(flatpakInstallation, m_cancellable, &localError); - if (!refs) { - qWarning() << "Failed to get list of installed refs for listing updates:" << localError->message; - return false; - } - - const QString pathExports = FlatpakResource::installationPath(flatpakInstallation) + QLatin1String("/exports/"); - const QString pathApps = pathExports + QLatin1String("share/applications/"); - - QVector resources; - for (uint i = 0; i < refs->len; i++) { - FlatpakInstalledRef *ref = FLATPAK_INSTALLED_REF(g_ptr_array_index(refs, i)); - - const auto name = QLatin1String(flatpak_ref_get_name(FLATPAK_REF(ref))); - if (name.endsWith(QLatin1String(".Debug")) || name.endsWith(QLatin1String(".Locale")) || name.endsWith(QLatin1String(".BaseApp")) - || name.endsWith(QLatin1String(".Docs"))) - continue; - - const auto res = getAppForInstalledRef(flatpakInstallation, ref); - if (res) { - res->setState(AbstractResource::Installed); - continue; - } - - AppStream::Component cid; - AppStream::Metadata metadata; - const QString fnDesktop = pathApps + name + QLatin1String(".desktop"); - AppStream::Metadata::MetadataError error = metadata.parseFile(fnDesktop, AppStream::Metadata::FormatKindDesktopEntry); - if (error != AppStream::Metadata::MetadataErrorNoError) { - if (QFile::exists(fnDesktop)) - qDebug() << "Failed to parse appstream metadata:" << error << fnDesktop; - - cid.setId(QString::fromLatin1(flatpak_ref_get_name(FLATPAK_REF(ref)))); -#if FLATPAK_CHECK_VERSION(1, 1, 2) - cid.setName(QString::fromUtf8(flatpak_installed_ref_get_appdata_name(ref))); -#endif - } else - cid = metadata.component(); - - FlatpakResource *resource = new FlatpakResource(cid, flatpakInstallation, this); - - resource->setIconPath(pathExports); - resource->setState(AbstractResource::Installed); - resource->setOrigin(QString::fromUtf8(flatpak_installed_ref_get_origin(ref))); - resource->updateFromRef(FLATPAK_REF(ref)); - - if (resource->resourceType() == FlatpakResource::Runtime) { - resources.prepend(resource); - } else { - resources.append(resource); - } - } - for (auto resource : qAsConst(resources)) - addResource(resource); - - return true; + fw->setFuture(QtConcurrent::run(&m_threadPool, pool, &AppStream::Pool::load)); } void FlatpakBackend::loadLocalUpdates(FlatpakInstallation *flatpakInstallation) @@ -825,48 +839,6 @@ void FlatpakBackend::loadLocalUpdates(FlatpakInstallation *flatpakInstallation) } } -void FlatpakBackend::loadRemoteUpdates(FlatpakInstallation *installation) -{ - auto fw = new QFutureWatcher(this); - connect(fw, &QFutureWatcher::finished, this, [this, installation, fw]() { - g_autoptr(GPtrArray) refs = fw->result(); - onFetchUpdatesFinished(installation, refs); - fw->deleteLater(); - acquireFetching(false); - }); - acquireFetching(true); - fw->setFuture(QtConcurrent::run(&m_threadPool, [installation, this]() -> GPtrArray * { - g_autoptr(GError) localError = nullptr; - if (g_cancellable_is_cancelled(m_cancellable)) { - qWarning() << "don't issue commands after cancelling"; - return {}; - } - GPtrArray *refs = flatpak_installation_list_installed_refs_for_update(installation, m_cancellable, &localError); - if (!refs) { - qWarning() << "Failed to get list of installed refs for listing updates: " << localError->message; - } - return refs; - })); -} - -void FlatpakBackend::onFetchUpdatesFinished(FlatpakInstallation *flatpakInstallation, GPtrArray *fetchedUpdates) -{ - if (!fetchedUpdates) { - qWarning() << "could not get updates for" << flatpakInstallation; - return; - } - - for (uint i = 0; i < fetchedUpdates->len; i++) { - FlatpakInstalledRef *ref = FLATPAK_INSTALLED_REF(g_ptr_array_index(fetchedUpdates, i)); - FlatpakResource *resource = getAppForInstalledRef(flatpakInstallation, ref); - if (resource) { - resource->setState(AbstractResource::Upgradeable); - updateAppSize(resource); - } else - qWarning() << "could not find updated resource" << flatpak_ref_get_name(FLATPAK_REF(ref)) << m_resources.size(); - } -} - bool FlatpakBackend::parseMetadataFromAppBundle(FlatpakResource *resource) { g_autoptr(FlatpakRef) ref = nullptr; @@ -1216,30 +1188,117 @@ ResultsStream *FlatpakBackend::search(const AbstractResourcesBackend::Filters &f return findResourceByPackageName(filter.resourceUrl); } else if (!filter.resourceUrl.isEmpty() || (!filter.extends.isEmpty() && !m_extends.contains(filter.extends))) return new ResultsStream(QStringLiteral("FlatpakStream-void"), {}); + else if (filter.state == AbstractResource::Upgradeable) { + auto stream = new ResultsStream(QStringLiteral("FlatpakStream-upgradeable")); + auto f = [this, stream] { + QVector resources; + for (auto installation : m_installations) { + g_autoptr(GError) localError = nullptr; + g_autoptr(GPtrArray) refs = flatpak_installation_list_installed_refs_for_update(installation, m_cancellable, &localError); + if (!refs) { + qWarning() << "Failed to get list of installed refs for listing updates:" << localError->message; + continue; + } + + resources.reserve(resources.size() + refs->len); + for (uint i = 0; i < refs->len; i++) { + FlatpakInstalledRef *ref = FLATPAK_INSTALLED_REF(g_ptr_array_index(refs, i)); + auto resource = getAppForInstalledRef(installation, ref); + resource->setState(AbstractResource::Upgradeable); + updateAppSize(resource); + if (resource->resourceType() == FlatpakResource::Runtime) { + resources.prepend(resource); + } else { + resources.append(resource); + } + } + } + if (!resources.isEmpty()) + Q_EMIT stream->resourcesFound(resources); + stream->finish(); + }; + + if (isFetching()) { + connect(this, &FlatpakBackend::initialized, stream, f); + } else { + QTimer::singleShot(0, this, f); + } + return stream; + } else if (filter.state == AbstractResource::Installed) { + auto stream = new ResultsStream(QStringLiteral("FlatpakStream-installed")); + auto f = [this, stream] { + QVector resources; + for (auto installation : m_installations) { + g_autoptr(GError) localError = nullptr; + g_autoptr(GPtrArray) refs = flatpak_installation_list_installed_refs(installation, m_cancellable, &localError); + if (!refs) { + qWarning() << "Failed to get list of installed refs for listing installed:" << localError->message; + continue; + } + + resources.reserve(resources.size() + refs->len); + for (uint i = 0; i < refs->len; i++) { + FlatpakInstalledRef *ref = FLATPAK_INSTALLED_REF(g_ptr_array_index(refs, i)); + QString name = QString::fromUtf8(flatpak_installed_ref_get_appdata_name(ref)); + if (name.endsWith(QLatin1String(".Debug")) || name.endsWith(QLatin1String(".Locale")) || name.endsWith(QLatin1String(".BaseApp")) + || name.endsWith(QLatin1String(".Docs"))) + continue; + + auto resource = getAppForInstalledRef(installation, ref); + if (resource->resourceType() == FlatpakResource::Runtime) { + resources.prepend(resource); + } else { + resources.append(resource); + } + } + } + if (!resources.isEmpty()) + Q_EMIT stream->resourcesFound(resources); + stream->finish(); + }; + + if (isFetching()) { + connect(this, &FlatpakBackend::initialized, stream, f); + } else { + QTimer::singleShot(0, this, f); + } + return stream; + } auto stream = new ResultsStream(QStringLiteral("FlatpakStream")); auto f = [this, stream, filter]() { QVector prioritary, rest; - for (auto r : qAsConst(m_resources)) { - const bool matchById = r->appstreamId().compare(filter.search, Qt::CaseInsensitive) == 0; - if (r->type() == AbstractResource::Technical && filter.state != AbstractResource::Upgradeable && !matchById) { - continue; + for (const auto &source : qAsConst(m_flatpakSources)) { + QVector resources; + if (source->m_pool) { + resources = kTransform>(source->m_pool->search(filter.search), [this, &source](const auto &comp) { + return resourceForComponent(comp, source); + }); + } else { + resources = source->m_resources.values().toVector(); } - if (r->state() < filter.state) - continue; - if (!filter.extends.isEmpty() && !r->extends().contains(filter.extends)) - continue; + for (auto r : resources) { + const bool matchById = r->appstreamId().compare(filter.search, Qt::CaseInsensitive) == 0; + if (r->type() == AbstractResource::Technical && filter.state != AbstractResource::Upgradeable && !matchById) { + continue; + } + if (r->state() < filter.state) + continue; + + if (!filter.extends.isEmpty() && !r->extends().contains(filter.extends)) + continue; - if (!filter.mimetype.isEmpty() && !r->mimetypes().contains(filter.mimetype)) - continue; + if (!filter.mimetype.isEmpty() && !r->mimetypes().contains(filter.mimetype)) + continue; - if (filter.search.isEmpty() || matchById) { - rest += r; - } else if (r->name().contains(filter.search, Qt::CaseInsensitive)) { - prioritary += r; - } else if (r->comment().contains(filter.search, Qt::CaseInsensitive)) { - rest += r; + if (filter.search.isEmpty() || matchById) { + rest += r; + } else if (r->name().contains(filter.search, Qt::CaseInsensitive)) { + prioritary += r; + } else if (r->comment().contains(filter.search, Qt::CaseInsensitive)) { + rest += r; + } } } auto f = [this](AbstractResource *l, AbstractResource *r) { @@ -1264,19 +1323,11 @@ QVector FlatpakBackend::resourcesByAppstreamName(const QStri { QVector resources; const QString nameWithDesktop = name + QLatin1String(".desktop"); - for (FlatpakResource *res : m_resources) { - if (QString::compare(res->appstreamId(), name, Qt::CaseInsensitive) == 0 - || QString::compare(res->appstreamId(), nameWithDesktop, Qt::CaseInsensitive) == 0) - resources << res; - else { - const auto alts = res->alternativeAppstreamIds(); - for (const auto &alt : alts) { - if (QString::compare(alt, name, Qt::CaseInsensitive) == 0 || QString::compare(alt, nameWithDesktop, Qt::CaseInsensitive) == 0) { - resources << res; - break; - } - } - } + for (const auto &source : m_flatpakSources) { + auto comps = source->m_pool->componentsById(name) + source->m_pool->componentsById(nameWithDesktop); + resources << kTransform>(comps, [this, source](const auto &comp) { + return resourceForComponent(comp, source); + }); } auto f = [this](AbstractResource *l, AbstractResource *r) { return flatpakResourceLessThan(l, r); @@ -1313,6 +1364,21 @@ ResultsStream *FlatpakBackend::findResourceByPackageName(const QUrl &url) return new ResultsStream(QStringLiteral("FlatpakStream-packageName-void"), {}); } +FlatpakResource *FlatpakBackend::resourceForComponent(const AppStream::Component &component, const QSharedPointer &source) const +{ + for (auto res : source->m_resources) { + if (res->appstreamId() == component.id()) { + return res; + } + } + + FlatpakResource *res = new FlatpakResource(component, source->installation(), const_cast(this)); + res->setOrigin(source->name()); + res->setIconPath(source->appstreamDir() + QLatin1String("/icons/")); + source->addResource(res); + return res; +} + AbstractBackendUpdater *FlatpakBackend::backendUpdater() const { return m_updater; @@ -1385,12 +1451,6 @@ void FlatpakBackend::checkForUpdates() // Load local updates, comparing current and latest commit loadLocalUpdates(installation); - if (g_cancellable_is_cancelled(m_cancellable)) - break; - - // Load updates from remote repositories - loadRemoteUpdates(installation); - if (g_cancellable_is_cancelled(m_cancellable)) break; } diff --git a/libdiscover/backends/FlatpakBackend/FlatpakBackend.h b/libdiscover/backends/FlatpakBackend/FlatpakBackend.h index a1d1216d1..26121044e 100644 --- a/libdiscover/backends/FlatpakBackend/FlatpakBackend.h +++ b/libdiscover/backends/FlatpakBackend/FlatpakBackend.h @@ -20,8 +20,15 @@ #include "flatpak-helper.h" class FlatpakSourcesBackend; +class FlatpakSource; class StandardBackendUpdater; class OdrsReviewsBackend; + +namespace AppStream +{ +class Pool; +} + class FlatpakBackend : public AbstractResourcesBackend { Q_OBJECT @@ -34,10 +41,6 @@ public: AbstractReviewsBackend *reviewsBackend() const override; ResultsStream *search(const AbstractResourcesBackend::Filters &search) override; ResultsStream *findResourceByPackageName(const QUrl &search); - QList resources() const - { - return m_resources.values(); - } bool isValid() const override; Transaction *installApplication(AbstractResource *app) override; @@ -74,12 +77,13 @@ public: private Q_SLOTS: void onFetchMetadataFinished(FlatpakResource *resource, const QByteArray &metadata); void onFetchSizeFinished(FlatpakResource *resource, guint64 downloadSize, guint64 installedSize); - void onFetchUpdatesFinished(FlatpakInstallation *flatpakInstallation, GPtrArray *updates); Q_SIGNALS: // for tests void initialized(); private: + friend class FlatpakSource; + void metadataRefreshed(); bool flatpakResourceLessThan(AbstractResource *l, AbstractResource *r) const; void announceRatingsReady(); @@ -90,14 +94,11 @@ private: void integrateRemote(FlatpakInstallation *flatpakInstallation, FlatpakRemote *remote); FlatpakRemote *getFlatpakRemoteByUrl(const QString &url, FlatpakInstallation *installation) const; FlatpakResource *getRuntimeForApp(FlatpakResource *resource) const; + FlatpakResource *resourceForComponent(const AppStream::Component &component, const QSharedPointer &source) const; - void addResource(FlatpakResource *resource); void loadAppsFromAppstreamData(); bool loadAppsFromAppstreamData(FlatpakInstallation *flatpakInstallation); - void loadInstalledApps(); - bool loadInstalledApps(FlatpakInstallation *flatpakInstallation); void loadLocalUpdates(FlatpakInstallation *flatpakInstallation); - void loadRemoteUpdates(FlatpakInstallation *flatpakInstallation); bool parseMetadataFromAppBundle(FlatpakResource *resource); void refreshAppstreamMetadata(FlatpakInstallation *installation, FlatpakRemote *remote); bool setupFlatpakInstallations(GError **error); @@ -107,11 +108,11 @@ private: bool updateAppMetadata(FlatpakResource *resource, const QString &path); bool updateAppSizeFromRemote(FlatpakResource *resource); void updateAppState(FlatpakResource *resource); + QSharedPointer findSource(FlatpakInstallation *installation, const QString &origin) const; QVector resourcesByAppstreamName(const QString &name) const; void acquireFetching(bool f); - QHash m_resources; StandardBackendUpdater *m_updater; FlatpakSourcesBackend *m_sources = nullptr; QSharedPointer m_reviews; @@ -122,6 +123,8 @@ private: GCancellable *m_cancellable; QVector m_installations; QThreadPool m_threadPool; + QVector> m_flatpakSources; + QSharedPointer m_localSource; }; #endif // FLATPAKBACKEND_H diff --git a/libdiscover/backends/FlatpakBackend/FlatpakResource.cpp b/libdiscover/backends/FlatpakBackend/FlatpakResource.cpp index 9fc5b9537..d93334509 100644 --- a/libdiscover/backends/FlatpakBackend/FlatpakResource.cpp +++ b/libdiscover/backends/FlatpakBackend/FlatpakResource.cpp @@ -47,6 +47,7 @@ FlatpakResource::FlatpakResource(const AppStream::Component &component, FlatpakI , m_propertyStates({{DownloadSize, NotKnownYet}, {InstalledSize, NotKnownYet}, {RequiredRuntime, NotKnownYet}}) , m_state(AbstractResource::None) { + qDebug() << "created..." << component.name() << installation; setObjectName(packageName()); // Start fetching remote icons during initialization @@ -180,17 +181,9 @@ QVariant FlatpakResource::icon() const for (const AppStream::Icon &icon : icons) { switch (icon.kind()) { case AppStream::Icon::KindLocal: - case AppStream::Icon::KindCached: { - const QString path = m_iconPath + icon.url().path(); - if (QFileInfo::exists(path)) { - ret.addFile(path, icon.size()); - } else { - const QString altPath = m_iconPath + QStringLiteral("%1x%2/").arg(icon.size().width()).arg(icon.size().height()) + icon.url().path(); - if (QFileInfo::exists(altPath)) { - ret.addFile(altPath, icon.size()); - } - } - } break; + case AppStream::Icon::KindCached: + ret.addFile(icon.url().toLocalFile(), icon.size()); + break; case AppStream::Icon::KindStock: { const auto ret = QIcon::fromTheme(icon.name()); if (!ret.isNull()) diff --git a/libdiscover/backends/FlatpakBackend/FlatpakResource.h b/libdiscover/backends/FlatpakBackend/FlatpakResource.h index ec35f400d..321020671 100644 --- a/libdiscover/backends/FlatpakBackend/FlatpakResource.h +++ b/libdiscover/backends/FlatpakBackend/FlatpakResource.h @@ -55,7 +55,7 @@ public: Q_ENUM(FlatpakFileType) struct Id { - FlatpakInstallation *const installation; + FlatpakInstallation *const installation; // TODO installation doesn't need to be part of the id anymore QString origin; FlatpakResource::ResourceType type; const QString id; -- GitLab From 5337bf5e64a97affa2568e44dce1275c5bd1f699 Mon Sep 17 00:00:00 2001 From: Aleix Pol Date: Tue, 3 Aug 2021 00:41:00 +0200 Subject: [PATCH 3/3] Bump AppStream dependency --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 26387b2f0..aae456456 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,7 @@ find_package(KF5 ${KF5_MIN_VERSION} REQUIRED CoreAddons Config Crash DBusAddons find_package(KF5Kirigami2 2.7.0) find_package(packagekitqt5 1.0.1 CONFIG) -find_package(AppStreamQt 0.12.8 CONFIG) +find_package(AppStreamQt 0.14.4 CONFIG) find_package(KF5Attica 5.23 CONFIG) find_package(KF5NewStuff 5.53 CONFIG) -- GitLab