Commit 2183d384 authored by Gilles Caulier's avatar Gilles Caulier 🗼
Browse files

Next stage to improve FuzzySearch management to apply patch #102381 from Mario Frank

When images are deleted, the ImageViewUtilities give
the ImageInfos to the DIO. In the ImageViewUtilities,
the ids of the deleted images are buffered in a list and given to the
AlbumManager via Signal-Slot-communication. I do that since in database
layer, only the URIs of the images are present and getting the image ids is
unnecessary effort. Since the AlbumManager already has all SAlbums, it is
easy to first get all SAlbums that represent DuplicateSearches (I implemented
a method to get all SAlbums by type) and then get all those albums that
contain the deleted images (SearchXmlReader is needed here).

In order to have a good performance and do no unnecessary work, the AlbumManager unites
all image ids from the SAlbums to update and subtracts the ids of the deleted images (set operations).
This way, I am able to communicate the set of images to rescan to the FindDuplicatesView.
Before notifying the FindDuplicatesView, I delete all SAlbums that should be updated.
The reason: Consider an SAlbum that contained two duplicates - the original image and the now deleted
image. If I start a normal duplicates search, no SAlbum will be created for the original image and
the SAlbum will thus not be updated and should not be in the final result set.

Since I now know the ids of the images to rescan, I notify the FindDuplicatesView.
But now I do not have albums to search in - and in fact this would not really be performant since
uninvolved images would be scanned. Thus, I extended the DuplicatesFinder with a new constructor
for searching for duplicates in a set of images. This could be applicable in other situations, too.
To make this work, I extended the DB-Jobinfo with a switch for update of duplicates where the
set of images is and not the albums and tags is used for duplicates search. This way, I can set the
image ids and the update flag in the job info. The DBJob triggers a new variant of rebuildDuplicates
in haariface where the image ids are taken directly.

With this approach, only the SAlbums that need to be updated are updated. SAlbums are only lost,
if no duplicates exist due to deletion which is appropriate. Previously not existing SAlbums cannot be created
since no new images are involved.

BUGS: 261417
FIXED-IN: 5.4.0
parent 0e716498
......@@ -60,6 +60,9 @@ ImageViewUtilities::ImageViewUtilities(QWidget* const parentWidget)
: QObject(parentWidget)
{
m_widget = parentWidget;
connect(this,SIGNAL(signalImagesDeleted(const QList<qlonglong>&)),
AlbumManager::instance(),SLOT(slotImagesDeleted(const QList<qlonglong>&)));
}
void ImageViewUtilities::setAsAlbumThumbnail(Album* album, const ImageInfo& imageInfo)
......@@ -106,10 +109,13 @@ bool ImageViewUtilities::deleteImages(const QList<ImageInfo>& infos, const Delet
QList<ImageInfo> deleteInfos = infos;
QList<QUrl> urlList;
QList<qlonglong> imageIds;
// Buffer the urls for deletion and imageids for notification of the AlbumManager
foreach(const ImageInfo& info, deleteInfos)
{
urlList << info.fileUrl();
imageIds << info.id();
}
DeleteDialog dialog(m_widget);
......@@ -129,6 +135,9 @@ bool ImageViewUtilities::deleteImages(const QList<ImageInfo>& infos, const Delet
const bool useTrash = !dialog.shouldDelete();
DIO::del(deleteInfos, useTrash);
// Signal the Albummanager about the ids of the deleted images.
emit signalImagesDeleted(imageIds);
return true;
}
......@@ -142,8 +151,18 @@ void ImageViewUtilities::deleteImagesDirectly(const QList<ImageInfo>& infos, con
return;
}
QList<qlonglong> imageIds;
foreach(const ImageInfo& info, infos)
{
imageIds << info.id();
}
const bool useTrash = (deleteMode == ImageViewUtilities::DeleteUseTrash);
DIO::del(infos, useTrash);
// Signal the Albummanager about the ids of the deleted images.
emit signalImagesDeleted(imageIds);
}
void ImageViewUtilities::notifyFileContentChanged(const QList<QUrl>& urls)
......
......@@ -82,6 +82,7 @@ public Q_SLOTS:
Q_SIGNALS:
void editorCurrentUrlChanged(const QUrl& url);
void signalImagesDeleted(const QList<qlonglong>& imageIds);
protected:
......
......@@ -84,6 +84,7 @@ extern "C"
#include "databaseserverstarter.h"
#include "coredbthumbinfoprovider.h"
#include "coredburl.h"
#include "coredbsearchxml.h"
#include "coredbwatch.h"
#include "dio.h"
#include "facetags.h"
......@@ -2071,6 +2072,20 @@ SAlbum* AlbumManager::findSAlbum(const QString& name) const
return 0;
}
QList<SAlbum*> AlbumManager::findSAlbumsBySearchType(int searchType) const
{
QList<SAlbum*> albums;
for (Album* album = d->rootSAlbum->firstChild(); album; album = album->next())
{
SAlbum* sAlbum = dynamic_cast<SAlbum*>(album);
if (sAlbum->searchType() == searchType)
{
albums.append(sAlbum);
}
}
return albums;
}
void AlbumManager::addGuardedPointer(Album* album, Album** pointer)
{
if (album)
......@@ -3468,6 +3483,54 @@ void AlbumManager::slotImageTagChange(const ImageTagChangeset& changeset)
}
}
void AlbumManager::slotImagesDeleted(const QList<qlonglong>& imageIds)
{
qCDebug(DIGIKAM_GENERAL_LOG) << "Got image deletion notification from ImageViewUtilities for " << imageIds.size() << " images.";
QSet<qlonglong> imagesToRescan;
QSet<SAlbum*> sAlbumsToDelete;
QList<SAlbum*> sAlbums = findSAlbumsBySearchType(DatabaseSearch::DuplicatesSearch);
foreach(SAlbum* sAlbum, sAlbums)
{
// Read the search query XML and save the image ids
SearchXmlReader reader(sAlbum->query());
SearchXml::Element element;
QSet<qlonglong> images;
while ((element = reader.readNext()) != SearchXml::End)
{
if ((element == SearchXml::Field) && (reader.fieldName().compare(QLatin1String("imageid")) == 0))
{
images = reader.valueToLongLongList().toSet();
}
}
// If the deleted images are part of the SAlbum,
// mark the album as ready for deletion and the images as ready for rescan.
if (images.intersects(imageIds.toSet()))
{
sAlbumsToDelete.insert(sAlbum);
imagesToRescan.unite(images);
}
}
if (!imagesToRescan.empty())
{
// Delete albums
foreach (SAlbum* sAlbum, sAlbumsToDelete)
{
deleteSAlbum(sAlbum);
}
// Remove the deleted images from the set of images for rescan.
imagesToRescan.subtract(imageIds.toSet());
qCDebug(DIGIKAM_GENERAL_LOG) << "Rescanning " << imagesToRescan.size() << " images for duplicates.";
emit signalUpdateDuplicatesAlbums(imagesToRescan.toList());
}
}
void AlbumManager::removeWatchedPAlbums(const PAlbum* const album)
{
d->albumWatch->removeWatchedPAlbums(album);
......
......@@ -258,6 +258,13 @@ public:
* @param name the name of the search
*/
SAlbum* findSAlbum(const QString& name) const;
/**
* @return SAlbums with given type, empty list if not found
* @param searchType the type of the search
*/
QList< SAlbum* > findSAlbumsBySearchType(int searchType) const;
//@}
/** @name Operations on PAlbum
......@@ -556,6 +563,7 @@ public:
* @param leadingSlash if <code>true</code> return name paths with a leading slash
*/
QStringList namePaths(const QList<QString>& tagNames, bool leadingSlash=true) const;
//@}
/** @name Accessors to counting maps
......@@ -645,6 +653,7 @@ Q_SIGNALS:
void signalDatesMapDirty(const QMap<QDateTime, int>&);
void signalTagPropertiesChanged(TAlbum* album);
void signalAlbumsUpdated(int type);
void signalUpdateDuplicatesAlbums(QList<qlonglong> imagesToRescan);
// Signals a change in this property. Please note that affected albums may appear or disappear after this signal has been emitted.
void signalShowOnlyAvailableAlbumsChanged(bool showsOnlyAvailableAlbums);
......@@ -666,6 +675,7 @@ private Q_SLOTS:
void slotSearchChange(const SearchChangeset& changeset);
void slotCollectionImageChange(const CollectionImageChangeset& changeset);
void slotImageTagChange(const ImageTagChangeset& changeset);
void slotImagesDeleted(const QList<qlonglong>& imageIds);
/**
* Scan albums directly from database and creates new PAlbums
......
......@@ -270,9 +270,9 @@ void SearchesJob::run()
}
else
{
if (m_jobInfo.albumsIds().isEmpty() && m_jobInfo.tagsIds().isEmpty())
if (m_jobInfo.albumsIds().isEmpty() && m_jobInfo.tagsIds().isEmpty() && m_jobInfo.imageIds().isEmpty())
{
qCDebug(DIGIKAM_DBJOB_LOG) << "No album ids passed for duplicates search";
qCDebug(DIGIKAM_DBJOB_LOG) << "No album, tag or image ids passed for duplicates search";
return;
}
......@@ -285,11 +285,21 @@ void SearchesJob::run()
// Rebuild the duplicate albums
HaarIface iface;
iface.rebuildDuplicatesAlbums(m_jobInfo.albumsIds(),
m_jobInfo.tagsIds(),
m_jobInfo.minThreshold(),
m_jobInfo.maxThreshold(),
&observer);
if (m_jobInfo.isAlbumUpdate())
{
iface.rebuildDuplicatesAlbums(m_jobInfo.imageIds(),
m_jobInfo.minThreshold(),
m_jobInfo.maxThreshold(),
&observer);
}
else
{
iface.rebuildDuplicatesAlbums(m_jobInfo.albumsIds(),
m_jobInfo.tagsIds(),
m_jobInfo.minThreshold(),
m_jobInfo.maxThreshold(),
&observer);
}
}
emit signalDone();
......
......@@ -197,6 +197,7 @@ SearchesDBJobInfo::SearchesDBJobInfo()
: DBJobInfo()
{
m_duplicates = false;
m_albumUpdate = false;
m_minThreshold = 0;
m_maxThreshold = 1;
m_searchIds = QList<int>();
......@@ -212,6 +213,16 @@ bool SearchesDBJobInfo::isDuplicatesJob() const
return m_duplicates;
}
void SearchesDBJobInfo::setAlbumUpdate()
{
m_albumUpdate = true;
}
bool SearchesDBJobInfo::isAlbumUpdate() const
{
return m_albumUpdate;
}
void SearchesDBJobInfo::setSearchId(int id)
{
m_searchIds = QList<int>() << id;
......@@ -257,6 +268,16 @@ QList<int> SearchesDBJobInfo::albumsIds() const
return m_albumsIds;
}
void SearchesDBJobInfo::setImageIds(const QList<qlonglong>& imageIds)
{
m_imageIds = imageIds;
}
QList<qlonglong> SearchesDBJobInfo::imageIds() const
{
return m_imageIds;
}
void SearchesDBJobInfo::setTagsIds(const QList<int>& tagsIds)
{
m_tagsIds = tagsIds;
......
......@@ -147,6 +147,9 @@ public:
void setDuplicatesJob();
bool isDuplicatesJob() const;
void setAlbumUpdate();
bool isAlbumUpdate() const;
void setSearchIds(QList<int> ids);
void setSearchId(int id);
QList<int> searchIds() const;
......@@ -160,17 +163,22 @@ public:
void setAlbumsIds(const QList<int>& albumsIds);
QList<int> albumsIds() const;
void setImageIds(const QList<qlonglong>& imageIds);
QList<qlonglong> imageIds() const;
void setTagsIds(const QList<int>& tagsIds);
QList<int> tagsIds() const;
public:
bool m_duplicates;
QList<int> m_searchIds;
double m_minThreshold;
double m_maxThreshold;
QList<int> m_albumsIds;
QList<int> m_tagsIds;
bool m_duplicates;
bool m_albumUpdate;
QList<int> m_searchIds;
double m_minThreshold;
double m_maxThreshold;
QList<int> m_albumsIds;
QList<qlonglong> m_imageIds;
QList<int> m_tagsIds;
};
// ---------------------------------------------
......
......@@ -766,17 +766,14 @@ void HaarIface::getBestAndWorstPossibleScore(Haar::SignatureData* const sig, Ske
*lowestAndBestScore = score;
}
void HaarIface::rebuildDuplicatesAlbums(const QList<int>& albums2Scan, const QList<int>& tags2Scan,
double requiredPercentage, double maximumPercentage, HaarProgressObserver* const observer)
{
// Carry out search. This takes long.
QMap< double,QMap< qlonglong,QList<qlonglong> > > results = findDuplicatesInAlbumsAndTags(albums2Scan, tags2Scan, requiredPercentage, maximumPercentage, observer);
QMap<QString, QString> HaarIface::writeSAlbumQueries(QMap< double,QMap< qlonglong,QList<qlonglong> > > searchResults)
{
// Build search XML from the results. Store list of ids of similar images.
QMap<QString, QString> queries;
// Iterate over the similarity
for (QMap< double,QMap< qlonglong,QList<qlonglong> > >::const_iterator similarity_it = results.constBegin(); similarity_it != results.constEnd(); ++similarity_it)
for (QMap< double,QMap< qlonglong,QList<qlonglong> > >::const_iterator similarity_it = searchResults.constBegin(); similarity_it != searchResults.constEnd(); ++similarity_it)
{
double similarity = similarity_it.key() * 100;
QMap<qlonglong,QList<qlonglong>> sameSimilarityMap = similarity_it.value();
......@@ -799,6 +796,39 @@ void HaarIface::rebuildDuplicatesAlbums(const QList<int>& albums2Scan, const QLi
}
}
return queries;
}
void HaarIface::rebuildDuplicatesAlbums(const QList<qlonglong>& imageIds, double requiredPercentage, double maximumPercentage,
HaarProgressObserver* const observer)
{
QMap< double,QMap< qlonglong,QList<qlonglong> > > results = findDuplicates(imageIds.toSet(), requiredPercentage, maximumPercentage, observer);
QMap<QString, QString> queries = writeSAlbumQueries(results);
// Write the new search albums to the database
{
CoreDbAccess access;
CoreDbTransaction transaction(&access);
// Update existing searches by deleting and adding them.
for (QMap<QString, QString>::const_iterator it = queries.constBegin(); it != queries.constEnd(); ++it)
{
access.db()->deleteSearch(it.key().toInt());
access.db()->addSearch(DatabaseSearch::DuplicatesSearch, it.key(), it.value());
}
}
}
void HaarIface::rebuildDuplicatesAlbums(const QList<int>& albums2Scan, const QList<int>& tags2Scan,
double requiredPercentage, double maximumPercentage, HaarProgressObserver* const observer)
{
// Carry out search. This takes long.
QMap< double,QMap< qlonglong,QList<qlonglong> > > results = findDuplicatesInAlbumsAndTags(albums2Scan, tags2Scan, requiredPercentage, maximumPercentage, observer);
// Build search XML from the results. Store list of ids of similar images.
QMap<QString, QString> queries = writeSAlbumQueries(results);
// Write search albums to database
{
CoreDbAccess access;
......
......@@ -136,6 +136,16 @@ public:
void rebuildDuplicatesAlbums(const QList<int>& albums2Scan, const QList<int>& tags2Scan,
double requiredPercentage, double maximumPercentage, HaarProgressObserver* const observer = 0);
/**
* This method rebuilds the given SAlbums by searching duplicates and replacing the SAlbums by the updated versions.
* @param imageIds The set of images to scan for duplicates.
* @param requiredPercentage The minimum similarity for duplicate recognition.
* @param maximumPercentage The maximum similarity for duplicate recognition.
* @param observer The progress observer.
*/
void rebuildDuplicatesAlbums(const QList<qlonglong>& imageIds, double requiredPercentage, double maximumPercentage,
HaarProgressObserver* const observer = 0);
/** Retrieve the Haar signature from database using image id.
* Return true if item signature exist else false.
*/
......@@ -153,6 +163,12 @@ private:
bool indexImage(qlonglong imageid);
/**
* This method writes the search results to the SearchXml structure.
* @param searchResults The results to write as XML.
*/
QMap<QString, QString> writeSAlbumQueries(QMap< double,QMap< qlonglong,QList<qlonglong> > > searchResults);
QList<qlonglong> bestMatches(Haar::SignatureData* const data, int numberOfResults, SketchType type);
QPair<double,QList<qlonglong>> bestMatchesWithThreshold(qlonglong imageid,Haar::SignatureData* const querySig,
double requiredPercentage, double maximumPercentage, SketchType type);
......
......@@ -181,6 +181,9 @@ FindDuplicatesView::FindDuplicatesView(QWidget* const parent)
this, SLOT(slotClear()));
connect(d->minSimilarity, SIGNAL(valueChanged(int)),this,SLOT(slotMinimumChanged(int)));
connect(AlbumManager::instance(),SIGNAL(signalUpdateDuplicatesAlbums(QList<qlonglong>)),
this,SLOT(slotUpdateDuplicates(QList<qlonglong>)));
}
FindDuplicatesView::~FindDuplicatesView()
......@@ -315,6 +318,20 @@ void FindDuplicatesView::slotFindDuplicates()
finder->start();
}
void FindDuplicatesView::slotUpdateDuplicates(const QList<qlonglong> imagesToRescan)
{
d->albumSelectors->saveState();
slotClear();
enableControlWidgets(false);
DuplicatesFinder* const finder = new DuplicatesFinder(imagesToRescan, d->minSimilarity->value(), d->maxSimilarity->value());
connect(finder, SIGNAL(signalComplete()),
this, SLOT(slotComplete()));
finder->start();
}
void FindDuplicatesView::slotMinimumChanged(int newValue)
{
// Set the new minimum value of the maximum similarity
......
......@@ -61,6 +61,7 @@ private Q_SLOTS:
void slotSearchUpdated(SAlbum* a);
void slotClear();
void slotFindDuplicates();
void slotUpdateDuplicates(const QList<qlonglong> imagesToRescan);
void slotDuplicatesAlbumActived();
void slotComplete();
void slotUpdateFingerPrints();
......
......@@ -54,24 +54,38 @@ public:
Private() :
minSimilarity(90),
maxSimilarity(100),
isAlbumUpdate(false),
job(0)
{
}
int minSimilarity;
int maxSimilarity;
bool isAlbumUpdate;
QList<int> albumsIdList;
QList<qlonglong> imageIdList;
QList<int> tagsIdList;
SearchesDBJobsThread* job;
};
DuplicatesFinder::DuplicatesFinder(const QList<qlonglong>& imageIds, int minSimilarity, int maxSimilarity, ProgressItem* const parent)
: MaintenanceTool(QLatin1String("DuplicatesFinder"), parent),
d(new Private)
{
d->minSimilarity = minSimilarity;
d->maxSimilarity = maxSimilarity;
d->isAlbumUpdate = true;
d->imageIdList = imageIds;
}
DuplicatesFinder::DuplicatesFinder(const AlbumList& albums, const AlbumList& tags, int minSimilarity, int maxSimilarity, ProgressItem* const parent)
: MaintenanceTool(QLatin1String("DuplicatesFinder"), parent),
d(new Private)
{
d->minSimilarity = minSimilarity;
d->maxSimilarity = maxSimilarity;
foreach(Album* const a, albums)
d->albumsIdList << a->id();
......@@ -85,7 +99,7 @@ DuplicatesFinder::DuplicatesFinder(const int minSimilarity, int maxSimilarity, P
{
d->minSimilarity = minSimilarity;
d->maxSimilarity = maxSimilarity;
foreach(Album* const a, AlbumManager::instance()->allPAlbums())
d->albumsIdList << a->id();
}
......@@ -109,6 +123,10 @@ void DuplicatesFinder::slotStart()
jobInfo.setMinThreshold(minThresh);
jobInfo.setMaxThreshold(maxThresh);
jobInfo.setAlbumsIds(d->albumsIdList);
jobInfo.setImageIds(d->imageIdList);
if (d->isAlbumUpdate)
jobInfo.setAlbumUpdate();
if (!d->tagsIdList.isEmpty())
jobInfo.setTagsIds(d->tagsIdList);
......
......@@ -44,6 +44,9 @@ class DuplicatesFinder : public MaintenanceTool
public:
/** Version to find all duplicates in the set of images
*/
DuplicatesFinder(const QList<qlonglong>& imageIds, int minSimilarity = 90, int maxSimilarity = 100, ProgressItem* const parent = 0);
/** Version to find all duplicates over a specific list to PAlbums and TAlbums
*/
DuplicatesFinder(const AlbumList& albums, const AlbumList& tags, int minSimilarity = 90, int maxSimilarity = 100, ProgressItem* const parent = 0);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment