Commit 924a4332 authored by Laurent Montel's avatar Laurent Montel 😁
Browse files

Start to implement RespectDiacriticAndAccents or not

parent f278fdc7
Pipeline #149067 canceled with stage
......@@ -6,6 +6,7 @@
*/
#include "abstractindexer.h"
#include "stringutil.h"
AbstractIndexer::AbstractIndexer() = default;
......@@ -24,3 +25,22 @@ void AbstractIndexer::updateFlags(const Akonadi::Item &item, const QSet<QByteArr
Q_UNUSED(addedFlags)
Q_UNUSED(removed)
}
bool AbstractIndexer::respectDiacriticAndAccents() const
{
return mRespectDiacriticAndAccents;
}
void AbstractIndexer::setRespectDiacriticAndAccents(bool newRespectDiacriticAndAccents)
{
mRespectDiacriticAndAccents = newRespectDiacriticAndAccents;
}
QString AbstractIndexer::normalizeString(const QString &str)
{
if (mRespectDiacriticAndAccents) {
return str;
} else {
return StringUtil::normalize(str);
}
}
......@@ -29,5 +29,12 @@ public:
virtual void move(Akonadi::Item::Id item, Akonadi::Collection::Id from, Akonadi::Collection::Id to);
virtual void updateFlags(const Akonadi::Item &item, const QSet<QByteArray> &addedFlags, const QSet<QByteArray> &removed);
Q_REQUIRED_RESULT bool respectDiacriticAndAccents() const;
void setRespectDiacriticAndAccents(bool newRespectDiacriticAndAccents);
protected:
bool mRespectDiacriticAndAccents = true;
Q_REQUIRED_RESULT QString normalizeString(const QString &str);
};
......@@ -47,14 +47,16 @@ AkonadiIndexingAgent::AkonadiIndexingAgent(const QString &id)
KConfigGroup cfg = config()->group("General");
const int agentIndexingVersion = cfg.readEntry("agentIndexingVersion", 0);
bool respectDiacriticAndAccents = cfg.readEntry("respectDiacriticAndAccents", true);
if (agentIndexingVersion < INDEXING_AGENT_VERSION) {
m_index.removeDatabase();
// Don't respect Diacritic and Accents in new Database so search will be more easy.
// TODO activate it in the future respectDiacriticAndAccents = false;
QTimer::singleShot(0, &m_scheduler, &Scheduler::scheduleCompleteSync);
cfg.writeEntry("agentIndexingVersion", INDEXING_AGENT_VERSION);
cfg.sync();
}
m_index.setRespectDiacriticAndAccents(respectDiacriticAndAccents);
if (!m_index.createIndexers()) {
Q_EMIT status(Broken, i18nc("@info:status", "No indexers available"));
setOnline(false);
......
......@@ -74,7 +74,7 @@ void AkonotesIndexer::process(const KMime::Message::Ptr &msg)
// (Give the subject a higher priority)
KMime::Headers::Subject *subject = msg->subject(false);
if (subject) {
const std::string str(subject->asUnicodeString().toStdString());
const std::string str(normalizeString(subject->asUnicodeString()).toStdString());
qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexing" << str.c_str();
m_termGen->index_text_without_positions(str, 1, "SU");
m_termGen->index_text_without_positions(str, 100);
......@@ -83,7 +83,7 @@ void AkonotesIndexer::process(const KMime::Message::Ptr &msg)
KMime::Content *mainBody = msg->mainBodyPart("text/plain");
if (mainBody) {
const std::string text(mainBody->decodedText().toStdString());
const std::string text(normalizeString(mainBody->decodedText()).toStdString());
m_termGen->index_text_without_positions(text);
m_termGen->index_text_without_positions(text, 1, "BO");
} else {
......@@ -115,7 +115,7 @@ void AkonotesIndexer::processPart(KMime::Content *content, KMime::Content *mainC
QTextDocument doc;
doc.setHtml(content->decodedText());
const std::string text(doc.toPlainText().toStdString());
const std::string text(normalizeString(doc.toPlainText()).toStdString());
m_termGen->index_text_without_positions(text);
}
}
......
......@@ -14,6 +14,7 @@ set(indexer_SRCS
../../search/email/agepostingsource.cpp
../../search/contact/contactsearchstore.cpp
../akonadi_indexer_agent_debug.cpp
../stringutil.cpp
)
set(indexer_LIBS
......
......@@ -124,8 +124,8 @@ void CalendarIndexer::indexEventItem(const Akonadi::Item &item, const KCalendarC
Akonadi::Search::XapianDocument doc;
doc.indexText(event->organizer().email(), QStringLiteral("O"));
doc.indexText(event->summary(), QStringLiteral("S"));
doc.indexText(event->location(), QStringLiteral("L"));
doc.indexText(normalizeString(event->summary()), QStringLiteral("S"));
doc.indexText(normalizeString(event->location()), QStringLiteral("L"));
const KCalendarCore::Attendee::List attendees = event->attendees();
KCalendarCore::Attendee::List::ConstIterator it;
KCalendarCore::Attendee::List::ConstIterator end(attendees.constEnd());
......
......@@ -8,6 +8,7 @@
#include "emailindexer.h"
#include "akonadi_indexer_agent_debug.h"
#include "stringutil.h"
#include <Akonadi/Collection>
#include <Akonadi/MessageFlags>
......@@ -182,7 +183,7 @@ void EmailIndexer::process(const KMime::Message::Ptr &msg)
// (Give the subject a higher priority)
KMime::Headers::Subject *subject = msg->subject(false);
if (subject) {
const std::string str(subject->asUnicodeString().toStdString());
const std::string str{normalizeString(subject->asUnicodeString()).toStdString()};
qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexing" << str.c_str();
m_termGen->index_text_without_positions(str, 1, "SU");
m_termGen->index_text_without_positions(str, 100);
......@@ -218,7 +219,7 @@ void EmailIndexer::process(const KMime::Message::Ptr &msg)
KMime::Content *mainBody = msg->mainBodyPart("text/plain");
if (mainBody) {
const std::string text(mainBody->decodedText().toStdString());
const std::string text(normalizeString(mainBody->decodedText()).toStdString());
m_termGen->index_text_without_positions(text);
m_termGen->index_text_without_positions(text, 1, "BO");
} else {
......@@ -249,7 +250,7 @@ void EmailIndexer::processPart(KMime::Content *content, KMime::Content *mainCont
QTextDocument doc;
doc.setHtml(content->decodedText());
const std::string text(doc.toPlainText().toStdString());
const std::string text(normalizeString(doc.toPlainText()).toStdString());
m_termGen->index_text_without_positions(text);
}
}
......
......@@ -229,6 +229,7 @@ bool Index::createIndexers()
QDir().mkpath(m_indexedItems->emailIndexingPath());
QDir().mkpath(m_indexedItems->emailContactsIndexingPath());
indexer = new EmailIndexer(m_indexedItems->emailIndexingPath(), m_indexedItems->emailContactsIndexingPath());
indexer->setRespectDiacriticAndAccents(mRespectDiacriticAndAccents);
addIndexer(indexer);
} catch (const Xapian::DatabaseError &e) {
delete indexer;
......@@ -241,6 +242,7 @@ bool Index::createIndexers()
try {
QDir().mkpath(m_indexedItems->contactIndexingPath());
indexer = new ContactIndexer(m_indexedItems->contactIndexingPath());
indexer->setRespectDiacriticAndAccents(mRespectDiacriticAndAccents);
addIndexer(indexer);
} catch (const Xapian::DatabaseError &e) {
delete indexer;
......@@ -253,6 +255,7 @@ bool Index::createIndexers()
try {
QDir().mkpath(m_indexedItems->akonotesIndexingPath());
indexer = new AkonotesIndexer(m_indexedItems->akonotesIndexingPath());
indexer->setRespectDiacriticAndAccents(mRespectDiacriticAndAccents);
addIndexer(indexer);
} catch (const Xapian::DatabaseError &e) {
delete indexer;
......@@ -265,6 +268,7 @@ bool Index::createIndexers()
try {
QDir().mkpath(m_indexedItems->calendarIndexingPath());
indexer = new CalendarIndexer(m_indexedItems->calendarIndexingPath());
indexer->setRespectDiacriticAndAccents(mRespectDiacriticAndAccents);
addIndexer(indexer);
} catch (const Xapian::DatabaseError &e) {
delete indexer;
......@@ -323,3 +327,8 @@ void Index::setOverrideDbPrefixPath(const QString &path)
{
m_indexedItems->setOverrideDbPrefixPath(path);
}
void Index::setRespectDiacriticAndAccents(bool b)
{
mRespectDiacriticAndAccents = b;
}
......@@ -55,6 +55,7 @@ public:
/// For testing
void setOverrideDbPrefixPath(const QString &path);
void setRespectDiacriticAndAccents(bool b);
public Q_SLOTS:
virtual void commit();
......@@ -68,5 +69,6 @@ private:
Akonadi::Search::PIM::IndexedItems *m_indexedItems = nullptr;
QTimer m_commitTimer;
CollectionIndexer *m_collectionIndexer = nullptr;
bool mRespectDiacriticAndAccents = true;
};
......@@ -2,7 +2,7 @@ include_directories(
${CMAKE_CURRENT_SOURCE_DIR}/..
)
add_executable(emailindexer emailtest.cpp ../emailindexer.cpp ../abstractindexer.cpp ../akonadi_indexer_agent_debug.cpp)
add_executable(emailindexer emailtest.cpp ../emailindexer.cpp ../abstractindexer.cpp ../akonadi_indexer_agent_debug.cpp ../stringutil.cpp)
target_link_libraries(emailindexer
Qt${QT_MAJOR_VERSION}::Test
......
......@@ -21,6 +21,7 @@ target_sources(searchplugintest PRIVATE
../../search/contact/contactsearchstore.cpp
../../search/calendar/calendarsearchstore.cpp
../../search/note/notesearchstore.cpp
../../agent/stringutil.cpp
${CMAKE_CURRENT_BINARY_DIR}/../../agent/akonadi_indexer_agent_debug.cpp
${CMAKE_CURRENT_BINARY_DIR}/../akonadiplugin_indexer_debug.cpp
)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment