Start to implement RespectDiacriticAndAccents or not

(cherry picked from commit 924a4332)

agent/abstractindexer.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "abstractindexer.h"
+#include "stringutil.h"
 
 AbstractIndexer::AbstractIndexer() = default;
 
@@ -24,3 +25,22 @@ void AbstractIndexer::updateFlags(const Akonadi::Item &item, const QSet<QByteArr
     Q_UNUSED(addedFlags)
     Q_UNUSED(removed)
 }
+
+bool AbstractIndexer::respectDiacriticAndAccents() const
+{
+    return mRespectDiacriticAndAccents;
+}
+
+void AbstractIndexer::setRespectDiacriticAndAccents(bool newRespectDiacriticAndAccents)
+{
+    mRespectDiacriticAndAccents = newRespectDiacriticAndAccents;
+}
+
+QString AbstractIndexer::normalizeString(const QString &str)
+{
+    if (mRespectDiacriticAndAccents) {
+        return str;
+    } else {
+        return StringUtil::normalize(str);
+    }
+}

agent/abstractindexer.h

@@ -29,5 +29,12 @@ public:
 
     virtual void move(Akonadi::Item::Id item, Akonadi::Collection::Id from, Akonadi::Collection::Id to);
     virtual void updateFlags(const Akonadi::Item &item, const QSet<QByteArray> &addedFlags, const QSet<QByteArray> &removed);
+
+    Q_REQUIRED_RESULT bool respectDiacriticAndAccents() const;
+    void setRespectDiacriticAndAccents(bool newRespectDiacriticAndAccents);
+
+protected:
+    bool mRespectDiacriticAndAccents = true;
+    Q_REQUIRED_RESULT QString normalizeString(const QString &str);
 };
 

agent/agent.cpp

@@ -47,14 +47,16 @@ AkonadiIndexingAgent::AkonadiIndexingAgent(const QString &id)
 
     KConfigGroup cfg = config()->group("General");
     const int agentIndexingVersion = cfg.readEntry("agentIndexingVersion", 0);
-
+    bool respectDiacriticAndAccents = cfg.readEntry("respectDiacriticAndAccents", true);
     if (agentIndexingVersion < INDEXING_AGENT_VERSION) {
         m_index.removeDatabase();
+        // Don't respect Diacritic and Accents in new Database so search will be more easy.
+        // TODO activate it in the future respectDiacriticAndAccents = false;
         QTimer::singleShot(0, &m_scheduler, &Scheduler::scheduleCompleteSync);
         cfg.writeEntry("agentIndexingVersion", INDEXING_AGENT_VERSION);
         cfg.sync();
     }
-
+    m_index.setRespectDiacriticAndAccents(respectDiacriticAndAccents);
     if (!m_index.createIndexers()) {
         Q_EMIT status(Broken, i18nc("@info:status", "No indexers available"));
         setOnline(false);

agent/akonotesindexer.cpp

@@ -74,7 +74,7 @@ void AkonotesIndexer::process(const KMime::Message::Ptr &msg)
     // (Give the subject a higher priority)
     KMime::Headers::Subject *subject = msg->subject(false);
     if (subject) {
-        const std::string str(subject->asUnicodeString().toStdString());
+        const std::string str(normalizeString(subject->asUnicodeString()).toStdString());
         qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexing" << str.c_str();
         m_termGen->index_text_without_positions(str, 1, "SU");
         m_termGen->index_text_without_positions(str, 100);
@@ -83,7 +83,7 @@ void AkonotesIndexer::process(const KMime::Message::Ptr &msg)
 
     KMime::Content *mainBody = msg->mainBodyPart("text/plain");
     if (mainBody) {
-        const std::string text(mainBody->decodedText().toStdString());
+        const std::string text(normalizeString(mainBody->decodedText()).toStdString());
         m_termGen->index_text_without_positions(text);
         m_termGen->index_text_without_positions(text, 1, "BO");
     } else {
@@ -115,7 +115,7 @@ void AkonotesIndexer::processPart(KMime::Content *content, KMime::Content *mainC
             QTextDocument doc;
             doc.setHtml(content->decodedText());
 
-            const std::string text(doc.toPlainText().toStdString());
+            const std::string text(normalizeString(doc.toPlainText()).toStdString());
             m_termGen->index_text_without_positions(text);
         }
     }

agent/autotests/CMakeLists.txt

@@ -14,6 +14,7 @@ set(indexer_SRCS
     ../../search/email/agepostingsource.cpp
     ../../search/contact/contactsearchstore.cpp
     ../akonadi_indexer_agent_debug.cpp
+    ../stringutil.cpp
 )
 
 set(indexer_LIBS

agent/calendarindexer.cpp

@@ -124,8 +124,8 @@ void CalendarIndexer::indexEventItem(const Akonadi::Item &item, const KCalendarC
     Akonadi::Search::XapianDocument doc;
 
     doc.indexText(event->organizer().email(), QStringLiteral("O"));
-    doc.indexText(event->summary(), QStringLiteral("S"));
-    doc.indexText(event->location(), QStringLiteral("L"));
+    doc.indexText(normalizeString(event->summary()), QStringLiteral("S"));
+    doc.indexText(normalizeString(event->location()), QStringLiteral("L"));
     const KCalendarCore::Attendee::List attendees = event->attendees();
     KCalendarCore::Attendee::List::ConstIterator it;
     KCalendarCore::Attendee::List::ConstIterator end(attendees.constEnd());

agent/emailindexer.cpp

@@ -8,6 +8,7 @@
 
 #include "emailindexer.h"
 #include "akonadi_indexer_agent_debug.h"
+#include "stringutil.h"
 #include <Akonadi/Collection>
 #include <Akonadi/MessageFlags>
 
@@ -182,7 +183,7 @@ void EmailIndexer::process(const KMime::Message::Ptr &msg)
     // (Give the subject a higher priority)
     KMime::Headers::Subject *subject = msg->subject(false);
     if (subject) {
-        const std::string str(subject->asUnicodeString().toStdString());
+        const std::string str{normalizeString(subject->asUnicodeString()).toStdString()};
         qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexing" << str.c_str();
         m_termGen->index_text_without_positions(str, 1, "SU");
         m_termGen->index_text_without_positions(str, 100);
@@ -218,7 +219,7 @@ void EmailIndexer::process(const KMime::Message::Ptr &msg)
 
     KMime::Content *mainBody = msg->mainBodyPart("text/plain");
     if (mainBody) {
-        const std::string text(mainBody->decodedText().toStdString());
+        const std::string text(normalizeString(mainBody->decodedText()).toStdString());
         m_termGen->index_text_without_positions(text);
         m_termGen->index_text_without_positions(text, 1, "BO");
     } else {
@@ -249,7 +250,7 @@ void EmailIndexer::processPart(KMime::Content *content, KMime::Content *mainCont
             QTextDocument doc;
             doc.setHtml(content->decodedText());
 
-            const std::string text(doc.toPlainText().toStdString());
+            const std::string text(normalizeString(doc.toPlainText()).toStdString());
             m_termGen->index_text_without_positions(text);
         }
     }

agent/index.cpp

@@ -229,6 +229,7 @@ bool Index::createIndexers()
         QDir().mkpath(m_indexedItems->emailIndexingPath());
         QDir().mkpath(m_indexedItems->emailContactsIndexingPath());
         indexer = new EmailIndexer(m_indexedItems->emailIndexingPath(), m_indexedItems->emailContactsIndexingPath());
+        indexer->setRespectDiacriticAndAccents(mRespectDiacriticAndAccents);
         addIndexer(indexer);
     } catch (const Xapian::DatabaseError &e) {
         delete indexer;
@@ -241,6 +242,7 @@ bool Index::createIndexers()
     try {
         QDir().mkpath(m_indexedItems->contactIndexingPath());
         indexer = new ContactIndexer(m_indexedItems->contactIndexingPath());
+        indexer->setRespectDiacriticAndAccents(mRespectDiacriticAndAccents);
         addIndexer(indexer);
     } catch (const Xapian::DatabaseError &e) {
         delete indexer;
@@ -253,6 +255,7 @@ bool Index::createIndexers()
     try {
         QDir().mkpath(m_indexedItems->akonotesIndexingPath());
         indexer = new AkonotesIndexer(m_indexedItems->akonotesIndexingPath());
+        indexer->setRespectDiacriticAndAccents(mRespectDiacriticAndAccents);
         addIndexer(indexer);
     } catch (const Xapian::DatabaseError &e) {
         delete indexer;
@@ -265,6 +268,7 @@ bool Index::createIndexers()
     try {
         QDir().mkpath(m_indexedItems->calendarIndexingPath());
         indexer = new CalendarIndexer(m_indexedItems->calendarIndexingPath());
+        indexer->setRespectDiacriticAndAccents(mRespectDiacriticAndAccents);
         addIndexer(indexer);
     } catch (const Xapian::DatabaseError &e) {
         delete indexer;
@@ -323,3 +327,8 @@ void Index::setOverrideDbPrefixPath(const QString &path)
 {
     m_indexedItems->setOverrideDbPrefixPath(path);
 }
+
+void Index::setRespectDiacriticAndAccents(bool b)
+{
+    mRespectDiacriticAndAccents = b;
+}

agent/index.h

@@ -55,6 +55,7 @@ public:
     /// For testing
     void setOverrideDbPrefixPath(const QString &path);
 
+    void setRespectDiacriticAndAccents(bool b);
 public Q_SLOTS:
     virtual void commit();
 
@@ -68,5 +69,6 @@ private:
     Akonadi::Search::PIM::IndexedItems *m_indexedItems = nullptr;
     QTimer m_commitTimer;
     CollectionIndexer *m_collectionIndexer = nullptr;
+    bool mRespectDiacriticAndAccents = true;
 };
 

agent/tests/CMakeLists.txt

@@ -2,7 +2,7 @@ include_directories(
   ${CMAKE_CURRENT_SOURCE_DIR}/..
 )
 
-add_executable(emailindexer emailtest.cpp ../emailindexer.cpp ../abstractindexer.cpp ../akonadi_indexer_agent_debug.cpp)
+add_executable(emailindexer emailtest.cpp ../emailindexer.cpp ../abstractindexer.cpp ../akonadi_indexer_agent_debug.cpp ../stringutil.cpp)
 
 target_link_libraries(emailindexer
   Qt${QT_MAJOR_VERSION}::Test

akonadiplugin/autotests/CMakeLists.txt

@@ -21,6 +21,7 @@ target_sources(searchplugintest PRIVATE
     ../../search/contact/contactsearchstore.cpp
     ../../search/calendar/calendarsearchstore.cpp
     ../../search/note/notesearchstore.cpp
+    ../../agent/stringutil.cpp
     ${CMAKE_CURRENT_BINARY_DIR}/../../agent/akonadi_indexer_agent_debug.cpp
     ${CMAKE_CURRENT_BINARY_DIR}/../akonadiplugin_indexer_debug.cpp
 )