Implement server-side batch fetching for entries

Yet to fix implementation wrt Akonadi
  - Use item streaming to supply batches to Akonadi

resources/etesync/basehandler.cpp

@@ -37,10 +37,10 @@ void BaseHandler::initialiseBaseDirectory()
     mResource->initialiseDirectory(baseDirectoryPath());
 }
 
-void BaseHandler::setupItems(EteSyncEntry **entries, Akonadi::Collection &collection)
+void BaseHandler::setupItems(std::vector<EteSyncEntryPtr> &entries, Akonadi::Collection &collection, QString &prevUid)
 {
     qCDebug(ETESYNC_LOG) << "BaseHandler: Setting up items";
-    QString prevUid = collection.remoteRevision();
+
     const QString journalUid = collection.remoteId();
 
     const bool isIncremental = (prevUid.isEmpty() || prevUid.isNull()) ? false : true;
@@ -99,7 +99,7 @@ void BaseHandler::slotItemsRetrieved(KJob *job)
         return;
     }
 
-    EteSyncEntry **entries = qobject_cast<EntriesFetchJob *>(job)->entries();
+    std::vector<EteSyncEntryPtr> entries = qobject_cast<EntriesFetchJob *>(job)->getEntries();
 
     Collection collection = qobject_cast<EntriesFetchJob *>(job)->collection();
 

resources/etesync/basehandler.h

@@ -38,7 +38,7 @@ public:
 
     virtual const QString mimeType() = 0;
 
-    virtual void setupItems(EteSyncEntry **entries, Akonadi::Collection &collection);
+    virtual void setupItems(std::vector<EteSyncEntryPtr> &entries, Akonadi::Collection &collection, QString &prevUid);
 
     virtual void itemAdded(const Akonadi::Item &item, const Akonadi::Collection &collection) = 0;
     virtual void itemChanged(const Akonadi::Item &item, const QSet<QByteArray> &parts) = 0;
@@ -61,7 +61,7 @@ protected:
 
     virtual QString baseDirectoryPath() const = 0;
     virtual const QString etesyncCollectionType() = 0;
-    virtual void getItemListFromEntries(EteSyncEntry **entries, Item::List &changedItems, Item::List &removedItems, Collection &collection, const QString &journalUid, QString &prevUid) = 0;
+    virtual void getItemListFromEntries(std::vector<EteSyncEntryPtr> &entries, Item::List &changedItems, Item::List &removedItems, Collection &collection, const QString &journalUid, QString &prevUid) = 0;
 
     bool handleConflictError(const Collection &collection);
 

resources/etesync/calendartaskbasehandler.cpp

@@ -36,15 +36,14 @@ CalendarTaskBaseHandler::CalendarTaskBaseHandler(EteSyncResource *resource) : Ba
     initialiseBaseDirectory();
 }
 
-void CalendarTaskBaseHandler::getItemListFromEntries(EteSyncEntry **entries, Item::List &changedItems, Item::List &removedItems, Collection &collection, const QString &journalUid, QString &prevUid)
+void CalendarTaskBaseHandler::getItemListFromEntries(std::vector<EteSyncEntryPtr> &entries, Item::List &changedItems, Item::List &removedItems, Collection &collection, const QString &journalUid, QString &prevUid)
 {
     const EteSyncJournalPtr &journal = mResource->getJournal(journalUid);
     EteSyncCryptoManagerPtr cryptoManager = etesync_journal_get_crypto_manager(journal.get(), mClientState->derived(), mClientState->keypair());
 
     QMap<QString, KCalendarCore::Incidence::Ptr> incidences;
 
-    for (EteSyncEntry **iter = entries; *iter; iter++) {
-        EteSyncEntryPtr entry(*iter);
+    for (auto &entry : entries) {
         EteSyncSyncEntryPtr syncEntry = etesync_entry_get_sync_entry(entry.get(), cryptoManager.get(), prevUid);
 
         CharPtr contentStr(etesync_sync_entry_get_content(syncEntry.get()));
@@ -72,8 +71,6 @@ void CalendarTaskBaseHandler::getItemListFromEntries(EteSyncEntry **entries, Ite
         prevUid = QStringFromCharPtr(CharPtr(etesync_entry_get_uid(entry.get())));
     }
 
-    free(entries);
-
     for (auto it = incidences.constBegin(); it != incidences.constEnd(); it++) {
         Item item;
         item.setMimeType(mimeType());

resources/etesync/calendartaskbasehandler.h

@@ -34,7 +34,7 @@ class CalendarTaskBaseHandler : public BaseHandler
 public:
     explicit CalendarTaskBaseHandler(EteSyncResource *resource);
 
-    void getItemListFromEntries(EteSyncEntry **entries, Item::List &changedItems, Item::List &removedItems, Collection &collection, const QString &journalUid, QString &prevUid) override;
+    void getItemListFromEntries(std::vector<EteSyncEntryPtr> &entries, Item::List &changedItems, Item::List &removedItems, Collection &collection, const QString &journalUid, QString &prevUid) override;
 
     void itemAdded(const Akonadi::Item &item, const Akonadi::Collection &collection) override;
     void itemChanged(const Akonadi::Item &item, const QSet<QByteArray> &parts) override;

resources/etesync/contacthandler.cpp

@@ -43,7 +43,7 @@ const QString ContactHandler::etesyncCollectionType()
     return QStringLiteral(ETESYNC_COLLECTION_TYPE_ADDRESS_BOOK);
 }
 
-void ContactHandler::getItemListFromEntries(EteSyncEntry **entries, Item::List &changedItems, Item::List &removedItems, Collection &collection, const QString &journalUid, QString &prevUid)
+void ContactHandler::getItemListFromEntries(std::vector<EteSyncEntryPtr> &entries, Item::List &changedItems, Item::List &removedItems, Collection &collection, const QString &journalUid, QString &prevUid)
 {
     const EteSyncJournalPtr &journal = mResource->getJournal(journalUid);
 
@@ -51,8 +51,7 @@ void ContactHandler::getItemListFromEntries(EteSyncEntry **entries, Item::List &
 
     QMap<QString, KContacts::Addressee> contacts;
 
-    for (EteSyncEntry **iter = entries; *iter; iter++) {
-        EteSyncEntryPtr entry(*iter);
+    for (auto &entry : entries) {
         EteSyncSyncEntryPtr syncEntry = etesync_entry_get_sync_entry(entry.get(), cryptoManager.get(), prevUid);
 
         KContacts::VCardConverter converter;
@@ -80,8 +79,6 @@ void ContactHandler::getItemListFromEntries(EteSyncEntry **entries, Item::List &
         prevUid = QStringFromCharPtr(CharPtr(etesync_entry_get_uid(entry.get())));
     }
 
-    free(entries);
-
     for (auto it = contacts.constBegin(); it != contacts.constEnd(); it++) {
         Item item;
         item.setMimeType(mimeType());

resources/etesync/contacthandler.h

@@ -37,7 +37,7 @@ public:
 
     const QString mimeType() override;
 
-    void getItemListFromEntries(EteSyncEntry **entries, Item::List &changedItems, Item::List &removedItems, Collection &collection, const QString &journalUid, QString &prevUid) override;
+    void getItemListFromEntries(std::vector<EteSyncEntryPtr> &entries, Item::List &changedItems, Item::List &removedItems, Collection &collection, const QString &journalUid, QString &prevUid) override;
 
     void itemAdded(const Akonadi::Item &item, const Akonadi::Collection &collection) override;
     void itemChanged(const Akonadi::Item &item, const QSet<QByteArray> &parts) override;

resources/etesync/entriesfetchjob.cpp

@@ -21,6 +21,7 @@
 
 #include <QTimer>
 
+#include "etesync_debug.h"
 #include "settings.h"
 
 using namespace EteSyncAPI;
@@ -38,15 +39,29 @@ void EntriesFetchJob::start()
 void EntriesFetchJob::fetchEntries()
 {
     const QString journalUid = mCollection.remoteId();
-    const QString prevUid = mCollection.remoteRevision();
-    EteSyncEntryManagerPtr entryManager(etesync_entry_manager_new(mClient, journalUid));
-    mEntries = etesync_entry_manager_list(entryManager.get(), prevUid, 0);
+    mPrevUid = mLastUid = mCollection.remoteRevision();
+    mEntryManager = etesync_entry_manager_new(mClient, journalUid);
 
-    if (!mEntries) {
+    while (fetchNextBatch()) {
+    }
+
+    if (etesync_get_error_code() != EteSyncErrorCode::ETESYNC_ERROR_CODE_NO_ERROR) {
         setError(UserDefinedError);
         CharPtr err(etesync_get_error_message());
         setErrorText(QStringFromCharPtr(err));
+        emitResult();
+        return;
     }
-
     emitResult();
 }
+
+bool EntriesFetchJob::fetchNextBatch()
+{
+    std::vector<EteSyncEntryPtr> entries = etesync_entry_manager_list(mEntryManager.get(), mLastUid, 50);
+    if (entries.empty()) {
+        return false;
+    }
+    mEntries.insert(mEntries.end(), std::make_move_iterator(entries.begin()), std::make_move_iterator(entries.end()));
+    mLastUid = QStringFromCharPtr(CharPtr(etesync_entry_get_uid(mEntries[mEntries.size() - 1].get())));
+    return true;
+}

resources/etesync/entriesfetchjob.h

@@ -33,9 +33,14 @@ namespace EteSyncAPI {
 
         void start() override;
 
-        EteSyncEntry **entries() const
+        std::vector<EteSyncEntryPtr> getEntries()
         {
-            return mEntries;
+            return std::move(mEntries);
+        }
+
+        QString getPrevUid()
+        {
+            return mPrevUid;
         }
 
         Akonadi::Collection collection() const
@@ -45,10 +50,14 @@ namespace EteSyncAPI {
 
     protected:
         void fetchEntries();
+        bool fetchNextBatch();
 
     private:
         const EteSync *mClient = nullptr;
-        EteSyncEntry **mEntries = nullptr;
+        QString mPrevUid;
+        QString mLastUid;
+        EteSyncEntryManagerPtr mEntryManager;
+        std::vector<EteSyncEntryPtr> mEntries;
         Akonadi::Collection mCollection;
     };
 }  // namespace EteSyncAPI

resources/etesync/etesyncadapter.cpp

@@ -77,10 +77,16 @@ qint32 etesync_entry_manager_create(const EteSyncEntryManager *entry_manager,
     return etesync_entry_manager_create(entry_manager, entries, charArrFromQString(prev_uid));
 }
 
-EteSyncEntry **etesync_entry_manager_list(const EteSyncEntryManager *entry_manager,
-                                          const QString &prev_uid, uintptr_t limit)
-{
-    return etesync_entry_manager_list(entry_manager, charArrFromQString(prev_uid), limit);
+std::vector<EteSyncEntryPtr> etesync_entry_manager_list(const EteSyncEntryManager *entry_manager,
+                                                        const QString &prev_uid, uintptr_t limit)
+{
+    auto entries = etesync_entry_manager_list(entry_manager, charArrFromQString(prev_uid), limit);
+    std::vector<EteSyncEntryPtr> rv;
+    while (*entries) {
+        rv.emplace_back(EteSyncEntryPtr{*entries});
+        ++entries;
+    }
+    return rv;
 }
 
 EteSyncEntryManagerPtr etesync_entry_manager_new(const EteSync *etesync,

resources/etesync/etesyncadapter.h

@@ -22,6 +22,7 @@
 
 #include <QString>
 #include <memory>
+#include <vector>
 
 #define charArrFromQString(str) str.isNull() ? NULL : qUtf8Printable(str)
 
@@ -118,8 +119,8 @@ qint32 etesync_entry_manager_create(const EteSyncEntryManager *entry_manager,
                                     const EteSyncEntry *const *entries,
                                     const QString &prev_uid);
 
-EteSyncEntry **etesync_entry_manager_list(const EteSyncEntryManager *entry_manager,
-                                          const QString &prev_uid, uintptr_t limit);
+std::vector<EteSyncEntryPtr> etesync_entry_manager_list(const EteSyncEntryManager *entry_manager,
+                                                        const QString &prev_uid, uintptr_t limit);
 
 EteSyncEntryManagerPtr etesync_entry_manager_new(const EteSync *etesync,
                                                  const QString &journal_uid);

resources/etesync/etesyncresource.cpp

@@ -256,7 +256,7 @@ void EteSyncResource::retrieveItems(const Akonadi::Collection &collection)
     if (timeSinceLastCacheUpdate <= 30) {
         const QString journalUid = collection.remoteId();
         const EteSyncJournalPtr &journal = getJournal(journalUid);
-        QString lastEntryUid = QStringFromCharPtr(CharPtr(etesync_journal_get_last_uid(journal.get())));
+        const QString lastEntryUid = QStringFromCharPtr(CharPtr(etesync_journal_get_last_uid(journal.get())));
         if (lastEntryUid == collection.remoteRevision()) {
             itemsRetrievalDone();
             return;
@@ -280,14 +280,16 @@ void EteSyncResource::slotItemsRetrieved(KJob *job)
     }
 
     qCDebug(ETESYNC_LOG) << "Retrieving entries";
-    EteSyncEntry **entries = qobject_cast<EntriesFetchJob *>(job)->entries();
+    auto entries = qobject_cast<EntriesFetchJob *>(job)->getEntries();
 
     Collection collection = qobject_cast<EntriesFetchJob *>(job)->collection();
 
+    QString prevUid = qobject_cast<EntriesFetchJob *>(job)->getPrevUid();
+
     auto handler = fetchHandlerForCollection(collection);
 
     if (handler) {
-        handler->setupItems(entries, collection);
+        handler->setupItems(entries, collection, prevUid);
     } else {
         qCWarning(ETESYNC_LOG) << "Unknown collection" << collection.name();
         itemsRetrieved({});