Commit 7802a783 authored by Christian Mollekopf's avatar Christian Mollekopf
Browse files

IMAP-Resource: Check available UID's first & retrieve 50 time more flags.

This way we avoid the UID fragmentation problem (we try to fetch a lot of
messages that are not available on the server).
It's also a nice first step towards only syncing a certain time-frame.

Retrieving 50 times more flags than full messages should still result in
low memory consumption while greatly reducing roundtrips.

BUG: 334280
parent 81f30e18
......@@ -44,6 +44,7 @@
#include <kimap/fetchjob.h>
#include <kimap/selectjob.h>
#include <kimap/session.h>
#include <kimap/searchjob.h>
/**
* A job that retrieves a set of messages in reverse-ordered batches.
......@@ -57,6 +58,7 @@ public:
virtual void start();
void fetchNextBatch();
void setUidBased(bool);
void setSearchTerm(const KIMAP::Term &);
Q_SIGNALS:
void itemsRetrieved(Akonadi::Item::List);
......@@ -67,6 +69,7 @@ private Q_SLOTS:
const QMap<qint64, KIMAP::MessageFlags> &flags,
const QMap<qint64, KIMAP::MessagePtr> &messages);
void onHeadersFetchDone(KJob *job);
void onUidSearchDone(KJob* job);
private:
//Batch fetching
......@@ -79,6 +82,7 @@ private:
const MessageHelper::Ptr m_messageHelper;
bool m_fetchInProgress;
bool m_continuationRequested;
KIMAP::Term m_searchTerm;
};
BatchFetcher::BatchFetcher(MessageHelper::Ptr messageHelper, const KIMAP::ImapSet &set, const KIMAP::FetchJob::FetchScope& scope, int batchSize, KIMAP::Session* session)
......@@ -104,15 +108,45 @@ void BatchFetcher::setUidBased(bool uidBased)
m_uidBased = uidBased;
}
void BatchFetcher::setSearchTerm(const KIMAP::Term &searchTerm)
{
m_searchTerm = searchTerm;
}
void BatchFetcher::start()
{
if (!m_searchTerm.isNull()) {
//Resolve the uid to sequence numbers
KIMAP::SearchJob *search = new KIMAP::SearchJob(m_session);
search->setUidBased(true);
search->setTerm(m_searchTerm);
connect(search, SIGNAL(result(KJob*)), this, SLOT(onUidSearchDone(KJob*)));
search->start();
} else {
fetchNextBatch();
}
}
void BatchFetcher::onUidSearchDone(KJob* job)
{
if (job->error()) {
kWarning() << "Search job failed: " << job->errorString();
setError(KJob::UserDefinedError);
emitResult();
return;
}
KIMAP::SearchJob *search = static_cast<KIMAP::SearchJob*>(job);
m_uidBased = search->isUidBased();
KIMAP::ImapSet set;
set.add(search->results());
m_currentSet = set;
fetchNextBatch();
}
void BatchFetcher::fetchNextBatch()
{
if (m_fetchInProgress) {
kWarning() << "fetchNextBatch called while fetch is in process";
m_continuationRequested = true;
return;
}
......@@ -125,38 +159,34 @@ void BatchFetcher::fetchNextBatch()
}
KIMAP::FetchJob *fetch = new KIMAP::FetchJob(m_session);
//In the most common case that we want optimized we use batch processing.
if (m_scope.changedSince == 0 && m_currentSet.intervals().size() == 1 && m_currentSet.intervals().first().hasDefinedEnd()) {
const KIMAP::ImapInterval interval = m_currentSet.intervals().first();
Q_ASSERT(interval.hasDefinedEnd());
//Reverse fetching would be great, because it gives you the most relevant (recent) messages first,
//but since we usually just check the latest uid, we wouldn't detect if an interval
//at the beginning is missing. Therefore this is disabled for now.
//get an interval of m_batchSize
// const qint64 end = interval.end();
// const qint64 begin = qMax(interval.begin(), end - m_batchSize + 1);
// const KIMAP::ImapSet intervalToFetch(begin, end);
// //fetch items in reverse order in chunks
// if (interval.begin() < (begin - 1)) {
// m_currentSet = KIMAP::ImapSet(interval.begin(), begin - 1);
// } else {
// m_currentSet = KIMAP::ImapSet();
// }
const qint64 begin = interval.begin();
const qint64 end = qMin(interval.end(), begin + m_batchSize - 1);
const KIMAP::ImapSet intervalToFetch(begin, end);
if (interval.end() > end) {
m_currentSet = KIMAP::ImapSet(end + 1, interval.end());
} else {
m_currentSet = KIMAP::ImapSet();
}
kDebug(5327) << "Fetching " << begin << " to " << end;
fetch->setSequenceSet(intervalToFetch);
} else {
kDebug(5327) << "Fetching all messages in one go.";
if (m_scope.changedSince != 0) {
kDebug(5327) << "Fetching all messages in one batch.";
fetch->setSequenceSet(m_currentSet);
m_currentSet = KIMAP::ImapSet();
} else {
KIMAP::ImapSet toFetch;
qint64 counter = 0;
KIMAP::ImapSet newSet;
//Take a chunk from the set
Q_FOREACH (const KIMAP::ImapInterval &interval, m_currentSet.intervals()) {
const qint64 wantedItems = m_batchSize - counter;
if (counter < m_batchSize) {
if (interval.size() <= wantedItems) {
counter += interval.size();
toFetch.add(interval);
} else {
counter += wantedItems;
toFetch.add(KIMAP::ImapInterval(interval.begin(), interval.begin() + wantedItems - 1));
newSet.add(KIMAP::ImapInterval(interval.begin() + wantedItems, interval.end()));
}
} else {
newSet.add(interval);
}
}
kDebug(5327) << "Fetching " << toFetch.intervals().size() << " intervals";
fetch->setSequenceSet(toFetch);
m_currentSet = newSet;
}
fetch->setUidBased(m_uidBased);
......@@ -585,6 +615,9 @@ void RetrieveItemsTask::retrieveItems(const KIMAP::ImapSet& set, const KIMAP::Fe
m_batchFetcher = new BatchFetcher(resourceState()->messageHelper(), set, scope, batchSize(), m_session);
m_batchFetcher->setUidBased(m_uidBasedFetch);
if (m_uidBasedFetch && set.intervals().size() == 1) {
m_batchFetcher->setSearchTerm(KIMAP::Term(KIMAP::Term::Uid, set));
}
m_batchFetcher->setProperty("alreadyFetched", set.intervals().first().begin());
connect(m_batchFetcher, SIGNAL(itemsRetrieved(Akonadi::Item::List)),
this, SLOT(onItemsRetrieved(Akonadi::Item::List)));
......@@ -645,7 +678,6 @@ void RetrieveItemsTask::onRetrievalDone(KJob *job)
listFlagsForImapSet(KIMAP::ImapSet(1, alreadyFetchedBegin - 1));
}
void RetrieveItemsTask::listFlagsForImapSet(const KIMAP::ImapSet& set)
{
kDebug(5327) << "Listing flags " << set.intervals().first().begin() << set.intervals().first().end();
......@@ -665,8 +697,11 @@ void RetrieveItemsTask::listFlagsForImapSet(const KIMAP::ImapSet& set)
}
}
m_batchFetcher = new BatchFetcher(resourceState()->messageHelper(), set, scope, batchSize(), m_session);
m_batchFetcher = new BatchFetcher(resourceState()->messageHelper(), set, scope, 50 * batchSize(), m_session);
m_batchFetcher->setUidBased(m_uidBasedFetch);
if (m_uidBasedFetch && scope.changedSince == 0 && set.intervals().size() == 1) {
m_batchFetcher->setSearchTerm(KIMAP::Term(KIMAP::Term::Uid, set));
}
connect(m_batchFetcher, SIGNAL(itemsRetrieved(Akonadi::Item::List)),
this, SLOT(onItemsRetrieved(Akonadi::Item::List)));
connect(m_batchFetcher, SIGNAL(result(KJob*)),
......
......@@ -60,7 +60,10 @@ private slots:
<< "S: * OK [ UIDVALIDITY 1149151135 ]"
<< "S: * OK [ UIDNEXT 9 ]"
<< "S: A000005 OK select done"
<< "C: A000006 UID FETCH 1:9 (RFC822.SIZE INTERNALDATE "
<< "C: A000006 UID SEARCH UID 1:9"
<< "S: * SEARCH 1 2 3 4 5 6 7 8 9"
<< "S: A000006 OK search done"
<< "C: A000007 UID FETCH 1:9 (RFC822.SIZE INTERNALDATE "
"BODY.PEEK[HEADER] "
"FLAGS UID)"
<< "S: * 1 FETCH ( FLAGS (\\Seen) UID 7 INTERNALDATE \"29-Jun-2010 15:26:42 +0200\" "
......@@ -70,7 +73,7 @@ private slots:
"Subject: Test Mail\r\n"
"\r\n"
" )"
<< "S: A000006 OK fetch done";
<< "S: A000007 OK fetch done";
callNames.clear();
callNames << "itemsRetrieved" << "applyCollectionChanges" << "itemsRetrievalDone" ;
......@@ -101,7 +104,10 @@ private slots:
<< "S: * OK [ UIDVALIDITY 1149151135 ]"
<< "S: * OK [ UIDNEXT 9 ]"
<< "S: A000005 OK select done"
<< "C: A000006 UID FETCH 1:9 (RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID)"
<< "C: A000006 UID SEARCH UID 1:9"
<< "S: * SEARCH 1 2 3 4 5 6 7 8 9"
<< "S: A000006 OK search done"
<< "C: A000007 UID FETCH 1:9 (RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID)"
<< "S: * 1 FETCH ( FLAGS (\\Seen) UID 7 INTERNALDATE \"29-Jun-2010 15:26:42 +0200\" "
"RFC822.SIZE 75 BODY[] {75}\r\n"
"From: Foo <foo@kde.org>\r\n"
......@@ -110,7 +116,7 @@ private slots:
"\r\n"
"Test\r\n"
" )"
<< "S: A000006 OK fetch done";
<< "S: A000007 OK fetch done";
callNames.clear();
callNames << "itemsRetrieved" << "applyCollectionChanges" << "itemsRetrievalDone";
......@@ -141,9 +147,12 @@ private slots:
<< "S: * OK [ UIDVALIDITY 1149151135 ]"
<< "S: * OK [ UIDNEXT 9 ]"
<< "S: A000005 OK select done"
<< "C: A000006 UID FETCH 1:9 (FLAGS UID)"
<< "C: A000006 UID SEARCH UID 1:9"
<< "S: * SEARCH 1 2 3 4 5 6 7 8 9"
<< "S: A000006 OK search done"
<< "C: A000007 UID FETCH 1:9 (FLAGS UID)"
<< "S: * 1 FETCH ( FLAGS (\\Seen) UID 7 )"
<< "S: A000006 OK fetch done";
<< "S: A000007 OK fetch done";
callNames.clear();
callNames << "itemsRetrievedIncremental" << "applyCollectionChanges" << "itemsRetrievedIncremental" << "itemsRetrievalDone";
......@@ -193,7 +202,10 @@ private slots:
<< "S: * OK [ UIDNEXT 9 ]"
<< "S: * OK [ HIGHESTMODSEQ 123456789 ]"
<< "S: A000005 OK select done"
<< "C: A000006 UID FETCH 8:9 (RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID)"
<< "C: A000006 UID SEARCH UID 8:9"
<< "S: * SEARCH 8 9"
<< "S: A000006 OK search done"
<< "C: A000007 UID FETCH 8:9 (RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID)"
<< "S: * 4 FETCH ( FLAGS (\\Seen) UID 8 INTERNALDATE \"29-Jun-2010 15:26:42 +0200\" "
"RFC822.SIZE 75 BODY[] {75}\r\n"
"From: Foo <foo@kde.org>\r\n"
......@@ -210,12 +222,15 @@ private slots:
"\r\n"
"Test\r\n"
" )"
<< "S: A000006 OK fetch done"
<< "C: A000007 UID FETCH 1:7 (FLAGS UID)"
<< "S: A000007 OK fetch done"
<< "C: A000008 UID SEARCH UID 1:7"
<< "S: * SEARCH 1 2 3 4 5 6 7"
<< "S: A000008 OK search done"
<< "C: A000009 UID FETCH 1:7 (FLAGS UID)"
<< "S: * 1 FETCH"
<< "S: * 2 FETCH"
<< "S: * 3 FETCH"
<< "S: A000007 OK fetch done";
<< "S: A000009 OK fetch done";
callNames.clear();
callNames << "itemsRetrieved" << "applyCollectionChanges" << "itemsRetrievalDone";
......@@ -304,9 +319,12 @@ private slots:
<< "S: * OK [ UIDNEXT 9 ]"
<< "S: * OK [ HIGHESTMODSEQ 123456789 ]"
<< "S: A000005 OK select done"
<< "C: A000006 UID FETCH 1:9 (FLAGS UID)"
<< "C: A000006 UID SEARCH UID 1:9"
<< "S: * SEARCH 1 2 3 4 5 6 7 8 9"
<< "S: A000006 OK search done"
<< "C: A000007 UID FETCH 1:9 (FLAGS UID)"
<< "S: * 5 FETCH ( UID 8 FLAGS () )"
<< "S: A000006 OK fetch done";
<< "S: A000007 OK fetch done";
callNames.clear();
callNames << "itemsRetrievedIncremental" << "applyCollectionChanges" << "itemsRetrievedIncremental" << "itemsRetrievalDone";
......@@ -335,7 +353,10 @@ private slots:
<< "S: * OK [ UIDVALIDITY 1149151135 ]"
<< "S: * OK [ UIDNEXT 9 ]"
<< "S: A000005 OK select done"
<< "C: A000006 UID FETCH 1:9 (RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID)"
<< "C: A000006 UID SEARCH UID 1:9"
<< "S: * SEARCH 1 2 3 4 5 6 7 8 9"
<< "S: A000006 OK search done"
<< "C: A000007 UID FETCH 1:9 (RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID)"
<< "S: * 1 FETCH ( FLAGS (\\Seen) UID 2321 )"
<< "S: * 1 FETCH ( FLAGS (\\Seen) UID 2321 INTERNALDATE \"29-Jun-2010 15:26:42 +0200\" "
"RFC822.SIZE 75 BODY[] {75}\r\n"
......@@ -345,7 +366,7 @@ private slots:
"\r\n"
"Test\r\n"
" )"
<< "S: A000006 OK fetch done";
<< "S: A000007 OK fetch done";
callNames.clear();
callNames << "itemsRetrieved" << "applyCollectionChanges" << "itemsRetrievalDone";
......@@ -373,7 +394,11 @@ private slots:
<< "S: * OK [ UIDVALIDITY 1149151135 ]"
<< "S: * OK [ UIDNEXT 120 ]"
<< "S: A000005 OK select done"
<< "C: A000006 UID FETCH 105:114 (RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID)"
<< "C: A000006 UID SEARCH UID 105:120"
//We asked for until 120 but only 119 is available (120 is uidnext)
<< "S: * SEARCH 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119"
<< "S: A000006 OK search done"
<< "C: A000007 UID FETCH 105:114 (RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID)"
<< "S: * 1 FETCH ( FLAGS (\\Seen) UID 105 INTERNALDATE \"29-Jun-2010 15:26:42 +0200\" "
"RFC822.SIZE 75 BODY[] {75}\r\n"
"From: Foo <foo@kde.org>\r\n"
......@@ -383,8 +408,8 @@ private slots:
"Test\r\n"
" )"
//9 more would follow but are excluded for clarity
<< "S: A000006 OK fetch done"
<< "C: A000007 UID FETCH 115:120 (RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID)"
<< "S: A000007 OK fetch done"
<< "C: A000008 UID FETCH 115:119 (RFC822.SIZE INTERNALDATE BODY.PEEK[] FLAGS UID)"
<< "S: * 1 FETCH ( FLAGS (\\Seen) UID 115 INTERNALDATE \"29-Jun-2010 15:26:42 +0200\" "
"RFC822.SIZE 75 BODY[] {75}\r\n"
"From: Foo <foo@kde.org>\r\n"
......@@ -394,18 +419,17 @@ private slots:
"Test\r\n"
" )"
//4 more would follow but are excluded for clarity
<< "S: A000007 OK fetch done"
<< "C: A000008 UID FETCH 1:100 (FLAGS UID)"
<< "S: * 1 FETCH ( FLAGS (\\Seen) UID 1 )"
//99 more would follow but are excluded for clarity
<< "S: A000008 OK fetch done"
<< "C: A000009 UID FETCH 101:104 (FLAGS UID)"
<< "S: * 1 FETCH ( FLAGS (\\Seen) UID 101 )"
<< "C: A000009 UID SEARCH UID 1:104"
<< "S: * SEARCH 1 2 99 100"
<< "S: A000009 OK search done"
<< "C: A000010 UID FETCH 1:2,99:100 (FLAGS UID)"
<< "S: * 1 FETCH ( FLAGS (\\Seen) UID 1 )"
//3 more would follow but are excluded for clarity
<< "S: A000009 OK fetch done";
<< "S: A000010 OK fetch done";
callNames.clear();
callNames << "itemsRetrieved" << "itemsRetrieved" << "itemsRetrieved" << "itemsRetrieved" << "applyCollectionChanges" << "itemsRetrievalDone";
callNames << "itemsRetrieved" << "itemsRetrieved" << "itemsRetrieved" << "applyCollectionChanges" << "itemsRetrievalDone";
QTest::newRow( "test batch processing" ) << collection << scenario << callNames;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment