itemretriever.cpp 15.3 KB
Newer Older
1
2
/*
    Copyright (c) 2009 Volker Krause <vkrause@kde.org>
3
    Copyright (c) 2010 Milian Wolff <mail@milianw.de>
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

    This library is free software; you can redistribute it and/or modify it
    under the terms of the GNU Library General Public License as published by
    the Free Software Foundation; either version 2 of the License, or (at your
    option) any later version.

    This library is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
    License for more details.

    You should have received a copy of the GNU Library General Public License
    along with this library; see the file COPYING.LIB.  If not, write to the
    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
    02110-1301, USA.
*/

#include "itemretriever.h"

23
#include "akonadi.h"
24
#include "connection.h"
25
#include "storage/datastore.h"
26
#include "storage/itemqueryhelper.h"
27
#include "storage/itemretrievalmanager.h"
28
#include "storage/itemretrievalrequest.h"
29
#include "storage/parthelper.h"
30
#include "storage/parttypehelper.h"
31
#include "storage/querybuilder.h"
32
#include "storage/selectquerybuilder.h"
33
#include "utils.h"
34

35
#include <shared/akranges.h>
36
#include <private/protocol_p.h>
37

38
39
#include <QEventLoop>

Laurent Montel's avatar
Laurent Montel committed
40
#include "akonadiserver_debug.h"
41
42

using namespace Akonadi;
43
using namespace Akonadi::Server;
44
using namespace AkRanges;
45

46
47
Q_DECLARE_METATYPE(ItemRetrievalResult)

48
ItemRetriever::ItemRetriever(ItemRetrievalManager &manager, Connection *connection, const CommandContext &context)
Daniel Vrátil's avatar
Daniel Vrátil committed
49
    : mScope()
50
    , mItemRetrievalManager(manager)
51
    , mConnection(connection)
52
    , mContext(context)
53
54
    , mFullPayload(false)
    , mRecursive(false)
Daniel Vrátil's avatar
Daniel Vrátil committed
55
    , mCanceled(false)
Guy Maurel's avatar
Guy Maurel committed
56
{
57
    qRegisterMetaType<ItemRetrievalResult>("Akonadi::Server::ItemRetrievalResult");
58
59
60
61
62
63
    if (mConnection) {
        connect(mConnection, &Connection::disconnected,
                this, [this]() {
                    mCanceled = true;
                });
    }
Guy Maurel's avatar
Guy Maurel committed
64
}
65

66
Connection *ItemRetriever::connection() const
67
{
68
    return mConnection;
69
70
}

71
void ItemRetriever::setRetrieveParts(const QVector<QByteArray> &parts)
72
{
73
    mParts = parts;
74
75
76
    std::sort(mParts.begin(), mParts.end());
    mParts.erase(std::unique(mParts.begin(), mParts.end()), mParts.end());

77
    // HACK, we need a full payload available flag in PimItem
78
79
    if (mFullPayload && !mParts.contains(AKONADI_PARAM_PLD_RFC822)) {
        mParts.append(AKONADI_PARAM_PLD_RFC822);
80
    }
81
82
}

83
void ItemRetriever::setItemSet(const ImapSet &set, const Collection &collection)
84
{
85
86
    mItemSet = set;
    mCollection = collection;
87
88
}

89
void ItemRetriever::setItemSet(const ImapSet &set, bool isUid)
90
{
91
92
    if (!isUid && mContext.collectionId() >= 0) {
        setItemSet(set, mContext.collection());
93
94
95
    } else {
        setItemSet(set);
    }
96
97
}

98
void ItemRetriever::setItem(const Entity::Id &id)
99
{
100
101
102
103
    ImapSet set;
    set.add(ImapInterval(id, id));
    mItemSet = set;
    mCollection = Collection();
104
105
}

106
void ItemRetriever::setRetrieveFullPayload(bool fullPayload)
107
{
108
109
    mFullPayload = fullPayload;
    // HACK, we need a full payload available flag in PimItem
110
111
    if (fullPayload && !mParts.contains(AKONADI_PARAM_PLD_RFC822)) {
        mParts.append(AKONADI_PARAM_PLD_RFC822);
112
    }
113
114
}

115
void ItemRetriever::setCollection(const Collection &collection, bool recursive)
116
{
117
118
119
    mCollection = collection;
    mItemSet = ImapSet();
    mRecursive = recursive;
120
121
}

122
void ItemRetriever::setScope(const Scope &scope)
123
{
124
    mScope = scope;
125
126
}

127
128
Scope ItemRetriever::scope() const
{
129
    return mScope;
130
131
}

132
void ItemRetriever::setChangedSince(const QDateTime &changedSince)
133
{
134
    mChangedSince = changedSince;
135
136
}

137
QVector<QByteArray> ItemRetriever::retrieveParts() const
138
{
139
    return mParts;
140
}
141

142
enum QueryColumns {
143
    PimItemIdColumn,
144

145
    CollectionIdColumn,
146
    ResourceIdColumn,
147

148
149
    PartTypeNameColumn,
    PartDatasizeColumn
150
};
151

152
153
QSqlQuery ItemRetriever::buildQuery() const
{
154
    QueryBuilder qb(PimItem::tableName());
155

156
    qb.addJoin(QueryBuilder::InnerJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
157

158
    qb.addJoin(QueryBuilder::LeftJoin, Part::tableName(), PimItem::idFullColumnName(), Part::pimItemIdFullColumnName());
159

160
161
162
163
164
    Query::Condition partTypeJoinCondition;
    partTypeJoinCondition.addColumnCondition(Part::partTypeIdFullColumnName(), Query::Equals, PartType::idFullColumnName());
    if (!mFullPayload && !mParts.isEmpty()) {
        partTypeJoinCondition.addCondition(PartTypeHelper::conditionFromFqNames(mParts));
    }
Laurent Montel's avatar
Laurent Montel committed
165
    partTypeJoinCondition.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("PLD"));
166
167
168
    qb.addJoin(QueryBuilder::LeftJoin, PartType::tableName(), partTypeJoinCondition);

    qb.addColumn(PimItem::idFullColumnName());
169
    qb.addColumn(PimItem::collectionIdFullColumnName());
170
    qb.addColumn(Collection::resourceIdFullColumnName());
171
172
173
    qb.addColumn(PartType::nameFullColumnName());
    qb.addColumn(Part::datasizeFullColumnName());

174
    if (!mItemSet.isEmpty() || mCollection.isValid()) {
175
        ItemQueryHelper::itemSetToQuery(mItemSet, qb, mCollection);
176
    } else {
177
        ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
178
    }
179

180
181
    // prevent a resource to trigger item retrieval from itself
    if (mConnection) {
182
183
184
185
        const Resource res = Resource::retrieveByName(QString::fromUtf8(mConnection->sessionId()));
        if (res.isValid()) {
            qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::NotEquals, res.id());
        }
186
    }
187

188
189
190
191
    if (mChangedSince.isValid()) {
        qb.addValueCondition(PimItem::datetimeFullColumnName(), Query::GreaterOrEqual,
                             mChangedSince.toUTC());
    }
192

193
    qb.addSortColumn(PimItem::idFullColumnName(), Query::Ascending);
194

195
196
197
198
    if (!qb.exec()) {
        mLastError = "Unable to retrieve items";
        throw ItemRetrieverException(mLastError);
    }
199

200
    qb.query().next();
201

202
    return qb.query();
203
204
}

Laurent Montel's avatar
Laurent Montel committed
205
206
namespace
{
207
static bool hasAllParts(const ItemRetrievalRequest &req, const QSet<QByteArray> &availableParts)
208
{
209
210
211
    return std::all_of(req.parts.begin(), req.parts.end(), [&availableParts](const auto &part) {
        return availableParts.contains(part);
    });
212
213
214
}
}

215
bool ItemRetriever::runItemRetrievalRequests(std::list<ItemRetrievalRequest> requests)
216
{
217
    QEventLoop eventLoop;
Daniel Vrátil's avatar
Daniel Vrátil committed
218
    std::vector<ItemRetrievalRequest::Id> pendingRequests;
219
220
    connect(&mItemRetrievalManager, &ItemRetrievalManager::requestFinished,
            this, [this, &eventLoop, &pendingRequests](const ItemRetrievalResult &result) {
Daniel Vrátil's avatar
Daniel Vrátil committed
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
                    const auto requestId = std::find(pendingRequests.begin(), pendingRequests.end(), result.request.id);
                    if (requestId != pendingRequests.end()) {
                        if (mCanceled) {
                            eventLoop.exit(1);
                        } else if (result.errorMsg.has_value()) {
                            mLastError = result.errorMsg->toUtf8();
                            eventLoop.exit(1);
                        } else {
                            Q_EMIT itemsRetrieved(result.request.ids);
                            pendingRequests.erase(requestId);
                            if (pendingRequests.empty()) {
                                eventLoop.quit();
                            }
                        }
                    }
236
237
238
239
240
    }, Qt::UniqueConnection);

    if (mConnection) {
        connect(mConnection, &Connection::connectionClosing,
                &eventLoop, [&eventLoop]() { eventLoop.exit(1); });
241
    }
242

243
244
245
246
    for (auto &&request : requests) {
        if ((!mFullPayload && request.parts.isEmpty()) || request.ids.isEmpty()) {
            continue;
        }
247

248
249
250
251
252
253
254
255
256
257
258
259
        // TODO: how should we handle retrieval errors here? so far they have been ignored,
        // which makes sense in some cases, do we need a command parameter for this?
        try {
            // Request is deleted inside ItemRetrievalManager, so we need to take
            // a copy here
            //const auto ids = request->ids;
            pendingRequests.push_back(request.id);
            mItemRetrievalManager.requestItemDelivery(std::move(request));
        } catch (const ItemRetrieverException &e) {
            qCCritical(AKONADISERVER_LOG) << e.type() << ": " << e.what();
            mLastError = e.what();
            return false;
260
        }
261
262
    }

263
264
265
266
267
268
269
270
271
272
273
    if (!pendingRequests.empty()) {
        if (eventLoop.exec()) {
            return false;
        }
    }

    return true;
}

akOptional<ItemRetriever::PreparedRequests> ItemRetriever::prepareRequests(QSqlQuery &query, const QByteArrayList &parts)
{
274
    QHash<qint64, QString> resourceIdNameCache;
275
276
277
    std::list<ItemRetrievalRequest> requests;
    QHash<qint64 /* collection */, decltype(requests)::iterator> colRequests;
    QHash<qint64 /* item */, decltype(requests)::iterator> itemRequests;
278
    QVector<qint64> readyItems;
279
    qint64 prevPimItemId = -1;
280
    QSet<QByteArray> availableParts;
281
    decltype(requests)::iterator lastRequest = requests.end();
282
283
    while (query.isValid()) {
        const qint64 pimItemId = query.value(PimItemIdColumn).toLongLong();
284
        const qint64 collectionId = query.value(CollectionIdColumn).toLongLong();
285
        const qint64 resourceId = query.value(ResourceIdColumn).toLongLong();
286
        const auto itemIter = itemRequests.constFind(pimItemId);
287

Daniel Vrátil's avatar
Daniel Vrátil committed
288
        if (Q_UNLIKELY(mCanceled)) {
289
            return nullopt;
Daniel Vrátil's avatar
Daniel Vrátil committed
290
291
        }

292
293
294
295
296
297
298
299
300
        if (pimItemId == prevPimItemId) {
            if (query.value(PartTypeNameColumn).isNull()) {
                // This is not the first part of the Item we saw, but LEFT JOIN PartTable
                // returned a null row - that means the row is an ATR part
                // which we don't care about
                query.next();
                continue;
            }
        } else {
301
302
            if (lastRequest != requests.end()) {
                if (hasAllParts(*lastRequest, availableParts)) {
303
304
305
306
307
308
309
310
311
312
                    // We went through all parts of a single item, if we have all
                    // parts available in the DB and they are not expired, then
                    // exclude this item from the retrieval
                    lastRequest->ids.removeOne(prevPimItemId);
                    itemRequests.remove(prevPimItemId);
                    readyItems.push_back(prevPimItemId);
                }
            }
            availableParts.clear();
            prevPimItemId = pimItemId;
313
314
        }

315
        if (itemIter != itemRequests.constEnd()) {
316
            lastRequest = itemIter.value();
317
        } else {
318
319
320
321
322
            const auto colIt = colRequests.find(collectionId);
            lastRequest = (colIt == colRequests.end()) ? requests.end() : colIt.value();
            if (lastRequest == requests.end() || lastRequest->ids.size() > 100) {
                requests.emplace_front(ItemRetrievalRequest{});
                lastRequest = requests.begin();
323
                lastRequest->ids.push_back(pimItemId);
324
325
326
327
                auto resIter = resourceIdNameCache.find(resourceId);
                if (resIter == resourceIdNameCache.end()) {
                    resIter = resourceIdNameCache.insert(resourceId, Resource::retrieveById(resourceId).name());
                }
328
329
                lastRequest->resourceId = *resIter;
                lastRequest->parts = parts;
330
                colRequests.insert(collectionId, lastRequest);
331
332
333
334
                itemRequests.insert(pimItemId, lastRequest);
            } else {
                lastRequest->ids.push_back(pimItemId);
                itemRequests.insert(pimItemId, lastRequest);
335
                colRequests.insert(collectionId, lastRequest);
336
            }
337
        }
338
        Q_ASSERT(lastRequest != requests.end());
339
340
341
342
343
344
345
346

        if (query.value(PartTypeNameColumn).isNull()) {
            // LEFT JOIN did not find anything, retrieve all parts
            query.next();
            continue;
        }

        qint64 datasize = query.value(PartDatasizeColumn).toLongLong();
347
348
        const QByteArray partName = Utils::variantToByteArray(query.value(PartTypeNameColumn));
        Q_ASSERT(!partName.startsWith(AKONADI_PARAM_PLD));
349
350
351
        if (datasize <= 0) {
            // request update for this part
            if (mFullPayload && !lastRequest->parts.contains(partName)) {
352
                lastRequest->parts.push_back(partName);
353
354
            }
        } else {
355
356
357
            // add the part to list of available parts, we will compare it with
            // the list of request parts once we handle all parts of this item
            availableParts.insert(partName);
358
359
        }
        query.next();
360
    }
361
    query.finish();
362

363
364
    // Post-check in case we only queried one item thus did not reach the check
    // at the beginning of the while() loop above
365
    if (lastRequest != requests.end() && hasAllParts(*lastRequest, availableParts)) {
366
367
368
369
        lastRequest->ids.removeOne(prevPimItemId);
        readyItems.push_back(prevPimItemId);
        // No need to update the hashtable at this point
    }
370

371
372
    return PreparedRequests{std::move(requests), std::move(readyItems)};
}
373

374
375
376
377
bool ItemRetriever::exec()
{
    if (mParts.isEmpty() && !mFullPayload) {
        return true;
378
379
    }

380
    verifyCache();
381

382
383
384
385
    QSqlQuery query = buildQuery();
    const auto parts = mParts | Views::filter([](const auto &part) { return part.startsWith(AKONADI_PARAM_PLD); })
                              | Views::transform([](const auto &part) { return part.mid(4); })
                              | Actions::toQList;
386

387
388
389
390
    const auto requests = prepareRequests(query, parts);
    if (!requests.has_value()) {
        return false;
    }
391

392
393
    if (!requests->readyItems.isEmpty()) {
        Q_EMIT itemsRetrieved(requests->readyItems);
394
    }
395

396
397
    if (!runItemRetrievalRequests(std::move(requests->requests))) {
        return false;
398
    }
399

400
401
402
403
    // retrieve items in child collections if requested
    bool result = true;
    if (mRecursive && mCollection.isValid()) {
        Q_FOREACH (const Collection &col, mCollection.children()) {
404
            ItemRetriever retriever(mItemRetrievalManager, mConnection, mContext);
405
406
407
            retriever.setCollection(col, mRecursive);
            retriever.setRetrieveParts(mParts);
            retriever.setRetrieveFullPayload(mFullPayload);
408
409
            connect(&retriever, &ItemRetriever::itemsRetrieved,
                    this, &ItemRetriever::itemsRetrieved);
410
411
412
413
414
            result = retriever.exec();
            if (!result) {
                break;
            }
        }
415
    }
416

417
    return result;
418
}
419
420
421

void ItemRetriever::verifyCache()
{
422
    if (!connection() || !connection()->verifyCacheOnRetrieval()) {
423
424
425
426
427
        return;
    }

    SelectQueryBuilder<Part> qb;
    qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName());
428
    qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External);
429
430
    qb.addValueCondition(Part::dataFullColumnName(), Query::IsNot, QVariant());
    if (mScope.scope() != Scope::Invalid) {
431
        ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
432
433
434
435
436
    } else {
        ItemQueryHelper::itemSetToQuery(mItemSet, qb, mCollection);
    }

    if (!qb.exec()) {
Laurent Montel's avatar
Laurent Montel committed
437
        mLastError = QByteArrayLiteral("Unable to query parts.");
438
439
440
441
        throw ItemRetrieverException(mLastError);
    }

    const Part::List externalParts = qb.result();
Laurent Montel's avatar
Laurent Montel committed
442
    for (Part part : externalParts) {
443
444
        PartHelper::verify(part);
    }
445
}
446
447
448

QByteArray ItemRetriever::lastError() const
{
449
    return mLastError;
450
}