extractorrepository.cpp 11.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/*
   Copyright (c) 2017 Volker Krause <vkrause@kde.org>

   This library is free software; you can redistribute it and/or modify it
   under the terms of the GNU Library General Public License as published by
   the Free Software Foundation; either version 2 of the License, or (at your
   option) any later version.

   This library is distributed in the hope that it will be useful, but WITHOUT
   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
   License for more details.

   You should have received a copy of the GNU Library General Public License
   along with this library; see the file COPYING.LIB.  If not, write to the
   Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
   02110-1301, USA.
*/

20
#include "config-kitinerary.h"
21
22
#include "extractorrepository.h"
#include "extractor.h"
23
#include "extractorfilter.h"
Volker Krause's avatar
Volker Krause committed
24
#include "logging.h"
25

26
#ifdef HAVE_KCAL
27
#include <KCalendarCore/Calendar>
28
#include <KCalendarCore/Event>
29
30
#endif

31
32
#include <KMime/Content>

33
#include <KPkPass/Pass>
34

35
#include <QDirIterator>
36
#include <QJsonArray>
Volker Krause's avatar
Volker Krause committed
37
38
#include <QJsonDocument>
#include <QJsonObject>
39
#include <QMetaProperty>
40
#include <QStandardPaths>
41

Volker Krause's avatar
Volker Krause committed
42
43
using namespace KItinerary;

44
45
46
static void initResources() // must be outside of a namespace
{
    Q_INIT_RESOURCE(extractors);
47
    Q_INIT_RESOURCE(vdv_certs);
48
49
}

50
51
52
namespace KItinerary {
class ExtractorRepositoryPrivate {
public:
53
    ExtractorRepositoryPrivate();
54
    void loadExtractors();
Volker Krause's avatar
Volker Krause committed
55
    void addExtractor(Extractor &&e);
56
57
    void extractorForTypeAndContent(ExtractorInput::Type type, const QString &content, std::vector<Extractor> &extractors) const;
    static void insertExtractor(const Extractor &ext, std::vector<Extractor> &extractors);
58
59

    std::vector<Extractor> m_extractors;
60
    QStringList m_extraSearchPaths;
61
62
63
};
}

64
ExtractorRepositoryPrivate::ExtractorRepositoryPrivate()
65
{
66
    initResources();
67
68
69
    loadExtractors();
}

70
void ExtractorRepositoryPrivate::extractorForTypeAndContent(ExtractorInput::Type type, const QString &content, std::vector<Extractor> &extractors) const
71
72
73
74
{
    for (auto it = m_extractors.begin(), end = m_extractors.end(); it != end; ++it) {
        for (const auto &filter : (*it).filters()) {
            if (filter.type() == type && filter.matches(content)) {
75
                insertExtractor(*it, extractors);
76
77
78
79
                break;
            }
        }
    }
80
}
81

82
83
84
85
86
87
88
89
90
91
// approximate set behavior on extractors, using the d pointers as a quick way to ensure uniqueness
void ExtractorRepositoryPrivate::insertExtractor(const Extractor &ext, std::vector<Extractor> &extractors)
{
    const auto it = std::lower_bound(extractors.begin(), extractors.end(), ext, [](const auto &lhs, const auto &rhs) {
        return lhs.d < rhs.d;
    });
    if (it != extractors.end() && (*it).d == ext.d) {
        return;
    }
    extractors.insert(it, ext);
92
93
}

94
95
96
97
98

ExtractorRepository::ExtractorRepository()
{
    static ExtractorRepositoryPrivate repo;
    d = &repo;
99
100
101
}

ExtractorRepository::~ExtractorRepository() = default;
102
ExtractorRepository::ExtractorRepository(KItinerary::ExtractorRepository &&) noexcept = default;
103

104
105
106
107
108
109
void ExtractorRepository::reload()
{
    d->m_extractors.clear();
    d->loadExtractors();
}

Volker Krause's avatar
Volker Krause committed
110
111
112
113
114
const std::vector<Extractor>& ExtractorRepository::allExtractors() const
{
    return d->m_extractors;
}

115
void ExtractorRepository::extractorsForMessage(KMime::Content *part, std::vector<Extractor> &extractors) const
116
{
Laurent Montel's avatar
Laurent Montel committed
117
    if (!part) {
118
        return;
Laurent Montel's avatar
Laurent Montel committed
119
    }
120

121
    for (auto it = d->m_extractors.begin(), end = d->m_extractors.end(); it != end; ++it) {
122
        for (const auto &filter : (*it).filters()) {
123
            if (filter.type() != ExtractorInput::Email) {
124
125
                continue;
            }
126
            auto header = part->headerByType(filter.fieldName().toUtf8().constData());
127
128
129
            auto ancestor = part;
            while (!header && ancestor->parent()) {
                ancestor = ancestor->parent();
130
                header = ancestor->headerByType(filter.fieldName().toUtf8().constData());
Laurent Montel's avatar
Laurent Montel committed
131
132
            }
            if (!header) {
133
                continue;
Laurent Montel's avatar
Laurent Montel committed
134
            }
135
136
            const auto headerData = header->asUnicodeString();
            if (filter.matches(headerData)) {
137
                ExtractorRepositoryPrivate::insertExtractor(*it, extractors);
138
139
140
141
142
143
                break;
            }
        }
    }
}

144
void ExtractorRepository::extractorsForPass(KPkPass::Pass *pass, std::vector<Extractor> &extractors) const
145
{
146
    if (pass->type() != KPkPass::Pass::BoardingPass && pass->type() != KPkPass::Pass::EventTicket) {
147
        return;
148
149
150
    }

    for (auto it = d->m_extractors.begin(), end = d->m_extractors.end(); it != end; ++it) {
151
        if ((*it).type() != ExtractorInput::PkPass) {
152
153
154
            continue;
        }
        for (const auto &filter : (*it).filters()) {
155
            if (filter.type() != ExtractorInput::PkPass) {
156
157
158
                continue;
            }

159
            QString value;
160
            if (filter.fieldName() == QLatin1String("passTypeIdentifier")) {
161
                value = pass->passTypeIdentifier();
162
163
164
165
            } else {
                continue;
            }
            if (filter.matches(value)) {
166
                ExtractorRepositoryPrivate::insertExtractor(*it, extractors);
167
168
169
170
171
172
                break;
            }
        }
    }
}

173
static QString valueForJsonPath(const QJsonObject &obj, const QString &path)
174
{
175
176
177
178
179
180
181
    const auto pathSections = path.splitRef(QLatin1Char('.'));
    QJsonValue v(obj);
    for (const auto &pathSection : pathSections) {
        if (!v.isObject()) {
            return {};
        }
        v = v.toObject().value(pathSection.toString());
182
    }
183
    return v.toString();
184
185
}

186
void ExtractorRepository::extractorsForJsonLd(const QJsonArray &data, std::vector<Extractor> &extractors) const
187
188
189
190
{
    for (const auto &val : data) {
        for (auto it = d->m_extractors.begin(), end = d->m_extractors.end(); it != end; ++it) {
            for (const auto &filter : (*it).filters()) {
191
                if (filter.type() != ExtractorInput::JsonLd) {
192
193
                    continue;
                }
194
195
                const auto value = valueForJsonPath(val.toObject(), filter.fieldName());
                if (value.isEmpty()) {
196
197
                    continue;
                }
198
                if (filter.matches(value)) {
199
                    ExtractorRepositoryPrivate::insertExtractor(*it, extractors);
200
201
202
203
204
205
206
                    break;
                }
            }
        }
    }
}

207
void ExtractorRepository::extractorsForBarcode(const QString &code, std::vector<Extractor> &extractors) const
208
{
209
    d->extractorForTypeAndContent(ExtractorInput::Barcode, code, extractors);
210
211
}

212
#ifdef HAVE_KCAL
213
void ExtractorRepository::extractorsForCalendar(const KCalendarCore::Calendar *cal, std::vector<Extractor> &extractors) const
214
215
216
{
    for (auto it = d->m_extractors.begin(), end = d->m_extractors.end(); it != end; ++it) {
        for (const auto &filter : (*it).filters()) {
217
            if (filter.type() != ExtractorInput::ICal) {
218
219
220
                continue;
            }

221
            const auto value = cal->property(filter.fieldName().toUtf8().constData());
222
            if (filter.matches(value.toString())) {
223
                ExtractorRepositoryPrivate::insertExtractor(*it, extractors);
224
225
226
227
228
                break;
            }
        }
    }
}
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250

void ExtractorRepository::extractorsForEvent(const KCalendarCore::Event *event, std::vector<Extractor> &extractors) const
{
    for (auto it = d->m_extractors.begin(), end = d->m_extractors.end(); it != end; ++it) {
        for (const auto &filter : (*it).filters()) {
            if (filter.type() != ExtractorInput::ICal) {
                continue;
            }

            const auto propIdx = KCalendarCore::Event::staticMetaObject.indexOfProperty(filter.fieldName().toUtf8().constData());
            if (propIdx < 0) {
                continue;
            }
            const auto prop = KCalendarCore::Event::staticMetaObject.property(propIdx);
            const auto value = prop.readOnGadget(event);
            if (filter.matches(value.toString())) {
                ExtractorRepositoryPrivate::insertExtractor(*it, extractors);
                break;
            }
        }
    }
}
251
252
#endif

253
void ExtractorRepository::extractorsForContent(const QString &content, std::vector<Extractor> &extractors) const
254
{
255
    d->extractorForTypeAndContent(ExtractorInput::Text, content, extractors);
256
257
}

Volker Krause's avatar
Volker Krause committed
258
Extractor ExtractorRepository::extractor(const QString &name) const
259
260
261
262
263
{
    auto it = std::lower_bound(d->m_extractors.begin(), d->m_extractors.end(), name, [](const auto &lhs, const auto &rhs) {
        return lhs.name() < rhs;
    });
    if (it != d->m_extractors.end() && (*it).name() == name) {
Volker Krause's avatar
Volker Krause committed
264
        return *it;
265
    }
Volker Krause's avatar
Volker Krause committed
266
    return {};
267
268
}

269
void ExtractorRepositoryPrivate::loadExtractors()
270
{
271
272
273
274
275
276
    auto searchDirs = m_extraSearchPaths;
    const auto qsp = QStandardPaths::standardLocations(QStandardPaths::GenericDataLocation);
    for (const auto &p : qsp) {
        searchDirs.push_back(p + QLatin1String("/kitinerary/extractors"));
    }
    searchDirs += QStringLiteral(":/org.kde.pim/kitinerary/extractors");
277
278

    for (const auto &dir : qAsConst(searchDirs)) {
279
        QDirIterator it(dir, QDir::Files);
280
        while (it.hasNext()) {
Volker Krause's avatar
Volker Krause committed
281
            const auto fileName = it.next();
282
283
284
285
            if (!fileName.endsWith(QLatin1String(".json"))) {
                continue;
            }

Volker Krause's avatar
Volker Krause committed
286
287
288
289
290
291
292
293
294
295
296
297
298
            QFile file(fileName);
            if (!file.open(QFile::ReadOnly)) {
                continue;
            }

            QJsonParseError error;
            const auto doc = QJsonDocument::fromJson(file.readAll(), &error);
            if (doc.isNull()) {
                qCWarning(Log) << "Extractor loading error:" << fileName << error.errorString();
                continue;
            }

            QFileInfo fi(fileName);
Volker Krause's avatar
Volker Krause committed
299
            const auto name = fi.fileName().left(fi.fileName().size() - 5);
Volker Krause's avatar
Volker Krause committed
300

301
302
303
            if (doc.isObject()) {
                const auto obj = doc.object();
                Extractor e;
304
                if (e.load(obj, fi.canonicalFilePath())) {
Volker Krause's avatar
Volker Krause committed
305
                    addExtractor(std::move(e));
306
307
                }
            } else if (doc.isArray()) {
308
                const auto extractorArray = doc.array();
Volker Krause's avatar
Volker Krause committed
309
                int i = 0;
310
311
                for (const auto &v : extractorArray) {
                    Extractor e;
312
                    if (e.load(v.toObject(), fi.canonicalFilePath(), extractorArray.size() == 1 ? -1 : i)) {
Volker Krause's avatar
Volker Krause committed
313
                        addExtractor(std::move(e));
314
                    }
Volker Krause's avatar
Volker Krause committed
315
                    ++i;
316
317
318
319
                }
            } else {
                qCWarning(Log) << "Invalid extractor meta-data:" << fileName;
                continue;
320
            }
Laurent Montel's avatar
Laurent Montel committed
321
        }
322
323
    }
}
Volker Krause's avatar
Volker Krause committed
324
325
326
327
328
329
330
331
332
333

void ExtractorRepositoryPrivate::addExtractor(Extractor &&e)
{
    auto it = std::lower_bound(m_extractors.begin(), m_extractors.end(), e, [](const auto &lhs, const auto &rhs) {
        return lhs.name() < rhs.name();
    });
    if (it == m_extractors.end() || (*it).name() != e.name()) {
        m_extractors.insert(it, std::move(e));
    }
}
334
335
336
337
338
339
340
341
342
343

QStringList ExtractorRepository::additionalSearchPaths() const
{
    return d->m_extraSearchPaths;
}

void ExtractorRepository::setAdditionalSearchPaths(const QStringList& searchPaths)
{
    d->m_extraSearchPaths = searchPaths;
}
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368

QJsonValue ExtractorRepository::extractorToJson(const Extractor &extractor) const
{
    QJsonArray a;
    bool added = false;
    for (const auto &e : d->m_extractors) {
        if (e.fileName() != extractor.fileName()) {
            continue;
        }
        if (extractor.name() == e.name()) {
            a.push_back(extractor.toJson());
            added = true;
        } else {
            a.push_back(e.toJson());
        }
    }
    if (!added) {
        a.push_back(extractor.toJson());
    }

    if (a.size() == 1) {
        return a.at(0);
    }
    return a;
}