Members of the KDE Community are recommended to subscribe to the kde-community mailing list at https://mail.kde.org/mailman/listinfo/kde-community to allow them to participate in important discussions and receive other important announcements

Commit 94437d7a authored by Volker Krause's avatar Volker Krause

Add filtering of incomplete/invalid data to post-processing

That way we will never show incomplete data if any of the extractors
fail to find mandatory information.
parent cf416ee4
......@@ -63,3 +63,8 @@ ecm_add_test(
NAME_PREFIX "messageviewerplugins-"
LINK_LIBRARIES Qt5::Test semantic_extractor
)
ecm_add_test(
postprocessortest.cpp
NAME_PREFIX "messageviewerplugins-"
LINK_LIBRARIES Qt5::Test semantic_extractor
)
[{
"@context": "http://schema.org",
"@type": "FlightReservation",
"reservationFor": {
"@type": "Flight",
"airline": {
"@type": "Airline",
"iataCode": "UA",
"name": "United"
},
"arrivalAirport": {
"@type": "Airport",
"geo": {
"@type": "GeoCoordinates",
"latitude": -77.45580291748047,
"longitude": 38.944400787353516
},
"iataCode": "IAD",
"name": "Washington Dulles International Airport"
},
"arrivalTime": "2027-03-05T06:30:00-05:00",
"departureAirport": {
"@type": "Airport",
"geo": {
"@type": "GeoCoordinates",
"latitude": -122.30899810791016,
"longitude": 47.44889831542969
},
"iataCode": "SEA",
"name": "Seattle-Tacoma International Airport"
},
"departureTime": "2027-03-04T20:15:00-08:00",
"flightNumber": "110"
},
"reservationNumber": "RXJ34P"
}]
[{
"@context": "http://schema.org",
"@type": "FlightReservation",
"reservationNumber": "RXJ34P",
"reservationFor": {
"@type": "Flight",
"flightNumber": "110",
"airline": {
"@type": "Airline",
"name": "United",
"iataCode": "UA"
},
"departureAirport": {
"@type": "Airport",
"name": "Seattle-Tacoma International Airport"
},
"departureTime": "2027-03-04T20:15:00",
"arrivalAirport": {
"@type": "Airport",
"name": "Washington Dulles International Airport"
},
"arrivalTime": "2027-03-05T06:30:00"
}
}]
[{
"@context": "http://schema.org",
"@type": "FlightReservation",
"reservationNumber": "RXJ34P",
"reservationFor": {
"@type": "Flight",
"flightNumber": "110",
"airline": {
"@type": "Airline",
"name": "United",
"iataCode": "UA"
},
"departureAirport": {
"@type": "Airport",
"name": "San Francisco Airport",
"iataCode": "SFO"
},
"departureTime": "2027-03-04T20:15:00-08:00",
"arrivalTime": "2027-03-05T06:30:00-05:00"
}
}, {
"@context": "http://schema.org",
"@type": "FlightReservation",
"reservationNumber": "XXX123"
}, {
"@context": "http://schema.org",
"@type": "FlightReservation",
"reservationNumber": "RXJ34P",
"reservationFor": {
"@type": "Flight",
"flightNumber": "110",
"airline": {
"@type": "Airline",
"name": "United",
"iataCode": "UA"
},
"departureAirport": {
"@type": "Airport",
"name": "San Francisco Airport",
"iataCode": "SFO"
},
"arrivalAirport": {
"@type": "Airport",
"name": "John F. Kennedy International Airport"
},
"arrivalTime": "2027-03-05T06:30:00-05:00"
}
}, {
"@context": "http://schema.org",
"@type": "FlightReservation",
"reservationNumber": "RXJ34P",
"reservationFor": {
"@type": "Flight",
"flightNumber": "110",
"airline": {
"@type": "Airline",
"name": "United",
"iataCode": "UA"
},
"departureAirport": {
"@type": "Airport",
"name": "San Francisco Airport",
"iataCode": "SFO"
},
"departureTime": "2027-03-04T20:15:00-08:00",
"arrivalAirport": {
"@type": "Airport",
"name": "John F. Kennedy International Airport"
}
}
}
]
/*
Copyright (c) 2017 Volker Krause <vkrause@kde.org>
This library is free software; you can redistribute it and/or modify it
under the terms of the GNU Library General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
This library is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
License for more details.
You should have received a copy of the GNU Library General Public License
along with this library; see the file COPYING.LIB. If not, write to the
Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
*/
#include "extractorpostprocessor.h"
#include "jsonlddocument.h"
#include <QDebug>
#include <QDir>
#include <QFile>
#include <QJsonArray>
#include <QJsonDocument>
#include <QObject>
#include <QTest>
class PostprocessorTest : public QObject
{
Q_OBJECT
private Q_SLOTS:
void testPostProc_data()
{
QTest::addColumn<QString>("preFile");
QTest::addColumn<QString>("postFile");
QDir dir(QStringLiteral(SOURCE_DIR "/postprocessordata"));
const auto lst = dir.entryList(QStringList(QStringLiteral("*.pre.json")), QDir::Files | QDir::Readable | QDir::NoSymLinks);
for (const auto &file : lst) {
const auto refFile = dir.path() + QLatin1Char('/') + file.left(file.size() - 8) + QStringLiteral("post.json");
if (!QFile::exists(refFile)) {
qDebug() << "reference file" << refFile << "does not exist, skipping test file" << file;
continue;
}
QTest::newRow(file.toLatin1()) << QString(dir.path() + QLatin1Char('/') + file) << refFile;
}
}
void testPostProc()
{
QFETCH(QString, preFile);
QFETCH(QString, postFile);
QFile f(preFile);
QVERIFY(f.open(QFile::ReadOnly));
const auto inArray = QJsonDocument::fromJson(f.readAll()).array();
QVERIFY(!inArray.isEmpty());
const auto preData = JsonLdDocument::fromJson(inArray);
QCOMPARE(inArray.size(), preData.size());
ExtractorPostprocessor postproc;
postproc.process(preData);
const auto outArray = JsonLdDocument::toJson(postproc.result());
QCOMPARE(outArray.size(), postproc.result().size());
QFile ref(postFile);
QVERIFY(ref.open(QFile::ReadOnly));
const auto refArray = QJsonDocument::fromJson(ref.readAll()).array();
if (outArray != refArray) {
qDebug().noquote() << QJsonDocument(outArray).toJson();
}
QCOMPARE(refArray.size(), postproc.result().size());
QCOMPARE(outArray, refArray);
}
};
QTEST_APPLESS_MAIN(PostprocessorTest)
#include "postprocessortest.moc"
......@@ -26,7 +26,7 @@
#define SEMANTIC_GADGET \
Q_GADGET \
Q_PROPERTY(QString className READ className CONSTANT) \
Q_PROPERTY(QString className READ className STORED false CONSTANT) \
inline QString className() const { return QString::fromUtf8(staticMetaObject.className()); }
#define SEMANTIC_PROPERTY(Type, Name) \
......@@ -100,8 +100,8 @@ class Flight
SEMANTIC_PROPERTY(Airport, arrivalAirport)
SEMANTIC_PROPERTY(QDateTime, arrivalTime)
Q_PROPERTY(QString departureTimeLocalized READ departureTimeLocalized CONSTANT)
Q_PROPERTY(QString arrivalTimeLocalized READ arrivalTimeLocalized CONSTANT)
Q_PROPERTY(QString departureTimeLocalized READ departureTimeLocalized STORED false CONSTANT)
Q_PROPERTY(QString arrivalTimeLocalized READ arrivalTimeLocalized STORED false CONSTANT)
private:
QString departureTimeLocalized() const;
QString arrivalTimeLocalized() const;
......@@ -124,8 +124,8 @@ class TrainTrip
SEMANTIC_PROPERTY(QDateTime, departureTime)
SEMANTIC_PROPERTY(QString, trainNumber)
Q_PROPERTY(QString departureTimeLocalized READ departureTimeLocalized CONSTANT)
Q_PROPERTY(QString arrivalTimeLocalized READ arrivalTimeLocalized CONSTANT)
Q_PROPERTY(QString departureTimeLocalized READ departureTimeLocalized STORED false CONSTANT)
Q_PROPERTY(QString arrivalTimeLocalized READ arrivalTimeLocalized STORED false CONSTANT)
private:
QString departureTimeLocalized() const;
......@@ -145,8 +145,8 @@ class LodgingReservation : protected Reservation
SEMANTIC_PROPERTY(QDateTime, checkinDate)
SEMANTIC_PROPERTY(QDateTime, checkoutDate)
Q_PROPERTY(QString checkinDateLocalized READ checkinDateLocalized CONSTANT)
Q_PROPERTY(QString checkoutDateLocalized READ checkoutDateLocalized CONSTANT)
Q_PROPERTY(QString checkinDateLocalized READ checkinDateLocalized STORED false CONSTANT)
Q_PROPERTY(QString checkoutDateLocalized READ checkoutDateLocalized STORED false CONSTANT)
private:
QString checkinDateLocalized() const;
QString checkoutDateLocalized() const;
......
......@@ -28,10 +28,12 @@
void ExtractorPostprocessor::process(const QVector<QVariant> &data)
{
m_data.reserve(data.size());
for (const auto &d : data) {
for (auto d : data) {
if (d.userType() == qMetaTypeId<FlightReservation>()) {
m_data.push_back(processFlightReservation(d));
} else {
d = processFlightReservation(d);
}
if (filterReservation(d)) {
m_data.push_back(d);
}
}
......@@ -123,3 +125,32 @@ void ExtractorPostprocessor::processFlightTime(QVariant &flight, const char *tim
JsonLdDocument::writeProperty(flight, timePropName, QDateTime());
JsonLdDocument::writeProperty(flight, timePropName, dt);
}
bool ExtractorPostprocessor::filterReservation(const QVariant &res) const
{
const auto resFor = JsonLdDocument::readProperty(res, "reservationFor");
if (resFor.isNull()) {
return false;
}
if (resFor.userType() == qMetaTypeId<Flight>()) {
return filterFlight(resFor);
}
return true;
}
bool ExtractorPostprocessor::filterFlight(const QVariant &flight) const
{
const auto depDt = JsonLdDocument::readProperty(flight, "departureTime").toDateTime();
const auto arrDt = JsonLdDocument::readProperty(flight, "arrivalTime").toDateTime();
return filterAirport(JsonLdDocument::readProperty(flight, "departureAirport"))
&& filterAirport(JsonLdDocument::readProperty(flight, "arrivalAirport"))
&& depDt.isValid() && arrDt.isValid();
}
bool ExtractorPostprocessor::filterAirport(const QVariant &airport) const
{
const auto iataCode = JsonLdDocument::readProperty(airport, "iataCode").toString();
const auto name = JsonLdDocument::readProperty(airport, "name").toString();
return !iataCode.isEmpty() || !name.isEmpty();
}
......@@ -38,6 +38,10 @@ private:
QVariant processAirport(QVariant airport) const;
void processFlightTime(QVariant &flight, const char *timePropName, const char *airportPropName) const;
bool filterReservation(const QVariant &res) const;
bool filterFlight(const QVariant &flight) const;
bool filterAirport(const QVariant &airport) const;
QVector<QVariant> m_data;
};
......
......@@ -114,6 +114,61 @@ QVector<QVariant> JsonLdDocument::fromJson(const QJsonArray &array)
return l;
}
static QJsonValue toJson(const QVariant &v)
{
const auto mo = QMetaType(v.userType()).metaObject();
if (!mo) {
// basic types
switch (v.type()) {
case QVariant::String:
return v.toString();
case QVariant::Double:
return v.toDouble();
case QVariant::Int:
return v.toInt();
case QVariant::DateTime:
return v.toDateTime().toString(Qt::ISODate);
default:
break;
}
if (v.userType() == qMetaTypeId<float>()) {
return v.toFloat();
}
qCDebug(SEMANTIC_LOG) << "unhandled value:" << v;
return {};
}
// composite types
QJsonObject obj;
obj.insert(QStringLiteral("@type"), QString::fromUtf8(mo->className()));
for (int i = 0; i < mo->propertyCount(); ++i) {
const auto prop = mo->property(i);
if (!prop.isStored()) {
continue;
}
const auto value = prop.readOnGadget(v.constData());
if (!value.isNull()) {
obj.insert(QString::fromUtf8(prop.name()), toJson(value));
}
}
return obj;
}
QJsonArray JsonLdDocument::toJson(const QVector<QVariant> &data)
{
QJsonArray a;
for (const auto &d : data) {
const auto value = toJson(d);
if (!value.isObject()) {
continue;
}
auto obj = value.toObject();
obj.insert(QStringLiteral("@context"), QStringLiteral("http://schema.org"));
a.push_back(obj);
}
return a;
}
QVariant JsonLdDocument::readProperty(const QVariant &obj, const char *name)
{
const auto mo = QMetaType(obj.userType()).metaObject();
......
......@@ -28,6 +28,7 @@ class QJsonArray;
/** Serialization/deserialization code for JSON-LD data. */
namespace JsonLdDocument {
QVector<QVariant> fromJson(const QJsonArray &array);
QJsonArray toJson(const QVector<QVariant> &data);
/** Read property @p name on object @p obj. */
QVariant readProperty(const QVariant &obj, const char *name);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment