Commit 27bbd9f3 authored by Volker Krause's avatar Volker Krause

Add semantic data post-processing step

This uses the new airport database to fill in missing IATA codes and
airport locations. We'll also fix arrival/departure timezones here in
the future.
parent dccfaec3
......@@ -14,6 +14,7 @@ set(semantic_lib_srcs
extractorengine.cpp
extractorfilter.cpp
extractorpreprocessor.cpp
extractorpostprocessor.cpp
extractorrepository.cpp
extractorrule.cpp
jsonlddocument.cpp
......
/*
Copyright (c) 2017 Volker Krause <vkrause@kde.org>
This library is free software; you can redistribute it and/or modify it
under the terms of the GNU Library General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
This library is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
License for more details.
You should have received a copy of the GNU Library General Public License
along with this library; see the file COPYING.LIB. If not, write to the
Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
*/
#include "extractorpostprocessor.h"
#include "datatypes.h"
#include "jsonlddocument.h"
#include "airportdb/airportdb.h"
void ExtractorPostprocessor::process(const QVector<QVariant> &data)
{
m_data.reserve(data.size());
for (const auto &d : data) {
if (d.userType() == qMetaTypeId<FlightReservation>()) {
m_data.push_back(processFlightReservation(d));
} else {
m_data.push_back(d);
}
}
}
QVector<QVariant> ExtractorPostprocessor::result() const
{
return m_data;
}
QVariant ExtractorPostprocessor::processProperty(QVariant obj, const char *name, QVariant (ExtractorPostprocessor::*processor)(QVariant) const) const
{
auto value = JsonLdDocument::readProperty(obj, name);
value = (this->*processor)(value);
JsonLdDocument::writeProperty(obj, name, value);
return obj;
}
QVariant ExtractorPostprocessor::processFlightReservation(QVariant res) const
{
return processProperty(res, "reservationFor", &ExtractorPostprocessor::processFlight);
}
QVariant ExtractorPostprocessor::processFlight(QVariant flight) const
{
flight = processProperty(flight, "departureAirport", &ExtractorPostprocessor::processAirport);
flight = processProperty(flight, "arrivalAirport", &ExtractorPostprocessor::processAirport);
// TODO fix timezones
return flight;
}
QVariant ExtractorPostprocessor::processAirport(QVariant airport) const
{
// complete missing IATA codes
auto iataCode = JsonLdDocument::readProperty(airport, "iataCode").toString();
if (iataCode.isEmpty()) {
iataCode = AirportDb::iataCodeFromName(JsonLdDocument::readProperty(airport, "name").toString()).toString();
if (!iataCode.isEmpty()) {
JsonLdDocument::writeProperty(airport, "iataCode", iataCode);
}
}
// complete missing geo coordinates
auto geo = JsonLdDocument::readProperty(airport, "geo");
if (!geo.value<GeoCoordinates>().isValid()) {
const auto coord = AirportDb::coordinateForAirport(AirportDb::IataCode{iataCode});
if (coord.isValid()) {
geo = QVariant::fromValue(GeoCoordinates());
JsonLdDocument::writeProperty(geo, "latitude", coord.latitude);
JsonLdDocument::writeProperty(geo, "longitude", coord.longitude);
JsonLdDocument::writeProperty(airport, "geo", geo);
}
}
return airport;
}
/*
Copyright (c) 2017 Volker Krause <vkrause@kde.org>
This library is free software; you can redistribute it and/or modify it
under the terms of the GNU Library General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
This library is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
License for more details.
You should have received a copy of the GNU Library General Public License
along with this library; see the file COPYING.LIB. If not, write to the
Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
*/
#ifndef EXTRACTORPOSTPROCESSOR_H
#define EXTRACTORPOSTPROCESSOR_H
#include <QVariant>
#include <QVector>
/** Post-process extracted data to filter out garbage and augment data from other sources. */
class ExtractorPostprocessor
{
public:
void process(const QVector<QVariant> &data);
QVector<QVariant> result() const;
private:
QVariant processProperty(QVariant obj, const char *name, QVariant(ExtractorPostprocessor::*processor)(QVariant) const) const;
QVariant processFlightReservation(QVariant res) const;
QVariant processFlight(QVariant flight) const;
QVariant processAirport(QVariant airport) const;
QVector<QVariant> m_data;
};
#endif // EXTRACTORPOSTPROCESSOR_H
......@@ -114,9 +114,9 @@ QVector<QVariant> JsonLdDocument::fromJson(const QJsonArray &array)
return l;
}
QVariant JsonLdDocument::readProperty(const QVariant &value, const char *name)
QVariant JsonLdDocument::readProperty(const QVariant &obj, const char *name)
{
const auto mo = QMetaType(value.userType()).metaObject();
const auto mo = QMetaType(obj.userType()).metaObject();
if (!mo) {
return {};
}
......@@ -127,5 +127,21 @@ QVariant JsonLdDocument::readProperty(const QVariant &value, const char *name)
}
const auto prop = mo->property(idx);
return prop.readOnGadget(value.constData());
return prop.readOnGadget(obj.constData());
}
void JsonLdDocument::writeProperty(QVariant &obj, const char *name, const QVariant &value)
{
const auto mo = QMetaType(obj.userType()).metaObject();
if (!mo) {
return;
}
const auto idx = mo->indexOfProperty(name);
if (idx < 0) {
return;
}
const auto prop = mo->property(idx);
prop.writeOnGadget(obj.data(), value);
}
......@@ -29,7 +29,11 @@ class QJsonArray;
namespace JsonLdDocument {
QVector<QVariant> fromJson(const QJsonArray &array);
QVariant readProperty(const QVariant &value, const char *name);
/** Read property @p name on object @p obj. */
QVariant readProperty(const QVariant &obj, const char *name);
/** Set property @p name on object @p obj to value @p value. */
void writeProperty(QVariant &obj, const char *name, const QVariant &value);
}
#endif // JSONLDDOCUMENT_H
......@@ -20,6 +20,7 @@
#include "semanticprocessor.h"
#include "extractorengine.h"
#include "extractorpreprocessor.h"
#include "extractorpostprocessor.h"
#include "jsonlddocument.h"
#include "structureddataextractor.h"
#include "semanticmemento.h"
......@@ -98,7 +99,10 @@ MimeTreeParser::MessagePart::Ptr SemanticProcessor::process(MimeTreeParser::Inte
}
}
// TODO postprocessor to filter incomplete/broken elements and merge duplicates
// postprocessor to filter incomplete/broken elements and merge duplicates
ExtractorPostprocessor postproc;
postproc.process(memento->data());
memento->setData(postproc.result());
qCDebug(SEMANTIC_LOG) << "-------------------------------------------- END SEMANTIC PARSING";
return {};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment