Commit 78513a3d authored by Volker Krause's avatar Volker Krause
Browse files

Break out the generic parts of JsonLdImportFilter

This is useful for the longer term goal of separating core and domain code
to support other domains alongside travel, and it's useful for implementing
type transformation code in C++ that is currently done ad hoc in JS.
parent 909055b1
......@@ -54,6 +54,8 @@ target_sources(KPimItinerary PRIVATE
jsapi/bytearray.cpp
jsapi/jsonld.cpp
json/jsonldfilterengine.cpp
knowledgedb/alphaid.cpp
knowledgedb/airportdb.cpp
knowledgedb/airportnametokenizer.cpp
......
/*
SPDX-FileCopyrightText: 2021 Volker Krause <vkrause@kde.org>
SPDX-License-Identifier: LGPL-2.0-or-later
*/
#include "jsonldfilterengine.h"
#include <QJsonArray>
#include <QJsonObject>
#include <cstring>
using namespace KItinerary;
void JsonLd::renameProperty(QJsonObject &obj, const char *oldName, const char *newName)
{
const auto value = obj.value(QLatin1String(oldName));
if (!value.isUndefined() && !obj.contains(QLatin1String(newName))) {
obj.insert(QLatin1String(newName), value);
obj.remove(QLatin1String(oldName));
}
}
JsonLdFilterEngine::JsonLdFilterEngine() = default;
JsonLdFilterEngine::~JsonLdFilterEngine() = default;
void JsonLdFilterEngine::filterRecursive(QJsonObject &obj)
{
auto type = obj.value(QLatin1String("@type")).toString().toUtf8();
// normalize type
if (m_typeMappings) {
const auto it = std::lower_bound(m_typeMappings, m_typeMappings + m_typeMappingsSize, type, [](const auto &lhs, const auto &rhs) {
return std::strcmp(lhs.fromType, rhs.constData()) < 0;
});
if (it != (m_typeMappings + m_typeMappingsSize) && std::strcmp((*it).fromType, type.constData()) == 0) {
type = it->toType;
obj.insert(QStringLiteral("@type"), QLatin1String(type));
}
}
for (auto it = obj.begin(); it != obj.end(); ++it) {
if ((*it).type() == QJsonValue::Object) {
QJsonObject subObj = (*it).toObject();
filterRecursive(subObj);
*it = subObj;
} else if ((*it).type() == QJsonValue::Array) {
QJsonArray array = (*it).toArray();
filterRecursive(array);
*it = array;
}
}
// rename properties
if (m_propertyMappings) {
const auto [pBegin, pEnd] = std::equal_range(m_propertyMappings, m_propertyMappings + m_propertyMappingsSize, type, [](const auto &lhs, const auto &rhs) {
if constexpr (std::is_same_v<std::decay_t<decltype(lhs)>, QByteArray>) {
return std::strcmp(lhs.constData(), rhs.type) < 0;
} else {
return std::strcmp(lhs.type, rhs.constData()) < 0;
}
});
for (auto it = pBegin; it != pEnd; ++it) {
JsonLd::renameProperty(obj, (*it).fromName, (*it).toName);
}
}
// apply filter functions
if (m_typeFilters) {
const auto filterIt = std::lower_bound(m_typeFilters, m_typeFilters + m_typeFiltersSize, type, [](const auto &lhs, const auto &rhs) {
return std::strcmp(lhs.type, rhs.constData()) < 0;
});
if (filterIt != (m_typeFilters + m_typeFiltersSize) && std::strcmp((*filterIt).type, type.constData()) == 0) {
(*filterIt).filterFunc(obj);
}
}
}
void JsonLdFilterEngine::filterRecursive(QJsonArray &array)
{
for (auto it = array.begin(); it != array.end(); ++it) {
if ((*it).type() == QJsonValue::Object) {
QJsonObject subObj = (*it).toObject();
filterRecursive(subObj);
*it = subObj;
} else if ((*it).type() == QJsonValue::Array) {
QJsonArray array = (*it).toArray();
filterRecursive(array);
*it = array;
}
}
}
void JsonLdFilterEngine::setTypeMappings(const JsonLdFilterEngine::TypeMapping *typeMappings, std::size_t count)
{
m_typeMappings = typeMappings;
m_typeMappingsSize = count;
}
void JsonLdFilterEngine::setTypeFilters(const JsonLdFilterEngine::TypeFilter *typeFilters, std::size_t count)
{
m_typeFilters = typeFilters;
m_typeFiltersSize = count;
}
void JsonLdFilterEngine::setPropertyMappings(const JsonLdFilterEngine::PropertyMapping *propertyMappings, std::size_t count)
{
m_propertyMappings = propertyMappings;
m_propertyMappingsSize = count;
}
/*
SPDX-FileCopyrightText: 2021 Volker Krause <vkrause@kde.org>
SPDX-License-Identifier: LGPL-2.0-or-later
*/
#ifndef KITINERARY_JSONLDFILTERENGINE_H
#define KITINERARY_JSONLDFILTERENGINE_H
#include <cstddef>
class QJsonArray;
class QJsonObject;
namespace KItinerary {
namespace JsonLd {
/** Rename a property, if present and the new name isn't in use already. */
void renameProperty(QJsonObject &obj, const char *oldName, const char *newName);
}
/** JSON-LD filtering for input normalization or type transforms. */
class JsonLdFilterEngine
{
public:
explicit JsonLdFilterEngine();
~JsonLdFilterEngine();
/** Recursively apply filtering rules to @p obj. */
void filterRecursive(QJsonObject &obj);
void filterRecursive(QJsonArray &array);
/** Type mappings.
* Has to be sorted by @c fromType.
*/
struct TypeMapping {
const char *fromType;
const char *toType;
};
void setTypeMappings(const TypeMapping *typeMappings, std::size_t count);
template <std::size_t N>
inline void setTypeMappings(const TypeMapping (&typeMappings)[N])
{
setTypeMappings(typeMappings, N);
}
/** Type filter functions.
* Has to be sorted by @c type.
*/
struct TypeFilter {
const char* type;
void(*filterFunc)(QJsonObject&);
};
void setTypeFilters(const TypeFilter *typeFilters, std::size_t count);
template <std::size_t N>
inline void setTypeFilters(const TypeFilter (&typeFilters)[N])
{
setTypeFilters(typeFilters, N);
}
/** Property mappings.
* Has to be sorted by @c type.
*/
struct PropertyMapping {
const char* type;
const char* fromName;
const char* toName;
};
void setPropertyMappings(const PropertyMapping *propertyMappings, std::size_t count);
template <std::size_t N>
inline void setPropertyMappings(const PropertyMapping (&propertyMappings)[N])
{
setPropertyMappings(propertyMappings, N);
}
private:
const TypeMapping *m_typeMappings = nullptr;
std::size_t m_typeMappingsSize;
const TypeFilter *m_typeFilters = nullptr;
std::size_t m_typeFiltersSize;
const PropertyMapping *m_propertyMappings = nullptr;
std::size_t m_propertyMappingsSize;
};
}
#endif // KITINERARY_JSONLDFILTERENGINE_H
......@@ -5,6 +5,7 @@
*/
#include "jsonldimportfilter.h"
#include "json/jsonldfilterengine.h"
#include "logging.h"
#include <QDebug>
......@@ -19,10 +20,7 @@ using namespace KItinerary;
// type normalization from full schema.org type hierarchy to our simplified subset
// IMPORTANT: keep alphabetically sorted by fromType!
static const struct {
const char* fromType;
const char* toType;
} type_mapping[] = {
static constexpr const JsonLdFilterEngine::TypeMapping type_mapping[] = {
{ "AutoDealer", "LocalBusiness" },
{ "AutoRepair", "LocalBusiness" },
{ "AutomotiveBusiness", "LocalBusiness" },
......@@ -67,15 +65,6 @@ static const struct {
{ "Winery", "FoodEstablishment" },
};
static void renameProperty(QJsonObject &obj, const char *oldName, const char *newName)
{
const auto value = obj.value(QLatin1String(oldName));
if (!value.isNull() && !obj.contains(QLatin1String(newName))) {
obj.insert(QLatin1String(newName), value);
obj.remove(QLatin1String(oldName));
}
}
static void migrateToAction(QJsonObject &obj, const char *propName, const char *typeName, bool remove)
{
const auto value = obj.value(QLatin1String(propName));
......@@ -101,24 +90,11 @@ static void migrateToAction(QJsonObject &obj, const char *propName, const char *
}
}
static void filterTrainTrip(QJsonObject &trip)
{
// move TrainTrip::trainCompany to TrainTrip::provider (as defined by schema.org)
renameProperty(trip, "trainCompany", "provider");
}
static void filterLodgingReservation(QJsonObject &res)
{
// check[in|out]Date -> check[in|out]Time (legacy Google format)
renameProperty(res, "checkinDate", "checkinTime");
renameProperty(res, "checkoutDate", "checkoutTime");
}
static void filterFlight(QJsonObject &res)
{
// move incomplete departureTime (ie. just ISO date, no time) to departureDay
if (res.value(QLatin1String("departureTime")).toString().size() == 10) {
renameProperty(res, "departureTime", "departureDay");
JsonLd::renameProperty(res, "departureTime", "departureDay");
}
}
......@@ -157,10 +133,10 @@ static void filterReservation(QJsonObject &res)
}
// legacy properties
renameProperty(res, "programMembership", "programMembershipUsed");
JsonLd::renameProperty(res, "programMembership", "programMembershipUsed");
// legacy potentialAction property
renameProperty(res, "action", "potentialAction");
JsonLd::renameProperty(res, "action", "potentialAction");
// move Google xxxUrl properties to Action instances
migrateToAction(res, "cancelReservationUrl", "CancelAction", true);
......@@ -170,17 +146,10 @@ static void filterReservation(QJsonObject &res)
migrateToAction(res, "url", "ViewAction", false);
// technically the wrong way (reservationId is the current schema.org standard), but hardly used anywhere (yet)
renameProperty(res, "reservationId", "reservationNumber");
JsonLd::renameProperty(res, "reservationId", "reservationNumber");
// "typos"
renameProperty(res, "Url", "url");
}
static void filterBusTrip(QJsonObject &trip)
{
renameProperty(trip, "arrivalStation", "arrivalBusStop");
renameProperty(trip, "departureStation", "departureBusStop");
renameProperty(trip, "busCompany", "provider");
JsonLd::renameProperty(res, "Url", "url");
}
static void filterFoodEstablishment(QJsonObject &restaurant)
......@@ -199,12 +168,6 @@ static void filterFoodEstablishment(QJsonObject &restaurant)
}
}
static void filterProgramMembership(QJsonObject &program)
{
renameProperty(program, "program", "programName");
renameProperty(program, "memberNumber", "membershipNumber");
}
static void filterActionTarget(QJsonObject &action)
{
QJsonArray targets;
......@@ -252,7 +215,7 @@ static void filterActionTarget(QJsonObject &action)
}
if (filteredTargetUrlString.isEmpty()) {
renameProperty(action, "url", "target");
JsonLd::renameProperty(action, "url", "target");
} else {
action.insert(QStringLiteral("target"), filteredTargetUrlString);
}
......@@ -279,68 +242,28 @@ static QJsonArray filterActions(const QJsonValue &v)
// filter functions applied to objects of the corresponding (already normalized) type
// IMPORTANT: keep alphabetically sorted by type!
static const struct {
const char* type;
void(*filterFunc)(QJsonObject&);
} type_filters[] = {
{ "BusTrip", filterBusTrip },
static constexpr const JsonLdFilterEngine::TypeFilter type_filters[] = {
{ "Flight", filterFlight },
{ "FoodEstablishment", filterFoodEstablishment },
{ "LodgingReservation", filterLodgingReservation },
{ "ProgramMembership", filterProgramMembership },
{ "TrainTrip", filterTrainTrip },
};
static void filterRecursive(QJsonObject &obj);
static void filterRecursive(QJsonArray &array)
{
for (auto it = array.begin(); it != array.end(); ++it) {
if ((*it).type() == QJsonValue::Object) {
QJsonObject subObj = (*it).toObject();
filterRecursive(subObj);
*it = subObj;
} else if ((*it).type() == QJsonValue::Array) {
QJsonArray array = (*it).toArray();
filterRecursive(array);
*it = array;
}
}
}
// property renaming
// IMPORTANT: keep alphabetically sorted by type!
static constexpr const JsonLdFilterEngine::PropertyMapping property_mappings[] = {
{ "BusTrip", "arrivalStation", "arrivalBusStop" },
{ "BusTrip", "busCompany", "provider" },
{ "BusTrip", "departureStation", "departureBusStop" },
static void filterRecursive(QJsonObject &obj)
{
auto type = obj.value(QLatin1String("@type")).toString().toUtf8();
// normalize type
const auto it = std::lower_bound(std::begin(type_mapping), std::end(type_mapping), type, [](const auto &lhs, const auto &rhs) {
return std::strcmp(lhs.fromType, rhs.constData()) < 0;
});
if (it != std::end(type_mapping) && std::strcmp((*it).fromType, type.constData()) == 0) {
type = it->toType;
obj.insert(QStringLiteral("@type"), QLatin1String(type));
}
// check[in|out]Date -> check[in|out]Time (legacy Google format)
{ "LodgingReservation", "checkinDate", "checkinTime" },
{ "LodgingReservation", "checkoutDate", "checkoutTime" },
for (auto it = obj.begin(); it != obj.end(); ++it) {
if ((*it).type() == QJsonValue::Object) {
QJsonObject subObj = (*it).toObject();
filterRecursive(subObj);
*it = subObj;
} else if ((*it).type() == QJsonValue::Array) {
QJsonArray array = (*it).toArray();
filterRecursive(array);
*it = array;
}
}
{ "ProgramMembership", "program", "programName" },
{ "ProgramMembership", "memberNumber", "membershipNumber" },
// apply filter functions
const auto filterIt = std::lower_bound(std::begin(type_filters), std::end(type_filters), type, [](const auto &lhs, const auto &rhs) {
return std::strcmp(lhs.type, rhs.constData()) < 0;
});
if (filterIt != std::end(type_filters) && std::strcmp((*filterIt).type, type.constData()) == 0) {
(*filterIt).filterFunc(obj);
}
}
// move TrainTrip::trainCompany to TrainTrip::provider (as defined by schema.org)
{ "TrainTrip", "trainCompany", "provider" },
};
static QJsonArray graphExpand(const QJsonObject &obj)
{
......@@ -377,10 +300,14 @@ QJsonArray JsonLdImportFilter::filterObject(const QJsonObject &obj)
QJsonArray results;
JsonLdFilterEngine filterEngine;
filterEngine.setTypeMappings(type_mapping);
filterEngine.setTypeFilters(type_filters);
filterEngine.setPropertyMappings(property_mappings);
for (const auto &type : types) {
QJsonObject res(obj);
res.insert(QStringLiteral("@type"), type);
filterRecursive(res);
filterEngine.filterRecursive(res);
if (type.endsWith(QLatin1String("Reservation"))) {
filterReservation(res);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment