Commit a2ecf044 authored by Volker Krause's avatar Volker Krause
Browse files

Add a database with timezones and locations of all airports

This is generated from Wikidata data, and when compiled is only about
200kb of read-only data.
parent de69ac09
/*
Copyright (c) 2017 Volker Krause <vkrause@kde.org>
This library is free software; you can redistribute it and/or modify it
under the terms of the GNU Library General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
This library is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
License for more details.
You should have received a copy of the GNU Library General Public License
along with this library; see the file COPYING.LIB. If not, write to the
Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
*/
#include "airportdb/airportdb.h"
#include <QDebug>
#include <QObject>
#include <QTest>
#include <QTimeZone>
#include <cmath>
class AirportDbTest : public QObject
{
Q_OBJECT
private Q_SLOTS:
void iataCodeTest()
{
const auto txl = AirportDb::IataCode{"TXL"};
QVERIFY(txl.isValid());
const auto invalid = AirportDb::IataCode{};
QVERIFY(!invalid.isValid());
QVERIFY(txl != invalid);
QVERIFY(!(txl == invalid));
QVERIFY(txl == txl);
QCOMPARE(invalid.toString(), QString());
const auto cdg = AirportDb::IataCode{"CDG"};
QVERIFY(cdg.isValid());
QVERIFY(cdg != txl);
QVERIFY(!(cdg == txl));
QVERIFY(cdg < txl);
QVERIFY(!(txl < cdg));
QVERIFY(AirportDb::IataCode{"ABC"} < AirportDb::IataCode{"CBA"});
QVERIFY(!(AirportDb::IataCode{"CBA"} < AirportDb::IataCode{"ABC"}));
}
void coordinateLookupTest()
{
auto coord = AirportDb::coordinateForAirport(AirportDb::IataCode{"TXL"});
QVERIFY(coord.isValid());
QCOMPARE((int)coord.latitude, 13);
QCOMPARE((int)coord.longitude, 52);
coord = AirportDb::coordinateForAirport(AirportDb::IataCode{"XXX"});
QVERIFY(!coord.isValid());
QVERIFY(std::isnan(coord.latitude));
QVERIFY(std::isnan(coord.longitude));
// test coordinate parsing corner cases
coord = AirportDb::coordinateForAirport(AirportDb::IataCode{"LCY"});
QCOMPARE((int)coord.latitude, 0);
QVERIFY(coord.latitude > 0.0f);
coord = AirportDb::coordinateForAirport(AirportDb::IataCode{"LHR"});
QCOMPARE((int)coord.latitude, 0);
QVERIFY(coord.latitude < 0.0f);
// Köln-Bonn is a hybrid civilian/military airport, so that should be included
coord = AirportDb::coordinateForAirport(AirportDb::IataCode{"CGN"});
QVERIFY(coord.isValid());
// Frankfurt-Hahn is a former military airport, should be included
coord = AirportDb::coordinateForAirport(AirportDb::IataCode{"HHN"});
QVERIFY(coord.isValid());
// Ramstein is a military airport that should not be included
coord = AirportDb::coordinateForAirport(AirportDb::IataCode{"RMS"});
QVERIFY(!coord.isValid());
// IATA codes that changed airports in various ways
QVERIFY(AirportDb::coordinateForAirport(AirportDb::IataCode{"DEN"}).isValid());
QVERIFY(AirportDb::coordinateForAirport(AirportDb::IataCode{"MUC"}).isValid());
QVERIFY(AirportDb::coordinateForAirport(AirportDb::IataCode{"GOT"}).isValid());
QVERIFY(AirportDb::coordinateForAirport(AirportDb::IataCode{"OSL"}).isValid());
// IATA codes of no longer active airports
QVERIFY(!AirportDb::coordinateForAirport(AirportDb::IataCode{"THF"}).isValid());
// IATA codes of civilian airports that match the primitive military filter
QVERIFY(AirportDb::coordinateForAirport(AirportDb::IataCode{"RAF"}).isValid());
QVERIFY(AirportDb::coordinateForAirport(AirportDb::IataCode{"CFB"}).isValid());
QVERIFY(AirportDb::coordinateForAirport(AirportDb::IataCode{"PAF"}).isValid());
// one airport with 3 IATA codes
coord = AirportDb::coordinateForAirport(AirportDb::IataCode{"BSL"});
QVERIFY(coord.isValid());
QCOMPARE(AirportDb::coordinateForAirport(AirportDb::IataCode{"BSL"}), AirportDb::coordinateForAirport(AirportDb::IataCode{"MLH"}));
QCOMPARE(AirportDb::coordinateForAirport(AirportDb::IataCode{"BSL"}), AirportDb::coordinateForAirport(AirportDb::IataCode{"EAP"}));
}
void timezoneLookupTest()
{
auto tz = AirportDb::timezoneForAirport(AirportDb::IataCode{"TXL"});
QVERIFY(tz.isValid());
QCOMPARE(tz.id(), QByteArray("Europe/Berlin"));
tz = AirportDb::timezoneForAirport(AirportDb::IataCode{"XXX"});
QVERIFY(!tz.isValid());
// tiny, make sure our lookup resolution is big enough for that
tz = AirportDb::timezoneForAirport(AirportDb::IataCode{"LUX"});
QCOMPARE(tz.id(), QByteArray("Europe/Luxembourg"));
}
void iataLookupTest()
{
QCOMPARE(AirportDb::iataCodeFromName(QStringLiteral("Flughafen Berlin-Tegel")), AirportDb::IataCode{"TXL"});
QCOMPARE(AirportDb::iataCodeFromName(QStringLiteral("TEGEL")), AirportDb::IataCode{"TXL"});
QCOMPARE(AirportDb::iataCodeFromName(QStringLiteral("Paris Charles de Gaulle")), AirportDb::IataCode{"CDG"});
QCOMPARE(AirportDb::iataCodeFromName(QStringLiteral("Zürich")), AirportDb::IataCode{"ZRH"});
QCOMPARE(AirportDb::iataCodeFromName(QStringLiteral("AMSTERDAM, NL (SCHIPHOL AIRPORT)")), AirportDb::IataCode{"AMS"});
QCOMPARE(AirportDb::iataCodeFromName(QStringLiteral("London Heathrow")), AirportDb::IataCode{"LHR"});
// not unique
QVERIFY(!AirportDb::iataCodeFromName(QStringLiteral("Flughafen Berlin")).isValid());
QVERIFY(!AirportDb::iataCodeFromName(QStringLiteral("Charles de Gaulle Orly")).isValid());
QVERIFY(!AirportDb::iataCodeFromName(QStringLiteral("Brussels Airport, BE")).isValid());
// would be nice of those would work, but needs more complex index
QVERIFY(!AirportDb::iataCodeFromName(QStringLiteral("DETROIT, MI (METROPOLITAN WAYNE CO), TERMINAL EM")).isValid());
}
};
QTEST_APPLESS_MAIN(AirportDbTest)
#include "airportdbtest.moc"
add_subdirectory(airportdb)
if(TARGET Poppler::Qt5)
set(HAVE_POPPLER ON)
endif()
......@@ -5,6 +7,7 @@ configure_file(config-semantic.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config-semant
# static lib for use by unit test
set(semantic_lib_srcs
airportdb/airportdb.cpp
datatypes.cpp
extractor.cpp
extractorcontext.cpp
......
timezones.png
timezones.colormap
timezones.qgs~
timezones.shapefile*
combined_shapefile*
wikidata_*.json
add_executable(generate-airportdb generatedb.cpp timezones.cpp)
target_link_libraries(generate-airportdb Qt5::Network Qt5::Gui)
find_program(XSLTPROC_EXECUTABLE xsltproc)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/timezones.colormap
COMMAND
${XSLTPROC_EXECUTABLE} --novalid
${CMAKE_CURRENT_SOURCE_DIR}/timezones.xsl
${CMAKE_CURRENT_SOURCE_DIR}/timezones.qgs
> ${CMAKE_CURRENT_SOURCE_DIR}/timezones.colormap
DEPENDS
${CMAKE_CURRENT_SOURCE_DIR}/timezones.xsl
${CMAKE_CURRENT_SOURCE_DIR}/timezones.qgs
)
add_custom_target(
rebuild-airportdb
COMMAND generate-airportdb -o ${CMAKE_CURRENT_SOURCE_DIR}/airportdb_p.cpp
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
DEPENDS
${CMAKE_CURRENT_SOURCE_DIR}/timezones.colormap
${CMAKE_CURRENT_SOURCE_DIR}/timezones.png
)
Updating the airport database
=============================
(1) Download timezone shapefile
Pick the latest timezones.shapefile.zip from
https://github.com/evansiroky/timezone-boundary-builder/releases
Extract combined_shapefile.shp from that zip file into this folder.
(2) Generate timezone lookup map
Open timezones.qgs in QGIS. Select "Project" > "Print Composers" > "timezone_lookup_map" and then
"Composer" > "Export as Image". Choose "timezones.png" in this folder and 2400dpi.
(3) Run the code generator
Run `make rebuild-airportdb` in the build dir of this folder.
/*
Copyright (c) 2017 Volker Krause <vkrause@kde.org>
This library is free software; you can redistribute it and/or modify it
under the terms of the GNU Library General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
This library is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
License for more details.
You should have received a copy of the GNU Library General Public License
along with this library; see the file COPYING.LIB. If not, write to the
Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
*/
#include "airportdb.h"
#include "airportdb_p.h"
#include "airportdb_p.cpp"
#include <QDebug>
#include <QRegularExpression>
#include <QTimeZone>
#include <algorithm>
#include <cstring>
namespace AirportDb {
static_assert(sizeof(IataCode) == sizeof(uint16_t), "IATA code changed size!");
static constexpr auto iata_table_size = sizeof(iata_table) / sizeof(IataCode);
static_assert(iata_table_size == sizeof(coordinate_table) / sizeof(Coordinate), "Airport coordinate table size mismatch!");
static_assert(iata_table_size == sizeof(timezone_table) / sizeof(uint16_t), "Airport timezone table size mismatch!");
bool Coordinate::isValid() const
{
return !std::isnan(latitude) && !std::isnan(longitude);
}
bool Coordinate::operator==(const Coordinate &other) const
{
return latitude == other.latitude && longitude == other.longitude;
}
IataCode::IataCode(const QString &iataStr) : IataCode()
{
if (iataStr.size() != 3)
return;
if (!iataStr.at(0).isUpper() || !iataStr.at(1).isUpper() || !iataStr.at(2).isUpper())
return;
m_letter0 = iataStr.at(0).toLatin1() - 'A';
m_letter1 = iataStr.at(1).toLatin1() - 'A';
m_letter2 = iataStr.at(2).toLatin1() - 'A';
m_valid = 1;
}
bool IataCode::isValid() const
{
return m_valid;
}
bool IataCode::operator<(IataCode rhs) const
{
return toUInt16() < rhs.toUInt16();
}
bool IataCode::operator==(IataCode other) const
{
return memcmp(this, &other, sizeof(IataCode)) == 0;
}
bool IataCode::operator!=(IataCode other) const
{
return memcmp(this, &other, sizeof(IataCode)) != 0;
}
QString IataCode::toString() const
{
if (!isValid())
return QString();
QString s;
s.reserve(3);
s.push_back(QLatin1Char(m_letter0 + 'A'));
s.push_back(QLatin1Char(m_letter1 + 'A'));
s.push_back(QLatin1Char(m_letter2 + 'A'));
return s;
}
uint16_t IataCode::toUInt16() const
{
return m_letter0 << 11 | m_letter1 << 6 | m_letter2 << 1 | m_valid;
}
static const IataCode* iataBegin()
{
return iata_table;
}
static const IataCode* iataEnd()
{
return iata_table + iata_table_size;
}
static int indexOfAirport(IataCode iataCode)
{
const auto iataIt = std::lower_bound(iataBegin(), iataEnd(), iataCode);
if (iataIt == iataEnd() || (*iataIt) != iataCode)
return -1;
return std::distance(iataBegin(), iataIt);
}
Coordinate coordinateForAirport(IataCode iataCode)
{
const auto iataIdx = indexOfAirport(iataCode);
if (iataIdx >= 0)
return coordinate_table[iataIdx];
return {};
}
QTimeZone timezoneForAirport(IataCode iataCode)
{
const auto iataIdx = indexOfAirport(iataCode);
if (iataIdx < 0)
return QTimeZone();
return QTimeZone(timezone_names + timezone_table[iataIdx]);
}
static const auto name_string_index_size = sizeof(name_string_index) / sizeof(NameIndex);
static const NameIndex* nameIndexBegin()
{
return name_string_index;
}
static const NameIndex* nameIndexEnd()
{
return name_string_index + name_string_index_size;
}
IataCode iataCodeFromName(const QString &name)
{
IataCode code;
int iataIdx = -1;
for (const auto &s : name.toCaseFolded().split(QRegularExpression(QStringLiteral("[ 0-9/'\"\\(\\)&\\,.–„-]")), QString::SkipEmptyParts)) {
const auto it = std::lower_bound(nameIndexBegin(), nameIndexEnd(), s.toUtf8(), [](const NameIndex &lhs, const QByteArray &rhs) {
const auto cmp = strncmp(name_string_table + lhs.offset(), rhs.constData(), std::min<int>(lhs.length, rhs.size()));
if (cmp == 0) {
return lhs.length < rhs.size();
}
return cmp < 0;
});
if (it == nameIndexEnd() || it->length != s.toUtf8().size() || strncmp(name_string_table + it->offset(), s.toUtf8().constData(), it->length) != 0)
continue;
if (iataIdx >= 0 && iataIdx != it->iataIndex)
return code; // not unique
iataIdx = it->iataIndex;
}
if (iataIdx > 0)
code = iata_table[iataIdx];
return code;
}
}
/*
Copyright (c) 2017 Volker Krause <vkrause@kde.org>
This library is free software; you can redistribute it and/or modify it
under the terms of the GNU Library General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
This library is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
License for more details.
You should have received a copy of the GNU Library General Public License
along with this library; see the file COPYING.LIB. If not, write to the
Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
*/
#ifndef AIRPORTDB_H
#define AIRPORTDB_H
#include <cmath>
#include <cstdint>
class QString;
class QTimeZone;
/** Database of all civilian airports, their locations and timezones. */
namespace AirportDb
{
/** Geographical coordinate. */
struct Coordinate {
inline constexpr Coordinate()
: latitude(NAN)
, longitude(NAN)
{
}
inline explicit constexpr Coordinate(float lat, float lng)
: latitude(lat)
, longitude(lng)
{
}
bool isValid() const;
bool operator==(const Coordinate &other) const;
float latitude;
float longitude;
};
/** IATA airport code. */
class IataCode
{
public:
inline constexpr IataCode()
: m_letter0(0)
, m_letter1(0)
, m_letter2(0)
, m_valid(0)
{
}
inline explicit constexpr IataCode(const char iataStr[4])
: m_letter0(iataStr[0] - 'A')
, m_letter1(iataStr[1] - 'A')
, m_letter2(iataStr[2] - 'A')
, m_valid(1)
{
}
explicit IataCode(const QString &iataStr);
bool isValid() const;
bool operator<(IataCode rhs) const;
bool operator==(IataCode other) const;
bool operator!=(IataCode other) const;
QString toString() const;
private:
uint16_t toUInt16() const;
uint16_t m_letter0 : 5;
uint16_t m_letter1 : 5;
uint16_t m_letter2 : 5;
uint16_t m_valid :1;
};
/** Returns the geographical coordinates the airport with IATA code @p iataCode is in. */
Coordinate coordinateForAirport(IataCode iataCode);
/** Returns the timezone the airport with IATA code @p iataCode is in. */
QTimeZone timezoneForAirport(IataCode iataCode);
/** Attempts to find the unique IATA code for the given airport name. */
IataCode iataCodeFromName(const QString &name);
}
#endif // AIRPORTDB_H
This diff is collapsed.
/*
Copyright (c) 2017 Volker Krause <vkrause@kde.org>
This library is free software; you can redistribute it and/or modify it
under the terms of the GNU Library General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
This library is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
License for more details.
You should have received a copy of the GNU Library General Public License
along with this library; see the file COPYING.LIB. If not, write to the
Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
*/
#ifndef AIRPORTDB_P_H
#define AIRPORTDB_P_H
#include <cstdint>
namespace AirportDb {
// pack 24 bit offset, 8 bit length and 16 bit IATA index into 48bit with 16bit alignment
struct NameIndex
{
inline explicit constexpr NameIndex(uint32_t offset, uint8_t len, uint16_t idx)
: offset1((offset & 0x00ffff00) >> 8)
, offset2(offset & 0x000000ff)
, length(len)
, iataIndex(idx)
{
}
inline constexpr uint32_t offset() const
{
return offset1 << 8 | offset2;
}
uint16_t offset1;
uint8_t offset2;
uint8_t length;
uint16_t iataIndex;
};
static_assert(sizeof(NameIndex) == 6, "NameIndex size changed!");
static_assert(alignof(NameIndex) <= 2, "NameIndex alignment changed!");
}
#endif
/*
Copyright (c) 2017 Volker Krause <vkrause@kde.org>
This library is free software; you can redistribute it and/or modify it
under the terms of the GNU Library General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
This library is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
License for more details.
You should have received a copy of the GNU Library General Public License
along with this library; see the file COPYING.LIB. If not, write to the
Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
*/
#include "timezones.h"
#include "airportdb_p.h"
#include <QByteArray>
#include <QCommandLineParser>
#include <QCoreApplication>
#include <QDebug>
#include <QFile>
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QMap>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <QRegularExpression>
#include <QUrlQuery>
#include <cmath>
using namespace AirportDb;
struct Airport
{
QUrl uri;
QString iataCode;
QString icaoCode;
QString label;
QString alias;
QByteArray tz;
int tzOffset;
float longitude = NAN;
float latitude = NAN;
};
static void parseCoordinate(const QString &value, Airport &a)
{
const auto idx = value.indexOf(QLatin1Char(' '));
bool latOk = false, longOk = false;
a.longitude = value.midRef(6, idx - 6).toFloat(&latOk);
a.latitude = value.midRef(idx + 1, value.size() - idx - 2).toFloat(&longOk);
if (!latOk || !longOk) {
a.longitude = NAN;
a.latitude = NAN;
}
}
static QJsonArray queryWikidata(const char *sparqlQuery, const QString &debugFileName)
{
QUrl url(QStringLiteral("https://query.wikidata.org/sparql"));
QUrlQuery query;
query.addQueryItem(QStringLiteral("query"), QString::fromUtf8(sparqlQuery).trimmed().simplified());
query.addQueryItem(QStringLiteral("format"), QStringLiteral("json"));
url.setQuery(query);
QNetworkAccessManager nam;
auto reply = nam.get(QNetworkRequest(url));
QObject::connect(reply, &QNetworkReply::finished, QCoreApplication::instance(), &QCoreApplication::quit);
QCoreApplication::instance()->exec();
if (reply->error() != QNetworkReply::NoError) {
qWarning() << reply->errorString();
return {};
}
<