Commit 14800de9 authored by Volker Krause's avatar Volker Krause
Browse files

Change the way we store timezones in the static database

So far this were offsets into the the IANA string table, now it's a flat
enum. The old way needed 13 bits per record, the new only needs 9 bit, at
the cost of an extra ~800 bytes for an offset table to get back to IANA
names. This however quickly pays of when storing large quantities, which
we do (~37k in the current database, more in the upcoming experiments for
an efficient geo coordinate to timezone mapping).
parent dc9ed833
......@@ -90,26 +90,26 @@ private Q_SLOTS:
{
auto station = KnowledgeDb::stationForIbnr(IBNR{1234567});
QVERIFY(!station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone());
QCOMPARE(toQTimeZone(station.timezone), QTimeZone());
station = KnowledgeDb::stationForIbnr({});
QVERIFY(!station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone());
QCOMPARE(toQTimeZone(station.timezone), QTimeZone());
station = KnowledgeDb::stationForIbnr(IBNR{8011160});
QVERIFY(station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone("Europe/Berlin"));
QCOMPARE(toQTimeZone(station.timezone), QTimeZone("Europe/Berlin"));
QCOMPARE(station.country, CountryId{"DE"});
station = KnowledgeDb::stationForIbnr(IBNR{8501687});
QVERIFY(station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone("Europe/Zurich"));
QCOMPARE(toQTimeZone(station.timezone), QTimeZone("Europe/Zurich"));
QCOMPARE(station.country, CountryId{"CH"});
// Aachen West, very close to the NL border, should be in DE timezone
station = KnowledgeDb::stationForIbnr(IBNR{8000404});
QVERIFY(station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone("Europe/Berlin"));
QCOMPARE(toQTimeZone(station.timezone), QTimeZone("Europe/Berlin"));
QCOMPARE(station.country, CountryId{"DE"});
}
......@@ -117,15 +117,15 @@ private Q_SLOTS:
{
auto station = KnowledgeDb::stationForUic(UICStation{1234567});
QVERIFY(!station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone());
QCOMPARE(toQTimeZone(station.timezone), QTimeZone());
station = KnowledgeDb::stationForUic({});
QVERIFY(!station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone());
QCOMPARE(toQTimeZone(station.timezone), QTimeZone());
station = KnowledgeDb::stationForUic(UICStation{1001332});
QVERIFY(station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone("Europe/Helsinki"));
QCOMPARE(toQTimeZone(station.timezone), QTimeZone("Europe/Helsinki"));
QCOMPARE(station.country, CountryId{"FI"});
}
......@@ -133,27 +133,27 @@ private Q_SLOTS:
{
auto station = KnowledgeDb::stationForGaresConnexionsId({});
QVERIFY(!station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone());
QCOMPARE(toQTimeZone(station.timezone), QTimeZone());
station = KnowledgeDb::stationForGaresConnexionsId(GaresConnexionsId{"XXXXX"});
QVERIFY(!station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone());
QCOMPARE(toQTimeZone(station.timezone), QTimeZone());
station = KnowledgeDb::stationForGaresConnexionsId(GaresConnexionsId{"FRAES"});
QVERIFY(station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone("Europe/Paris"));
QCOMPARE(toQTimeZone(station.timezone), QTimeZone("Europe/Paris"));
QCOMPARE(station.country, CountryId{"FR"});
station = KnowledgeDb::stationForGaresConnexionsId(GaresConnexionsId{QStringLiteral("FRXYT")});
QVERIFY(station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone("Europe/Paris"));
QCOMPARE(toQTimeZone(station.timezone), QTimeZone("Europe/Paris"));
QCOMPARE(station.country, CountryId{"FR"});
station = KnowledgeDb::stationForGaresConnexionsId(GaresConnexionsId{"CHGVA"});
QEXPECT_FAIL("", "Wikidata does not supply ids for non-French stations yet", Continue);
QVERIFY(station.coordinate.isValid());
QEXPECT_FAIL("", "Wikidata does not supply ids for non-French stations yet", Continue);
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone("Europe/Zurich"));
QCOMPARE(toQTimeZone(station.timezone), QTimeZone("Europe/Zurich"));
QEXPECT_FAIL("", "Wikidata does not supply ids for non-French stations yet", Continue);
QCOMPARE(station.country, CountryId{"CH"});
}
......@@ -219,9 +219,9 @@ private Q_SLOTS:
{
using namespace KnowledgeDb;
QCOMPARE(KnowledgeDb::timezoneForCountry(CountryId{"DE"}).toQTimeZone(), QTimeZone("Europe/Berlin"));
QCOMPARE(KnowledgeDb::timezoneForCountry(CountryId{"FR"}).toQTimeZone(), QTimeZone("Europe/Paris"));
QCOMPARE(KnowledgeDb::timezoneForCountry(CountryId{"BR"}).toQTimeZone(), QTimeZone());
QCOMPARE(toQTimeZone(timezoneForCountry(CountryId{"DE"})), QTimeZone("Europe/Berlin"));
QCOMPARE(toQTimeZone(timezoneForCountry(CountryId{"FR"})), QTimeZone("Europe/Paris"));
QCOMPARE(toQTimeZone(timezoneForCountry(CountryId{"BR"})), QTimeZone());
}
void testUICCountryCodeLookup()
......@@ -244,27 +244,27 @@ private Q_SLOTS:
{
auto station = KnowledgeDb::stationForIndianRailwaysStationCode(QString());
QVERIFY(!station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone());
QCOMPARE(toQTimeZone(station.timezone), QTimeZone());
station = KnowledgeDb::stationForIndianRailwaysStationCode(QStringLiteral("NDLS"));
QVERIFY(station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone("Asia/Kolkata"));
QCOMPARE(toQTimeZone(station.timezone), QTimeZone("Asia/Kolkata"));
QCOMPARE(station.country, CountryId{"IN"});
station = KnowledgeDb::stationForIndianRailwaysStationCode(QStringLiteral("ndls"));
QVERIFY(!station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone());
QCOMPARE(toQTimeZone(station.timezone), QTimeZone());
}
void testFinishStationCodeLookup()
{
auto station = KnowledgeDb::stationForVRStationCode(VRStationCode(QStringLiteral("HKI")));
QVERIFY(station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone("Europe/Helsinki"));
QCOMPARE(toQTimeZone(station.timezone), QTimeZone("Europe/Helsinki"));
station = KnowledgeDb::stationForVRStationCode(VRStationCode(QStringLiteral("BLÄ")));
QVERIFY(!station.coordinate.isValid());
QCOMPARE(station.timezone.toQTimeZone(), QTimeZone());
QCOMPARE(toQTimeZone(station.timezone), QTimeZone());
}
};
......
......@@ -284,12 +284,12 @@ QDateTime ExtractorPostprocessorPrivate::processTrainTripTime(QDateTime dt, cons
QTimeZone tz;
if (station.identifier().startsWith(QLatin1String("sncf:"))) {
const auto record = KnowledgeDb::stationForGaresConnexionsId(KnowledgeDb::GaresConnexionsId{station.identifier().mid(5)});
tz = record.timezone.toQTimeZone();
tz = KnowledgeDb::toQTimeZone(record.timezone);
} else if (station.identifier().startsWith(QLatin1String("ibnr:"))) {
const auto record = KnowledgeDb::stationForIbnr(KnowledgeDb::IBNR{station.identifier().mid(5).toUInt()});
tz = record.timezone.toQTimeZone();
tz = KnowledgeDb::toQTimeZone(record.timezone);
} else if (!station.address().addressCountry().isEmpty()) {
tz = KnowledgeDb::timezoneForCountry(KnowledgeDb::CountryId{station.address().addressCountry()}).toQTimeZone();
tz = KnowledgeDb::toQTimeZone(KnowledgeDb::timezoneForCountry(KnowledgeDb::CountryId{station.address().addressCountry()}));
}
if (!tz.isValid()) {
return dt;
......@@ -565,7 +565,7 @@ QDateTime ExtractorPostprocessorPrivate::processTimeForLocation(QDateTime dt, co
QTimeZone tz;
if (!place.address().addressCountry().isEmpty()) {
tz = KnowledgeDb::timezoneForCountry(KnowledgeDb::CountryId{place.address().addressCountry()}).toQTimeZone();
tz = KnowledgeDb::toQTimeZone(KnowledgeDb::timezoneForCountry(KnowledgeDb::CountryId{place.address().addressCountry()}));
}
if (!tz.isValid()) {
return dt;
......
......@@ -62,7 +62,7 @@ function(generate_db dbtype outfile)
endfunction()
generate_db(country countrydb_data.cpp)
generate_db(timezone timezonedb_data.cpp)
generate_db(timezoneheader timezonedb_data_p.h)
generate_db(timezoneheader timezonedb_data.h)
generate_db(airport airportdb_data.cpp ${OSM_PLANET_DIR}/airports-bbox.osm)
generate_db(trainstation trainstationdb_data.cpp)
......
......@@ -325,7 +325,7 @@ bool AirportDbGenerator::generate(QIODevice* out)
#include "airportdb_p.h"
#include "knowledgedb.h"
#include "timezonedb.h"
#include "timezonedb_data_p.h"
#include "timezonedb_data.h"
#include <limits>
......
......@@ -57,7 +57,7 @@ void CodeGen::writeCountryIsoCode(QIODevice *out, const QString &isoCode)
void CodeGen::writeTimezone(QIODevice *out, const QByteArray &tzName)
{
if (tzName.isEmpty()) {
out->write("Timezone{}");
out->write("Tz::Undefined");
} else {
out->write("Tz::");
writeTimezoneEnum(out, tzName);
......
......@@ -32,7 +32,7 @@ void TimezoneDbGenerator::generate(QIODevice *out)
out->write(R"(
#include "timezonedb_p.h"
#include "timezonedb_data_p.h"
#include "timezonedb_data.h"
namespace KItinerary {
namespace KnowledgeDb {
......@@ -40,7 +40,7 @@ namespace KnowledgeDb {
// timezone name strings
static const char timezone_names[] =
)");
// timezone string tables
// timezone string table
for (const auto &tz : tzDb.m_zones) {
out->write(" ");
out->write("\"");
......@@ -49,6 +49,22 @@ static const char timezone_names[] =
}
out->write(R"(;
static constexpr const uint16_t timezone_names_offsets[] = {
)");
out->write(QByteArray::number(tzDb.m_zones.front().size()));
out->write(", // Undefined\n");
// offsets into timezone string table
for (const auto &tz : tzDb.m_zones) {
out->write(" ");
out->write(QByteArray::number(tzDb.offset(tz)));
out->write(", // ");
out->write(tz);
out->write("\n");
}
out->write(R"(};
static constexpr const CountryTimezoneMap country_timezone_map[] = {
)");
......@@ -76,23 +92,22 @@ void TimezoneDbGenerator::generateHeader(QIODevice *out)
Timezones tzDb;
out->write(R"(
#ifndef KITINERARY_KNOWLEDGEDB_TIMEZONEDB_DATA_P_H
#define KITINERARY_KNOWLEDGEDB_TIMEZONEDB_DATA_P_H
#ifndef KITINERARY_KNOWLEDGEDB_TIMEZONEDB_DATA_H
#define KITINERARY_KNOWLEDGEDB_TIMEZONEDB_DATA_H
#include <cstdint>
namespace KItinerary {
namespace KnowledgeDb {
/** Enum representing all timezones, values match the offsets into the timezone name string table. */
/** Enum representing all timezones. */
enum class Tz : uint16_t {
Undefined,
)");
for (const auto &tz : tzDb.m_zones) {
out->write(" ");
CodeGen::writeTimezoneEnum(out, tz);
out->write(" = ");
out->write(QByteArray::number(tzDb.offset(tz)));
out->write(",\n");
}
out->write(R"(};
......
......@@ -62,7 +62,7 @@ bool TrainStationDbGenerator::generate(QIODevice *out)
#include "knowledgedb.h"
#include "timezonedb.h"
#include "trainstationdb.h"
#include "timezonedb_data_p.h"
#include "timezonedb_data.h"
namespace KItinerary {
namespace KnowledgeDb {
......
......@@ -60,7 +60,7 @@ QTimeZone timezoneForAirport(IataCode iataCode)
return {};
}
return (*it).timezone.toQTimeZone();
return KnowledgeDb::toQTimeZone((*it).timezone);
}
KnowledgeDb::CountryId countryForAirport(IataCode iataCode)
......
......@@ -38,7 +38,7 @@ namespace KnowledgeDb {
struct Airport {
IataCode iataCode;
CountryId country;
Timezone timezone;
Tz timezone;
};
/** Returns the geographical coordinates the airport with IATA code @p iataCode is in. */
......
......@@ -9,7 +9,7 @@
#include "airportdb_p.h"
#include "knowledgedb.h"
#include "timezonedb.h"
#include "timezonedb_data_p.h"
#include "timezonedb_data.h"
#include <limits>
......@@ -21,22 +21,27 @@
#include <QTimeZone>
using namespace KItinerary::KnowledgeDb;
using namespace KItinerary;
QTimeZone Timezone::toQTimeZone() const
const char* KnowledgeDb::tzId(KnowledgeDb::Tz tz)
{
if (offset > sizeof(timezone_names)) {
return timezone_names + timezone_names_offsets[static_cast<std::underlying_type<KnowledgeDb::Tz>::type>(tz)];
}
QTimeZone KnowledgeDb::toQTimeZone(Tz tz)
{
if (tz == Tz::Undefined) {
return {};
}
return QTimeZone(timezone_names + offset);
return QTimeZone(tzId(tz));
}
Timezone KItinerary::KnowledgeDb::timezoneForCountry(CountryId country)
KnowledgeDb::Tz KnowledgeDb::timezoneForCountry(CountryId country)
{
const auto it = std::lower_bound(std::begin(country_timezone_map), std::end(country_timezone_map), country);
if (it != std::end(country_timezone_map) && (*it).country == country) {
return (*it).timezone;
}
return {};
return Tz::Undefined;
}
......@@ -20,6 +20,7 @@
#include <kitinerary_export.h>
#include "countrydb.h"
#include "timezonedb_data.h"
#include <cstdint>
#include <limits>
......@@ -28,28 +29,16 @@ class QTimeZone;
namespace KItinerary {
namespace KnowledgeDb {
enum class Tz : uint16_t;
/** Returns the IANA timezone id for @p tz. */
const char* tzId(Tz tz);
/** Timezone type as used in the database.
* @note Do not use in API/ABI.
*/
struct Timezone {
inline constexpr Timezone() = default;
inline constexpr Timezone(Tz tz)
: offset(static_cast<uint16_t>(tz))
{}
/** Returns the corresponding QTimeZone. */
KITINERARY_EXPORT QTimeZone toQTimeZone() const;
// offset into timezone name string table
uint16_t offset = std::numeric_limits<uint16_t>::max();
};
/** Returns the corresponding QTimeZone. */
KITINERARY_EXPORT QTimeZone toQTimeZone(Tz tz);
/** Returns the timezone for the given country, as long as there is exactly
* one timezone used in that country.
*/
KITINERARY_EXPORT Timezone timezoneForCountry(CountryId country);
KITINERARY_EXPORT Tz timezoneForCountry(CountryId country);
}
}
......
/*
* This code is auto-generated from Wikidata data. Licensed under CC0.
* SPDX-License-Identifier: ODbL-1.0
*
* This code is auto-generated from OpenStreetMap (licensed under ODbL) and Wikidata (licensed under CC0), do not edit!
*/
#include "timezonedb_p.h"
#include "timezonedb_data_p.h"
#include "timezonedb_data.h"
namespace KItinerary {
namespace KnowledgeDb {
......@@ -426,6 +428,424 @@ static const char timezone_names[] =
"Pacific/Wallis\0"
;
static constexpr const uint16_t timezone_names_offsets[] = {
14, // Undefined
0, // Africa/Abidjan
15, // Africa/Accra
28, // Africa/Addis_Ababa
47, // Africa/Algiers
62, // Africa/Asmara
76, // Africa/Bamako
90, // Africa/Bangui
104, // Africa/Banjul
118, // Africa/Bissau
132, // Africa/Blantyre
148, // Africa/Brazzaville
167, // Africa/Bujumbura
184, // Africa/Cairo
197, // Africa/Casablanca
215, // Africa/Ceuta
228, // Africa/Conakry
243, // Africa/Dakar
256, // Africa/Dar_es_Salaam
277, // Africa/Djibouti
293, // Africa/Douala
307, // Africa/El_Aaiun
323, // Africa/Freetown
339, // Africa/Gaborone
355, // Africa/Harare
369, // Africa/Johannesburg
389, // Africa/Juba
401, // Africa/Kampala
416, // Africa/Khartoum
432, // Africa/Kigali
446, // Africa/Kinshasa
462, // Africa/Lagos
475, // Africa/Libreville
493, // Africa/Lome
505, // Africa/Luanda
519, // Africa/Lubumbashi
537, // Africa/Lusaka
551, // Africa/Malabo
565, // Africa/Maputo
579, // Africa/Maseru
593, // Africa/Mbabane
608, // Africa/Mogadishu
625, // Africa/Monrovia
641, // Africa/Nairobi
656, // Africa/Ndjamena
672, // Africa/Niamey
686, // Africa/Nouakchott
704, // Africa/Ouagadougou
723, // Africa/Porto-Novo
741, // Africa/Sao_Tome
757, // Africa/Tripoli
772, // Africa/Tunis
785, // Africa/Windhoek
801, // America/Adak
814, // America/Anchorage
832, // America/Anguilla
849, // America/Antigua
865, // America/Araguaina
883, // America/Argentina/Buenos_Aires
914, // America/Argentina/Catamarca
942, // America/Argentina/Cordoba
968, // America/Argentina/Jujuy
992, // America/Argentina/La_Rioja
1019, // America/Argentina/Mendoza
1045, // America/Argentina/Rio_Gallegos
1076, // America/Argentina/Salta
1100, // America/Argentina/San_Juan
1127, // America/Argentina/San_Luis
1154, // America/Argentina/Tucuman
1180, // America/Argentina/Ushuaia
1206, // America/Aruba
1220, // America/Asuncion
1237, // America/Atikokan
1254, // America/Bahia
1268, // America/Bahia_Banderas
1291, // America/Barbados
1308, // America/Belem
1322, // America/Belize
1337, // America/Blanc-Sablon
1358, // America/Boa_Vista
1376, // America/Bogota
1391, // America/Boise
1405, // America/Cambridge_Bay
1427, // America/Campo_Grande
1448, // America/Cancun
1463, // America/Caracas
1479, // America/Cayenne
1495, // America/Cayman
1510, // America/Chicago
1526, // America/Chihuahua
1544, // America/Costa_Rica
1563, // America/Creston
1579, // America/Cuiaba
1594, // America/Curacao
1610, // America/Danmarkshavn
1631, // America/Dawson
1646, // America/Dawson_Creek
1667, // America/Denver
1682, // America/Detroit
1698, // America/Dominica
1715, // America/Edmonton
1732, // America/Eirunepe
1749, // America/El_Salvador
1769, // America/Fort_Nelson
1789, // America/Fortaleza
1807, // America/Glace_Bay
1825, // America/Godthab
1841, // America/Goose_Bay
1859, // America/Grand_Turk
1878, // America/Grenada
1894, // America/Guadeloupe
1913, // America/Guatemala
1931, // America/Guayaquil
1949, // America/Guyana
1964, // America/Halifax
1980, // America/Havana
1995, // America/Hermosillo
2014, // America/Indiana/Indianapolis
2043, // America/Indiana/Knox
2064, // America/Indiana/Marengo
2088, // America/Indiana/Petersburg
2115, // America/Indiana/Tell_City
2141, // America/Indiana/Vevay
2163, // America/Indiana/Vincennes
2189, // America/Indiana/Winamac
2213, // America/Inuvik
2228, // America/Iqaluit
2244, // America/Jamaica
2260, // America/Juneau
2275, // America/Kentucky/Louisville
2303, // America/Kentucky/Monticello
2331, // America/Kralendijk
2350, // America/La_Paz
2365, // America/Lima
2378, // America/Los_Angeles
2398, // America/Lower_Princes
2420, // America/Maceio
2435, // America/Managua
2451, // America/Manaus
2466, // America/Marigot
2482, // America/Martinique
2501, // America/Matamoros
2519, // America/Mazatlan
2536, // America/Menominee
2554, // America/Merida
2569, // America/Metlakatla
2588, // America/Mexico_City
2608, // America/Miquelon
2625, // America/Moncton
2641, // America/Monterrey
2659, // America/Montevideo
2678, // America/Montserrat
2697, // America/Nassau
2712, // America/New_York
2729, // America/Nipigon
2745, // America/Nome
2758, // America/Noronha
2774, // America/North_Dakota/Beulah
2802, // America/North_Dakota/Center
2830, // America/North_Dakota/New_Salem
2861, // America/Ojinaga
2877, // America/Panama
2892, // America/Pangnirtung
2912, // America/Paramaribo
2931, // America/Phoenix
2947, // America/Port-au-Prince
2970, // America/Port_of_Spain
2992, // America/Porto_Velho
3012, // America/Puerto_Rico
3032, // America/Punta_Arenas
3053, // America/Rainy_River
3073, // America/Rankin_Inlet
3094, // America/Recife
3109, // America/Regina
3124, // America/Resolute
3141, // America/Rio_Branco
3160, // America/Santarem
3177, // America/Santiago
3194, // America/Santo_Domingo
3216, // America/Sao_Paulo
3234, // America/Scoresbysund
3255, // America/Sitka
3269, // America/St_Barthelemy
3291, // America/St_Johns
3308, // America/St_Kitts
3325, // America/St_Lucia
3342, // America/St_Thomas
3360, // America/St_Vincent
3379, // America/Swift_Current
3401, // America/Tegucigalpa
3421, // America/Thule
3435, // America/Thunder_Bay
3455, // America/Tijuana
3471, // America/Toronto
3487, // America/Tortola
3503, // America/Vancouver
3521, // America/Whitehorse
3540, // America/Winnipeg
3557, // America/Yakutat
3573, // America/Yellowknife
3593, // Antarctica/Macquarie
3614, // Arctic/Longyearbyen
3634, // Asia/Aden
3644, // Asia/Almaty
3656, // Asia/Amman
3667, // Asia/Anadyr
3679, // Asia/Aqtau
3690, // Asia/Aqtobe
3702, // Asia/Ashgabat
3716, // Asia/Atyrau
3728, // Asia/Baghdad
3741, // Asia/Bahrain
3754, // Asia/Baku
3764, // Asia/Bangkok