Commit 2814c9a3 authored by Volker Krause's avatar Volker Krause
Browse files

Complete the port to KI18nLocaleData

This removes a lot of now obsolete timezone and country lookup code and
data tables.
parent 974665a4
Pipeline #98830 passed with stage
in 13 minutes and 35 seconds
......@@ -6,6 +6,7 @@
#include <config-kitinerary.h>
#include <knowledgedb/airportdb.h>
#include <knowledgedb/timezonedb.h>
#include <KItinerary/LocationUtil>
......
......@@ -8,6 +8,7 @@
#include <config-kitinerary.h>
#include <knowledgedb/alphaid.h>
#include <knowledgedb/timezonedb.h>
#include <knowledgedb/trainstationdb.h>
#include <QDebug>
......@@ -291,11 +292,7 @@ private Q_SLOTS:
QCOMPARE(timezoneForLocation(32.55783, -117.04773, u"US"), QTimeZone("America/Los_Angeles"));
// Cordoba (AR), AR has several sub-zones that are all equivalent
#if HAVE_KI18N_LOCALE_DATA
QCOMPARE(timezoneForLocation(-31.4, -64.2, u"AR"), QTimeZone("America/Argentina/Buenos_Aires"));
#else
QCOMPARE(timezoneForLocation(-31.4, -64.2, u"AR"), QTimeZone("America/Argentina/Cordoba"));
#endif
// polar regions
QCOMPARE(timezoneForLocation(-90.0, 0.0, {}), QTimeZone());
......@@ -308,38 +305,6 @@ private Q_SLOTS:
QCOMPARE(timezoneForLocation(NAN, NAN, u"LU"), QTimeZone("Europe/Luxembourg"));
}
void testCountryFromCoordinate()
{
using namespace KnowledgeDb;
// basic tests
QCOMPARE(countryForCoordinate(52.4, 13.1), QLatin1String{"DE"});
QCOMPARE(countryForCoordinate(-8.0, -35.0), QLatin1String{"BR"});
QCOMPARE(countryForCoordinate(-36.5, 175.0), QLatin1String{"NZ"});
QCOMPARE(countryForCoordinate(44.0, -79.5), QLatin1String{"CA"});
// ambiguous locations
QCOMPARE(countryForCoordinate(51.44344, 4.93373), QString());
#if HAVE_KI18N_LOCALE_DATA
// special case: northern Vietnam has a non-VN timezone (not the case anywhere else in the world up to 2020a)
QCOMPARE(countryForCoordinate(21.0, 106.0), QLatin1String("VN"));
QCOMPARE(countryForCoordinate(10.5, 107.0), QLatin1String{"VN"});
QCOMPARE(countryForCoordinate(13.7, 100.4), QLatin1String("TH"));
#else
// special case: northern Vietnam has a non-VN timezone (not the case anywhere else in the world up to 2020a)
QCOMPARE(countryForCoordinate(21.0, 106.0), QString());
QCOMPARE(countryForCoordinate(10.5, 107.0), QLatin1String{"VN"});
QCOMPARE(countryForCoordinate(13.7, 100.4), QString());
// disputed areas
QCOMPARE(countryForCoordinate(45.0, 34.0), QString());
#endif
// overseas territories with separate ISO 3166-1 codes
QCOMPARE(countryForCoordinate(4.8, -52.3), QLatin1String{"GF"}); // could also be "FR"
}
void testUICCountryCodeLookup()
{
using namespace KnowledgeDb;
......@@ -348,16 +313,6 @@ private Q_SLOTS:
QCOMPARE(KnowledgeDb::countryIdForUicCode(0), CountryId{});
}
#if !HAVE_KI18N_LOCALE_DATA
void testIso3Lookup()
{
using namespace KnowledgeDb;
QCOMPARE(KnowledgeDb::countryIdFromIso3166_1alpha3(CountryId3{"ITA"}), CountryId{"IT"});
QCOMPARE(KnowledgeDb::countryIdFromIso3166_1alpha3(CountryId3{"FOO"}), CountryId{});
}
#endif
void testIndianRailwaysStationCodeLookup()
{
auto station = KnowledgeDb::stationForIndianRailwaysStationCode(QString());
......
......@@ -9,7 +9,6 @@ add_executable(generate-knowledgedb
airportdbgenerator.cpp
countrydbgenerator.cpp
osmairportdb.cpp
timezonedbgenerator.cpp
trainstationdbgenerator.cpp
util.cpp
../lib/stringutil.cpp
......@@ -49,8 +48,6 @@ function(generate_db dbtype outfile)
set(outfiles ${outfiles} PARENT_SCOPE)
endfunction()
generate_db(country countrydb_data.cpp)
generate_db(timezone timezonedb_data.cpp)
generate_db(timezoneheader timezonedb_data.h)
generate_db(airport airportdb_data.cpp ${OSM_PLANET_DIR}/airports-bbox.osm)
generate_db(trainstation trainstationdb_data.cpp)
......
Updating the airport database
=============================
(1) Download timezone shapefile
(1) Configure with -DOSM_PLANT_DIR=<dir> pointing to a location
with plenty of free space for a full OSM DB download (200+GB).
Pick the latest timezones.shapefile.zip from
https://github.com/evansiroky/timezone-boundary-builder/releases
Extract the zip file into this folder.
(2) Generate the timezone index
- If necessary, adjust the output path at the end of qgis/generate-z-order-curve-spatial-index.py.
- Open timezones.qgs in QGIS.
- Select "Plugins" > "Python Console".
- In the Python Console, click on the "Show Editor" toolbar icon.
- In the Python script editor, select "Open Script..." in the toolbar, and open qgis/generate-z-order-curve-spatial-index.py.
- In the Python script editor, select "Run Script".
- Wait.
- Once the script has finished, an updated timezone index can be found in the output file set in the first step.
(3) Run the code generator
(2) Run the code generator
Run `make rebuild-knowledgedb` in the build dir of this folder.
......
......@@ -296,8 +296,6 @@ bool AirportDbGenerator::generate(QIODevice* out)
#include "airportdb.h"
#include "airportdb_p.h"
#include "knowledgedb.h"
#include "timezonedb.h"
#include "timezonedb_data.h"
#include <limits>
......
......@@ -47,7 +47,6 @@ namespace KItinerary {
namespace KnowledgeDb {
)");
writeCountryTable(out);
writeIso3CodeTable(out);
writeUicCodeTable(out);
out->write(R"(
}
......@@ -61,10 +60,9 @@ namespace KnowledgeDb {
bool CountryDbGenerator::fetchCountryList()
{
const auto countryArray = WikiData::query(R"(
SELECT DISTINCT ?country ?countryLabel ?isoCode ?iso3Code ?demolished WHERE {
SELECT DISTINCT ?country ?countryLabel ?isoCode ?demolished WHERE {
?country (wdt:P31/wdt:P279*) wd:Q6256.
?country wdt:P297 ?isoCode.
?country wdt:P298 ?iso3Code.
OPTIONAL { ?country wdt:P576 ?demolished. }
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
} ORDER BY (?country))", "wikidata_country.json");
......@@ -93,13 +91,6 @@ bool CountryDbGenerator::fetchCountryList()
} else {
m_isoCodeMap[isoCode] = uri;
}
const auto iso3Code = countryObj.value(QLatin1String("iso3Code")).toObject().value(QLatin1String("value")).toString().toUpper();
if (iso3Code.size() != 3 || !Util::containsOnlyLetters(iso3Code)) {
qWarning() << "ISO 3166-1 alpha 3 format violation" << iso3Code << uri;
continue;
}
m_iso3CodeMap[iso3Code] = isoCode;
}
return true;
......@@ -275,19 +266,6 @@ void CountryDbGenerator::writeCountryTable(QIODevice *out)
out->write("};\n\n");
}
void Generator::CountryDbGenerator::writeIso3CodeTable(QIODevice *out)
{
out->write("static const IsoCountryCodeMapping iso_country_code_table[] = {\n");
for (const auto &kv : m_iso3CodeMap) {
out->write(" { CountryId3{\"");
out->write(kv.first.toUtf8());
out->write("\"}, CountryId{\"");
out->write(kv.second.toUtf8());
out->write("\"}},\n");
}
out->write("};\n\n");
}
void Generator::CountryDbGenerator::writeUicCodeTable(QIODevice *out)
{
out->write("static const UicCountryCodeMapping uic_country_code_table[] = {\n");
......
......@@ -40,14 +40,12 @@ private:
bool fetchUicCountryCodes();
QUrl insertOrMerge(const QJsonObject &obj);
void writeCountryTable(QIODevice *out);
void writeIso3CodeTable(QIODevice *out);
void writeUicCodeTable(QIODevice *out);
void printSummary();
std::vector<Country> m_countries;
std::map<QString, QUrl> m_isoCodeMap;
std::map<uint16_t, QString> m_uicCodeMap;
std::map<QString, QString> m_iso3CodeMap;
int m_isoCodeConflicts = 0;
};
......
......@@ -6,7 +6,6 @@
#include "airportdbgenerator.h"
#include "countrydbgenerator.h"
#include "timezonedbgenerator.h"
#include "trainstationdbgenerator.h"
#include <QCommandLineParser>
......@@ -42,12 +41,6 @@ int main(int argc, char **argv)
} else if (parser.value(dbOpt) == QLatin1String("country")) {
CountryDbGenerator gen;
return gen.generate(&out) ? 0 : 1;
} else if (parser.value(dbOpt) == QLatin1String("timezone")) {
TimezoneDbGenerator gen;
gen.generate(&out);
} else if (parser.value(dbOpt) == QLatin1String("timezoneheader")) {
TimezoneDbGenerator gen;
gen.generateHeader(&out);
} else if (parser.value(dbOpt) == QLatin1String("trainstation")) {
TrainStationDbGenerator gen;
return gen.generate(&out) ? 0 : 1;
......
#
# SPDX-FileCopyrightText: Volker Krause <vkrause@kde.org>
#
# SPDX-License-Identifier: LGPL-2.0-or-later
#
import functools
import datetime
import time
import pytz
import qgis.core
#
# parameters for the spatial index
#
featureAreaRatioThreshold = 0.02 # 1% at zDepth 11 is ~150m
zDepth = 11 # minimum tile size is 1/(2^zdepth), amount of bits needed to store z index is 2*zDepth
#
# z-order curve coordinate primitives
#
xStart = -180
xRange = 360
# cut out artic regions (starting at 65°S and 80°N), that saves about 20% z-order curve coverage which we
# can better use to increase precision in more relevant areas
yStart = -65
yRange = 145
xStep = xRange / (1 << zDepth)
yStep = yRange / (1 << zDepth)
def z2x(z):
x = 0
for i in range(0, zDepth):
x += (z & (1 << i * 2)) >> i
return x
def z2y(z):
y = 0
for i in range(0, zDepth):
y += (z & (1 << (1 + i * 2))) >> (i + 1)
return y
def rectForZ(z, depth):
mask = (1 << (2*(zDepth - depth))) - 1
x = z2x(z & ~mask) * xStep + xStart
y = z2y(z & ~mask) * yStep + yStart
xSize = xRange / (1 << depth)
ySize = yRange / (1 << depth)
return QgsRectangle(x, y, x + xSize, y + ySize)
#
# Parallelized spatial index computation of a single sub-tile
#
LOG_CATEGORY = 'SpatialIndexBuilder'
class SpatialIndexerSubTask(QgsTask):
def __init__(self, layer, zStart, zStartDepth):
super().__init__('Compute spatial index sub-tile ' + str(zStart), QgsTask.CanCancel)
self.layer = layer
self.zStart = zStart
self.zStartDepth = zStartDepth
self.lastFeature = []
self.exception = None
self.result = []
def run(self):
try:
self.computeTile(self.zStart, self.zStartDepth)
except Exception as e:
self.exception = e
QgsMessageLog.logMessage('Exception in task "{}"'.format(self.exception), LOG_CATEGORY, Qgis.Info)
return True
def computeTile(self, zStart, depth):
if self.isCanceled() or depth < 1:
return
z = zStart
d = depth - 1
zIncrement = 1 << (2*d)
for i in range(0, 4):
# find features in the input vector layer inside our current tile
layerFeatures = []
for f in self.layer.getFeatures(rectForZ(z, zDepth -d)):
layerFeatures.append(f)
feature = []
featureCount = len(layerFeatures)
# recurse on conflicts
if depth > 1 and featureCount > 1:
self.computeTile(z, d)
# leaf tile, process the result
else:
# translate this into our result format: a list of (feature,areaRatio) tuples
if featureCount == 1: # we can skip the expensive area ratio computation in this case
feature = [(layerFeatures[0]['tzid'], 1)]
elif featureCount > 1:
rectGeo = QgsGeometry.fromRect(rectForZ(z, zDepth - d))
for f in layerFeatures:
featureArea = f.geometry().intersection(rectGeo).area()
feature.append((f['tzid'], featureArea / rectGeo.area()))
feature = self.normalizeAndFilter(feature)
# if there's a change to the previous value, propagate to the result output
if self.lastFeature != feature and feature != []:
self.result.append((z, feature))
self.lastFeature = feature
z += zIncrement
def isValidFeature(self, f):
return not f.startswith("Etc/")
def normalizeAndFilter(self, r):
if len(r) == 0:
return r
r = list(filter(lambda x: x[1] > featureAreaRatioThreshold, r))
r = list(filter(lambda x: self.isValidFeature(x[0]), r))
if len(r) < 1:
return r
n = functools.reduce(lambda n, f: n + f[1], r, 0)
r = [(k, v/n) for (k, v) in r]
r.sort(key = lambda x: x[1], reverse = True)
return r
def finished(self, result):
if not result and self.exception != None:
QgsMessageLog.logMessage('Task "{name}" Exception: {exception}'.format(name=self.description(), exception=self.exception), LOG_CATEGORY, Qgis.Critical)
raise self.exception
#
# Tasks for spawning the sub-tasks doing the actual work, and accumulating the result
#
class SpatialIndexerTask(QgsTask):
def __init__(self, layer, outputFileName):
super().__init__('Compute spatial index', QgsTask.CanCancel)
self.setDependentLayers([layer])
self.tasks = []
self.outputFileName = outputFileName
self.exception = None
self.conflictTiles = 0
self.hardConflictTiles = 0
self.startTime = time.time()
startDepth = 4
startIncrement = 1 << (2 * (zDepth - startDepth))
for i in range(0, (1 << (2 * startDepth))):
task = SpatialIndexerSubTask(layer, i * startIncrement, zDepth - startDepth)
self.addSubTask(task, [], QgsTask.ParentDependsOnSubTask)
self.tasks.append(task)
def run(self):
try:
QgsMessageLog.logMessage('Aggregating results...', LOG_CATEGORY, Qgis.Info)
out = open(self.outputFileName, "w")
out.write("""/*
* SPDX-License-Identifier: ODbL-1.0
* SPDX-FileCopyrightText: OpenStreetMap contributors
*
* Autogenerated spatial index generated using QGIS.
*/
#include "timezonedb_p.h"
namespace KItinerary {
namespace KnowledgeDb {
""")
out.write('static constexpr const TimezoneZIndexParams timezone_index_params = {{ {xStart}, {xRange}, {yStart}, {yRange}, {zDepth} }};\n\n'
.format(xStart = xStart, xRange = xRange, yStart = yStart, yRange = yRange, zDepth = zDepth));
out.write('static constexpr TimezoneZIndexEntry timezone_index[] = {\n')
prevFeature = ""
prevAmbiguous = False
for task in self.tasks:
for (z,res) in task.result:
feature = ""
isAmbiguous = False
if len(res) > 1:
self.conflictTiles += 1
isAmbiguous = True
if len(res) > 1 and self.isConflict(res):
feature = "Undefined"
self.hardConflictTiles += 1
else:
feature = res[0][0].replace('/', '_').replace('-', '_')
coverage = res[0][1]
if prevFeature == feature and prevAmbiguous == isAmbiguous:
continue
prevFeature = feature
prevAmbiguous = isAmbiguous
if isAmbiguous:
out.write(" { " + str(z) + ", Tz::" + feature + ", true }, // " + str(coverage) + "\n")
else:
out.write(" { " + str(z) + ", Tz::" + feature + ", false },\n")
out.write("};\n}\n}\n")
out.close()
return True
except Exception as e:
self.exception = e
QgsMessageLog.logMessage('Exception in task "{}"'.format(self.exception), LOG_CATEGORY, Qgis.Info)
return False
def isConflict(self, r):
tz = pytz.timezone(r[0][0])
return not all(self.isSameTimezone(tz, pytz.timezone(x[0])) for x in r[1:])
def isSameTimezone(self, lhs, rhs):
try:
# hacky tz comparison, lacking access to the rules for comparing actual DST transition times
dt = datetime.datetime.today().toordinal()
return all(lhs.utcoffset(datetime.datetime.fromordinal(dt + 30*x)) == rhs.utcoffset(datetime.datetime.fromordinal(dt + 30*x))
and lhs.tzname(datetime.datetime.fromordinal(dt + 30*x)) == rhs.tzname(datetime.datetime.fromordinal(dt + 30*x)) for x in range(0, 11))
except:
return False
def finished(self, result):
QgsMessageLog.logMessage('Finished task "{}"'.format(self.description()), LOG_CATEGORY, Qgis.Info)
QgsMessageLog.logMessage(' "{}" of the area is conflicting'.format(str(self.conflictTiles / (1 << (2 * zDepth)))), LOG_CATEGORY, Qgis.Info)
QgsMessageLog.logMessage(' "{}" of the area is not covered'.format(str(self.hardConflictTiles / (1 << (2 * zDepth)))), LOG_CATEGORY, Qgis.Info)
QgsMessageLog.logMessage(' computation took "{}" seconds'.format(str(time.time() - self.startTime)), LOG_CATEGORY, Qgis.Info)
if not result and self.exception != None:
QgsMessageLog.logMessage('Task "{name}" Exception: {exception}'.format(name=self.description(), exception=self.exception), LOG_CATEGORY, Qgis.Critical)
raise self.exception
#
# actually launch things
#
task = SpatialIndexerTask(iface.activeLayer(), '/k/kde5/src/kitinerary/src/knowledgedb/timezone_zindex.cpp')
QgsApplication.taskManager().addTask(task)
/*
SPDX-FileCopyrightText: 2018 Volker Krause <vkrause@kde.org>
SPDX-License-Identifier: LGPL-2.0-or-later
*/
#include "timezonedbgenerator.h"
#include "codegen.h"
#include "timezones.h"
#include <QDebug>
#include <QIODevice>
using namespace KItinerary::Generator;
void TimezoneDbGenerator::generate(QIODevice *out)
{
CodeGen::writeLicenseHeaderOSM(out);
Timezones tzDb;
out->write(R"(
#include "timezonedb_p.h"
#include "timezonedb_data.h"
namespace KItinerary {
namespace KnowledgeDb {
// timezone name strings
static const char timezone_names[] =
)");
// timezone string table
for (const auto &tz : tzDb.m_zones) {
out->write(" ");
out->write("\"");
out->write(tz);
out->write("\\0\"\n");
}
out->write(R"(;
static constexpr const uint16_t timezone_names_offsets[] = {
)");
out->write(QByteArray::number(tzDb.m_zones.front().size()));
out->write(", // Undefined\n");
// offsets into timezone string table
for (const auto &tz : tzDb.m_zones) {
out->write(" ");
out->write(QByteArray::number(tzDb.offset(tz)));
out->write(", // ");
out->write(tz);
out->write("\n");
}
out->write(R"(};
static constexpr const CountryTimezoneMap country_timezone_map[] = {
)");
for (const auto &map : tzDb.m_countryZones) {
if (map.second.size() != 1) {
continue;
}
out->write(" { ");
CodeGen::writeCountryIsoCode(out, map.first);
out->write(", ");
CodeGen::writeTimezone(out, map.second.at(0));
out->write(" },\n");
}
out->write(R"(};
static constexpr const CountryId timezone_country_map[] = {
CountryId{}, // Undefined
)");
for (const auto &tz : tzDb.m_zones) {
out->write(" ");
CodeGen::writeCountryIsoCode(out, tzDb.m_countryForZone[tz]);
out->write(", // ");
out->write(tz);
out->write("\n");
}
out->write(R"(};
}
}
)");
}
void TimezoneDbGenerator::generateHeader(QIODevice *out)
{
CodeGen::writeLicenseHeaderOSM(out);
Timezones tzDb;
out->write(R"(
#ifndef KITINERARY_KNOWLEDGEDB_TIMEZONEDB_DATA_H
#define KITINERARY_KNOWLEDGEDB_TIMEZONEDB_DATA_H
#include <cstdint>
namespace KItinerary {
namespace KnowledgeDb {
/** Enum representing all timezones. */
enum class Tz : uint16_t {
Undefined,
)");
for (const auto &tz : tzDb.m_zones) {
out->write(" ");
CodeGen::writeTimezoneEnum(out, tz);