Complete the port to KI18nLocaleData

This removes a lot of now obsolete timezone and country lookup code and
data tables.

autotests/airportdbtest.cpp

@@ -6,6 +6,7 @@
 
 #include <config-kitinerary.h>
 #include <knowledgedb/airportdb.h>
+#include <knowledgedb/timezonedb.h>
 
 #include <KItinerary/LocationUtil>
 

autotests/knowledgedbtest.cpp

@@ -8,6 +8,7 @@
 
 #include <config-kitinerary.h>
 #include <knowledgedb/alphaid.h>
+#include <knowledgedb/timezonedb.h>
 #include <knowledgedb/trainstationdb.h>
 
 #include <QDebug>
@@ -291,11 +292,7 @@ private Q_SLOTS:
         QCOMPARE(timezoneForLocation(32.55783, -117.04773, u"US"), QTimeZone("America/Los_Angeles"));
 
         // Cordoba (AR), AR has several sub-zones that are all equivalent
-#if HAVE_KI18N_LOCALE_DATA
         QCOMPARE(timezoneForLocation(-31.4, -64.2, u"AR"), QTimeZone("America/Argentina/Buenos_Aires"));
-#else
-        QCOMPARE(timezoneForLocation(-31.4, -64.2, u"AR"), QTimeZone("America/Argentina/Cordoba"));
-#endif
 
         // polar regions
         QCOMPARE(timezoneForLocation(-90.0, 0.0, {}), QTimeZone());
@@ -308,38 +305,6 @@ private Q_SLOTS:
         QCOMPARE(timezoneForLocation(NAN, NAN, u"LU"), QTimeZone("Europe/Luxembourg"));
     }
 
-    void testCountryFromCoordinate()
-    {
-        using namespace KnowledgeDb;
-
-        // basic tests
-        QCOMPARE(countryForCoordinate(52.4, 13.1), QLatin1String{"DE"});
-        QCOMPARE(countryForCoordinate(-8.0, -35.0), QLatin1String{"BR"});
-        QCOMPARE(countryForCoordinate(-36.5, 175.0), QLatin1String{"NZ"});
-        QCOMPARE(countryForCoordinate(44.0, -79.5), QLatin1String{"CA"});
-
-        // ambiguous locations
-        QCOMPARE(countryForCoordinate(51.44344, 4.93373), QString());
-
-#if HAVE_KI18N_LOCALE_DATA
-        // special case: northern Vietnam has a non-VN timezone (not the case anywhere else in the world up to 2020a)
-        QCOMPARE(countryForCoordinate(21.0, 106.0), QLatin1String("VN"));
-        QCOMPARE(countryForCoordinate(10.5, 107.0), QLatin1String{"VN"});
-        QCOMPARE(countryForCoordinate(13.7, 100.4), QLatin1String("TH"));
-#else
-        // special case: northern Vietnam has a non-VN timezone (not the case anywhere else in the world up to 2020a)
-        QCOMPARE(countryForCoordinate(21.0, 106.0), QString());
-        QCOMPARE(countryForCoordinate(10.5, 107.0), QLatin1String{"VN"});
-        QCOMPARE(countryForCoordinate(13.7, 100.4), QString());
-
-        // disputed areas
-        QCOMPARE(countryForCoordinate(45.0, 34.0), QString());
-#endif
-
-        // overseas territories with separate ISO 3166-1 codes
-        QCOMPARE(countryForCoordinate(4.8, -52.3), QLatin1String{"GF"}); // could also be "FR"
-    }
-
     void testUICCountryCodeLookup()
     {
         using namespace KnowledgeDb;
@@ -348,16 +313,6 @@ private Q_SLOTS:
         QCOMPARE(KnowledgeDb::countryIdForUicCode(0), CountryId{});
     }
 
-#if !HAVE_KI18N_LOCALE_DATA
-    void testIso3Lookup()
-    {
-        using namespace KnowledgeDb;
-
-        QCOMPARE(KnowledgeDb::countryIdFromIso3166_1alpha3(CountryId3{"ITA"}), CountryId{"IT"});
-        QCOMPARE(KnowledgeDb::countryIdFromIso3166_1alpha3(CountryId3{"FOO"}), CountryId{});
-    }
-#endif
-
     void testIndianRailwaysStationCodeLookup()
     {
         auto station = KnowledgeDb::stationForIndianRailwaysStationCode(QString());

src/knowledgedb-generator/CMakeLists.txt

@@ -9,7 +9,6 @@ add_executable(generate-knowledgedb
     airportdbgenerator.cpp
     countrydbgenerator.cpp
     osmairportdb.cpp
-    timezonedbgenerator.cpp
     trainstationdbgenerator.cpp
     util.cpp
     ../lib/stringutil.cpp
@@ -49,8 +48,6 @@ function(generate_db dbtype outfile)
     set(outfiles ${outfiles} PARENT_SCOPE)
 endfunction()
 generate_db(country countrydb_data.cpp)
-generate_db(timezone timezonedb_data.cpp)
-generate_db(timezoneheader timezonedb_data.h)
 generate_db(airport airportdb_data.cpp ${OSM_PLANET_DIR}/airports-bbox.osm)
 generate_db(trainstation trainstationdb_data.cpp)
 

src/knowledgedb-generator/README

 Updating the airport database
 =============================
 
-(1) Download timezone shapefile
+(1) Configure with -DOSM_PLANT_DIR=<dir> pointing to a location
+with plenty of free space for a full OSM DB download (200+GB).
 
-Pick the latest timezones.shapefile.zip from
-https://github.com/evansiroky/timezone-boundary-builder/releases
-
-Extract the zip file into this folder.
-
-(2) Generate the timezone index
-
-- If necessary, adjust the output path at the end of qgis/generate-z-order-curve-spatial-index.py.
-- Open timezones.qgs in QGIS.
-- Select "Plugins" > "Python Console".
-- In the Python Console, click on the "Show Editor" toolbar icon.
-- In the Python script editor, select "Open Script..." in the toolbar, and open qgis/generate-z-order-curve-spatial-index.py.
-- In the Python script editor, select "Run Script".
-- Wait.
-- Once the script has finished, an updated timezone index can be found in the output file set in the first step.
-
-(3) Run the code generator
+(2) Run the code generator
 
 Run `make rebuild-knowledgedb` in the build dir of this folder.
 

src/knowledgedb-generator/airportdbgenerator.cpp

@@ -296,8 +296,6 @@ bool AirportDbGenerator::generate(QIODevice* out)
 #include "airportdb.h"
 #include "airportdb_p.h"
 #include "knowledgedb.h"
-#include "timezonedb.h"
-#include "timezonedb_data.h"
 
 #include <limits>
 

src/knowledgedb-generator/countrydbgenerator.cpp

@@ -47,7 +47,6 @@ namespace KItinerary {
 namespace KnowledgeDb {
 )");
     writeCountryTable(out);
-    writeIso3CodeTable(out);
     writeUicCodeTable(out);
     out->write(R"(
 }
@@ -61,10 +60,9 @@ namespace KnowledgeDb {
 bool CountryDbGenerator::fetchCountryList()
 {
     const auto countryArray = WikiData::query(R"(
-        SELECT DISTINCT ?country ?countryLabel ?isoCode ?iso3Code ?demolished WHERE {
+        SELECT DISTINCT ?country ?countryLabel ?isoCode ?demolished WHERE {
             ?country (wdt:P31/wdt:P279*) wd:Q6256.
             ?country wdt:P297 ?isoCode.
-            ?country wdt:P298 ?iso3Code.
             OPTIONAL { ?country wdt:P576 ?demolished. }
             SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
         } ORDER BY (?country))", "wikidata_country.json");
@@ -93,13 +91,6 @@ bool CountryDbGenerator::fetchCountryList()
         } else {
             m_isoCodeMap[isoCode] = uri;
         }
-
-        const auto iso3Code = countryObj.value(QLatin1String("iso3Code")).toObject().value(QLatin1String("value")).toString().toUpper();
-        if (iso3Code.size() != 3 || !Util::containsOnlyLetters(iso3Code)) {
-            qWarning() << "ISO 3166-1 alpha 3 format violation" << iso3Code << uri;
-            continue;
-        }
-        m_iso3CodeMap[iso3Code] = isoCode;
     }
 
     return true;
@@ -275,19 +266,6 @@ void CountryDbGenerator::writeCountryTable(QIODevice *out)
     out->write("};\n\n");
 }
 
-void Generator::CountryDbGenerator::writeIso3CodeTable(QIODevice *out)
-{
-    out->write("static const IsoCountryCodeMapping iso_country_code_table[] = {\n");
-    for (const auto &kv : m_iso3CodeMap) {
-        out->write("    { CountryId3{\"");
-        out->write(kv.first.toUtf8());
-        out->write("\"}, CountryId{\"");
-        out->write(kv.second.toUtf8());
-        out->write("\"}},\n");
-    }
-    out->write("};\n\n");
-}
-
 void Generator::CountryDbGenerator::writeUicCodeTable(QIODevice *out)
 {
     out->write("static const UicCountryCodeMapping uic_country_code_table[] = {\n");

src/knowledgedb-generator/countrydbgenerator.h

@@ -40,14 +40,12 @@ private:
     bool fetchUicCountryCodes();
     QUrl insertOrMerge(const QJsonObject &obj);
     void writeCountryTable(QIODevice *out);
-    void writeIso3CodeTable(QIODevice *out);
     void writeUicCodeTable(QIODevice *out);
     void printSummary();
 
     std::vector<Country> m_countries;
     std::map<QString, QUrl> m_isoCodeMap;
     std::map<uint16_t, QString> m_uicCodeMap;
-    std::map<QString, QString> m_iso3CodeMap;
 
     int m_isoCodeConflicts = 0;
 };

src/knowledgedb-generator/main.cpp

@@ -6,7 +6,6 @@
 
 #include "airportdbgenerator.h"
 #include "countrydbgenerator.h"
-#include "timezonedbgenerator.h"
 #include "trainstationdbgenerator.h"
 
 #include <QCommandLineParser>
@@ -42,12 +41,6 @@ int main(int argc, char **argv)
     } else if (parser.value(dbOpt) == QLatin1String("country")) {
         CountryDbGenerator gen;
         return gen.generate(&out) ? 0 : 1;
-    } else if (parser.value(dbOpt) == QLatin1String("timezone")) {
-        TimezoneDbGenerator gen;
-        gen.generate(&out);
-    } else if (parser.value(dbOpt) == QLatin1String("timezoneheader")) {
-        TimezoneDbGenerator gen;
-        gen.generateHeader(&out);
     } else if (parser.value(dbOpt) == QLatin1String("trainstation")) {
         TrainStationDbGenerator gen;
         return gen.generate(&out) ? 0 : 1;

src/knowledgedb-generator/qgis/generate-z-order-curve-spatial-index.py

-#
-# SPDX-FileCopyrightText: Volker Krause <vkrause@kde.org>
-#
-# SPDX-License-Identifier: LGPL-2.0-or-later
-#
-
-import functools
-import datetime
-import time
-import pytz
-import qgis.core
-
-#
-# parameters for the spatial index
-#
-
-featureAreaRatioThreshold = 0.02 # 1% at zDepth 11 is ~150m
-zDepth = 11 # minimum tile size is 1/(2^zdepth), amount of bits needed to store z index is 2*zDepth
-
-
-#
-# z-order curve coordinate primitives
-#
-xStart = -180
-xRange = 360
-# cut out artic regions (starting at 65°S and 80°N), that saves about 20% z-order curve coverage which we
-# can better use to increase precision in more relevant areas
-yStart = -65
-yRange = 145
-
-xStep = xRange / (1 << zDepth)
-yStep = yRange / (1 << zDepth)
-
-def z2x(z):
-    x = 0
-    for i in range(0, zDepth):
-        x += (z & (1 << i * 2)) >> i
-    return x
-
-def z2y(z):
-    y = 0
-    for i in range(0, zDepth):
-        y += (z & (1 << (1 + i * 2))) >> (i + 1)
-    return y
-
-def rectForZ(z, depth):
-    mask = (1 << (2*(zDepth - depth))) - 1
-    x = z2x(z & ~mask) * xStep + xStart
-    y = z2y(z & ~mask) * yStep + yStart
-    xSize = xRange / (1 << depth)
-    ySize = yRange / (1 << depth)
-    return QgsRectangle(x, y, x + xSize, y + ySize)
-
-
-#
-# Parallelized spatial index computation of a single sub-tile
-#
-LOG_CATEGORY = 'SpatialIndexBuilder'
-
-class SpatialIndexerSubTask(QgsTask):
-    def __init__(self, layer, zStart, zStartDepth):
-        super().__init__('Compute spatial index sub-tile ' + str(zStart), QgsTask.CanCancel)
-        self.layer = layer
-        self.zStart = zStart
-        self.zStartDepth = zStartDepth
-        self.lastFeature = []
-        self.exception = None
-        self.result = []
-
-    def run(self):
-        try:
-            self.computeTile(self.zStart, self.zStartDepth)
-        except Exception as e:
-            self.exception = e
-            QgsMessageLog.logMessage('Exception in task "{}"'.format(self.exception), LOG_CATEGORY, Qgis.Info)
-        return True
-
-    def computeTile(self, zStart, depth):
-        if self.isCanceled() or depth < 1:
-            return
-        z = zStart
-        d = depth - 1
-        zIncrement = 1 << (2*d)
-        for i in range(0, 4):
-            # find features in the input vector layer inside our current tile
-            layerFeatures = []
-            for f in self.layer.getFeatures(rectForZ(z, zDepth -d)):
-                layerFeatures.append(f)
-
-            feature = []
-            featureCount = len(layerFeatures)
-
-            # recurse on conflicts
-            if depth > 1 and featureCount > 1:
-                self.computeTile(z, d)
-            # leaf tile, process the result
-            else:
-                # translate this into our result format: a list of (feature,areaRatio) tuples
-                if featureCount == 1: # we can skip the expensive area ratio computation in this case
-                    feature = [(layerFeatures[0]['tzid'], 1)]
-                elif featureCount > 1:
-                    rectGeo = QgsGeometry.fromRect(rectForZ(z, zDepth - d))
-                    for f in layerFeatures:
-                        featureArea = f.geometry().intersection(rectGeo).area()
-                        feature.append((f['tzid'], featureArea / rectGeo.area()))
-                feature = self.normalizeAndFilter(feature)
-
-                # if there's a change to the previous value, propagate to the result output
-                if self.lastFeature != feature and feature != []:
-                    self.result.append((z, feature))
-                    self.lastFeature = feature
-
-            z += zIncrement
-
-    def isValidFeature(self, f):
-        return not f.startswith("Etc/")
-
-    def normalizeAndFilter(self, r):
-        if len(r) == 0:
-            return r
-        r = list(filter(lambda x: x[1] > featureAreaRatioThreshold, r))
-        r = list(filter(lambda x: self.isValidFeature(x[0]), r))
-
-        if len(r) < 1:
-            return r
-        n = functools.reduce(lambda n, f: n + f[1], r, 0)
-        r = [(k, v/n) for (k, v) in r]
-        r.sort(key = lambda x: x[1], reverse = True)
-        return r
-
-    def finished(self, result):
-        if not result and self.exception != None:
-            QgsMessageLog.logMessage('Task "{name}" Exception: {exception}'.format(name=self.description(), exception=self.exception), LOG_CATEGORY, Qgis.Critical)
-            raise self.exception
-
-
-#
-# Tasks for spawning the sub-tasks doing the actual work, and accumulating the result
-#
-class SpatialIndexerTask(QgsTask):
-    def __init__(self, layer, outputFileName):
-        super().__init__('Compute spatial index', QgsTask.CanCancel)
-        self.setDependentLayers([layer])
-        self.tasks = []
-        self.outputFileName = outputFileName
-        self.exception = None
-        self.conflictTiles = 0
-        self.hardConflictTiles = 0
-        self.startTime = time.time()
-
-        startDepth = 4
-        startIncrement = 1 << (2 * (zDepth - startDepth))
-        for i in range(0, (1 << (2 * startDepth))):
-            task = SpatialIndexerSubTask(layer, i * startIncrement, zDepth - startDepth)
-            self.addSubTask(task, [], QgsTask.ParentDependsOnSubTask)
-            self.tasks.append(task)
-
-    def run(self):
-        try:
-            QgsMessageLog.logMessage('Aggregating results...', LOG_CATEGORY, Qgis.Info)
-
-            out = open(self.outputFileName, "w")
-            out.write("""/*
- * SPDX-License-Identifier: ODbL-1.0
- * SPDX-FileCopyrightText: OpenStreetMap contributors
- *
- * Autogenerated spatial index generated using QGIS.
- */
-
-#include "timezonedb_p.h"
-
-namespace KItinerary {
-namespace KnowledgeDb {
-
-""")
-            out.write('static constexpr const TimezoneZIndexParams timezone_index_params = {{ {xStart}, {xRange}, {yStart}, {yRange}, {zDepth} }};\n\n'
-               .format(xStart = xStart, xRange = xRange, yStart = yStart, yRange = yRange, zDepth = zDepth));
-            out.write('static constexpr TimezoneZIndexEntry timezone_index[] = {\n')
-
-            prevFeature = ""
-            prevAmbiguous = False
-            for task in self.tasks:
-                for (z,res) in task.result:
-                    feature = ""
-
-                    isAmbiguous = False
-                    if len(res) > 1:
-                        self.conflictTiles += 1
-                        isAmbiguous = True
-                    if len(res) > 1 and self.isConflict(res):
-                        feature = "Undefined"
-                        self.hardConflictTiles += 1
-                    else:
-                        feature = res[0][0].replace('/', '_').replace('-', '_')
-                    coverage = res[0][1]
-
-                    if prevFeature == feature and prevAmbiguous == isAmbiguous:
-                        continue
-                    prevFeature = feature
-                    prevAmbiguous = isAmbiguous
-                    if isAmbiguous:
-                        out.write("    { " + str(z) + ", Tz::" + feature + ", true }, // " + str(coverage) + "\n")
-                    else:
-                        out.write("    { " + str(z) + ", Tz::" + feature + ", false },\n")
-
-            out.write("};\n}\n}\n")
-            out.close()
-            return True
-
-        except Exception as e:
-            self.exception = e
-            QgsMessageLog.logMessage('Exception in task "{}"'.format(self.exception), LOG_CATEGORY, Qgis.Info)
-            return False
-
-    def isConflict(self, r):
-        tz = pytz.timezone(r[0][0])
-        return not all(self.isSameTimezone(tz, pytz.timezone(x[0])) for x in r[1:])
-
-    def isSameTimezone(self, lhs, rhs):
-        try:
-            # hacky tz comparison, lacking access to the rules for comparing actual DST transition times
-            dt = datetime.datetime.today().toordinal()
-            return all(lhs.utcoffset(datetime.datetime.fromordinal(dt + 30*x)) == rhs.utcoffset(datetime.datetime.fromordinal(dt + 30*x))
-                   and lhs.tzname(datetime.datetime.fromordinal(dt + 30*x)) == rhs.tzname(datetime.datetime.fromordinal(dt + 30*x)) for x in range(0, 11))
-        except:
-            return False
-
-    def finished(self, result):
-        QgsMessageLog.logMessage('Finished task "{}"'.format(self.description()), LOG_CATEGORY, Qgis.Info)
-        QgsMessageLog.logMessage(' "{}" of the area is conflicting'.format(str(self.conflictTiles / (1 << (2 * zDepth)))), LOG_CATEGORY, Qgis.Info)
-        QgsMessageLog.logMessage(' "{}" of the area is not covered'.format(str(self.hardConflictTiles / (1 << (2 * zDepth)))), LOG_CATEGORY, Qgis.Info)
-        QgsMessageLog.logMessage(' computation took "{}" seconds'.format(str(time.time() - self.startTime)), LOG_CATEGORY, Qgis.Info)
-        if not result and self.exception != None:
-            QgsMessageLog.logMessage('Task "{name}" Exception: {exception}'.format(name=self.description(), exception=self.exception), LOG_CATEGORY, Qgis.Critical)
-            raise self.exception
-
-
-#
-# actually launch things
-#
-task = SpatialIndexerTask(iface.activeLayer(), '/k/kde5/src/kitinerary/src/knowledgedb/timezone_zindex.cpp')
-QgsApplication.taskManager().addTask(task)

src/knowledgedb-generator/timezonedbgenerator.cpp

-/*
-    SPDX-FileCopyrightText: 2018 Volker Krause <vkrause@kde.org>
-
-    SPDX-License-Identifier: LGPL-2.0-or-later
-*/
-
-#include "timezonedbgenerator.h"
-#include "codegen.h"
-#include "timezones.h"
-
-#include <QDebug>
-#include <QIODevice>
-
-using namespace KItinerary::Generator;
-
-void TimezoneDbGenerator::generate(QIODevice *out)
-{
-    CodeGen::writeLicenseHeaderOSM(out);
-
-    Timezones tzDb;
-
-    out->write(R"(
-#include "timezonedb_p.h"
-#include "timezonedb_data.h"
-
-namespace KItinerary {
-namespace KnowledgeDb {
-
-// timezone name strings
-static const char timezone_names[] =
-)");
-    // timezone string table
-    for (const auto &tz : tzDb.m_zones) {
-        out->write("    ");
-        out->write("\"");
-        out->write(tz);
-        out->write("\\0\"\n");
-    }
-    out->write(R"(;
-
-static constexpr const uint16_t timezone_names_offsets[] = {
-    )");
-    out->write(QByteArray::number(tzDb.m_zones.front().size()));
-    out->write(", // Undefined\n");
-
-    // offsets into timezone string table
-    for (const auto &tz : tzDb.m_zones) {
-        out->write("    ");
-        out->write(QByteArray::number(tzDb.offset(tz)));
-        out->write(", // ");
-        out->write(tz);
-        out->write("\n");
-    }
-
-    out->write(R"(};
-
-static constexpr const CountryTimezoneMap country_timezone_map[] = {
-)");
-
-    for (const auto &map : tzDb.m_countryZones) {
-        if (map.second.size() != 1) {
-            continue;
-        }
-        out->write("    { ");
-        CodeGen::writeCountryIsoCode(out, map.first);
-        out->write(", ");
-        CodeGen::writeTimezone(out, map.second.at(0));
-        out->write(" },\n");
-    }
-
-    out->write(R"(};
-
-static constexpr const CountryId timezone_country_map[] = {
-    CountryId{}, // Undefined
-)");
-
-    for (const auto &tz : tzDb.m_zones) {
-        out->write("    ");
-        CodeGen::writeCountryIsoCode(out, tzDb.m_countryForZone[tz]);
-        out->write(", // ");
-        out->write(tz);
-        out->write("\n");
-    }
-
-    out->write(R"(};
-
-}
-}
-)");
-}
-
-void TimezoneDbGenerator::generateHeader(QIODevice *out)
-{
-    CodeGen::writeLicenseHeaderOSM(out);
-
-    Timezones tzDb;
-    out->write(R"(
-#ifndef KITINERARY_KNOWLEDGEDB_TIMEZONEDB_DATA_H
-#define KITINERARY_KNOWLEDGEDB_TIMEZONEDB_DATA_H
-
-#include <cstdint>
-
-namespace KItinerary {
-namespace KnowledgeDb {
-
-/** Enum representing all timezones. */
-enum class Tz : uint16_t {
-    Undefined,
-)");
-
-    for (const auto &tz : tzDb.m_zones) {
-        out->write("    ");
-        CodeGen::writeTimezoneEnum(out, tz