Commit b19fd67f authored by Volker Krause's avatar Volker Krause
Browse files

Enable TileDirectory to load input data from an OSMX file

This is considerably more efficient to read and to update compared to the
manually maintained o5m pyramid used so far, at the cost of a lot of disk
space.

The OSMX usage is extremely primitive so far, relying solely on its command
line tool. We could gain a bit more performance still by using OSMX API
directly and avoiding the PBF file indirection, but that is in the ~1%
range, and technically not entirely straightforward due to how OSMX is
built and its bundled dependencies.

This code will be used by the upcoming Tirex backend for on-demand tile
generation, existing code is only touched for removing a small bit of
dead code.
parent 4b979623
......@@ -24,6 +24,7 @@
#include <QUrl>
#include <QNetworkRequest>
#include <QNetworkReply>
#include <QTemporaryFile>
#include <QThread>
#include <iostream>
......@@ -35,15 +36,9 @@ namespace Marble {
QMap<int, TagsFilter::Tags> TileDirectory::m_tags;
TileDirectory::TileDirectory(TileType tileType, const QString &cacheDir, ParsingRunnerManager &manager, QString const &extension, int maxZoomLevel) :
TileDirectory::TileDirectory(TileType tileType, const QString &cacheDir, ParsingRunnerManager &manager, int maxZoomLevel) :
m_cacheDir(cacheDir),
m_baseDir(),
m_manager(manager),
m_zoomLevel(-1),
m_tileX(-1),
m_tileY(-1),
m_tagZoomLevel(-1),
m_extension(extension),
m_tileType(tileType),
m_landmassFile("land-polygons-split-4326.zip"),
m_maxZoomLevel(maxZoomLevel)
......@@ -72,6 +67,16 @@ TileDirectory::TileDirectory(TileType tileType, const QString &cacheDir, Parsing
QDir().mkpath(m_baseDir);
}
TileDirectory::TileDirectory(const QString &cacheDir, const QString &osmxFile, ParsingRunnerManager &manager, int maxZoomLevel, int loadZoomLevel) :
m_cacheDir(cacheDir),
m_osmxFile(osmxFile),
m_manager(manager),
m_zoomLevel(loadZoomLevel),
m_tileType(OpenStreetMap),
m_maxZoomLevel(maxZoomLevel)
{
}
TileId TileDirectory::tileFor(int zoomLevel, int tileX, int tileY) const
{
int const zoomDiff = zoomLevel - m_zoomLevel;
......@@ -86,11 +91,38 @@ QSharedPointer<GeoDataDocument> TileDirectory::load(int zoomLevel, int tileX, in
if (tile.x() == m_tileX && tile.y() == m_tileY) {
return m_landmass;
}
m_tileX = tile.x();
m_tileY = tile.y();
QString const filename = QString("%1/%2/%3.%4").arg(m_baseDir).arg(tile.x()).arg(tile.y()).arg("o5m");
m_landmass = open(filename, m_manager);
if (!m_osmxFile.isEmpty()) {
const auto tileBox = m_tileProjection.geoCoordinates(tile);
const QString bbox = QString::number(tileBox.south(GeoDataCoordinates::Degree))
+ QLatin1Char(',') + QString::number(tileBox.west(GeoDataCoordinates::Degree))
+ QLatin1Char(',') + QString::number(tileBox.north(GeoDataCoordinates::Degree))
+ QLatin1Char(',') + QString::number(tileBox.east(GeoDataCoordinates::Degree));
// TODO the following could be optimized by directly reading via OSMX API
QTemporaryFile tempPbfFile(m_cacheDir + "/tmp/XXXXXX.osm.pbf");
if (!tempPbfFile.open()) {
qCritical() << "Failed to open temporary file!" << tempPbfFile.errorString() << m_cacheDir;
return {};
}
QProcess osmx;
osmx.start("osmx", QStringList({ "extract" , (m_cacheDir + QLatin1Char('/') + m_osmxFile), tempPbfFile.fileName(), "--noUserData", "--bbox", bbox }));
osmx.waitForFinished(5*60*1000);
if (osmx.exitCode() != 0) {
qWarning() << osmx.readAllStandardError();
qWarning() << "osmx failed: " << osmx.errorString() << osmx.exitStatus() << osmx.exitCode();
return {};
}
m_landmass = open(tempPbfFile.fileName(), m_manager);
} else {
QString const filename = QString("%1/%2/%3.%4").arg(m_baseDir).arg(tile.x()).arg(tile.y()).arg("o5m");
m_landmass = open(filename, m_manager);
}
if (m_landmass) {
PeakAnalyzer::determineZoomLevel(m_landmass->placemarkList());
}
......
......@@ -56,7 +56,15 @@ public:
OpenStreetMap
};
TileDirectory(TileType tileType, const QString &cacheDir, ParsingRunnerManager &manager, const QString &extension, int maxZoomLevel);
TileDirectory(TileType tileType, const QString &cacheDir, ParsingRunnerManager &manager, int maxZoomLevel);
/** Create a tile directory for loading data from an OSMX file.
* @param maxZoomLevel The output zoom level.
* @param loadZoomLevel The zoom level at which the input data should be loaded.
* This must be smaller or equal to maxZoomLevel. Using a smaller value can be more efficient when
* generating a larger batch of tiles that fall within a lower zoom level tile, but comes at a greater
* cost for memory and clipping operations.
*/
TileDirectory(const QString &cacheDir, const QString &osmxFile, ParsingRunnerManager &manager, int maxZoomLevel, int loadZoomLevel);
QSharedPointer<GeoDataDocument> load(int zoomLevel, int tileX, int tileY);
void setInputFile(const QString &filename);
......@@ -88,13 +96,13 @@ private:
QString m_cacheDir;
QString m_baseDir;
QString m_osmxFile;
ParsingRunnerManager &m_manager;
QSharedPointer<GeoDataDocument> m_landmass;
int m_zoomLevel;
int m_tileX;
int m_tileY;
int m_tagZoomLevel;
QString m_extension;
int m_zoomLevel = -1;
int m_tileX = -1;
int m_tileY = -1;
int m_tagZoomLevel = -1;
QSharedPointer<VectorClipper> m_clipper;
QSharedPointer<TagsFilter> m_tagsFilter;
TileType m_tileType;
......
......@@ -118,8 +118,8 @@ int main(int argc, char *argv[])
parser.showHelp(1);
}
TileDirectory mapTiles(TileDirectory::OpenStreetMap, cacheDirectory, manager, extension, centerTile.zoomLevel());
TileDirectory landTiles(TileDirectory::Landmass, cacheDirectory, manager, extension, centerTile.zoomLevel());
TileDirectory mapTiles(TileDirectory::OpenStreetMap, cacheDirectory, manager, centerTile.zoomLevel());
TileDirectory landTiles(TileDirectory::Landmass, cacheDirectory, manager, centerTile.zoomLevel());
int const offset = 3;
int const N = pow(2,centerTile.zoomLevel());
......
......@@ -43,7 +43,7 @@ int main(int argc, char **argv)
}
GeoDataLatLonBox world(85.0, -85.0, 180.0, -180.0, GeoDataCoordinates::Degree);
TileDirectory loader(TileDirectory::Landmass, cacheDirectory, manager, QStringLiteral("o5m"), 17);
TileDirectory loader(TileDirectory::Landmass, cacheDirectory, manager, 17);
loader.setBoundingBox(world);
loader.createTiles();
return 0;
......
......@@ -234,12 +234,12 @@ int main(int argc, char *argv[])
} else {
QString const region = QFileInfo(inputFileName).fileName();
QString const regionDir = QString("%1/%2").arg(cacheDirectory).arg(QFileInfo(inputFileName).baseName());
TileDirectory mapTiles(TileDirectory::OpenStreetMap, regionDir, manager, extension, maxZoomLevel);
TileDirectory mapTiles(TileDirectory::OpenStreetMap, regionDir, manager, maxZoomLevel);
mapTiles.setInputFile(inputFileName);
mapTiles.createTiles();
auto const boundingBox = mapTiles.boundingBox();
TileDirectory loader(TileDirectory::Landmass, cacheDirectory, manager, extension, maxZoomLevel);
TileDirectory loader(TileDirectory::Landmass, cacheDirectory, manager, maxZoomLevel);
loader.setBoundingBox(boundingBox);
loader.createTiles();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment