Commit 955c2548 authored by Volker Krause's avatar Volker Krause

Add support for loading OSM PBF files

This enables the tile generator to consume osmx output directly, without
having to convert that to o5m first.
parent d202b912
......@@ -98,6 +98,9 @@ endif()
#######################################################
# Find Qt dependencies
find_package(Protobuf)
set_package_properties(Protobuf PROPERTIES TYPE OPTIONAL PURPOSE "Parsing of OSM PBF files.")
if(CMAKE_SYSTEM_NAME STREQUAL Android)
set(REQUIRED_QT_VERSION 5.7.0)
# TODO: still needed with ECM toolchain?
......
......@@ -6,8 +6,18 @@ INCLUDE_DIRECTORIES(
${CMAKE_CURRENT_SOURCE_DIR}/writers
${CMAKE_CURRENT_SOURCE_DIR}/translators
${CMAKE_CURRENT_BINARY_DIR}
${Protobuf_INCLUDE_DIRS}
)
if (Protobuf_FOUND)
PROTOBUF_GENERATE_CPP(pbf_srcs pbf_hdrs
${CMAKE_SOURCE_DIR}/tools/osm-addresses/pbf/fileformat.proto
${CMAKE_SOURCE_DIR}/tools/osm-addresses/pbf/osmformat.proto
)
set(EXTRA_LIBS ${Protobuf_LIBRARIES} ${ZLIB_LIBRARIES})
add_definitions(-DHAVE_PROTOBUF)
endif()
set( osm_writers_SRCS
writers/OsmObjectAttributeWriter.cpp
writers/OsmNodeTagWriter.cpp
......@@ -30,10 +40,12 @@ set( osm_SRCS
OsmWay.cpp
OsmRelation.cpp
OsmElementDictionary.cpp
OsmPbfParser.cpp
${pbf_srcs}
)
marble_add_plugin( OsmPlugin ${osm_SRCS} ${osm_writers_SRCS} ${osm_translators_SRCS} )
target_link_libraries(OsmPlugin o5mreader)
target_link_libraries(OsmPlugin o5mreader ${EXTRA_LIBS})
find_package(ECM ${REQUIRED_ECM_VERSION} QUIET)
if(NOT ECM_FOUND)
......
......@@ -20,6 +20,7 @@
#include "GeoDataPolyStyle.h"
#include <MarbleZipReader.h>
#include "o5mreader.h"
#include "OsmPbfParser.h"
#include <QColor>
#include <QFile>
......@@ -39,6 +40,8 @@ GeoDataDocument *OsmParser::parse(const QString &filename, QString &error)
if (fileInfo.suffix() == QLatin1String("o5m")) {
return parseO5m(filename, error);
} else if (filename.endsWith(QLatin1String(".osm.pbf"))) {
return parseOsmPbf(filename, error);
} else {
return parseXml(filename, error);
}
......@@ -200,6 +203,20 @@ GeoDataDocument* OsmParser::parseXml(const QString &filename, QString &error)
return createDocument(m_nodes, m_ways, m_relations);
}
GeoDataDocument* OsmParser::parseOsmPbf(const QString &filename, QString &error)
{
QFile f(filename);
if (!f.open(QFile::ReadOnly)) {
error = f.errorString();
return nullptr;
}
const auto data = f.map(0, f.size());
OsmPbfParser p;
p.parse(data, f.size());
return createDocument(p.m_nodes, p.m_ways, p.m_relations);
}
GeoDataDocument *OsmParser::createDocument(OsmNodes &nodes, OsmWays &ways, OsmRelations &relations)
{
GeoDataDocument* document = new GeoDataDocument;
......
......@@ -32,6 +32,7 @@ public:
private:
static GeoDataDocument* parseXml(const QString &filename, QString &error);
static GeoDataDocument* parseO5m(const QString &filename, QString &error);
static GeoDataDocument* parseOsmPbf(const QString &filename, QString &error);
static GeoDataDocument *createDocument(OsmNodes &nodes, OsmWays &way, OsmRelations &relations);
};
......
/*
SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org>
SPDX-License-Identifier: LGPL-2.0-or-later
*/
#include "OsmPbfParser.h"
#ifdef HAVE_PROTOBUF
#include "fileformat.pb.h"
#include "osmformat.pb.h"
#endif
#include <QDebug>
#include <QtEndian>
#include <zlib.h>
#include <cstdint>
#include <cstring>
using namespace Marble;
void OsmPbfParser::parse(const uint8_t *data, std::size_t len)
{
#ifdef HAVE_PROTOBUF
const uint8_t *it = data;
const uint8_t *end = data + len;
while (parseBlob(it, end));
#endif
}
#ifdef HAVE_PROTOBUF
bool OsmPbfParser::parseBlob(const uint8_t *&it, const uint8_t *end)
{
if (std::distance(it, end) < (int)sizeof(int32_t)) {
return false;
}
int32_t blobHeaderSize = 0;
std::memcpy(&blobHeaderSize, it, sizeof(int32_t));
blobHeaderSize = qFromBigEndian(blobHeaderSize);
it += sizeof(int32_t);
if (blobHeaderSize < 0 || std::distance(it, end) < blobHeaderSize) {
return false;
}
OSMPBF::BlobHeader blobHeader;
if (!blobHeader.ParseFromArray(it, blobHeaderSize)) {
return false;
}
it += blobHeaderSize;
OSMPBF::Blob blob;
if (std::distance(it, end) < blobHeader.datasize() || !blob.ParseFromArray(it, blobHeader.datasize())) {
return false;
}
const uint8_t *dataBegin = nullptr;
if (blob.has_raw()) {
dataBegin = reinterpret_cast<const uint8_t*>(blob.raw().data());
} else if (blob.has_zlib_data()) {
m_zlibBuffer.resize(blob.raw_size());
z_stream zStream;
zStream.next_in = (uint8_t*)blob.zlib_data().data();
zStream.avail_in = blob.zlib_data().size();
zStream.next_out = (uint8_t*)m_zlibBuffer.data();
zStream.avail_out = blob.raw_size();
zStream.zalloc = nullptr;
zStream.zfree = nullptr;
zStream.opaque = nullptr;
auto result = inflateInit(&zStream);
if (result != Z_OK) {
return false;
}
result = inflate(&zStream, Z_FINISH);
if (result != Z_STREAM_END) {
return false;
}
result = inflateEnd( &zStream );
dataBegin = reinterpret_cast<const uint8_t*>(m_zlibBuffer.constData());
} else {
return false;
}
if (std::strcmp(blobHeader.type().c_str(), "OSMData") == 0) {
parsePrimitiveBlock(dataBegin, blob.raw_size());
}
m_zlibBuffer.clear();
it += blobHeader.datasize();
return true;
}
void OsmPbfParser::parsePrimitiveBlock(const uint8_t *data, std::size_t len)
{
OSMPBF::PrimitiveBlock block;
if (!block.ParseFromArray(data, len)) {
return;
}
for (int i = 0; i < block.primitivegroup_size(); ++i) {
const auto &group = block.primitivegroup(i);
if (group.nodes_size()) {
qWarning() << "non-dense nodes - not implemented yet!";
} else if (group.has_dense()) {
parseDenseNodes(block, group);
} else if (group.ways_size()) {
parseWays(block, group);
} else if (group.relations_size()) {
parseRelations(block, group);
}
}
}
void OsmPbfParser::parseDenseNodes(const OSMPBF::PrimitiveBlock &block, const OSMPBF::PrimitiveGroup &group)
{
int64_t idDelta = 0;
int64_t latDelta = 0;
int64_t lonDelta = 0;
int tagIdx = 0;
const auto dense = group.dense();
for (int i = 0; i < dense.id_size(); ++i) {
idDelta += dense.id(i);
latDelta += dense.lat(i);
lonDelta += dense.lon(i);
auto &node = m_nodes[idDelta];
node.osmData().setId(idDelta);
node.setCoordinates(GeoDataCoordinates(lonDelta * 1.0e-7, latDelta * 1.0e-7, 0.0, GeoDataCoordinates::Degree));
while (tagIdx < dense.keys_vals_size()) {
const auto keyIdx = dense.keys_vals(tagIdx++);
if (keyIdx == 0) {
break;
}
const auto valIdx = dense.keys_vals(tagIdx++);
const QString keyString = *m_stringPool.insert(QString::fromUtf8(block.stringtable().s(keyIdx).data()));
const QString valueString = *m_stringPool.insert(QString::fromUtf8(block.stringtable().s(valIdx).data()));
node.osmData().addTag(keyString, valueString);
}
}
}
void OsmPbfParser::parseWays(const OSMPBF::PrimitiveBlock &block, const OSMPBF::PrimitiveGroup &group)
{
for (int i = 0; i < group.ways_size(); ++i) {
const auto &w = group.ways(i);
auto &way = m_ways[w.id()];
way.osmData().setId(w.id());
int64_t idDelta = 0;
for (int j = 0; j < w.refs_size(); ++j) {
idDelta += w.refs(j);
way.addReference(idDelta);
}
for (int j = 0; j < w.keys_size(); ++j) {
const QString keyString = *m_stringPool.insert(QString::fromUtf8(block.stringtable().s(w.keys(j)).data()));
const QString valueString = *m_stringPool.insert(QString::fromUtf8(block.stringtable().s(w.vals(j)).data()));
way.osmData().addTag(keyString, valueString);
}
}
}
void OsmPbfParser::parseRelations(const OSMPBF::PrimitiveBlock &block, const OSMPBF::PrimitiveGroup &group)
{
for (int i = 0; i < group.relations_size(); ++i) {
const auto &r = group.relations(i);
auto &rel = m_relations[r.id()];
rel.osmData().setId(r.id());
int64_t idDelta = 0;
for (int j = 0; j < r.memids_size(); ++j) {
idDelta += r.memids(j);
const QString role = *m_stringPool.insert(QString::fromUtf8(block.stringtable().s(r.roles_sid(j)).data()));
QString typeName;
const auto type = r.types(j);
switch (type) {
case OSMPBF::Relation_MemberType_NODE: typeName = QStringLiteral("node"); break;
case OSMPBF::Relation_MemberType_WAY: typeName = QStringLiteral("way"); break;
case OSMPBF::Relation_MemberType_RELATION: typeName = QStringLiteral("relation"); break;
}
rel.addMember(idDelta, role, typeName);
}
for (int j = 0; j < r.keys_size(); ++j) {
const QString keyString = *m_stringPool.insert(QString::fromUtf8(block.stringtable().s(r.keys(j)).data()));
const QString valueString = *m_stringPool.insert(QString::fromUtf8(block.stringtable().s(r.vals(j)).data()));
rel.osmData().addTag(keyString, valueString);
}
}
}
#endif
/*
SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org>
SPDX-License-Identifier: LGPL-2.0-or-later
*/
#ifndef MARBLE_OSMPBFPARSER_H
#define MARBLE_OSMPBFPARSER_H
#include "OsmNode.h"
#include "OsmWay.h"
#include "OsmRelation.h"
namespace OSMPBF {
class PrimitiveBlock;
class PrimitiveGroup;
}
namespace Marble {
class OsmPbfParser
{
public:
void parse(const uint8_t *data, std::size_t len);
OsmNodes m_nodes;
OsmWays m_ways;
OsmRelations m_relations;
private:
bool parseBlob(const uint8_t *&it, const uint8_t *end);
void parsePrimitiveBlock(const uint8_t *data, std::size_t len);
void parseDenseNodes(const OSMPBF::PrimitiveBlock &block, const OSMPBF::PrimitiveGroup &group);
void parseWays(const OSMPBF::PrimitiveBlock &block, const OSMPBF::PrimitiveGroup &group);
void parseRelations(const OSMPBF::PrimitiveBlock &block, const OSMPBF::PrimitiveGroup &group);
QByteArray m_zlibBuffer;
QSet<QString> m_stringPool;
};
}
#endif // MARBLE_OSMPBFPARSER_H
......@@ -57,7 +57,7 @@ QString OsmPlugin::fileFormatDescription() const
QStringList OsmPlugin::fileExtensions() const
{
return QStringList() << QStringLiteral("osm") << QStringLiteral("osm.zip") << QStringLiteral("o5m");
return QStringList() << QStringLiteral("osm") << QStringLiteral("osm.zip") << QStringLiteral("o5m") << QStringLiteral("osm.pbf");
}
ParsingRunner* OsmPlugin::newRunner() const
......
......@@ -22,7 +22,6 @@ add_subdirectory( stars )
add_subdirectory( sentineltile )
add_subdirectory( vectorosm-tilecreator )
find_package(Protobuf)
find_package(ZLIB)
if(PROTOBUF_FOUND AND ZLIB_FOUND)
# add_subdirectory( osm-addresses )
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment