main.cpp 12.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
//
// This file is part of the Marble Virtual Globe.
//
// This program is free software licensed under the GNU LGPL. You can
// find a copy of this license in LICENSE.txt in the top directory of
// the source code.
//
// Copyright 2016      David Kolozsvari <freedawson@gmail.com>
//

11
#include "GeoDataTypes.h"
12
#include "GeoDataDocumentWriter.h"
13
14
#include "MarbleModel.h"
#include "ParsingRunnerManager.h"
15
#include "GeoDataGeometry.h"
16
#include "GeoDataPlacemark.h"
17
#include "GeoDataLatLonAltBox.h"
18
#include "TileId.h"
19
#include "MarbleDirs.h"
20

21
#include <QApplication>
22
#include <QCommandLineParser>
23
24
#include <QDebug>
#include <QFileInfo>
25
26
#include <QDir>
#include <QString>
27
#include <QElapsedTimer>
28
#include <QSharedPointer>
29
#include <QFileInfo>
30
#include <QUrl>
31
#include <QBuffer>
32
33

#include <QMessageLogContext>
34
#include <QProcess>
35

36
#include "VectorClipper.h"
37
#include "NodeReducer.h"
38
#include "WayConcatenator.h"
39
#include "TileIterator.h"
40
#include "TileDirectory.h"
41
#include "MbTileWriter.h"
42

Dennis Nienhüser's avatar
Dennis Nienhüser committed
43
44
#include <iostream>

45
46
using namespace Marble;

47
48
49
50
GeoDataDocument* mergeDocuments(GeoDataDocument* map1, GeoDataDocument* map2)
{
    GeoDataDocument* mergedMap = new GeoDataDocument(*map1);

51
52
    OsmPlacemarkData marbleLand;
    marbleLand.addTag("marble_land","landmass");
53
54
    foreach (auto placemark, map2->placemarkList()) {
        GeoDataPlacemark* land = new GeoDataPlacemark(*placemark);
55
56
57
        if(land->geometry()->nodeType() == GeoDataTypes::GeoDataPolygonType) {
            land->setOsmData(marbleLand);
        }
58
        mergedMap->append(land);
59
60
61
62
63
    }

    return mergedMap;
}

64
QString tileFileName(const QCommandLineParser &parser, int x, int y, int zoomLevel)
Dennis Nienhüser's avatar
Dennis Nienhüser committed
65
66
{
    QString const extension = parser.value("extension");
67
    QString const outputDir = QString("%1/%2/%3").arg(parser.value("output")).arg(zoomLevel).arg(x);
Dennis Nienhüser's avatar
Dennis Nienhüser committed
68
    QString const outputFile = QString("%1/%2.%3").arg(outputDir).arg(y).arg(extension);
69
70
71
    return outputFile;
}

72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
void writeBoundaryTile(GeoDataDocument* tile, const QString &region, const QCommandLineParser &parser, int x, int y, int zoomLevel)
{
    QString const extension = parser.value("extension");
    QString const outputDir = QString("%1/boundaries/%2/%3/%4").arg(parser.value("cache-directory")).arg(region).arg(zoomLevel).arg(x);
    QString const outputFile = QString("%1/%2.%3").arg(outputDir).arg(y).arg(extension);
    QDir().mkpath(outputDir);
    GeoDataDocumentWriter::write(outputFile, *tile);
}

QSharedPointer<GeoDataDocument> mergeBoundaryTiles(const QSharedPointer<GeoDataDocument> &background, ParsingRunnerManager &manager, const QCommandLineParser &parser, int x, int y, int zoomLevel)
{
    GeoDataDocument* mergedMap = new GeoDataDocument;
    OsmPlacemarkData marbleLand;
    marbleLand.addTag("marble_land","landmass");
    foreach (auto placemark, background->placemarkList()) {
        GeoDataPlacemark* land = new GeoDataPlacemark(*placemark);
        if(land->geometry()->nodeType() == GeoDataTypes::GeoDataPolygonType) {
            land->setOsmData(marbleLand);
        }
        mergedMap->append(land);
    }

    QString const extension = parser.value("extension");
    QString const boundaryDir = QString("%1/boundaries").arg(parser.value("cache-directory"));
    foreach(auto const &dir, QDir(boundaryDir).entryList(QDir::Dirs | QDir::NoDotAndDotDot)) {
        QString const file = QString("%1/%2/%3/%4/%5.%6").arg(boundaryDir).arg(dir).arg(zoomLevel).arg(x).arg(y).arg(extension);
        if (QFileInfo(file).exists()) {
            auto tile = TileDirectory::open(file, manager);
            if (tile) {
                foreach (auto placemark, tile->placemarkList()) {
                    mergedMap->append(new GeoDataPlacemark(*placemark));
                }
            }
        }
    }

    return QSharedPointer<GeoDataDocument>(mergedMap);
}

111
112
bool writeTile(GeoDataDocument* tile, const QString &outputFile)
{
113
    QDir().mkpath(QFileInfo(outputFile).path());
Dennis Nienhüser's avatar
Dennis Nienhüser committed
114
    if (!GeoDataDocumentWriter::write(outputFile, *tile)) {
115
        qWarning() << "Could not write the file " << outputFile;
Dennis Nienhüser's avatar
Dennis Nienhüser committed
116
117
118
119
120
        return false;
    }
    return true;
}

121
122
int main(int argc, char *argv[])
{
123
    QCoreApplication app(argc, argv);
124

125
126
    QCoreApplication::setApplicationName("marble-vectorosm-tilecreator");
    QCoreApplication::setApplicationVersion("0.1");
127
128
129
130
131

    QCommandLineParser parser;
    parser.setApplicationDescription("A tool for Marble, which is used to reduce the details of osm maps.");
    parser.addHelpOption();
    parser.addVersionOption();
132
    parser.addPositionalArgument("input", "The input .osm or .shp file.");
133
134

    parser.addOptions({
135
                          {{"t", "osmconvert"}, "Tile data using osmconvert."},
136
                          {"conflict-resolution", "How to deal with existing tiles: overwrite, skip or merge", "mode", "overwrite"},
137
                          {{"c", "cache-directory"}, "Directory for temporary data.", "cache", "cache"},
138
                          {{"m", "mbtile"}, "Store tiles at level 15 onwards in a mbtile database.", "mbtile"},
139
                          {{"z", "zoom-level"}, "Zoom level according to which OSM information has to be processed.", "levels", "11,13,15,17"},
140
                          {{"o", "output"}, "Output file or directory", "output", QString("%1/maps/earth/vectorosm").arg(MarbleDirs::localPath())},
141
                          {{"e", "extension"}, "Output file type: o5m (default), osm or kml", "file extension", "o5m"}
Dennis Nienhüser's avatar
Dennis Nienhüser committed
142
                      });
143
144
145
146
147

    // Process the actual command line arguments given by the user
    parser.process(app);

    const QStringList args = parser.positionalArguments();
148
149
150
151
    if (args.isEmpty()) {
        parser.showHelp();
        return 0;
    }
152
153
    // input is args.at(0), output is args.at(1)

154
    QString const extension = parser.value("extension");
155
    QString inputFileName = args.at(0);
156
157
    auto const levels = parser.value("zoom-level").split(',');
    QVector<unsigned int> zoomLevels;
158
    int maxZoomLevel = 0;
159
    foreach(auto const &level, levels) {
160
161
162
        int const zoomLevel = level.toInt();
        maxZoomLevel = qMax(zoomLevel, maxZoomLevel);
        zoomLevels << zoomLevel;
163
    }
164

165
166
167
168
169
    if (zoomLevels.isEmpty()) {
        parser.showHelp(1);
        return 1;
    }

170
171
172
173
174
175
176
177
178
    bool const overwriteTiles = parser.value("conflict-resolution") == "overwrite";
    QSharedPointer<MbTileWriter> mbtileWriter;
    if (parser.isSet("mbtile")) {
        QString const mbtile = parser.value("mbtile");
        mbtileWriter = QSharedPointer<MbTileWriter>(new MbTileWriter(mbtile, extension));
        mbtileWriter->setReportProgress(false);
        mbtileWriter->setCommitInterval(500);
    }

179
180
    MarbleModel model;
    ParsingRunnerManager manager(model.pluginManager());
181
182
183
184
185
186
187
188
    QString const cacheDirectory = parser.value("cache-directory");
    QDir().mkpath(cacheDirectory);
    if (!QFileInfo(cacheDirectory).isWritable()) {
        qWarning() << "Cannot write to cache directory" << cacheDirectory;
        parser.showHelp(1);
    }

    if (*zoomLevels.cbegin() <= 9) {
189
        auto map = TileDirectory::open(inputFileName, manager);
190
        VectorClipper processor(map.data(), maxZoomLevel);
191
        GeoDataLatLonBox world(85.0, -85.0, 180.0, -180.0, GeoDataCoordinates::Degree);
192
193
194
195
196
197
        foreach(auto zoomLevel, zoomLevels) {
            TileIterator iter(world, zoomLevel);
            qint64 count = 0;
            qint64 const total = iter.total();
            foreach(auto const &tileId, iter) {
                ++count;
198
                QString const filename = tileFileName(parser, tileId.x(), tileId.y(), zoomLevel);
199
200
201
                if (!overwriteTiles && QFileInfo(filename).exists()) {
                    continue;
                }
202
                GeoDataDocument* tile = processor.clipTo(zoomLevel, tileId.x(), tileId.y());
203
                NodeReducer nodeReducer(tile, zoomLevel+1);
204
                if (!writeTile(tile, filename)) {
205
206
207
                    return 4;
                }
                std::cout << "Tile " << count << "/" << total << " (" << tile->name().toStdString() << ") done.";
208
209
                double const reduction = nodeReducer.removedNodes() / qMax(1.0, double(nodeReducer.remainingNodes() + nodeReducer.removedNodes()));
                std::cout << " Node reduction: " << qRound(reduction * 100.0) << "%";
210
                std::cout << "      \r";
211
212
                std::cout.flush();
                delete tile;
213
214
            }
        }
215
    } else {
216
        QString const region = QFileInfo(inputFileName).fileName();
217
        QString const regionDir = QString("%1/%2").arg(cacheDirectory).arg(QFileInfo(inputFileName).baseName());
218
        TileDirectory mapTiles(TileDirectory::OpenStreetMap, regionDir, manager, extension, maxZoomLevel);
219
220
        mapTiles.setInputFile(inputFileName);
        mapTiles.createTiles();
221
222
        auto const boundingBox = mapTiles.boundingBox();

223
        TileDirectory loader(TileDirectory::Landmass, cacheDirectory, manager, extension, maxZoomLevel);
224
225
        loader.setBoundingBox(boundingBox);
        loader.createTiles();
226

227
228
229
230
        typedef QMap<QString, QVector<TileId> > Tiles;
        Tiles tiles;

        qint64 total = 0;
231
        QSet<QString> boundaryTiles;
232
233
234
235
        foreach(auto zoomLevel, zoomLevels) {
            // @todo FIXME Assumes placemark ownership
            //WayConcatenator concatenator(tagsFilter.accepted(), QStringList() << "highway=*", false);

236
            TileIterator iter(mapTiles.boundingBox(), zoomLevel);
237
            total += iter.total();
238
            foreach(auto const &tileId, iter) {
239
                auto const tile = TileId(QString(), zoomLevel, tileId.x(), tileId.y());
240
241
                int const innerNodes = mapTiles.innerNodes(tile);
                if (innerNodes > 0) {
242
243
244
                    auto const mapTile = mapTiles.tileFor(zoomLevel, tileId.x(), tileId.y());
                    auto const name = QString("%1/%2/%3").arg(mapTile.zoomLevel()).arg(mapTile.x()).arg(mapTile.y());
                    tiles[name] << tile;
245
246
247
                    if (innerNodes < 4) {
                        boundaryTiles << name;
                    }
248
                }
249
250
251
252
            }
        }

        qint64 count = 0;
253
254
        for (auto iter = tiles.begin(), end = tiles.end(); iter != end; ++iter) {
            foreach(auto const &tileId, iter.value()) {
255
                ++count;
256
                int const zoomLevel = tileId.zoomLevel();
257
                QString const filename = tileFileName(parser, tileId.x(), tileId.y(), zoomLevel);
258
259
260
261
262
263
                if (!overwriteTiles) {
                    if (zoomLevel > 13 && mbtileWriter && mbtileWriter->hasTile(tileId.x(), tileId.y(), zoomLevel)) {
                        continue;
                    } else if (QFileInfo(filename).exists()) {
                        continue;
                    }
264
                }
265
266
267
268
269
270
271
272
273
274
275
276
277

                typedef QSharedPointer<GeoDataDocument> GeoDocPtr;
                GeoDocPtr tile1 = GeoDocPtr(mapTiles.clip(zoomLevel, tileId.x(), tileId.y()));
                TagsFilter::removeAnnotationTags(tile1.data());
                GeoDocPtr tile2 = GeoDocPtr(loader.clip(zoomLevel, tileId.x(), tileId.y()));
                GeoDocPtr combined = GeoDocPtr(mergeDocuments(tile1.data(), tile2.data()));
                NodeReducer nodeReducer(combined.data(), zoomLevel);

                if (boundaryTiles.contains(iter.key())) {
                    writeBoundaryTile(tile1.data(), region, parser, tileId.x(), tileId.y(), zoomLevel);
                    combined = mergeBoundaryTiles(tile2, manager, parser, tileId.x(), tileId.y(), zoomLevel);
                }

278
279
280
281
282
283
284
285
286
287
                if (zoomLevel > 13 && mbtileWriter) {
                    QBuffer buffer;
                    buffer.open(QBuffer::ReadWrite);
                    if (GeoDataDocumentWriter::write(&buffer, *combined, extension)) {
                        buffer.seek(0);
                        mbtileWriter->addTile(&buffer, tileId.x(), tileId.y(), zoomLevel);
                    } else {
                        qWarning() << "Could not write the tile " << combined->name();
                    }
                } else {
288
                    if (!writeTile(combined.data(), filename)) {
289
290
                        return 4;
                    }
291
292
                }

293
294
295
                TileDirectory::printProgress(count / double(total));
                std::cout << "  Tile " << count << "/" << total << " (";
                std::cout << combined->name().toStdString() << ").";
296
297
                double const reduction = nodeReducer.removedNodes() / qMax(1.0, double(nodeReducer.remainingNodes() + nodeReducer.removedNodes()));
                std::cout << " Node reduction: " << qRound(reduction * 100.0) << "%";
298
                std::cout << "      \r";
299
                std::cout.flush();
300
301
            }
        }
302
303
        TileDirectory::printProgress(1.0);
        std::cout << "  Vector OSM tiles complete." << std::string(30, ' ') << std::endl;
304
305
    }

306
307
    return 0;
}