Implement writing Tirex metatiles and sending positive replies

This produces textual dummy content still, but it demonstrates the entire
roundtrip from mod_tile -> Tirex -> our code -> HTTP response with the
right content now.

tools/vectorosm-tilecreator/CMakeLists.txt

@@ -47,6 +47,5 @@ target_link_libraries(marble-vectorosm-process-land-polygons vectorosm-toolchain
 add_executable(marble-vectorosm-tirex-backend
     vectorosm-tirex-backend.cpp
     tirexbackend.cpp
-    tirexmetatile.cpp
 )
 target_link_libraries(marble-vectorosm-tirex-backend vectorosm-toolchain)

tools/vectorosm-tilecreator/tirexbackend.cpp

@@ -6,8 +6,8 @@
 
 #include "tirexbackend.h"
 
-#include <QTimer>
-#include <QUdpSocket>
+#include <QDir>
+#include <QSettings>
 
 #include <cstdlib>
 #include <cstring>
@@ -15,42 +15,48 @@
 
 TirexBackend::TirexBackend(QObject *parent)
     : QObject(parent)
-    , m_commandSocket(new QUdpSocket(this))
-    , m_heartbeatTimer(new QTimer(this))
 {
     // setup command socket
     const auto socketFd = getenv("TIREX_BACKEND_SOCKET_FILENO");
     if (!socketFd) {
         qFatal("TIREX_BACKEND_SOCKET_FILENO not set!");
     }
-    const auto cmdSocketResult = m_commandSocket->setSocketDescriptor(std::atoi(socketFd));
+    const auto cmdSocketResult = m_commandSocket.setSocketDescriptor(std::atoi(socketFd));
     if (!cmdSocketResult) {
-        qFatal("Failed to use command socket fd: %s", qPrintable(m_commandSocket->errorString()));
+        qFatal("Failed to use command socket fd: %s", qPrintable(m_commandSocket.errorString()));
     }
-    connect(m_commandSocket, &QUdpSocket::readyRead, this, &TirexBackend::commandReadyRead);
+    connect(&m_commandSocket, &QUdpSocket::readyRead, this, &TirexBackend::commandReadyRead);
 
     // setup heartbeat pipe and timer
     const auto pipeFd = getenv("TIREX_BACKEND_PIPE_FILENO");
     const auto aliveTimeout = getenv("TIREX_BACKEND_ALIVE_TIMEOUT");
     if (pipeFd && aliveTimeout) {
         m_heartbeatFd = std::atoi(pipeFd);
-        m_heartbeatTimer->setInterval(std::chrono::seconds(std::atoi(aliveTimeout)));
-        m_heartbeatTimer->setSingleShot(false);
-        connect(m_heartbeatTimer, &QTimer::timeout, this, [this]() {
+        m_heartbeatTimer.setInterval(std::chrono::seconds(std::atoi(aliveTimeout)));
+        m_heartbeatTimer.setSingleShot(false);
+        connect(&m_heartbeatTimer, &QTimer::timeout, this, [this]() {
             write(m_heartbeatFd, "alive", 5);
         });
-        m_heartbeatTimer->start();
+        m_heartbeatTimer.start();
     } else {
         qWarning() << "not using heartbeat timer";
     }
+
+    // read map configuration
+    const auto configFiles = getenv("TIREX_BACKEND_MAP_CONFIGS");
+    if (configFiles) {
+        QSettings settings(QString::fromUtf8(configFiles), QSettings::IniFormat);
+        m_tileDir = settings.value(QLatin1String("tiledir")).toString();
+    }
 }
 
 TirexBackend::~TirexBackend() = default;
 
 void TirexBackend::commandReadyRead()
 {
-    while (m_commandSocket->hasPendingDatagrams()) {
-        const auto dgram = m_commandSocket->receiveDatagram();
+    while (m_commandSocket.hasPendingDatagrams()) {
+        m_renderTime.restart();
+        const auto dgram = m_commandSocket.receiveDatagram();
 
         TirexMetatileRequest req;
         int nextIdx = 0;
@@ -96,7 +102,7 @@ void TirexBackend::commandReadyRead()
             errorMsg += "type=" + QByteArray(type, typeLen) + "\n";
             errorMsg += "result=error\nerrormsg=unsupported requested\n";
             auto msg = dgram.makeReply(errorMsg);
-            m_commandSocket->writeDatagram(msg);
+            m_commandSocket.writeDatagram(msg);
             continue;
         }
 
@@ -107,7 +113,15 @@ void TirexBackend::commandReadyRead()
 
 void TirexBackend::tileDone(const TirexMetatileRequest &req)
 {
-    // TODO
+    QByteArray msg = "id=" + req.id
+        + "\ntype=metatile_render_request\nresult=ok\nx=" + QByteArray::number(req.tile.x)
+        + "\ny=" + QByteArray::number(req.tile.y)
+        + "\nz=" + QByteArray::number(req.tile.z)
+        + "\nmetatile=" + metatileFileName(req).toUtf8()
+        + "\nrender_time=" + QByteArray::number(m_renderTime.elapsed()) + "\n";
+    auto reply = std::move(req.reply);
+    reply.setData(msg);
+    m_commandSocket.writeDatagram(reply);
 }
 
 void TirexBackend::tileError(const TirexMetatileRequest &req, const QString &errMsg)
@@ -115,5 +129,88 @@ void TirexBackend::tileError(const TirexMetatileRequest &req, const QString &err
     QByteArray msg = "id=" + req.id + "\ntype=metatile_render_request\nresult=error\nerrmsg=" + errMsg.toUtf8() + "\n";
     auto reply = std::move(req.reply);
     reply.setData(msg);
-    m_commandSocket->writeDatagram(reply);
+    m_commandSocket.writeDatagram(reply);
+}
+
+QString TirexBackend::tileDir() const
+{
+    return m_tileDir;
+}
+
+QString TirexBackend::metatileFileName(const TirexMetatileRequest &req)
+{
+    auto x = req.tile.x;
+    auto y = req.tile.y;
+    uint8_t hash[5];
+    for (auto i = 0; i < 5; i++) {
+        hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
+        x >>= 4;
+        y >>= 4;
+    }
+
+    QString path = m_tileDir + QLatin1Char('/') + QString::number(req.tile.z) + QLatin1Char('/') +
+        QString::number(hash[4]) + QLatin1Char('/') +
+        QString::number(hash[3]) + QLatin1Char('/') +
+        QString::number(hash[2]) + QLatin1Char('/') +
+        QString::number(hash[1]) + QLatin1Char('/');
+    QDir().mkpath(path);
+    path += QString::number(hash[0]) + QLatin1String(".meta");
+    return path;
+}
+
+int TirexBackend::metatileColumns() const
+{
+    return m_metatileCols;
+}
+
+int TirexBackend::metatileRows() const
+{
+    return m_metatileRows;
+}
+
+struct TirexMetatileHeader {
+    char magic[4];
+    int count;
+    int x;
+    int y;
+    int z;
+};
+
+struct TirexMetatileEntry {
+    int offset;
+    int size;
+};
+
+void TirexBackend::writeMetatileHeader(QIODevice *io, const TirexMetatile &tile) const
+{
+    TirexMetatileHeader header;
+    header.magic[0] = 'M';
+    header.magic[1] = 'E';
+    header.magic[2] = 'T';
+    header.magic[3] = 'A';
+    header.count = m_metatileRows * m_metatileCols;
+    header.x = tile.x;
+    header.y = tile.y;
+    header.z = tile.z;
+    io->write(reinterpret_cast<const char*>(&header), sizeof(header));
+
+    TirexMetatileEntry entry;
+    entry.offset = 0;
+    entry.size = 0;
+    for (int i = 0; i < header.count; ++i) {
+        io->write(reinterpret_cast<const char*>(&entry), sizeof(entry));
+    }
+}
+
+void TirexBackend::writeMetatileEntry(QIODevice *io, int entryIdx, int offset, int size) const
+{
+    const auto seekPos = io->pos();
+
+    io->seek(sizeof(TirexMetatileHeader) + entryIdx * sizeof(TirexMetatileEntry));
+    TirexMetatileEntry entry;
+    entry.offset = offset;
+    entry.size = size;
+    io->write(reinterpret_cast<const char*>(&entry), sizeof(TirexMetatileEntry));
+
+    io->seek(seekPos);
 }

tools/vectorosm-tilecreator/tirexbackend.h

@@ -7,14 +7,20 @@
 #ifndef TIREXBACKEND_H
 #define TIREXBACKEND_H
 
-#include "tirexmetatile.h"
-
+#include <QElapsedTimer>
 #include <QNetworkDatagram>
 #include <QObject>
+#include <QTimer>
+#include <QUdpSocket>
 
-class TirexMetatile;
-class QUdpSocket;
-class QTimer;
+/** A Tirex meta tile. */
+class TirexMetatile
+{
+public:
+    int x = -1;
+    int y = -1;
+    int z = -1;
+};
 
 /** A render request for a meta-tile from Tirex. */
 class TirexMetatileRequest
@@ -37,18 +43,46 @@ public:
     explicit TirexBackend(QObject *parent = nullptr);
     ~TirexBackend();
 
+    /** Report a rendered request as done to Tirex. */
     void tileDone(const TirexMetatileRequest &req);
+    /** Indicate a render request failed. */
     void tileError(const TirexMetatileRequest &req, const QString &errMsg);
 
+    /** The base directory for tile output. */
+    QString tileDir() const;
+    /** Returns the full file name of the requested meta tile. */
+    QString metatileFileName(const TirexMetatileRequest &req);
+
+    /** Amount of rows in a single metatile. */
+    int metatileRows() const;
+    /** Amount of columns in a single metatile. */
+    int metatileColumns() const;
+
+    /** Writes the meta tile header structures.
+     *  QIODevice is positioned at the end for writing the first content tile afterwards.
+     */
+    void writeMetatileHeader(QIODevice *io, const TirexMetatile &tile) const;
+    /** Updates the tile offset and size at the given index.
+     * The QIODevice seek position will not change by this.
+     */
+    void writeMetatileEntry(QIODevice *io, int entryIdx, int offset, int size) const;
+
 Q_SIGNALS:
-    void tileRequested(const TirexMetatileRequest &tile);
+    /** Emitted when a new tile is requested.
+     *  Respond to this by calling tileDone or tileError.
+     */
+    void tileRequested(const TirexMetatileRequest &req);
 
 private:
     void commandReadyRead();
 
-    QUdpSocket *m_commandSocket = nullptr;
-    QTimer *m_heartbeatTimer = nullptr;
+    QUdpSocket m_commandSocket;
+    QTimer m_heartbeatTimer;
+    QString m_tileDir;
+    QElapsedTimer m_renderTime;
     int m_heartbeatFd = -1;
+    static constexpr const int m_metatileRows = 8; // TODO read from config eventually
+    static constexpr const int m_metatileCols = 8;
 };
 
 #endif // TIREXBACKEND_H

tools/vectorosm-tilecreator/tirexmetatile.cpp

-/*
-    SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org>
-
-    SPDX-License-Identifier: LGPL-2.0-or-later
-*/
-
-#include "tirexmetatile.h"
-
-TirexMetatile::TirexMetatile() = default;
-TirexMetatile::~TirexMetatile() = default;

tools/vectorosm-tilecreator/tirexmetatile.h

-/*
-    SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org>
-
-    SPDX-License-Identifier: LGPL-2.0-or-later
-*/
-
-#ifndef TIREXMETATILE_H
-#define TIREXMETATILE_H
-
-/** A Tirex meta tile. */
-class TirexMetatile
-{
-public:
-    TirexMetatile();
-    ~TirexMetatile();
-
-    int x = -1;
-    int y = -1;
-    int z = -1;
-};
-
-#endif // TIREXMETATILE_H

tools/vectorosm-tilecreator/vectorosm-tirex-backend.cpp

@@ -5,10 +5,10 @@
 */
 
 #include "tirexbackend.h"
-#include "tirexmetatile.h"
 
 #include <QCoreApplication>
 #include <QDebug>
+#include <QFile>
 
 int main(int argc, char **argv)
 {
@@ -18,9 +18,24 @@ int main(int argc, char **argv)
     TirexBackend backend;
 
     QObject::connect(&backend, &TirexBackend::tileRequested, &app, [&backend](const TirexMetatileRequest &req) {
-        qDebug() << req.tile.x << req.tile.y << req.tile.z;
-//         backend.tileDone(req);
-        backend.tileError(req, QStringLiteral("not yet implemented"));
+        qDebug() << req.tile.x << req.tile.y << req.tile.z << backend.metatileFileName(req);
+
+        QFile f(backend.metatileFileName(req));
+        if (!f.open(QFile::WriteOnly)) {
+            backend.tileError(req, f.errorString());
+            return;
+        }
+
+        backend.writeMetatileHeader(&f, req.tile);
+        for (int x = 0; x < backend.metatileColumns(); ++x) {
+            for (int y = 0; y < backend.metatileRows(); ++y) {
+                QByteArray b = "TEST: " + QByteArray::number(x + req.tile.x) + "x" + QByteArray::number(y + req.tile.y);
+                const auto offset = f.pos();
+                f.write(b);
+                backend.writeMetatileEntry(&f, x * backend.metatileColumns() + y, offset, b.size());
+            }
+        }
+        backend.tileDone(req);
     });
 
     return app.exec();