pw_framebuffer.cpp 33.9 KB
Newer Older
1
/* This file is part of the KDE project
Jan Grulich's avatar
Jan Grulich committed
2
   Copyright (C) 2018-2021 Jan Grulich <jgrulich@redhat.com>
3
4
5
6
7
8
9
10
   Copyright (C) 2018 Oleg Chernovskiy <kanedias@xaker.ru>

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public
   License as published by the Free Software Foundation; either
   version 3 of the License, or (at your option) any later version.
*/

Jan Grulich's avatar
Jan Grulich committed
11
12
#include "config-krfb.h"

13
14
15
16
17
18
19
20
21
22
23
24
25
// system
#include <sys/mman.h>
#include <cstring>

// Qt
#include <QCoreApplication>
#include <QGuiApplication>
#include <QScreen>
#include <QSocketNotifier>
#include <QDebug>
#include <QRandomGenerator>

// pipewire
Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
26
#include <sys/ioctl.h>
27
28

#include <spa/param/format-utils.h>
29
30
#include <spa/param/video/format-utils.h>
#include <spa/param/props.h>
Jan Grulich's avatar
Jan Grulich committed
31
#include <spa/utils/result.h>
32
33
34

#include <pipewire/pipewire.h>

35
#include <climits>
36
37
38
39

#include "pw_framebuffer.h"
#include "xdp_dbus_screencast_interface.h"
#include "xdp_dbus_remotedesktop_interface.h"
40
#include "krfb_fb_pipewire_debug.h"
41

42
#ifdef HAVE_DMA_BUF
Jan Grulich's avatar
Jan Grulich committed
43
44
45
46
47
48
49
50
51
#include <fcntl.h>
#include <unistd.h>

#include <gbm.h>
#include <epoxy/egl.h>
#include <epoxy/gl.h>
#endif /* HAVE_DMA_BUF */

static const int BYTES_PER_PIXEL = 4;
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
static const uint MIN_SUPPORTED_XDP_KDE_SC_VERSION = 1;

Q_DECLARE_METATYPE(PWFrameBuffer::Stream);
Q_DECLARE_METATYPE(PWFrameBuffer::Streams);

const QDBusArgument &operator >> (const QDBusArgument &arg, PWFrameBuffer::Stream &stream)
{
    arg.beginStructure();
    arg >> stream.nodeId;

    arg.beginMap();
    while (!arg.atEnd()) {
        QString key;
        QVariant map;
        arg.beginMapEntry();
        arg >> key >> map;
        arg.endMapEntry();
        stream.map.insert(key, map);
    }
    arg.endMap();
    arg.endStructure();

    return arg;
}

Jan Grulich's avatar
Jan Grulich committed
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
const char * formatGLError(GLenum err)
{
    switch(err) {
    case GL_NO_ERROR:
        return "GL_NO_ERROR";
    case GL_INVALID_ENUM:
        return "GL_INVALID_ENUM";
    case GL_INVALID_VALUE:
        return "GL_INVALID_VALUE";
    case GL_INVALID_OPERATION:
        return "GL_INVALID_OPERATION";
    case GL_STACK_OVERFLOW:
        return "GL_STACK_OVERFLOW";
    case GL_STACK_UNDERFLOW:
        return "GL_STACK_UNDERFLOW";
    case GL_OUT_OF_MEMORY:
        return "GL_OUT_OF_MEMORY";
    default:
        return (QLatin1String("0x") + QString::number(err, 16)).toLocal8Bit().constData();
    }
}
Jan Grulich's avatar
Jan Grulich committed
98

99
100
101
102
103
104
105
106
107
108
109
110
/**
 * @brief The PWFrameBuffer::Private class - private counterpart of PWFramebuffer class. This is the entity where
 *        whole logic resides, for more info search for "d-pointer pattern" information.
 */
class PWFrameBuffer::Private {
public:
    Private(PWFrameBuffer *q);
    ~Private();

private:
    friend class PWFrameBuffer;

111
112
113
    static void onCoreError(void *data, uint32_t id, int seq, int res, const char *message);
    static void onStreamParamChanged(void *data, uint32_t id, const struct spa_pod *format);
    static void onStreamStateChanged(void *data, pw_stream_state old, pw_stream_state state, const char *error_message);
114
115
116
117
118
119
120
121
122
123
124
125
    static void onStreamProcess(void *data);

    void initDbus();
    void initPw();

    // dbus handling
    void handleSessionCreated(quint32 &code, QVariantMap &results);
    void handleDevicesSelected(quint32 &code, QVariantMap &results);
    void handleSourcesSelected(quint32 &code, QVariantMap &results);
    void handleRemoteDesktopStarted(quint32 &code, QVariantMap &results);

    // pw handling
126
    pw_stream *createReceivingStream();
127
128
129
130
131
132
    void handleFrame(pw_buffer *pwBuffer);

    // link to public interface
    PWFrameBuffer *q;

    // pipewire stuff
133
    struct pw_context *pwContext = nullptr;
134
135
136
    struct pw_core *pwCore = nullptr;
    struct pw_stream *pwStream = nullptr;
    struct pw_thread_loop *pwMainLoop = nullptr;
137
138
139
140
141
142
143
144
145

    // wayland-like listeners
    // ...of events that happen in pipewire server
    spa_hook coreListener = {};
    spa_hook streamListener = {};

    // event handlers
    pw_core_events pwCoreEvents = {};
    pw_stream_events pwStreamEvents = {};
146

147
    uint pwStreamNodeId = 0;
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162

    // negotiated video format
    spa_video_info_raw *videoFormat = nullptr;

    // requests a session from XDG Desktop Portal
    // auto-generated and compiled from xdp_dbus_interface.xml file
    QScopedPointer<OrgFreedesktopPortalScreenCastInterface> dbusXdpScreenCastService;
    QScopedPointer<OrgFreedesktopPortalRemoteDesktopInterface> dbusXdpRemoteDesktopService;

    // XDP screencast session handle
    QDBusObjectPath sessionPath;
    // Pipewire file descriptor
    QDBusUnixFileDescriptor pipewireFd;

    // screen geometry holder
Jan Grulich's avatar
Jan Grulich committed
163
164
    QSize streamSize;
    QSize videoSize;
165
166

    // Allowed devices
167
    uint devices = 0;
168
169
170

    // sanity indicator
    bool isValid = true;
Jan Grulich's avatar
Jan Grulich committed
171
172
173
174
175
176
177
178
179
180
181
182
183
184

#if HAVE_DMA_BUF
    struct EGLStruct {
        QList<QByteArray> extensions;
        EGLDisplay display = EGL_NO_DISPLAY;
        EGLContext context = EGL_NO_CONTEXT;
    };

    bool m_eglInitialized = false;
    qint32 m_drmFd = 0; // for GBM buffer mmap
    gbm_device *m_gbmDevice = nullptr; // for passed GBM buffer retrieval

    EGLStruct m_egl;
#endif /* HAVE_DMA_BUF */
185
186
187
188
};

PWFrameBuffer::Private::Private(PWFrameBuffer *q) : q(q)
{
189
190
191
192
193
194
195
    pwCoreEvents.version = PW_VERSION_CORE_EVENTS;
    pwCoreEvents.error = &onCoreError;

    pwStreamEvents.version = PW_VERSION_STREAM_EVENTS;
    pwStreamEvents.state_changed = &onStreamStateChanged;
    pwStreamEvents.param_changed = &onStreamParamChanged;
    pwStreamEvents.process = &onStreamProcess;
Jan Grulich's avatar
Jan Grulich committed
196

Jan Grulich's avatar
Jan Grulich committed
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
#if HAVE_DMA_BUF
    m_drmFd = open("/dev/dri/renderD128", O_RDWR);

    if (m_drmFd < 0) {
        qCWarning(KRFB_FB_PIPEWIRE) << "Failed to open drm render node: " << strerror(errno);
        return;
    }

    m_gbmDevice = gbm_create_device(m_drmFd);

    if (!m_gbmDevice) {
        qCWarning(KRFB_FB_PIPEWIRE) << "Cannot create GBM device: " << strerror(errno);
        return;
    }

    // Get the list of client extensions
    const char* clientExtensionsCString = eglQueryString(EGL_NO_DISPLAY, EGL_EXTENSIONS);
    const QByteArray clientExtensionsString = QByteArray::fromRawData(clientExtensionsCString, qstrlen(clientExtensionsCString));
    if (clientExtensionsString.isEmpty()) {
        // If eglQueryString() returned NULL, the implementation doesn't support
        // EGL_EXT_client_extensions. Expect an EGL_BAD_DISPLAY error.
        qCWarning(KRFB_FB_PIPEWIRE) << "No client extensions defined! " << formatGLError(eglGetError());
        return;
    }

    m_egl.extensions = clientExtensionsString.split(' ');

    // Use eglGetPlatformDisplayEXT() to get the display pointer
    // if the implementation supports it.
    if (!m_egl.extensions.contains(QByteArrayLiteral("EGL_EXT_platform_base")) ||
            !m_egl.extensions.contains(QByteArrayLiteral("EGL_MESA_platform_gbm"))) {
        qCWarning(KRFB_FB_PIPEWIRE) << "One of required EGL extensions is missing";
        return;
    }

    m_egl.display = eglGetPlatformDisplayEXT(EGL_PLATFORM_GBM_MESA, m_gbmDevice, nullptr);

    if (m_egl.display == EGL_NO_DISPLAY) {
        qCWarning(KRFB_FB_PIPEWIRE) << "Error during obtaining EGL display: " << formatGLError(eglGetError());
        return;
    }

    EGLint major, minor;
    if (eglInitialize(m_egl.display, &major, &minor) == EGL_FALSE) {
        qCWarning(KRFB_FB_PIPEWIRE) << "Error during eglInitialize: " << formatGLError(eglGetError());
        return;
    }

    if (eglBindAPI(EGL_OPENGL_API) == EGL_FALSE) {
        qCWarning(KRFB_FB_PIPEWIRE) << "bind OpenGL API failed";
        return;
    }

    m_egl.context = eglCreateContext(m_egl.display, nullptr, EGL_NO_CONTEXT, nullptr);

    if (m_egl.context == EGL_NO_CONTEXT) {
        qCWarning(KRFB_FB_PIPEWIRE) << "Couldn't create EGL context: " << formatGLError(eglGetError());
        return;
    }

    qCDebug(KRFB_FB_PIPEWIRE) << "Egl initialization succeeded";
    qCDebug(KRFB_FB_PIPEWIRE) << QStringLiteral("EGL version: %1.%2").arg(major).arg(minor);

    m_eglInitialized = true;
#endif /* HAVE_DMA_BUF */
262
263
264
265
266
267
268
269
270
271
}

/**
 * @brief PWFrameBuffer::Private::initDbus - initialize D-Bus connectivity with XDG Desktop Portal.
 *        Based on XDG_CURRENT_DESKTOP environment variable it will give us implementation that we need,
 *        in case of KDE it is xdg-desktop-portal-kde binary.
 */
void PWFrameBuffer::Private::initDbus()
{
    qInfo() << "Initializing D-Bus connectivity with XDG Desktop Portal";
272
273
    dbusXdpScreenCastService.reset(new OrgFreedesktopPortalScreenCastInterface(QStringLiteral("org.freedesktop.portal.Desktop"),
                                                                     QStringLiteral("/org/freedesktop/portal/desktop"),
274
                                                                     QDBusConnection::sessionBus()));
275
276
    dbusXdpRemoteDesktopService.reset(new OrgFreedesktopPortalRemoteDesktopInterface(QStringLiteral("org.freedesktop.portal.Desktop"),
                                                                     QStringLiteral("/org/freedesktop/portal/desktop"),
277
278
279
                                                                     QDBusConnection::sessionBus()));
    auto version = dbusXdpScreenCastService->version();
    if (version < MIN_SUPPORTED_XDP_KDE_SC_VERSION) {
280
        qCWarning(KRFB_FB_PIPEWIRE) << "Unsupported XDG Portal screencast interface version:" << version;
281
282
283
284
285
286
        isValid = false;
        return;
    }

    // create session
    auto sessionParameters = QVariantMap {
287
288
        { QStringLiteral("session_handle_token"), QStringLiteral("krfb%1").arg(QRandomGenerator::global()->generate()) },
        { QStringLiteral("handle_token"), QStringLiteral("krfb%1").arg(QRandomGenerator::global()->generate()) }
289
290
291
292
293
294
295
296
297
298
299
300
    };
    auto sessionReply = dbusXdpRemoteDesktopService->CreateSession(sessionParameters);
    sessionReply.waitForFinished();
    if (!sessionReply.isValid()) {
        qWarning("Couldn't initialize XDP-KDE screencast session");
        isValid = false;
        return;
    }

    qInfo() << "DBus session created: " << sessionReply.value().path();
    QDBusConnection::sessionBus().connect(QString(),
                                          sessionReply.value().path(),
301
302
                                          QStringLiteral("org.freedesktop.portal.Request"),
                                          QStringLiteral("Response"),
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
                                          this->q,
                                          SLOT(handleXdpSessionCreated(uint, QVariantMap)));
}

void PWFrameBuffer::handleXdpSessionCreated(quint32 code, QVariantMap results)
{
    d->handleSessionCreated(code, results);
}

/**
 * @brief PWFrameBuffer::Private::handleSessionCreated - handle creation of ScreenCast session.
 *        XDG Portal answers with session path if it was able to successfully create the screencast.
 *
 * @param code return code for dbus call. Zero is success, non-zero means error
 * @param results map with results of call.
 */
void PWFrameBuffer::Private::handleSessionCreated(quint32 &code, QVariantMap &results)
{
    if (code != 0) {
322
        qCWarning(KRFB_FB_PIPEWIRE) << "Failed to create session: " << code;
323
324
325
326
        isValid = false;
        return;
    }

327
    sessionPath = QDBusObjectPath(results.value(QStringLiteral("session_handle")).toString());
328
329
330
331

    // select sources for the session
    auto selectionOptions = QVariantMap {
        // We have to specify it's an uint, otherwise xdg-desktop-portal will not forward it to backend implementation
332
333
        { QStringLiteral("types"), QVariant::fromValue<uint>(7) }, // request all (KeyBoard, Pointer, TouchScreen)
        { QStringLiteral("handle_token"), QStringLiteral("krfb%1").arg(QRandomGenerator::global()->generate()) }
334
335
336
337
    };
    auto selectorReply = dbusXdpRemoteDesktopService->SelectDevices(sessionPath, selectionOptions);
    selectorReply.waitForFinished();
    if (!selectorReply.isValid()) {
338
        qCWarning(KRFB_FB_PIPEWIRE) << "Couldn't select devices for the remote-desktop session";
339
340
341
342
343
        isValid = false;
        return;
    }
    QDBusConnection::sessionBus().connect(QString(),
                                          selectorReply.value().path(),
344
345
                                          QStringLiteral("org.freedesktop.portal.Request"),
                                          QStringLiteral("Response"),
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
                                          this->q,
                                          SLOT(handleXdpDevicesSelected(uint, QVariantMap)));
}

void PWFrameBuffer::handleXdpDevicesSelected(quint32 code, QVariantMap results)
{
    d->handleDevicesSelected(code, results);
}

/**
 * @brief PWFrameBuffer::Private::handleDevicesCreated - handle selection of devices we want to use for remote desktop
 *
 * @param code return code for dbus call. Zero is success, non-zero means error
 * @param results map with results of call.
 */
void PWFrameBuffer::Private::handleDevicesSelected(quint32 &code, QVariantMap &results)
{
Alexey Min's avatar
Alexey Min committed
363
    Q_UNUSED(results)
364
    if (code != 0) {
365
        qCWarning(KRFB_FB_PIPEWIRE) << "Failed to select devices: " << code;
366
367
368
369
370
371
        isValid = false;
        return;
    }

    // select sources for the session
    auto selectionOptions = QVariantMap {
372
373
374
        { QStringLiteral("types"), QVariant::fromValue<uint>(1) }, // only MONITOR is supported
        { QStringLiteral("multiple"), false },
        { QStringLiteral("handle_token"), QStringLiteral("krfb%1").arg(QRandomGenerator::global()->generate()) }
375
376
377
378
    };
    auto selectorReply = dbusXdpScreenCastService->SelectSources(sessionPath, selectionOptions);
    selectorReply.waitForFinished();
    if (!selectorReply.isValid()) {
379
        qCWarning(KRFB_FB_PIPEWIRE) << "Couldn't select sources for the screen-casting session";
380
381
382
383
384
        isValid = false;
        return;
    }
    QDBusConnection::sessionBus().connect(QString(),
                                          selectorReply.value().path(),
385
386
                                          QStringLiteral("org.freedesktop.portal.Request"),
                                          QStringLiteral("Response"),
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
                                          this->q,
                                          SLOT(handleXdpSourcesSelected(uint, QVariantMap)));
}

void PWFrameBuffer::handleXdpSourcesSelected(quint32 code, QVariantMap results)
{
    d->handleSourcesSelected(code, results);
}

/**
 * @brief PWFrameBuffer::Private::handleSourcesSelected - handle Screencast sources selection.
 *        XDG Portal shows a dialog at this point which allows you to select monitor from the list.
 *        This function is called after you make a selection.
 *
 * @param code return code for dbus call. Zero is success, non-zero means error
 * @param results map with results of call.
 */
void PWFrameBuffer::Private::handleSourcesSelected(quint32 &code, QVariantMap &)
{
    if (code != 0) {
407
        qCWarning(KRFB_FB_PIPEWIRE) << "Failed to select sources: " << code;
408
409
410
411
412
413
        isValid = false;
        return;
    }

    // start session
    auto startParameters = QVariantMap {
414
        { QStringLiteral("handle_token"), QStringLiteral("krfb%1").arg(QRandomGenerator::global()->generate()) }
415
416
417
418
419
    };
    auto startReply = dbusXdpRemoteDesktopService->Start(sessionPath, QString(), startParameters);
    startReply.waitForFinished();
    QDBusConnection::sessionBus().connect(QString(),
                                          startReply.value().path(),
420
421
                                          QStringLiteral("org.freedesktop.portal.Request"),
                                          QStringLiteral("Response"),
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
                                          this->q,
                                          SLOT(handleXdpRemoteDesktopStarted(uint, QVariantMap)));
}


void PWFrameBuffer::handleXdpRemoteDesktopStarted(quint32 code, QVariantMap results)
{
    d->handleRemoteDesktopStarted(code, results);
}

/**
 * @brief PWFrameBuffer::Private::handleScreencastStarted - handle Screencast start.
 *        At this point there shall be ready pipewire stream to consume.
 *
 * @param code return code for dbus call. Zero is success, non-zero means error
 * @param results map with results of call.
 */
void PWFrameBuffer::Private::handleRemoteDesktopStarted(quint32 &code, QVariantMap &results)
{
    if (code != 0) {
442
        qCWarning(KRFB_FB_PIPEWIRE) << "Failed to start screencast: " << code;
443
444
445
446
447
        isValid = false;
        return;
    }

    // there should be only one stream
448
    Streams streams = qdbus_cast<Streams>(results.value(QStringLiteral("streams")));
449
450
    if (streams.isEmpty()) {
        // maybe we should check deeper with qdbus_cast but this suffices for now
451
        qCWarning(KRFB_FB_PIPEWIRE) << "Failed to get screencast streams";
452
453
454
455
456
457
458
        isValid = false;
        return;
    }

    auto streamReply = dbusXdpScreenCastService->OpenPipeWireRemote(sessionPath, QVariantMap());
    streamReply.waitForFinished();
    if (!streamReply.isValid()) {
459
        qCWarning(KRFB_FB_PIPEWIRE) << "Couldn't open pipewire remote for the screen-casting session";
460
461
462
463
464
465
        isValid = false;
        return;
    }

    pipewireFd = streamReply.value();
    if (!pipewireFd.isValid()) {
466
        qCWarning(KRFB_FB_PIPEWIRE) << "Couldn't get pipewire connection file descriptor";
467
468
469
470
        isValid = false;
        return;
    }

471
    devices = results.value(QStringLiteral("types")).toUInt();
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487

    pwStreamNodeId = streams.first().nodeId;

    initPw();
}

/**
 * @brief PWFrameBuffer::Private::initPw - initialize Pipewire socket connectivity.
 *        pipewireFd should be pointing to existing file descriptor that was passed by D-Bus at this point.
 */
void PWFrameBuffer::Private::initPw() {
    qInfo() << "Initializing Pipewire connectivity";

    // init pipewire (required)
    pw_init(nullptr, nullptr); // args are not used anyways

488
    pwMainLoop = pw_thread_loop_new("pipewire-main-loop", nullptr);
Jan Grulich's avatar
Jan Grulich committed
489
490
    pw_thread_loop_lock(pwMainLoop);

491
492
    pwContext = pw_context_new(pw_thread_loop_get_loop(pwMainLoop), nullptr, 0);
    if (!pwContext) {
493
        qCWarning(KRFB_FB_PIPEWIRE) << "Failed to create PipeWire context";
494
495
496
497
498
        return;
    }

    pwCore = pw_context_connect(pwContext, nullptr, 0);
    if (!pwCore) {
499
        qCWarning(KRFB_FB_PIPEWIRE) << "Failed to connect PipeWire context";
500
501
502
503
504
505
506
        return;
    }

    pw_core_add_listener(pwCore, &coreListener, &pwCoreEvents, this);

    pwStream = createReceivingStream();
    if (!pwStream) {
507
        qCWarning(KRFB_FB_PIPEWIRE) << "Failed to create PipeWire stream";
508
509
        return;
    }
510
511

    if (pw_thread_loop_start(pwMainLoop) < 0) {
512
        qCWarning(KRFB_FB_PIPEWIRE) << "Failed to start main PipeWire loop";
513
514
        isValid = false;
    }
Jan Grulich's avatar
Jan Grulich committed
515

Jan Grulich's avatar
Jan Grulich committed
516
    pw_thread_loop_unlock(pwMainLoop);
517
518
}

519
520
521
522
523
524
525
526
527
void PWFrameBuffer::Private::onCoreError(void *data, uint32_t id, int seq, int res, const char *message)
{
    Q_UNUSED(data);
    Q_UNUSED(id);
    Q_UNUSED(seq);
    Q_UNUSED(res);

    qInfo() << "core error: " << message;
}
528
529
530
531
532
533
534
535
536

/**
 * @brief PWFrameBuffer::Private::onStreamStateChanged - called whenever stream state changes on pipewire server
 * @param data pointer that you have set in pw_stream_add_listener call's last argument
 * @param state new state that stream has changed to
 * @param error_message optional error message, is set to non-null if state is error
 */
void PWFrameBuffer::Private::onStreamStateChanged(void *data, pw_stream_state /*old*/, pw_stream_state state, const char *error_message)
{
Jan Grulich's avatar
Jan Grulich committed
537
    Q_UNUSED(data);
538

Jan Grulich's avatar
Jan Grulich committed
539
    qInfo() << "Stream state changed: " << pw_stream_state_as_string(state);
Jan Grulich's avatar
Jan Grulich committed
540

541
542
    switch (state) {
    case PW_STREAM_STATE_ERROR:
543
        qCWarning(KRFB_FB_PIPEWIRE) << "pipewire stream error: " << error_message;
544
545
546
547
548
549
550
        break;
    case PW_STREAM_STATE_PAUSED:
    case PW_STREAM_STATE_STREAMING:
    case PW_STREAM_STATE_UNCONNECTED:
    case PW_STREAM_STATE_CONNECTING:
        break;
    }
551
552
553
554
555
556
557
558
}

/**
 * @brief PWFrameBuffer::Private::onStreamFormatChanged - being executed after stream is set to active
 *        and after setup has been requested to connect to it. The actual video format is being negotiated here.
 * @param data pointer that you have set in pw_stream_add_listener call's last argument
 * @param format format that's being proposed
 */
559
void PWFrameBuffer::Private::onStreamParamChanged(void *data, uint32_t id, const struct spa_pod *format)
560
561
562
563
{
    qInfo() << "Stream format changed";
    auto *d = static_cast<PWFrameBuffer::Private *>(data);

564
    if (!format || id != SPA_PARAM_Format) {
565
566
567
568
569
570
571
        return;
    }

    d->videoFormat = new spa_video_info_raw();
    spa_format_video_raw_parse(format, d->videoFormat);
    auto width = d->videoFormat->size.width;
    auto height = d->videoFormat->size.height;
Jan Grulich's avatar
Jan Grulich committed
572
    auto stride = SPA_ROUND_UP_N(width * BYTES_PER_PIXEL, 4);
573
    auto size = height * stride;
Jan Grulich's avatar
Jan Grulich committed
574
    d->streamSize = QSize(width, height);
575
576
577
578
579

    uint8_t buffer[1024];
    auto builder = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer));

    // setup buffers and meta header for new format
Jan Grulich's avatar
Jan Grulich committed
580
581
582
583
584
585
586
587
    const struct spa_pod *params[3];

#if HAVE_DMA_BUF
    const auto bufferTypes = d->m_eglInitialized ? (1 << SPA_DATA_DmaBuf) | (1 << SPA_DATA_MemFd) | (1 << SPA_DATA_MemPtr) :
                                                   (1 << SPA_DATA_MemFd) | (1 << SPA_DATA_MemPtr);
#else
    const auto bufferTypes = (1 << SPA_DATA_MemFd) | (1 << SPA_DATA_MemPtr);
#endif /* HAVE_DMA_BUF */
588
589
590

    params[0] = reinterpret_cast<spa_pod *>(spa_pod_builder_add_object(&builder,
                SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers,
591
592
593
594
                SPA_PARAM_BUFFERS_size, SPA_POD_Int(size),
                SPA_PARAM_BUFFERS_stride, SPA_POD_Int(stride),
                SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(8, 1, 32),
                SPA_PARAM_BUFFERS_blocks, SPA_POD_Int(1),
Jan Grulich's avatar
Jan Grulich committed
595
596
                SPA_PARAM_BUFFERS_align, SPA_POD_Int(16),
                SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int(bufferTypes)));
597
598
    params[1] = reinterpret_cast<spa_pod *>(spa_pod_builder_add_object(&builder,
                SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta,
599
600
                SPA_PARAM_META_type, SPA_POD_Id(SPA_META_Header),
                SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_header))));
Jan Grulich's avatar
Jan Grulich committed
601
602
603
604
605
    params[2] = reinterpret_cast<spa_pod*>(spa_pod_builder_add_object(&builder,
                SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type,
                SPA_POD_Id(SPA_META_VideoCrop), SPA_PARAM_META_size,
                SPA_POD_Int(sizeof(struct spa_meta_region))));
    pw_stream_update_params(d->pwStream, params, 3);
606
607
608
609
610
611
612
613
614
615
616
}

/**
 * @brief PWFrameBuffer::Private::onNewBuffer - called when new buffer is available in pipewire stream
 * @param data pointer that you have set in pw_stream_add_listener call's last argument
 * @param id
 */
void PWFrameBuffer::Private::onStreamProcess(void *data)
{
    auto *d = static_cast<PWFrameBuffer::Private *>(data);

Jan Grulich's avatar
Jan Grulich committed
617
618
    pw_buffer* next_buffer;
    pw_buffer* buffer = nullptr;
619

Jan Grulich's avatar
Jan Grulich committed
620
621
622
623
    next_buffer = pw_stream_dequeue_buffer(d->pwStream);
    while (next_buffer) {
        buffer = next_buffer;
        next_buffer = pw_stream_dequeue_buffer(d->pwStream);
624

Jan Grulich's avatar
Jan Grulich committed
625
626
        if (next_buffer) {
            pw_stream_queue_buffer(d->pwStream, buffer);
Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
627
628
        }
    }
Jan Grulich's avatar
Jan Grulich committed
629
630
631
632
633
634
635
636

    if (!buffer) {
        return;
    }

    d->handleFrame(buffer);

    pw_stream_queue_buffer(d->pwStream, buffer);
Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
637
638
}

639
640
641
void PWFrameBuffer::Private::handleFrame(pw_buffer *pwBuffer)
{
    auto *spaBuffer = pwBuffer->buffer;
Jan Grulich's avatar
Jan Grulich committed
642
    uint8_t *src = nullptr;
643

Jan Grulich's avatar
Jan Grulich committed
644
    if (spaBuffer->datas[0].chunk->size == 0) {
645
        qCDebug(KRFB_FB_PIPEWIRE)  << "discarding null buffer";
646
647
648
        return;
    }

Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
649
    std::function<void()> cleanup;
Jan Grulich's avatar
Jan Grulich committed
650
    if (spaBuffer->datas->type == SPA_DATA_MemFd) {
Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
651
652
653
654
655
        uint8_t *map = static_cast<uint8_t*>(mmap(
            nullptr, spaBuffer->datas->maxsize + spaBuffer->datas->mapoffset,
            PROT_READ, MAP_PRIVATE, spaBuffer->datas->fd, 0));

        if (map == MAP_FAILED) {
656
            qCWarning(KRFB_FB_PIPEWIRE) << "Failed to mmap the memory: " << strerror(errno);
Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
657
658
659
660
661
662
663
            return;
        }
        src = SPA_MEMBER(map, spaBuffer->datas[0].mapoffset, uint8_t);

        cleanup = [map, spaBuffer] {
            munmap(map, spaBuffer->datas->maxsize + spaBuffer->datas->mapoffset);
        };
Jan Grulich's avatar
Jan Grulich committed
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
    } else if (spaBuffer->datas[0].type == SPA_DATA_MemPtr) {
        src = static_cast<uint8_t*>(spaBuffer->datas[0].data);
    }
#if HAVE_DMA_BUF
    else if (spaBuffer->datas->type == SPA_DATA_DmaBuf) {
        if (!m_eglInitialized) {
            // Shouldn't reach this
            qCWarning(KRFB_FB_PIPEWIRE) << "Failed to process DMA buffer.";
            return;
        }

        gbm_import_fd_data importInfo = {static_cast<int>(spaBuffer->datas->fd), static_cast<uint32_t>(streamSize.width()),
                                         static_cast<uint32_t>(streamSize.height()), static_cast<uint32_t>(spaBuffer->datas[0].chunk->stride), GBM_BO_FORMAT_ARGB8888};
        gbm_bo *imported = gbm_bo_import(m_gbmDevice, GBM_BO_IMPORT_FD, &importInfo, GBM_BO_USE_SCANOUT);
        if (!imported) {
            qCWarning(KRFB_FB_PIPEWIRE) << "Failed to process buffer: Cannot import passed GBM fd - " << strerror(errno);
            return;
        }

        // bind context to render thread
        eglMakeCurrent(m_egl.display, EGL_NO_SURFACE, EGL_NO_SURFACE, m_egl.context);

        // create EGL image from imported BO
        EGLImageKHR image = eglCreateImageKHR(m_egl.display, nullptr, EGL_NATIVE_PIXMAP_KHR, imported, nullptr);

        if (image == EGL_NO_IMAGE_KHR) {
            qCWarning(KRFB_FB_PIPEWIRE) << "Failed to record frame: Error creating EGLImageKHR - " << formatGLError(glGetError());
            gbm_bo_destroy(imported);
            return;
        }

        // create GL 2D texture for framebuffer
        GLuint texture;
        glGenTextures(1, &texture);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
        glBindTexture(GL_TEXTURE_2D, texture);
        glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, image);

        src = static_cast<uint8_t*>(malloc(streamSize.width() * streamSize.height() * BYTES_PER_PIXEL));

        GLenum glFormat = GL_BGRA;
        switch (videoFormat->format) {
            case SPA_VIDEO_FORMAT_RGBx:
                glFormat = GL_RGBA;
                break;
            case SPA_VIDEO_FORMAT_RGBA:
                glFormat = GL_RGBA;
                break;
            case SPA_VIDEO_FORMAT_BGRx:
                glFormat = GL_BGRA;
                break;
            case SPA_VIDEO_FORMAT_RGB:
                glFormat = GL_RGB;
                break;
            case SPA_VIDEO_FORMAT_BGR:
                glFormat = GL_BGR;
                break;
            default:
                glFormat = GL_BGRA;
                break;
        }
        glGetTexImage(GL_TEXTURE_2D, 0, glFormat, GL_UNSIGNED_BYTE, src);

        if (!src) {
            qCWarning(KRFB_FB_PIPEWIRE) << "Failed to get image from DMA buffer.";
            gbm_bo_destroy(imported);
            return;
        }

        cleanup = [src] {
            free(src);
        };

        glDeleteTextures(1, &texture);
        eglDestroyImageKHR(m_egl.display, image);

        gbm_bo_destroy(imported);
Jan Grulich's avatar
Jan Grulich committed
744
    }
Jan Grulich's avatar
Jan Grulich committed
745
#endif /* HAVE_DMA_BUF */
Jan Grulich's avatar
Jan Grulich committed
746

Jan Grulich's avatar
Jan Grulich committed
747
748
749
750
751
752
753
    struct spa_meta_region* videoMetadata =
    static_cast<struct spa_meta_region*>(spa_buffer_find_meta_data(
        spaBuffer, SPA_META_VideoCrop, sizeof(*videoMetadata)));

    if (videoMetadata && (videoMetadata->region.size.width > static_cast<uint32_t>(streamSize.width()) ||
                          videoMetadata->region.size.height > static_cast<uint32_t>(streamSize.height()))) {
        qCWarning(KRFB_FB_PIPEWIRE) << "Stream metadata sizes are wrong!";
Jan Grulich's avatar
Jan Grulich committed
754
        return;
Jan Grulich's avatar
Jan Grulich committed
755
756
    }

Jan Grulich's avatar
Jan Grulich committed
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
    // Use video metadata when video size from metadata is set and smaller than
    // video stream size, so we need to adjust it.
    bool videoFullWidth = true;
    bool videoFullHeight = true;
    if (videoMetadata && videoMetadata->region.size.width != 0 &&
        videoMetadata->region.size.height != 0) {
        if (videoMetadata->region.size.width < static_cast<uint32_t>(streamSize.width())) {
            videoFullWidth = false;
        } else if (videoMetadata->region.size.height < static_cast<uint32_t>(streamSize.height())) {
            videoFullHeight = false;
        }
    }

    QSize prevVideoSize = videoSize;
    if (!videoFullHeight || !videoFullWidth) {
        videoSize = QSize(videoMetadata->region.size.width, videoMetadata->region.size.height);
    } else {
        videoSize = streamSize;
    }

    if (!q->fb || videoSize != prevVideoSize) {
        if (q->fb) {
            free(q->fb);
        }
        q->fb = static_cast<char*>(malloc(videoSize.width() * videoSize.height() * BYTES_PER_PIXEL));

        if (!q->fb) {
            qCWarning(KRFB_FB_PIPEWIRE) << "Failed to allocate buffer";
            isValid = false;
            return;
        }

        Q_EMIT q->frameBufferChanged();
    }

    const qint32 dstStride = videoSize.width() * BYTES_PER_PIXEL;
    const qint32 srcStride = spaBuffer->datas[0].chunk->stride;

    if (!videoFullHeight && (videoMetadata->region.position.y + videoSize.height() <=  streamSize.height())) {
        src += srcStride * videoMetadata->region.position.y;
    }

    const int xOffset = !videoFullWidth && (videoMetadata->region.position.x + videoSize.width() <= streamSize.width())
                            ? videoMetadata->region.position.x * BYTES_PER_PIXEL : 0;

    char *dst = q->fb;
    for (int i = 0; i < videoSize.height(); ++i) {
        // Adjust source content based on crop video position if needed
        src += xOffset;
        std::memcpy(dst, src, dstStride);
Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
807

Jan Grulich's avatar
Jan Grulich committed
808
809
810
        if (videoFormat->format == SPA_VIDEO_FORMAT_BGRA || videoFormat->format == SPA_VIDEO_FORMAT_BGRx) {
            for (int j = 0; j < dstStride; j += 4) {
                std::swap(dst[j], dst[j + 2]);
Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
811
812
            }
        }
Jan Grulich's avatar
Jan Grulich committed
813
814
815
816
817
818
819
820
821
822
823

        src += srcStride - xOffset;
        dst += dstStride;
    }

    if (spaBuffer->datas->type == SPA_DATA_MemFd ||
        spaBuffer->datas->type == SPA_DATA_DmaBuf) {
        cleanup();
    }

    if (videoFormat->format != SPA_VIDEO_FORMAT_RGB) {
Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
824
825
826
827
        const QImage::Format format = videoFormat->format == SPA_VIDEO_FORMAT_BGR  ? QImage::Format_BGR888
                                    : videoFormat->format == SPA_VIDEO_FORMAT_RGBx ? QImage::Format_RGBX8888
                                                                                   : QImage::Format_RGB32;

Jan Grulich's avatar
Jan Grulich committed
828
        QImage img((uchar*) q->fb, videoSize.width(), videoSize.height(), dstStride, format);
Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
829
830
        img.convertTo(QImage::Format_RGB888);
    }
Jan Grulich's avatar
Jan Grulich committed
831
832

    q->tiles.append(QRect(0, 0, videoSize.width(), videoSize.height()));
833
834
835
836
837
838
839
}

/**
 * @brief PWFrameBuffer::Private::createReceivingStream - create a stream that will consume Pipewire buffers
 *        and copy the framebuffer to the existing image that we track. The state of the stream and configuration
 *        are later handled by the corresponding listener.
 */
840
pw_stream *PWFrameBuffer::Private::createReceivingStream()
841
842
{
    spa_rectangle pwMinScreenBounds = SPA_RECTANGLE(1, 1);
Jan Grulich's avatar
Jan Grulich committed
843
    spa_rectangle pwMaxScreenBounds = SPA_RECTANGLE(UINT32_MAX, UINT32_MAX);
844
845
846
847

    spa_fraction pwFramerateMin = SPA_FRACTION(0, 1);
    spa_fraction pwFramerateMax = SPA_FRACTION(60, 1);

Jan Grulich's avatar
Jan Grulich committed
848
849
850
    pw_properties* reuseProps = pw_properties_new_string("pipewire.client.reuse=1");

    auto stream = pw_stream_new(pwCore, "krfb-fb-consume-stream", reuseProps);
Jan Grulich's avatar
Jan Grulich committed
851

852
853
854
855
856
857
    uint8_t buffer[1024] = {};
    const spa_pod *params[1];
    auto builder = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer));

    params[0] = reinterpret_cast<spa_pod *>(spa_pod_builder_add_object(&builder,
                SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat,
858
859
                SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video),
                SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw),
Aleix Pol Gonzalez's avatar
Aleix Pol Gonzalez committed
860
861
862
863
                SPA_FORMAT_VIDEO_format, SPA_POD_CHOICE_ENUM_Id(6,
                                                    SPA_VIDEO_FORMAT_RGBx, SPA_VIDEO_FORMAT_RGBA,
                                                    SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_BGRA,
                                                    SPA_VIDEO_FORMAT_RGB, SPA_VIDEO_FORMAT_BGR),
864
865
866
                SPA_FORMAT_VIDEO_size, SPA_POD_CHOICE_RANGE_Rectangle(&pwMaxScreenBounds, &pwMinScreenBounds, &pwMaxScreenBounds),
                SPA_FORMAT_VIDEO_framerate, SPA_POD_Fraction(&pwFramerateMin),
                SPA_FORMAT_VIDEO_maxFramerate, SPA_POD_CHOICE_RANGE_Fraction(&pwFramerateMax, &pwFramerateMin, &pwFramerateMax)));
Jan Grulich's avatar
Jan Grulich committed
867

Jan Grulich's avatar
Jan Grulich committed
868
869
    pw_stream_add_listener(stream, &streamListener, &pwStreamEvents, this);

Jan Grulich's avatar
Jan Grulich committed
870
    if (pw_stream_connect(stream, PW_DIRECTION_INPUT, pwStreamNodeId, PW_STREAM_FLAG_AUTOCONNECT, params, 1) != 0) {
871
872
        isValid = false;
    }
873
874

    return stream;
875
876
877
878
879
880
881
882
883
884
885
886
}

PWFrameBuffer::Private::~Private()
{
    if (pwMainLoop) {
        pw_thread_loop_stop(pwMainLoop);
    }

    if (pwStream) {
        pw_stream_destroy(pwStream);
    }

887
888
889
    if (pwCore) {
        pw_core_disconnect(pwCore);
    }
890

891
892
893
    if (pwContext) {
        pw_context_destroy(pwContext);
    }
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923

    if (pwMainLoop) {
        pw_thread_loop_destroy(pwMainLoop);
    }
}

PWFrameBuffer::PWFrameBuffer(WId winid, QObject *parent)
    : FrameBuffer (winid, parent),
      d(new Private(this))
{
    // D-Bus is most important in init chain, no toys for us if something is wrong with XDP
    // PipeWire connectivity is initialized after D-Bus session is started
    d->initDbus();

    fb = nullptr;
}

PWFrameBuffer::~PWFrameBuffer()
{
    free(fb);
    fb = nullptr;
}

int PWFrameBuffer::depth()
{
    return 32;
}

int PWFrameBuffer::height()
{
Jan Grulich's avatar
Jan Grulich committed
924
    return d->videoSize.height();
925
926
927
928
}

int PWFrameBuffer::width()
{
Jan Grulich's avatar
Jan Grulich committed
929
    return d->videoSize.width();
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
}

int PWFrameBuffer::paddedWidth()
{
    return width() * 4;
}

void PWFrameBuffer::getServerFormat(rfbPixelFormat &format)
{
    format.bitsPerPixel = 32;
    format.depth = 32;
    format.trueColour = true;
    format.bigEndian = false;
}

void PWFrameBuffer::startMonitor()
{

}

void PWFrameBuffer::stopMonitor()
{

}

QVariant PWFrameBuffer::customProperty(const QString &property) const
{
    if (property == QLatin1String("stream_node_id")) {
        return QVariant::fromValue<uint>(d->pwStreamNodeId);
    } if (property == QLatin1String("session_handle")) {
        return QVariant::fromValue<QDBusObjectPath>(d->sessionPath);
    }

    return FrameBuffer::customProperty(property);
}

bool PWFrameBuffer::isValid() const
{
    return d->isValid;
}