Commit ef8a9ac5 authored by Jean-Baptiste Mardelle's avatar Jean-Baptiste Mardelle
Browse files

Refactor audio recording: allow pause/resume with space bar, display live waveform on record

Fixes #752
parent 68a90dda
......@@ -105,7 +105,7 @@ void AudioLevelWidget::drawBackground(int channels)
m_pixmap.fill(Qt::transparent);
int totalWidth;
if (channels < 2) {
m_channelWidth = newSize.width() / 2;
m_channelWidth = newSize.width();
totalWidth = m_channelWidth;
} else {
m_channelWidth = (newSize.width() - (channels - 1)) / channels;
......@@ -150,7 +150,7 @@ void AudioLevelWidget::drawBackground(int channels)
for (int i = 0; i < channels; i++) {
p.drawLine(m_offset + i * (m_channelWidth + m_channelDistance), 0, i * (m_channelWidth + m_channelDistance), rect.height() - 1);
}
} else {
} else if (channels > 1) {
m_channelDistance = 2;
m_channelFillWidth = m_channelWidth - 2;
for (int i = 0; i < channels; i++) {
......@@ -159,6 +159,10 @@ void AudioLevelWidget::drawBackground(int channels)
p.fillRect(m_offset + i * (m_channelWidth + m_channelDistance) - 2, 0, 2, rect.height(), Qt::transparent);
}
}
} else {
m_channelDistance = 0;
m_channelFillWidth = m_channelWidth;
p.drawRect(m_offset, 0, m_channelWidth - 1, rect.height() - 1);
}
p.end();
}
......
......@@ -10,6 +10,8 @@
#include "mixerwidget.hpp"
#include "timeline2/model/timelineitemmodel.hpp"
#include "effects/effectsrepository.hpp"
#include "capture/mediacapture.h"
#include "mlt++/MltService.h"
#include "mlt++/MltTractor.h"
......@@ -30,6 +32,7 @@ MixerManager::MixerManager(QWidget *parent)
, m_visibleMixerManager(false)
, m_expandedWidth(-1)
, m_recommendedWidth(300)
, m_monitorTrack(-1)
, m_filterIsV2(false)
{
m_masterBox = new QHBoxLayout;
......@@ -58,6 +61,34 @@ void MixerManager::checkAudioLevelVersion()
m_filterIsV2 = EffectsRepository::get()->exists(QStringLiteral("audiolevel")) && EffectsRepository::get()->getVersion(QStringLiteral("audiolevel")) > 100;
}
void MixerManager::monitorAudio(int tid, bool monitor)
{
if (!monitor) {
if (m_mixers.count(tid) > 0) {
m_mixers[tid]->monitorAudio(false);
}
m_monitorTrack = -1;
pCore->getAudioDevice()->switchMonitorState(false);
return;
}
// We want to monitor audio
if (m_monitorTrack > -1) {
// Another track is monitoring
if (m_mixers.count(m_monitorTrack) > 0) {
m_mixers[m_monitorTrack]->monitorAudio(false);
}
m_monitorTrack = -1;
} else {
pCore->getAudioDevice()->switchMonitorState(true);
}
if (m_mixers.count(tid) > 0) {
m_mixers[tid]->monitorAudio(true);
} else {
return;
}
m_monitorTrack = tid;
}
void MixerManager::registerTrack(int tid, std::shared_ptr<Mlt::Tractor> service, const QString &trackTag, const QString &trackName)
{
if (m_mixers.count(tid) > 0) {
......@@ -184,6 +215,7 @@ void MixerManager::recordStateChanged(int tid, bool recording)
if (m_mixers.count(tid) > 0) {
m_mixers[tid]->setRecordState(recording);
}
emit pCore->switchTimelineRecord(recording);
}
void MixerManager::connectMixer(bool doConnect)
......
......@@ -40,6 +40,8 @@ public:
void unsetModel();
/** @brief Some features rely on a specific version of MLT's audiolevel filter, so check it */
void checkAudioLevelVersion();
/** @brief Enable/disable audio monitoring on a track */
void monitorAudio(int tid, bool monitor);
public slots:
void recordStateChanged(int tid, bool recording);
......@@ -71,6 +73,7 @@ private:
int m_expandedWidth;
QVector <int> m_soloMuted;
int m_recommendedWidth;
int m_monitorTrack;
bool m_filterIsV2;
};
......@@ -94,9 +94,9 @@ MixerWidget::MixerWidget(int tid, std::shared_ptr<Mlt::Tractor> service, QString
, m_solo(nullptr)
, m_record(nullptr)
, m_collapse(nullptr)
, m_monitor(nullptr)
, m_lastVolume(0)
, m_listener(nullptr)
, m_recording(false)
, m_trackTag(std::move(trackTag))
{
buildUI(service.get(), trackName);
......@@ -116,6 +116,7 @@ MixerWidget::MixerWidget(int tid, Mlt::Tractor *service, QString trackTag, const
, m_solo(nullptr)
, m_record(nullptr)
, m_collapse(nullptr)
, m_monitor(nullptr)
, m_lastVolume(0)
, m_listener(nullptr)
, m_recording(false)
......@@ -153,7 +154,11 @@ void MixerWidget::buildUI(Mlt::Tractor *service, const QString &trackName)
m_volumeSpin->setFrame(false);
connect(m_volumeSpin, static_cast<void (QDoubleSpinBox::*)(double)>(&QDoubleSpinBox::valueChanged), this, [&](double val) {
m_volumeSlider->setValue(fromDB(val));
if (m_recording || (m_monitor && m_monitor->isChecked())) {
m_volumeSlider->setValue(val);
} else {
m_volumeSlider->setValue(fromDB(val));
}
});
QLabel *labelLeft = nullptr;
......@@ -287,7 +292,20 @@ void MixerWidget::buildUI(Mlt::Tractor *service, const QString &trackName)
m_record->setToolTip(i18n("Record"));
m_record->setCheckable(true);
m_record->setAutoRaise(true);
m_monitor = new QToolButton(this);
m_monitor->setIcon(QIcon::fromTheme("audio-input-microphone"));
m_monitor->setToolTip(i18n("Monitor audio input"));
m_monitor->setCheckable(true);
m_monitor->setAutoRaise(true);
connect(m_monitor, &QToolButton::toggled, this, [&](bool toggled) {
m_manager->monitorAudio(m_tid, toggled);
});
connect(m_record, &QToolButton::clicked, this, [&]() {
if (!m_monitor->isChecked()) {
// Start monitoring first
emit m_manager->monitorAudio(m_tid, true);
}
emit m_manager->recordAudio(m_tid);
});
} else {
......@@ -304,17 +322,9 @@ void MixerWidget::buildUI(Mlt::Tractor *service, const QString &trackName)
});
}
auto *showEffects = new QToolButton(this);
showEffects->setIcon(QIcon::fromTheme("autocorrection"));
showEffects->setToolTip(i18n("Open Effect Stack"));
showEffects->setAutoRaise(true);
connect(showEffects, &QToolButton::clicked, this, [&]() {
emit m_manager->showEffectStack(m_tid);
});
connect(m_volumeSlider, &QSlider::valueChanged, this, [&](int value) {
QSignalBlocker bk(m_volumeSpin);
if (m_recording) {
if (m_recording || (m_monitor && m_monitor->isChecked())) {
m_volumeSpin->setValue(value);
KdenliveSettings::setAudiocapturevolume(value);
emit m_manager->updateRecVolume();
......@@ -365,7 +375,9 @@ void MixerWidget::buildUI(Mlt::Tractor *service, const QString &trackName)
if (m_record) {
buttonslay->addWidget(m_record);
}
buttonslay->addWidget(showEffects);
if (m_monitor) {
buttonslay->addWidget(m_monitor);
}
lay->addLayout(buttonslay);
if (m_balanceSlider) {
auto *balancelay = new QGridLayout;
......@@ -429,6 +441,10 @@ void MixerWidget::updateLabel()
QPalette pal = m_trackLabel->palette();
pal.setColor(QPalette::Window, Qt::red);
m_trackLabel->setPalette(pal);
} else if (m_monitor && m_monitor->isChecked()) {
QPalette pal = m_trackLabel->palette();
pal.setColor(QPalette::Window, Qt::darkBlue);
m_trackLabel->setPalette(pal);
} else if (m_muteAction->isActive()) {
QPalette pal = m_trackLabel->palette();
pal.setColor(QPalette::Window, QColor(0xff8c00));
......@@ -489,7 +505,7 @@ void MixerWidget::gotRecLevels(QVector<qreal>levels)
m_audioMeterWidget->setAudioValues({-100, -100});
break;
case 1:
m_audioMeterWidget->setAudioValues({levels[0], -100});
m_audioMeterWidget->setAudioValues({levels[0]});
break;
default:
m_audioMeterWidget->setAudioValues({levels[0], levels[1]});
......@@ -497,13 +513,11 @@ void MixerWidget::gotRecLevels(QVector<qreal>levels)
}
}
void MixerWidget::setRecordState(bool recording)
void MixerWidget::updateMonitorState()
{
m_recording = recording;
m_record->setChecked(m_recording);
QSignalBlocker bk(m_volumeSpin);
QSignalBlocker bk2(m_volumeSlider);
if (m_recording) {
if (m_recording || (m_monitor && m_monitor->isChecked())) {
connect(pCore->getAudioDevice(), &MediaCapture::audioLevels, this, &MixerWidget::gotRecLevels);
if (m_balanceSlider) {
m_balanceSlider->setEnabled(false);
......@@ -514,12 +528,12 @@ void MixerWidget::setRecordState(bool recording)
m_volumeSpin->setValue(KdenliveSettings::audiocapturevolume());
m_volumeSlider->setValue(KdenliveSettings::audiocapturevolume());
} else {
disconnect(pCore->getAudioDevice(), &MediaCapture::audioLevels, this, &MixerWidget::gotRecLevels);
if (m_balanceSlider) {
m_balanceSlider->setEnabled(true);
m_balanceSpin->setEnabled(true);
}
int level = m_levelFilter->get_int("level");
disconnect(pCore->getAudioDevice(), &MediaCapture::audioLevels, this, &MixerWidget::gotRecLevels);
m_volumeSpin->setRange(-100, 60);
m_volumeSpin->setSuffix(i18n("dB"));
m_volumeSpin->setValue(level);
......@@ -528,6 +542,26 @@ void MixerWidget::setRecordState(bool recording)
updateLabel();
}
void MixerWidget::monitorAudio(bool monitor)
{
QSignalBlocker bk(m_monitor);
if (monitor) {
m_monitor->setChecked(true);
updateMonitorState();
} else {
m_monitor->setChecked(false);
updateMonitorState();
reset();
}
}
void MixerWidget::setRecordState(bool recording)
{
m_recording = recording;
m_record->setChecked(m_recording);
updateMonitorState();
}
void MixerWidget::connectMixer(bool doConnect, bool filterV2)
{
if (doConnect) {
......
......@@ -56,6 +56,10 @@ public:
void connectMixer(bool doConnect, bool filterV2);
/** @brief Disable/enable monitoring by disabling/enabling filter */
void pauseMonitoring(bool pause);
/** @brief Update widget to reflect state (monitor/record/none) */
void updateMonitorState();
/** @brief Enable/disable audio monitoring on this mixer */
void monitorAudio(bool monitor);
protected:
void mousePressEvent(QMouseEvent *event) override;
......@@ -87,6 +91,7 @@ private:
QToolButton *m_solo;
QToolButton *m_record;
QToolButton *m_collapse;
QToolButton *m_monitor;
KSqueezedTextLabel *m_trackLabel;
QMutex m_storeMutex;
double m_lastVolume;
......
......@@ -28,10 +28,10 @@ AudioDevInfo::AudioDevInfo(const QAudioFormat &format, QObject *parent)
case 8:
switch (m_format.sampleType()) {
case QAudioFormat::UnSignedInt:
m_maxAmplitude = 255;
maxAmplitude = 255;
break;
case QAudioFormat::SignedInt:
m_maxAmplitude = 127;
maxAmplitude = 127;
break;
default:
break;
......@@ -40,10 +40,10 @@ AudioDevInfo::AudioDevInfo(const QAudioFormat &format, QObject *parent)
case 16:
switch (m_format.sampleType()) {
case QAudioFormat::UnSignedInt:
m_maxAmplitude = 65535;
maxAmplitude = 65535;
break;
case QAudioFormat::SignedInt:
m_maxAmplitude = 32767;
maxAmplitude = 32767;
break;
default:
break;
......@@ -53,13 +53,13 @@ AudioDevInfo::AudioDevInfo(const QAudioFormat &format, QObject *parent)
case 32:
switch (m_format.sampleType()) {
case QAudioFormat::UnSignedInt:
m_maxAmplitude = 0xffffffff;
maxAmplitude = 0xffffffff;
break;
case QAudioFormat::SignedInt:
m_maxAmplitude = 0x7fffffff;
maxAmplitude = 0x7fffffff;
break;
case QAudioFormat::Float:
m_maxAmplitude = 0x7fffffff; // Kind of
maxAmplitude = 0x7fffffff; // Kind of
default:
break;
}
......@@ -79,7 +79,7 @@ qint64 AudioDevInfo::readData(char *data, qint64 maxSize)
qint64 AudioDevInfo::writeData(const char *data, qint64 len)
{
if (m_maxAmplitude) {
if (maxAmplitude) {
Q_ASSERT(m_format.sampleSize() % 8 == 0);
const int channelBytes = m_format.sampleSize() / 8;
const int sampleBytes = m_format.channelCount() * channelBytes;
......@@ -127,11 +127,14 @@ qint64 AudioDevInfo::writeData(const char *data, qint64 len)
}
}
QVector<qreal> dbLevels;
QVector<qreal> recLevels;
for (int j = 0; j < m_format.channelCount(); ++j) {
qreal val = qMin(levels.at(j), m_maxAmplitude);
val = 20. * log10(val / m_maxAmplitude);
qreal val = qMin(levels.at(j), maxAmplitude);
val = 20. * log10(val / maxAmplitude);
recLevels << val;
dbLevels << IEC_ScaleMax(val, 0);
}
emit levelRecChanged(recLevels);
emit levelChanged(dbLevels);
}
return len;
......@@ -142,7 +145,6 @@ MediaCapture::MediaCapture(QObject *parent)
, currentState(-1)
, m_audioInput(nullptr)
, m_audioInfo(nullptr)
, m_probe(nullptr)
, m_audioDevice("default:")
, m_path(QUrl())
, m_recordState(0)
......@@ -152,6 +154,11 @@ MediaCapture::MediaCapture(QObject *parent)
connect(&m_resetTimer, &QTimer::timeout, this, &MediaCapture::resetIfUnused);
}
void MediaCapture::switchMonitorState(int tid, bool run)
{
pCore->mixer()->monitorAudio(tid, run);
}
void MediaCapture::switchMonitorState(bool run)
{
if (run) {
......@@ -183,9 +190,33 @@ void MediaCapture::switchMonitorState(bool run)
m_audioInput = std::make_unique<QAudioInput>(deviceInfo, format, this);
QObject::connect(m_audioInfo.data(), &AudioDevInfo::levelChanged, m_audioInput.get(), [&] (const QVector<qreal> &level) {
m_levels = level;
if (m_recordState == QMediaRecorder::RecordingState) {
// Get the frame number
int currentPos = qRound(m_recTimer.elapsed() / 1000. * pCore->getCurrentFps());
if (currentPos > m_lastPos) {
// Only store 1 value per frame
switch (level.count()) {
case 2:
for (int i = 0; i < currentPos - m_lastPos; i++) {
m_recLevels << qMax(level.first(), level.last());
}
break;
default:
for (int i = 0; i < currentPos - m_lastPos; i++) {
m_recLevels << level.first();
}
break;
}
m_lastPos = currentPos;
}
}
emit levelsChanged();
});
m_audioInput->setVolume(KdenliveSettings::audiocapturevolume()/100.0);
QObject::connect(m_audioInfo.data(), &AudioDevInfo::levelRecChanged, this, &MediaCapture::audioLevels);
qreal linearVolume = QAudio::convertVolume(KdenliveSettings::audiocapturevolume() / 100.0,
QAudio::LogarithmicVolumeScale,
QAudio::LinearVolumeScale);
m_audioInput->setVolume(linearVolume);
m_audioInfo->open(QIODevice::WriteOnly);
m_audioInput->start(m_audioInfo.data());
} else {
......@@ -197,6 +228,11 @@ void MediaCapture::switchMonitorState(bool run)
}
}
const QVector<double> MediaCapture::recLevels() const
{
return m_recLevels;
}
bool MediaCapture::isMonitoring() const
{
return m_audioInput != nullptr;
......@@ -223,15 +259,13 @@ void MediaCapture::recordAudio(int tid, bool record)
QMutexLocker lk(&m_recMutex);
if (!m_audioRecorder) {
m_audioRecorder = std::make_unique<QAudioRecorder>(this);
if (!m_probe) {
m_probe = std::make_unique<QAudioProbe>(this);
connect(m_probe.get(), &QAudioProbe::audioBufferProbed, this, &MediaCapture::processBuffer);
}
m_probe->setSource(m_audioRecorder.get());
connect(m_audioRecorder.get(), &QAudioRecorder::stateChanged, this, [&, tid] (QMediaRecorder::State state) {
m_recordState = state;
if (m_recordState == QMediaRecorder::StoppedState) {
m_resetTimer.start();
m_recLevels.clear();
m_recTimer.invalidate();
m_lastPos = -1;
emit audioLevels(QVector <qreal>());
emit pCore->finalizeRecording(getCaptureOutputLocation().toLocalFile());
}
......@@ -243,7 +277,10 @@ void MediaCapture::recordAudio(int tid, bool record)
setAudioCaptureDevice();
m_audioRecorder->setAudioInput(m_audioDevice);
setCaptureOutputLocation();
m_audioRecorder->setVolume(KdenliveSettings::audiocapturevolume()/100.0);
qreal linearVolume = QAudio::convertVolume(KdenliveSettings::audiocapturevolume() / 100.0,
QAudio::LogarithmicVolumeScale,
QAudio::LinearVolumeScale);
m_audioRecorder->setVolume(linearVolume);
//qDebug()<<"START AREC: "<<m_path<<"\n; CODECS: "<<m_audioRecorder->supportedAudioCodecs();
connect(m_audioRecorder.get(), static_cast<void (QAudioRecorder::*)(QMediaRecorder::Error)>(&QAudioRecorder::error),
this, &MediaCapture::displayErrorMessage);
......@@ -254,11 +291,17 @@ void MediaCapture::recordAudio(int tid, bool record)
audioSettings.setChannelCount(KdenliveSettings::audiocapturechannels());
m_audioRecorder->setEncodingSettings(audioSettings);
m_audioRecorder->setOutputLocation(m_path);
m_lastPos = -1;
m_recLevels.clear();
m_audioRecorder->record();
} else if (m_audioRecorder->state() != QMediaRecorder::PausedState) {
m_recTimer.start();
} else if (!record) {
m_audioRecorder->stop();
m_recTimer.invalidate();
} else {
m_audioRecorder->record();
m_lastPos = -1;
m_recTimer.restart();
}
}
......@@ -344,8 +387,14 @@ void MediaCapture::setAudioCaptureDevice()
void MediaCapture::setAudioVolume()
{
qreal linearVolume = QAudio::convertVolume(KdenliveSettings::audiocapturevolume() / 100.0,
QAudio::LogarithmicVolumeScale,
QAudio::LinearVolumeScale);
if (m_audioRecorder) {
m_audioRecorder->setVolume(KdenliveSettings::audiocapturevolume()/100.0);
m_audioRecorder->setVolume(linearVolume);
}
if (m_audioInput) {
m_audioInput->setVolume(linearVolume);
}
}
......@@ -360,133 +409,6 @@ int MediaCapture::getState()
return currentState;
}
template <class T> QVector<qreal> getBufferLevels(const T *buffer, int frames, int channels)
{
QVector<qreal> max_values;
max_values.fill(0, channels);
for (int i = 0; i < frames; ++i) {
for (int j = 0; j < channels; ++j) {
qreal value = qAbs(qreal(buffer[i * channels + j]));
if (value > max_values.at(j)) {
max_values.replace(j, value);
}
}
}
return max_values;
}
// This function returns the maximum possible sample value for a given audio format
qreal getPeakValue(const QAudioFormat &format)
{
// Note: Only the most common sample formats are supported
if (!format.isValid()) {
return qreal(0);
}
if (format.codec() != "audio/pcm") {
return qreal(0);
}
switch (format.sampleType()) {
case QAudioFormat::Unknown:
break;
case QAudioFormat::Float:
if (format.sampleSize() != 32) { // other sample formats are not supported
return qreal(0);
}
return qreal(1.00003);
case QAudioFormat::SignedInt:
if (format.sampleSize() == 32) {
return qreal(INT_MAX);
}
if (format.sampleSize() == 16) {
return qreal(SHRT_MAX);
}
if (format.sampleSize() == 8) {
return qreal(CHAR_MAX);
}
break;
case QAudioFormat::UnSignedInt:
if (format.sampleSize() == 32) {
return qreal(UINT_MAX);
}
if (format.sampleSize() == 16) {
return qreal(USHRT_MAX);
}
if (format.sampleSize() == 8) {
return qreal(UCHAR_MAX);
}
break;
}
return qreal(0);
}
QVector<qreal> getBufferLevels(const QAudioBuffer &buffer)
{
QVector<qreal> values;
if (!buffer.format().isValid() || buffer.format().byteOrder() != QAudioFormat::LittleEndian) {
return values;
}
if (buffer.format().codec() != "audio/pcm") {
return values;
}
int channelCount = buffer.format().channelCount();
values.fill(0, channelCount);
qreal peak_value = getPeakValue(buffer.format());
if (qFuzzyCompare(peak_value, qreal(0))) {
return values;
}
switch (buffer.format().sampleType()) {
case QAudioFormat::Unknown:
case QAudioFormat::UnSignedInt:
if (buffer.format().sampleSize() == 32) {
values = getBufferLevels(buffer.constData<quint32>(), buffer.frameCount(), channelCount);
}
if (buffer.format().sampleSize() == 16) {
values = getBufferLevels(buffer.constData<quint16>(), buffer.frameCount(), channelCount);
}
if (buffer.format().sampleSize() == 8) {
values = getBufferLevels(buffer.constData<quint8>(), buffer.frameCount(), channelCount);
}
break;
case QAudioFormat::Float:
if (buffer.format().sampleSize() == 32) {
values = getBufferLevels(buffer.constData<float>(), buffer.frameCount(), channelCount);
}
break;
case QAudioFormat::SignedInt:
if (buffer.format().sampleSize() == 32) {
values = getBufferLevels(buffer.constData<qint32>(), buffer.frameCount(), channelCount);
}
if (buffer.format().sampleSize() == 16) {
values = getBufferLevels(buffer.constData<qint16>(), buffer.frameCount(), channelCount);
}
if (buffer.format().sampleSize() == 8) {
values = getBufferLevels(buffer.constData<qint8>(), buffer.frameCount(), channelCount);
}
break;
}
std::transform(values.begin(), values.end(), values.begin(),
[peak_value](double val) { return (20. * log10(val / peak_value)); });
return values;
}
void MediaCapture::processBuffer(const QAudioBuffer &buffer)
{
QVector <qreal> levels = getBufferLevels(buffer);
emit audioLevels(levels);
/*for (auto &l : m_levels) {
l = IEC_ScaleMax(l, 0);
}
emit levelsChanged();*/
}