diff --git a/src/plugins/idletime/poller.cpp b/src/plugins/idletime/poller.cpp
index 3ac43e1860fad5a1a26056eed2c04b5fa05220a2..7a49aa7347979180c2852a33ecc42cb27978f3c3 100644
--- a/src/plugins/idletime/poller.cpp
+++ b/src/plugins/idletime/poller.cpp
@@ -33,6 +33,7 @@ bool KWinIdleTimePoller::isAvailable()
 bool KWinIdleTimePoller::setUpPoller()
 {
     connect(waylandServer()->idle(), &KWaylandServer::IdleInterface::inhibitedChanged, this, &KWinIdleTimePoller::onInhibitedChanged);
+    connect(waylandServer()->seat(), &KWaylandServer::SeatInterface::timestampChanged, this, &KWinIdleTimePoller::onTimestampChanged);
 
     return true;
 }
@@ -41,12 +42,12 @@ void KWinIdleTimePoller::unloadPoller()
 {
     if (waylandServer() && waylandServer()->idle()) {
         disconnect(waylandServer()->idle(), &KWaylandServer::IdleInterface::inhibitedChanged, this, &KWinIdleTimePoller::onInhibitedChanged);
+        disconnect(waylandServer()->idle(), &KWaylandServer::IdleInterface::timestampChanged, this, &KWinIdleTimePoller::onTimestampChanged);
     }
 
     qDeleteAll(m_timeouts);
     m_timeouts.clear();
 
-    m_started = false;
     m_idling = false;
 }
 
@@ -67,12 +68,11 @@ void KWinIdleTimePoller::addTimeout(int newTimeout)
     m_timeouts.insert(newTimeout, timer);
 
     if (!waylandServer()->idle()->isInhibited()) {
-        m_started = true;
         timer->start();
     }
 }
 
-void KWinIdleTimePoller::onActivity()
+void KWinIdleTimePoller::processActivity()
 {
     if (m_idling) {
         Q_EMIT resumingFromIdle();
@@ -86,15 +86,9 @@ void KWinIdleTimePoller::onActivity()
 
 void KWinIdleTimePoller::onInhibitedChanged()
 {
-    if (!m_started) {
-        // if timers were not on, nothing to do
-        return;
-    }
     if (waylandServer()->idle()->isInhibited()) {
         // must stop the timers
         stopCatchingIdleEvents();
-        // keep started state from before inhibition
-        m_started = true;
     } else {
         // resume the timers
         catchIdleEvent();
@@ -104,11 +98,15 @@ void KWinIdleTimePoller::onInhibitedChanged()
     }
 }
 
-void KWinIdleTimePoller::catchIdleEvent()
+void KWinIdleTimePoller::onTimestampChanged()
 {
-    m_started = true;
-    connect(waylandServer()->seat(), &KWaylandServer::SeatInterface::timestampChanged, this, &KWinIdleTimePoller::onActivity);
+    if (!waylandServer()->idle()->isInhibited()) {
+        processActivity();
+    }
+}
 
+void KWinIdleTimePoller::catchIdleEvent()
+{
     for (QTimer *timer : qAsConst(m_timeouts)) {
         timer->start();
     }
@@ -116,9 +114,6 @@ void KWinIdleTimePoller::catchIdleEvent()
 
 void KWinIdleTimePoller::stopCatchingIdleEvents()
 {
-    m_started = false;
-    disconnect(waylandServer()->seat(), &KWaylandServer::SeatInterface::timestampChanged, this, &KWinIdleTimePoller::onActivity);
-
     for (QTimer *timer : qAsConst(m_timeouts)) {
         timer->stop();
     }
@@ -129,7 +124,7 @@ void KWinIdleTimePoller::simulateUserActivity()
     if (waylandServer()->idle()->isInhibited()) {
         return ;
     }
-    onActivity();
+    processActivity();
     waylandServer()->simulateUserActivity();
 }
 
diff --git a/src/plugins/idletime/poller.h b/src/plugins/idletime/poller.h
index 679e6d1738d348b6b5642bc539f192fb4e7102a7..0f28aeacd446eb4c89eb1b59266ee975f34076d3 100644
--- a/src/plugins/idletime/poller.h
+++ b/src/plugins/idletime/poller.h
@@ -40,13 +40,13 @@ public Q_SLOTS:
     void simulateUserActivity() override;
 
 private Q_SLOTS:
-    void onActivity();
     void onInhibitedChanged();
+    void onTimestampChanged();
 
 private:
+    void processActivity();
     QHash<int, QTimer *> m_timeouts;
     bool m_idling = false;
-    bool m_started = false;
 };
 
 }