Fix result of resolution if mixed protocols are needed but not allowed

If mixed protocols are not allowed, but for some recipients there are
only OpenPGP keys and for others there are only S/MIME certificates,
then the resolution should fail.

GnuPG-bug-id: 5283

autotests/keyresolvercoretest.cpp

@@ -216,6 +216,21 @@ private Q_SLOTS:
         QCOMPARE(resolver.encryptionKeys().value(UnknownProtocol).size(), 0);
     }
 
+    void test_reports_failure_if_mixed_protocols_are_not_allowed_but_needed()
+    {
+        KeyResolverCore resolver(/*encrypt=*/ true, /*sign=*/ false);
+        resolver.setAllowMixedProtocols(false);
+        resolver.setRecipients({"sender-openpgp@example.net", "sender-smime@example.net"});
+
+        const bool success = resolver.resolve();
+        QVERIFY(!success);
+        QCOMPARE(resolver.encryptionKeys().value(OpenPGP).size(), 1);
+        QVERIFY(resolver.encryptionKeys().value(OpenPGP).contains("sender-openpgp@example.net"));
+        QCOMPARE(resolver.encryptionKeys().value(CMS).size(), 1);
+        QVERIFY(resolver.encryptionKeys().value(CMS).contains("sender-smime@example.net"));
+        QCOMPARE(resolver.encryptionKeys().value(UnknownProtocol).size(), 0);
+    }
+
     void test_openpgp_overrides_are_used_if_both_protocols_are_allowed()
     {
         const QString override = testKey("prefer-openpgp@example.net", OpenPGP).primaryFingerprint();

src/kleo/keyresolvercore.cpp

@@ -436,6 +436,17 @@ QStringList KeyResolverCore::Private::unresolvedRecipients(GpgME::Protocol proto
     return result;
 }
 
+namespace
+{
+bool hasUnresolvedRecipients(const QMap<QString, QMap<Protocol, std::vector<Key>>> &encryptionKeys, Protocol protocol)
+{
+    return std::any_of(std::cbegin(encryptionKeys), std::cend(encryptionKeys),
+                       [protocol] (const auto &protocolKeysMap) {
+                           return protocolKeysMap.value(protocol).empty();
+                       });
+}
+}
+
 bool KeyResolverCore::Private::resolve()
 {
     qCDebug(LIBKLEO_LOG) << "Starting ";
@@ -452,34 +463,28 @@ bool KeyResolverCore::Private::resolve()
         resolveSign(OpenPGP);
         resolveEnc(OpenPGP);
     }
-    const QStringList unresolvedPGP = unresolvedRecipients(OpenPGP);
-    bool pgpOnly = unresolvedPGP.empty() && (!mSign || mSigKeys.contains(OpenPGP));
+    const bool pgpOnly = !hasUnresolvedRecipients(mEncKeys, OpenPGP) && (!mSign || mSigKeys.contains(OpenPGP));
 
     if (mFormat != OpenPGP) {
         resolveSign(CMS);
         resolveEnc(CMS);
     }
-    const QStringList unresolvedCMS = unresolvedRecipients(CMS);
-    bool cmsOnly = unresolvedCMS.empty() && (!mSign || mSigKeys.contains(CMS));
+    const bool cmsOnly = !hasUnresolvedRecipients(mEncKeys, CMS) && (!mSign || mSigKeys.contains(CMS));
 
     if (mAllowMixed && mFormat == UnknownProtocol) {
         mergeEncryptionKeys();
     }
-    const QStringList unresolvedMerged = unresolvedRecipients(UnknownProtocol);
+    const bool hasUnresolvedMerged = mAllowMixed && mFormat == UnknownProtocol && hasUnresolvedRecipients(mEncKeys, UnknownProtocol);
 
     // Check if we need the user to select different keys.
     bool needsUser = false;
     if (!pgpOnly && !cmsOnly) {
         if (mAllowMixed && mFormat == UnknownProtocol) {
-            needsUser = !unresolvedMerged.empty();
+            needsUser = hasUnresolvedMerged;
         } else {
-            for (const auto &unresolved: unresolvedPGP) {
-                if (unresolvedCMS.contains(unresolved)) {
-                    // We have at least one unresolvable key.
-                    needsUser = true;
-                    break;
-                }
-            }
+            needsUser = (mFormat == UnknownProtocol)
+                        || (mFormat == OpenPGP && !pgpOnly)
+                        || (mFormat == CMS && !cmsOnly);
         }
         if (mSign) {
             // So every recipient could be resolved through