SSH: Work around issue with dynamic_cast.

It has been observed that on MacOs, a dynamic_cast from
Botan::Public_Key to Botan::RSA_PublicKey reproducibly fails even though
it should definitely succeed. This happens with both gcc and clang on
different Macs, but on no other platform. The problem could not be
reproduced with an example project.
The workaround is to move the allocation of the respective object from
the client side to the Botan library itself. In addition, the following
actions were taken to guard against similar problems in the future:
    - Also move to Botan the allocations of all other objects that are
potentially dynamically cast.
    - Use shared pointers for these objects, so the deallocation also
happens inside Botan.

Change-Id: Ie595a56a239a41e2629b6ff631de59910b8244dd
Reviewed-by: Eike Ziller <eike.ziller@digia.com>

src/libs/3rdparty/botan/botan.cpp

@@ -47182,3 +47182,44 @@ u32bit version_minor() { return BOTAN_VERSION_MINOR; }
 u32bit version_patch() { return BOTAN_VERSION_PATCH; }
 
 }
+
+namespace Botan {
+PublicKeyPtr createRsaPublicKey(const BigInt &e, const BigInt &n)
+{
+    return PublicKeyPtr(new RSA_PublicKey(e, n));
+}
+
+PublicKeyPtr createDsaPublicKey(const DL_Group &group, const BigInt &y)
+{
+    return PublicKeyPtr(new DSA_PublicKey(group, y));
+}
+
+PrivateKeyPtr createRsaPrivateKey(RandomNumberGenerator &rng, const BigInt &p, const BigInt &q,
+        const BigInt &e, const BigInt &d, const BigInt &n)
+{
+    return PrivateKeyPtr(new RSA_PrivateKey(rng, p, q, e, d, n));
+}
+
+PrivateKeyPtr createRsaPrivateKey(RandomNumberGenerator &rng, size_t bits, size_t exp)
+{
+    return PrivateKeyPtr(new RSA_PrivateKey(rng, bits, exp));
+}
+
+PrivateKeyPtr createDsaPrivateKey(RandomNumberGenerator &rng, const DL_Group &group,
+        const BigInt &private_key)
+{
+    return PrivateKeyPtr(new DSA_PrivateKey(rng, group, private_key));
+}
+
+PrivateKeyPtr loadPkcs8PrivateKey(DataSource& source, RandomNumberGenerator& rng,
+        const User_Interface& ui)
+{
+    return PrivateKeyPtr(PKCS8::load_key(source, rng, ui));
+}
+
+DhPrivateKeyPtr createDhPrivateKey(RandomNumberGenerator &rng, const DL_Group &grp, const BigInt &x)
+{
+    return DhPrivateKeyPtr(new DH_PrivateKey(rng, grp, x));
+}
+
+}

src/libs/3rdparty/botan/botan.h

@@ -9,6 +9,7 @@
 #define BOTAN_AMALGAMATION_H__
 
 #include <QtGlobal>
+#include <QSharedPointer>
 
 #include <iosfwd>
 #include <map>
@@ -16181,7 +16182,26 @@ class BOTAN_DLL ANSI_X919_MAC : public MessageAuthenticationCode
       SecureVector<byte> state;
       size_t position;
    };
+}
 
+namespace Botan {
+typedef QSharedPointer<Public_Key> PublicKeyPtr;
+BOTAN_DLL PublicKeyPtr createRsaPublicKey(const BigInt &e, const BigInt &n);
+BOTAN_DLL PublicKeyPtr createDsaPublicKey(const DL_Group& group, const BigInt& y);
+
+typedef QSharedPointer<Private_Key> PrivateKeyPtr;
+BOTAN_DLL PrivateKeyPtr createRsaPrivateKey(RandomNumberGenerator& rng, const BigInt& p,
+        const BigInt& q, const BigInt& e, const BigInt& d = 0, const BigInt& n = 0);
+BOTAN_DLL PrivateKeyPtr createRsaPrivateKey(RandomNumberGenerator& rng, size_t bits,
+        size_t exp = 65537);
+BOTAN_DLL PrivateKeyPtr createDsaPrivateKey(RandomNumberGenerator& rng, const DL_Group& group,
+        const BigInt& private_key = 0);
+BOTAN_DLL PrivateKeyPtr loadPkcs8PrivateKey(DataSource& source, RandomNumberGenerator& rng,
+        const User_Interface& ui);
+
+typedef QSharedPointer<DH_PrivateKey> DhPrivateKeyPtr;
+BOTAN_DLL DhPrivateKeyPtr createDhPrivateKey(RandomNumberGenerator& rng, const DL_Group& grp,
+        const BigInt& x = 0);
 }
 
 

src/libs/ssh/sshcryptofacility.cpp

@@ -218,16 +218,14 @@ bool SshEncryptionFacility::createAuthenticationKeyFromPKCS8(const QByteArray &p
     try {
         Pipe pipe;
         pipe.process_msg(convertByteArray(privKeyFileContents), privKeyFileContents.size());
-        Private_Key * const key = PKCS8::load_key(pipe, m_rng, SshKeyPasswordRetriever());
-        if (DSA_PrivateKey * const dsaKey = dynamic_cast<DSA_PrivateKey *>(key)) {
+        const PrivateKeyPtr authKey = loadPkcs8PrivateKey(pipe, m_rng, SshKeyPasswordRetriever());
+        if (DSA_PrivateKey * const dsaKey = dynamic_cast<DSA_PrivateKey *>(authKey.data())) {
             m_authKeyAlgoName = SshCapabilities::PubKeyDss;
-            m_authKey.reset(dsaKey);
             pubKeyParams << dsaKey->group_p() << dsaKey->group_q()
                          << dsaKey->group_g() << dsaKey->get_y();
             allKeyParams << pubKeyParams << dsaKey->get_x();
-        } else if (RSA_PrivateKey * const rsaKey = dynamic_cast<RSA_PrivateKey *>(key)) {
+        } else if (RSA_PrivateKey * const rsaKey = dynamic_cast<RSA_PrivateKey *>(authKey.data())) {
             m_authKeyAlgoName = SshCapabilities::PubKeyRsa;
-            m_authKey.reset(rsaKey);
             pubKeyParams << rsaKey->get_e() << rsaKey->get_n();
             allKeyParams << pubKeyParams << rsaKey->get_p() << rsaKey->get_q()
                          << rsaKey->get_d();
@@ -235,6 +233,7 @@ bool SshEncryptionFacility::createAuthenticationKeyFromPKCS8(const QByteArray &p
             qWarning("%s: Unexpected code flow, expected success or exception.", Q_FUNC_INFO);
             return false;
         }
+        m_authKey = authKey;
     } catch (const Botan::Exception &ex) {
         error = QLatin1String(ex.what());
         return false;
@@ -291,15 +290,13 @@ bool SshEncryptionFacility::createAuthenticationKeyFromOpenSSL(const QByteArray
         if (m_authKeyAlgoName == SshCapabilities::PubKeyDss) {
             BigInt p, q, g, y, x;
             sequence.decode (p).decode (q).decode (g).decode (y).decode (x);
-            DSA_PrivateKey * const dsaKey = new DSA_PrivateKey(m_rng, DL_Group(p, q, g), x);
-            m_authKey.reset(dsaKey);
+            m_authKey = createDsaPrivateKey(m_rng, DL_Group(p, q, g), x);
             pubKeyParams << p << q << g << y;
             allKeyParams << pubKeyParams << x;
         } else {
             BigInt p, q, e, d, n;
             sequence.decode(n).decode(e).decode(d).decode(p).decode(q);
-            RSA_PrivateKey * const rsaKey = new RSA_PrivateKey(m_rng, p, q, e, d, n);
-            m_authKey.reset(rsaKey);
+            m_authKey = createRsaPrivateKey(m_rng, p, q, e, d, n);
             pubKeyParams << e << n;
             allKeyParams << pubKeyParams << p << q << d;
         }

src/libs/ssh/sshcryptofacility_p.h

@@ -117,7 +117,7 @@ private:
     QByteArray m_authKeyAlgoName;
     QByteArray m_authPubKeyBlob;
     QByteArray m_cachedPrivKeyContents;
-    QScopedPointer<Botan::Private_Key> m_authKey;
+    QSharedPointer<Botan::Private_Key> m_authKey;
     mutable Botan::AutoSeeded_RNG m_rng;
 };
 

src/libs/ssh/sshkeyexchange.cpp

@@ -136,8 +136,7 @@ bool SshKeyExchange::sendDhInitPacket(const SshIncomingPacket &serverKexInit)
         kexInitParams.compressionAlgorithmsServerToClient.names);
 
     AutoSeeded_RNG rng;
-    m_dhKey.reset(new DH_PrivateKey(rng,
-        DL_Group(botanKeyExchangeAlgoName(keyAlgo))));
+    m_dhKey = createDhPrivateKey(rng, DL_Group(botanKeyExchangeAlgoName(keyAlgo)));
 
     m_serverKexInitPayload = serverKexInit.payLoad();
     m_sendFacility.sendKeyDhInitPacket(m_dhKey->get_y());
@@ -184,28 +183,24 @@ void SshKeyExchange::sendNewKeysPacket(const SshIncomingPacket &dhReply,
     printData("H", m_h);
 #endif // CREATOR_SSH_DEBUG
 
-    QScopedPointer<Public_Key> sigKey;
-    QScopedPointer<PK_Verifier> verifier;
+    QSharedPointer<Public_Key> publicKey;
+    QByteArray algorithm;
     if (m_serverHostKeyAlgo == SshCapabilities::PubKeyDss) {
         const DL_Group group(reply.parameters.at(0), reply.parameters.at(1),
             reply.parameters.at(2));
-        DSA_PublicKey * const dsaKey
-            = new DSA_PublicKey(group, reply.parameters.at(3));
-        sigKey.reset(dsaKey);
-        verifier.reset(new PK_Verifier(*dsaKey, botanEmsaAlgoName(SshCapabilities::PubKeyDss)));
+        publicKey = createDsaPublicKey(group, reply.parameters.at(3));
+        algorithm = SshCapabilities::PubKeyDss;
     } else if (m_serverHostKeyAlgo == SshCapabilities::PubKeyRsa) {
-        RSA_PublicKey * const rsaKey
-            = new RSA_PublicKey(reply.parameters.at(1), reply.parameters.at(0));
-        sigKey.reset(rsaKey);
-        verifier.reset(new PK_Verifier(*rsaKey, botanEmsaAlgoName(SshCapabilities::PubKeyRsa)));
+        publicKey = createRsaPublicKey(reply.parameters.at(1), reply.parameters.at(0));
+        algorithm = SshCapabilities::PubKeyRsa;
     } else {
         Q_ASSERT(!"Impossible: Neither DSS nor RSA!");
     }
     const byte * const botanH = convertByteArray(m_h);
     const Botan::byte * const botanSig
         = convertByteArray(reply.signatureBlob);
-    if (!verifier->verify_message(botanH, m_h.size(), botanSig,
-        reply.signatureBlob.size())) {
+    if (!PK_Verifier(*publicKey, botanEmsaAlgoName(algorithm)).verify_message(botanH, m_h.size(),
+            botanSig, reply.signatureBlob.size())) {
         throw SSH_SERVER_EXCEPTION(SSH_DISCONNECT_KEY_EXCHANGE_FAILED,
             "Invalid signature in SSH_MSG_KEXDH_REPLY packet.");
     }

src/libs/ssh/sshkeyexchange_p.h

@@ -33,6 +33,7 @@
 
 #include <QByteArray>
 #include <QScopedPointer>
+#include <QSharedPointer>
 
 namespace Botan {
 class DH_PrivateKey;
@@ -71,7 +72,7 @@ private:
     QByteArray m_serverId;
     QByteArray m_clientKexInitPayload;
     QByteArray m_serverKexInitPayload;
-    QScopedPointer<Botan::DH_PrivateKey> m_dhKey;
+    QSharedPointer<Botan::DH_PrivateKey> m_dhKey;
     QByteArray m_k;
     QByteArray m_h;
     QByteArray m_serverHostKeyAlgo;

src/libs/ssh/sshkeygenerator.cpp

@@ -60,9 +60,9 @@ bool SshKeyGenerator::generateKeys(KeyType type, PrivateKeyFormat format, int ke
         AutoSeeded_RNG rng;
         KeyPtr key;
         if (m_type == Rsa)
-            key = KeyPtr(new RSA_PrivateKey(rng, keySize));
+            key = createRsaPrivateKey(rng, keySize);
         else
-            key = KeyPtr(new DSA_PrivateKey(rng, DL_Group(rng, DL_Group::DSA_Kosherizer, keySize)));
+            key = createDsaPrivateKey(rng, DL_Group(rng, DL_Group::DSA_Kosherizer, keySize));
         switch (format) {
         case Pkcs8:
             generatePkcs8KeyStrings(key, rng);