SSH: Produce "mixed" key files by default.

I.e. the private key in PKCS8 format, the public key in a format
suitable for OpenSSH's authorized_keys file. Seems like the most
pragmatic solution for now.

Change-Id: I02d9bf4d98d836d9fc33453bff0b3f4834e7e173
Reviewed-on: http://codereview.qt.nokia.com/2350
Reviewed-by: Christian Kandeler <christian.kandeler@nokia.com>

src/libs/utils/ssh/sshkeygenerator.cpp

@@ -52,17 +52,13 @@ namespace Utils {
 using namespace Botan;
 using namespace Internal;
 
-SshKeyGenerator::SshKeyGenerator()
+SshKeyGenerator::SshKeyGenerator() : m_type(Rsa)
-    : m_type(Rsa)
-    , m_format(OpenSsl)
 {
 }
 
-bool SshKeyGenerator::generateKeys(KeyType type, PrivateKeyFormat format,
+bool SshKeyGenerator::generateKeys(KeyType type, PrivateKeyFormat format, int keySize)
-    int keySize)
 {
     m_type = type;
-    m_format = format;
 
     try {
         AutoSeeded_RNG rng;
@@ -70,24 +66,33 @@ bool SshKeyGenerator::generateKeys(KeyType type, PrivateKeyFormat format,
         if (m_type == Rsa)
             key = KeyPtr(new RSA_PrivateKey(rng, keySize));
         else
-            key = KeyPtr(new DSA_PrivateKey(rng, DL_Group(rng, DL_Group::DSA_Kosherizer,
+            key = KeyPtr(new DSA_PrivateKey(rng, DL_Group(rng, DL_Group::DSA_Kosherizer, keySize)));
-                keySize)));
+        switch (format) {
-        return m_format == Pkcs8
+        case Pkcs8:
-            ? generatePkcs8Keys(key) : generateOpenSslKeys(key);
+            generatePkcs8KeyStrings(key);
+            break;
+        case OpenSsl:
+            generateOpenSslKeyStrings(key);
+            break;
+        case Mixed:
+        default:
+            generatePkcs8KeyString(key, true);
+            generateOpenSslPublicKeyString(key);
+        }
+        return true;
     } catch (Botan::Exception &e) {
         m_error = tr("Error generating key: %1").arg(e.what());
         return false;
     }
 }
 
-bool SshKeyGenerator::generatePkcs8Keys(const KeyPtr &key)
+void SshKeyGenerator::generatePkcs8KeyStrings(const KeyPtr &key)
 {
-    generatePkcs8Key(key, false);
+    generatePkcs8KeyString(key, false);
-    generatePkcs8Key(key, true);
+    generatePkcs8KeyString(key, true);
-    return true;
 }
 
-void SshKeyGenerator::generatePkcs8Key(const KeyPtr &key, bool privateKey)
+void SshKeyGenerator::generatePkcs8KeyString(const KeyPtr &key, bool privateKey)
 {
     Pipe pipe;
     pipe.start_msg();
@@ -105,45 +110,61 @@ void SshKeyGenerator::generatePkcs8Key(const KeyPtr &key, bool privateKey)
         pipe.message_count() - 1);
 }
 
-bool SshKeyGenerator::generateOpenSslKeys(const KeyPtr &key)
+void SshKeyGenerator::generateOpenSslKeyStrings(const KeyPtr &key)
 {
-    QList<BigInt> publicParams;
+    generateOpenSslPublicKeyString(key);
-    QList<BigInt> allParams;
+    generateOpenSslPrivateKeyString(key);
+}
+
+void SshKeyGenerator::generateOpenSslPublicKeyString(const KeyPtr &key)
+{
+    QList<BigInt> params;
     QByteArray keyId;
     if (m_type == Rsa) {
-        const QSharedPointer<RSA_PrivateKey> rsaKey
+        const QSharedPointer<RSA_PrivateKey> rsaKey = key.dynamicCast<RSA_PrivateKey>();
-            = key.dynamicCast<RSA_PrivateKey>();
+        params << rsaKey->get_e() << rsaKey->get_n();
-        publicParams << rsaKey->get_e() << rsaKey->get_n();
-        allParams << rsaKey->get_n() << rsaKey->get_e() << rsaKey->get_d()
-            << rsaKey->get_p() << rsaKey->get_q();
         keyId = SshCapabilities::PubKeyRsa;
     } else {
-        const QSharedPointer<DSA_PrivateKey> dsaKey
+        const QSharedPointer<DSA_PrivateKey> dsaKey = key.dynamicCast<DSA_PrivateKey>();
-            = key.dynamicCast<DSA_PrivateKey>();
+        params << dsaKey->group_p() << dsaKey->group_q() << dsaKey->group_g() << dsaKey->get_y();
-        publicParams << dsaKey->group_p() << dsaKey->group_q()
-            << dsaKey->group_g() << dsaKey->get_y();
-        allParams << publicParams << dsaKey->get_x();
         keyId = SshCapabilities::PubKeyDss;
     }
 
     QByteArray publicKeyBlob = AbstractSshPacket::encodeString(keyId);
-    foreach (const BigInt &b, publicParams)
+    foreach (const BigInt &b, params)
         publicKeyBlob += AbstractSshPacket::encodeMpInt(b);
     publicKeyBlob = publicKeyBlob.toBase64();
     const QByteArray id = "QtCreator/"
         + QDateTime::currentDateTime().toString(Qt::ISODate).toUtf8();
     m_publicKey = keyId + ' ' + publicKeyBlob + ' ' + id;
+}
+
+void SshKeyGenerator::generateOpenSslPrivateKeyString(const KeyPtr &key)
+{
+    QList<BigInt> params;
+    QByteArray keyId;
+    const char *label;
+    if (m_type == Rsa) {
+        const QSharedPointer<RSA_PrivateKey> rsaKey
+            = key.dynamicCast<RSA_PrivateKey>();
+        params << rsaKey->get_n() << rsaKey->get_e() << rsaKey->get_d() << rsaKey->get_p()
+            << rsaKey->get_q();
+        keyId = SshCapabilities::PubKeyRsa;
+        label = "RSA PRIVATE KEY";
+    } else {
+        const QSharedPointer<DSA_PrivateKey> dsaKey = key.dynamicCast<DSA_PrivateKey>();
+        params << dsaKey->group_p() << dsaKey->group_q() << dsaKey->group_g() << dsaKey->get_y()
+            << dsaKey->get_x();
+        keyId = SshCapabilities::PubKeyDss;
+        label = "DSA PRIVATE KEY";
+    }
 
     DER_Encoder encoder;
     encoder.start_cons(SEQUENCE).encode(0U);
-    foreach (const BigInt &b, allParams)
+    foreach (const BigInt &b, params)
         encoder.encode(b);
     encoder.end_cons();
-    const char * const label
+    m_privateKey = QByteArray(PEM_Code::encode (encoder.get_contents(), label).c_str());
-        = m_type == Rsa ? "RSA PRIVATE KEY" : "DSA PRIVATE KEY";
-    m_privateKey
-        = QByteArray(PEM_Code::encode (encoder.get_contents(), label).c_str());
-    return true;
 }
 
 } // namespace Utils

src/libs/utils/ssh/sshkeygenerator.h

@@ -49,28 +49,29 @@ class QTCREATOR_UTILS_EXPORT SshKeyGenerator
     Q_DECLARE_TR_FUNCTIONS(SshKeyGenerator)
 public:
     enum KeyType { Rsa, Dsa };
-    enum PrivateKeyFormat { Pkcs8, OpenSsl };
+    enum PrivateKeyFormat { Pkcs8, OpenSsl, Mixed };
 
     SshKeyGenerator();
     bool generateKeys(KeyType type, PrivateKeyFormat format, int keySize);
+
     QString error() const { return m_error; }
     QByteArray privateKey() const { return m_privateKey; }
     QByteArray publicKey() const { return m_publicKey; }
     KeyType type() const { return m_type; }
-    PrivateKeyFormat format() const { return m_format; }
 
 private:
     typedef QSharedPointer<Botan::Private_Key> KeyPtr;
 
-    bool generatePkcs8Keys(const KeyPtr &key);
+    void generatePkcs8KeyStrings(const KeyPtr &key);
-    void generatePkcs8Key(const KeyPtr &key, bool privateKey);
+    void generatePkcs8KeyString(const KeyPtr &key, bool privateKey);
-    bool generateOpenSslKeys(const KeyPtr &key);
+    void generateOpenSslKeyStrings(const KeyPtr &key);
+    void generateOpenSslPrivateKeyString(const KeyPtr &key);
+    void generateOpenSslPublicKeyString(const KeyPtr &key);
 
     QString m_error;
     QByteArray m_publicKey;
     QByteArray m_privateKey;
     KeyType m_type;
-    PrivateKeyFormat m_format;
 };
 
 } // namespace Utils

src/plugins/remotelinux/maemodeviceconfigwizard.cpp

@@ -346,7 +346,7 @@ private:
         m_ui->statusLabel->setText(tr("Creating keys ... "));
         SshKeyGenerator keyGenerator;
         if (!keyGenerator.generateKeys(SshKeyGenerator::Rsa,
-             SshKeyGenerator::OpenSsl, 1024)) {
+             SshKeyGenerator::Mixed, 1024)) {
             QMessageBox::critical(this, tr("Cannot Create Keys"),
                 tr("Key creation failed: %1").arg(keyGenerator.error()));
             enableInput();

src/plugins/remotelinux/sshkeycreationdialog.cpp

@@ -78,7 +78,7 @@ void SshKeyCreationDialog::generateSshKey()
 
     QApplication::setOverrideCursor(Qt::BusyCursor);
 
-    if (m_keyGenerator->generateKeys(keyType, SshKeyGenerator::OpenSsl,
+    if (m_keyGenerator->generateKeys(keyType, SshKeyGenerator::Mixed,
                                      m_ui.comboBox->currentText().toUShort())) {
         m_ui.plainTextEdit->setPlainText(m_keyGenerator->publicKey());
         m_ui.savePublicKey->setEnabled(true);