[Qemu-devel] [PATCH v2 12/15] nbd: implement TLS support in the protocol negotiation

["Daniel P. Berrange" <berrange@redhat.com>]

This extends the NBD protocol handling code so that it is capable
of negotiating TLS support during the connection setup. This involves
requesting the STARTTLS protocol option before any other NBD options.

Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
---
 block/nbd-client.c  |  12 ++-
 blockdev-nbd.c      |   2 +-
 include/block/nbd.h |   8 ++
 nbd.c               | 289 +++++++++++++++++++++++++++++++++++++++++++++++++---
 qemu-nbd.c          |   4 +-
 5 files changed, 296 insertions(+), 19 deletions(-)

diff --git a/block/nbd-client.c b/block/nbd-client.c
index ed69d4b..f7bacfd 100644
--- a/block/nbd-client.c
+++ b/block/nbd-client.c
@@ -389,7 +389,10 @@ int nbd_client_init(BlockDriverState *bs, QIOChannelSocket *sioc,
     qio_channel_set_blocking(QIO_CHANNEL(sioc), true, NULL);
 
     ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), export,
-                                &client->nbdflags, &client->size, errp);
+                                &client->nbdflags,
+                                NULL, NULL,
+                                &client->ioc,
+                                &client->size, errp);
     if (ret < 0) {
         logout("Failed to negotiate with the NBD server\n");
         return ret;
@@ -399,8 +402,11 @@ int nbd_client_init(BlockDriverState *bs, QIOChannelSocket *sioc,
     qemu_co_mutex_init(&client->free_sema);
     client->sioc = sioc;
     object_ref(OBJECT(client->sioc));
-    client->ioc = QIO_CHANNEL(sioc);
-    object_ref(OBJECT(client->ioc));
+
+    if (!client->ioc) {
+        client->ioc = QIO_CHANNEL(sioc);
+        object_ref(OBJECT(client->ioc));
+    }
 
     /* Now that we're connected, set the socket to be non-blocking and
      * kick the reply mechanism.  */
diff --git a/blockdev-nbd.c b/blockdev-nbd.c
index 7d4c415..6af1684 100644
--- a/blockdev-nbd.c
+++ b/blockdev-nbd.c
@@ -33,7 +33,7 @@ static gboolean nbd_accept(QIOChannel *ioc, GIOCondition condition,
         return TRUE;
     }
 
-    nbd_client_new(NULL, cioc, nbd_client_put);
+    nbd_client_new(NULL, cioc, NULL, NULL, nbd_client_put);
     object_unref(OBJECT(cioc));
     return TRUE;
 }
diff --git a/include/block/nbd.h b/include/block/nbd.h
index 0230c7f..f2d8ecf 100644
--- a/include/block/nbd.h
+++ b/include/block/nbd.h
@@ -24,6 +24,7 @@
 #include "qemu-common.h"
 #include "qemu/option.h"
 #include "io/channel-socket.h"
+#include "crypto/tlscreds.h"
 
 struct nbd_request {
     uint32_t magic;
@@ -56,7 +57,10 @@ struct nbd_reply {
 #define NBD_REP_ACK             (1)             /* Data sending finished. */
 #define NBD_REP_SERVER          (2)             /* Export description. */
 #define NBD_REP_ERR_UNSUP       ((UINT32_C(1) << 31) | 1) /* Unknown option. */
+#define NBD_REP_ERR_POLICY      ((UINT32_C(1) << 31) | 2) /* Server denies request */
 #define NBD_REP_ERR_INVALID     ((UINT32_C(1) << 31) | 3) /* Invalid length. */
+#define NBD_REP_ERR_TLS_REQD    ((UINT32_C(1) << 31) | 5) /* TLS required */
+
 
 #define NBD_CMD_MASK_COMMAND	0x0000ffff
 #define NBD_CMD_FLAG_FUA	(1 << 16)
@@ -81,6 +85,8 @@ ssize_t nbd_wr_syncv(QIOChannel *ioc,
                      size_t length,
                      bool do_read);
 int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
+                          QCryptoTLSCreds *tlscreds, const char *hostname,
+                          QIOChannel **outioc,
                           off_t *size, Error **errp);
 int nbd_init(int fd, QIOChannelSocket *sioc, uint32_t flags, off_t size);
 ssize_t nbd_send_request(QIOChannel *ioc, struct nbd_request *request);
@@ -106,6 +112,8 @@ void nbd_export_close_all(void);
 
 NBDClient *nbd_client_new(NBDExport *exp,
                           QIOChannelSocket *sioc,
+                          QCryptoTLSCreds *tlscreds,
+                          const char *tlsaclname,
                           void (*close)(NBDClient *));
 void nbd_client_get(NBDClient *client);
 void nbd_client_put(NBDClient *client);
diff --git a/nbd.c b/nbd.c
index 0f6c755..daab170 100644
--- a/nbd.c
+++ b/nbd.c
@@ -18,6 +18,7 @@
 
 #include "block/nbd.h"
 #include "sysemu/block-backend.h"
+#include "io/channel-tls.h"
 
 #include "qemu/coroutine.h"
 #include "qemu/iov.h"
@@ -85,6 +86,8 @@
 #define NBD_OPT_EXPORT_NAME     (1)
 #define NBD_OPT_ABORT           (2)
 #define NBD_OPT_LIST            (3)
+#define NBD_OPT_PEEK_EXPORT     (4)
+#define NBD_OPT_STARTTLS        (5)
 
 /* NBD errors are based on errno numbers, so there is a 1:1 mapping,
  * but only a limited set of errno values is specified in the protocol.
@@ -171,6 +174,8 @@ struct NBDClient {
     void (*close)(NBDClient *client);
 
     NBDExport *exp;
+    QCryptoTLSCreds *tlscreds;
+    char *tlsaclname;
     QIOChannelSocket *sioc; /* The underlying data channel */
     QIOChannel *ioc; /* The current I/O channel which may differ (eg TLS) */
 
@@ -365,6 +370,8 @@ static int nbd_send_rep(QIOChannel *ioc, uint32_t type, uint32_t opt)
     uint64_t magic;
     uint32_t len;
 
+    TRACE("Reply opt=%x type=%x", type, opt);
+
     magic = cpu_to_be64(NBD_REP_MAGIC);
     if (write_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
         LOG("write failed (rep magic)");
@@ -483,6 +490,74 @@ fail:
     return rc;
 }
 
+
+struct NBDTLSHandshakeData {
+    GMainLoop *loop;
+    bool complete;
+    Error *error;
+};
+
+static void nbd_tls_handshake(Object *src,
+                              Error *err,
+                              void *opaque)
+{
+    struct NBDTLSHandshakeData *data = opaque;
+
+    if (err) {
+        TRACE("TLS failed %s", error_get_pretty(err));
+        data->error = error_copy(err);
+    }
+    data->complete = true;
+    g_main_loop_quit(data->loop);
+}
+
+static QIOChannel *nbd_handle_starttls(NBDClient *client, uint32_t length)
+{
+    QIOChannel *ioc;
+    QIOChannelTLS *tioc;
+    struct NBDTLSHandshakeData data = { 0 };
+
+    TRACE("Setting up TLS");
+    ioc = client->ioc;
+    if (length) {
+        if (drop_sync(ioc, length) != length) {
+            return NULL;
+        }
+        nbd_send_rep(ioc, NBD_REP_ERR_INVALID, NBD_OPT_STARTTLS);
+        return NULL;
+    }
+
+    nbd_send_rep(client->ioc, NBD_REP_ACK, NBD_OPT_STARTTLS);
+
+    tioc = qio_channel_tls_new_server(ioc,
+                                      client->tlscreds,
+                                      client->tlsaclname,
+                                      NULL);
+    if (!tioc) {
+        return NULL;
+    }
+
+    TRACE("Starting TLS handshake");
+    data.loop = g_main_loop_new(g_main_context_default(), FALSE);
+    qio_channel_tls_handshake(tioc,
+                              nbd_tls_handshake,
+                              &data,
+                              NULL);
+
+    if (!data.complete) {
+        g_main_loop_run(data.loop);
+    }
+    g_main_loop_unref(data.loop);
+    if (data.error) {
+        object_unref(OBJECT(tioc));
+        error_free(data.error);
+        return NULL;
+    }
+
+    return QIO_CHANNEL(tioc);
+}
+
+
 static int nbd_receive_options(NBDClient *client)
 {
     QIOChannel *ioc = client->ioc;
@@ -548,7 +623,28 @@ static int nbd_receive_options(NBDClient *client)
         length = be32_to_cpu(length);
 
         TRACE("Checking option 0x%x", clientflags);
-        if (fixedNewstyle) {
+        if (client->tlscreds &&
+            ioc == (QIOChannel *)client->sioc) {
+            if (!fixedNewstyle) {
+                TRACE("Unsupported option 0x%x", clientflags);
+                return -EINVAL;
+            }
+            switch (clientflags) {
+            case NBD_OPT_STARTTLS:
+                ioc = nbd_handle_starttls(client, length);
+                if (!ioc) {
+                    return -EIO;
+                }
+                object_unref(OBJECT(client->ioc));
+                client->ioc = QIO_CHANNEL(ioc);
+                break;
+
+            default:
+                TRACE("Option 0x%x not permitted before TLS", clientflags);
+                nbd_send_rep(client->ioc, NBD_REP_ERR_TLS_REQD, clientflags);
+                return -EINVAL;
+            }
+        } else if (fixedNewstyle) {
             switch (clientflags) {
             case NBD_OPT_LIST:
                 ret = nbd_handle_list(client, length);
@@ -563,6 +659,15 @@ static int nbd_receive_options(NBDClient *client)
             case NBD_OPT_EXPORT_NAME:
                 return nbd_handle_export_name(client, length);
 
+            case NBD_OPT_STARTTLS:
+                if (client->tlscreds) {
+                    TRACE("TLS already enabled");
+                    nbd_send_rep(client->ioc, NBD_REP_ERR_INVALID, clientflags);
+                } else {
+                    TRACE("TLS not configured");
+                    nbd_send_rep(client->ioc, NBD_REP_ERR_POLICY, clientflags);
+                }
+                return -EINVAL;
             default:
                 TRACE("Unsupported option 0x%x", clientflags);
                 nbd_send_rep(client->ioc, NBD_REP_ERR_UNSUP, clientflags);
@@ -587,13 +692,13 @@ static int nbd_receive_options(NBDClient *client)
 
 static int nbd_send_negotiate(NBDClient *client)
 {
-    QIOChannel *ioc = client->ioc;
     char buf[8 + 8 + 8 + 128];
     int rc;
     const int myflags = (NBD_FLAG_HAS_FLAGS | NBD_FLAG_SEND_TRIM |
                          NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA);
+    bool oldStyle;
 
-    /* Negotiation header without options:
+    /* Old style negotiation header without options:
         [ 0 ..   7]   passwd       ("NBDMAGIC")
         [ 8 ..  15]   magic        (NBD_CLIENT_MAGIC)
         [16 ..  23]   size
@@ -601,24 +706,25 @@ static int nbd_send_negotiate(NBDClient *client)
         [26 ..  27]   export flags
         [28 .. 151]   reserved     (0)
 
-       Negotiation header with options, part 1:
+       New style negotiation header with options:
         [ 0 ..   7]   passwd       ("NBDMAGIC")
         [ 8 ..  15]   magic        (NBD_OPTS_MAGIC)
         [16 ..  17]   server flags (0)
-
-       part 2 (after options are sent):
+        ....options sent....
         [18 ..  25]   size
         [26 ..  27]   export flags
         [28 .. 151]   reserved     (0)
      */
 
-    qio_channel_set_blocking(ioc, true, NULL);
+    qio_channel_set_blocking(client->ioc, true, NULL);
     rc = -EINVAL;
 
     TRACE("Beginning negotiation.");
     memset(buf, 0, sizeof(buf));
     memcpy(buf, "NBDMAGIC", 8);
-    if (client->exp) {
+
+    oldStyle = client->exp != NULL && !client->tlscreds;
+    if (oldStyle) {
         assert ((client->exp->nbdflags & ~65535) == 0);
         cpu_to_be64w((uint64_t*)(buf + 8), NBD_CLIENT_MAGIC);
         cpu_to_be64w((uint64_t*)(buf + 16), client->exp->size);
@@ -628,13 +734,17 @@ static int nbd_send_negotiate(NBDClient *client)
         cpu_to_be16w((uint16_t *)(buf + 16), NBD_FLAG_FIXED_NEWSTYLE);
     }
 
-    if (client->exp) {
-        if (write_sync(ioc, buf, sizeof(buf)) != sizeof(buf)) {
+    if (oldStyle) {
+        if (client->tlscreds) {
+            TRACE("TLS cannot be enabled with oldstyle protocol");
+            goto fail;
+        }
+        if (write_sync(client->ioc, buf, sizeof(buf)) != sizeof(buf)) {
             LOG("write failed");
             goto fail;
         }
     } else {
-        if (write_sync(ioc, buf, 18) != 18) {
+        if (write_sync(client->ioc, buf, 18) != 18) {
             LOG("write failed");
             goto fail;
         }
@@ -647,7 +757,8 @@ static int nbd_send_negotiate(NBDClient *client)
         assert ((client->exp->nbdflags & ~65535) == 0);
         cpu_to_be64w((uint64_t*)(buf + 18), client->exp->size);
         cpu_to_be16w((uint16_t*)(buf + 26), client->exp->nbdflags | myflags);
-        if (write_sync(ioc, buf + 18, sizeof(buf) - 18) != sizeof(buf) - 18) {
+        if (write_sync(client->ioc, buf + 18, sizeof(buf) - 18) !=
+            sizeof(buf) - 18) {
             LOG("write failed");
             goto fail;
         }
@@ -656,7 +767,7 @@ static int nbd_send_negotiate(NBDClient *client)
     TRACE("Negotiation succeeded.");
     rc = 0;
 fail:
-    qio_channel_set_blocking(ioc, false, NULL);
+    qio_channel_set_blocking(client->ioc, false, NULL);
     return rc;
 }
 
@@ -672,10 +783,18 @@ static int nbd_handle_reply_err(uint32_t opt, uint32_t type, Error **errp)
         error_setg(errp, "Unsupported option type %x", opt);
         break;
 
+    case NBD_REP_ERR_POLICY:
+        error_setg(errp, "Denied by server for option %x", opt);
+        break;
+
     case NBD_REP_ERR_INVALID:
         error_setg(errp, "Invalid data length for option %x", opt);
         break;
 
+    case NBD_REP_ERR_TLS_REQD:
+        error_setg(errp, "TLS negotiation required before option %x", opt);
+        break;
+
     default:
         error_setg(errp, "Unknown error code when asking for option %x", opt);
         break;
@@ -834,7 +953,109 @@ static char *nbd_receive_pick_export(QIOChannel *ioc,
     return chosenname;
 }
 
+
+static QIOChannel *nbd_receive_starttls(QIOChannel *ioc,
+                                        QCryptoTLSCreds *tlscreds,
+                                        const char *hostname, Error **errp)
+{
+    uint64_t magic = cpu_to_be64(NBD_OPTS_MAGIC);
+    uint32_t opt = cpu_to_be32(NBD_OPT_STARTTLS);
+    uint32_t length = 0;
+    uint32_t type;
+    QIOChannelTLS *tioc;
+    struct NBDTLSHandshakeData data = { 0 };
+
+    TRACE("Requesting TLS from server");
+    if (write_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "Failed to send option magic");
+        return NULL;
+    }
+
+    if (write_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "Failed to send option number");
+        return NULL;
+    }
+
+    if (write_sync(ioc, &length, sizeof(length)) != sizeof(length)) {
+        error_setg(errp, "Failed to send option length");
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server1");
+    if (read_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "failed to read option magic");
+        return NULL;
+    }
+    magic = be64_to_cpu(magic);
+    if (magic != NBD_REP_MAGIC) {
+        error_setg(errp, "Unexpected option magic");
+        return NULL;
+    }
+    TRACE("Getting TLS reply from server2");
+    if (read_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "failed to read option");
+        return NULL;
+    }
+    opt = be32_to_cpu(opt);
+    if (opt != NBD_OPT_STARTTLS) {
+        error_setg(errp, "Unexpected option type %x expected %x",
+                   opt, NBD_OPT_STARTTLS);
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server");
+    if (read_sync(ioc, &type, sizeof(type)) != sizeof(type)) {
+        error_setg(errp, "failed to read option type");
+        return NULL;
+    }
+    type = be32_to_cpu(type);
+    if (type != NBD_REP_ACK) {
+        error_setg(errp, "Server rejected request to start TLS %x",
+                   type);
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server");
+    if (read_sync(ioc, &length, sizeof(length)) != sizeof(length)) {
+        error_setg(errp, "failed to read option length");
+        return NULL;
+    }
+    length = be32_to_cpu(length);
+    if (length != 0) {
+        error_setg(errp, "Start TLS reponse was not zero %x",
+                   length);
+        return NULL;
+    }
+
+    TRACE("TLS request approved, setting up TLS");
+    tioc = qio_channel_tls_new_client(ioc, tlscreds, hostname, errp);
+    if (!tioc) {
+        return NULL;
+    }
+    data.loop = g_main_loop_new(g_main_context_default(), FALSE);
+    TRACE("Starting TLS hanshake");
+    qio_channel_tls_handshake(tioc,
+                              nbd_tls_handshake,
+                              &data,
+                              NULL);
+
+    if (!data.complete) {
+        g_main_loop_run(data.loop);
+    }
+    g_main_loop_unref(data.loop);
+    if (data.error) {
+        error_propagate(errp, data.error);
+        object_unref(OBJECT(tioc));
+        return NULL;
+    }
+
+    return QIO_CHANNEL(tioc);
+}
+
+
 int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
+                          QCryptoTLSCreds *tlscreds, const char *hostname,
+                          QIOChannel **outioc,
                           off_t *size, Error **errp)
 {
     char buf[256];
@@ -842,10 +1063,19 @@ int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
     int rc;
     char *autoname = NULL;
 
-    TRACE("Receiving negotiation.");
+    TRACE("Receiving negotiation tlscreds=%p hostname=%s.",
+          tlscreds, hostname ? hostname : "<null>");
 
     rc = -EINVAL;
 
+    if (outioc) {
+        *outioc = NULL;
+    }
+    if (tlscreds && !outioc) {
+        error_setg(errp, "Output I/O channel required for TLS");
+        goto fail;
+    }
+
     if (read_sync(ioc, buf, 8) != 8) {
         error_setg(errp, "Failed to read data");
         goto fail;
@@ -907,6 +1137,18 @@ int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
             error_setg(errp, "Failed to send clientflags field");
             goto fail;
         }
+        if (tlscreds) {
+            if (fixedNewStyle) {
+                *outioc = nbd_receive_starttls(ioc, tlscreds, hostname, errp);
+                if (!*outioc) {
+                    goto fail;
+                }
+                ioc = *outioc;
+            } else {
+                error_setg(errp, "Server does not support STARTTLS");
+                goto fail;
+            }
+        }
         /* write the export name */
         if (fixedNewStyle) {
             autoname = nbd_receive_pick_export(ioc, name, errp);
@@ -961,6 +1203,10 @@ int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
             error_setg(errp, "Server does not support export names");
             goto fail;
         }
+        if (tlscreds) {
+            error_setg(errp, "Server does not support STARTTLS");
+            goto fail;
+        }
 
         if (read_sync(ioc, &s, sizeof(s)) != sizeof(s)) {
             error_setg(errp, "Failed to read export length");
@@ -1247,6 +1493,10 @@ void nbd_client_put(NBDClient *client)
         nbd_unset_handlers(client);
         object_unref(OBJECT(client->sioc));
         object_unref(OBJECT(client->ioc));
+        if (client->tlscreds) {
+            object_unref(OBJECT(client->tlscreds));
+        }
+        g_free(client->tlsaclname);
         if (client->exp) {
             QTAILQ_REMOVE(&client->exp->clients, client, next);
             nbd_export_put(client->exp);
@@ -1757,12 +2007,19 @@ static void nbd_update_can_read(NBDClient *client)
 
 NBDClient *nbd_client_new(NBDExport *exp,
                           QIOChannelSocket *sioc,
+                          QCryptoTLSCreds *tlscreds,
+                          const char *tlsaclname,
                           void (*close)(NBDClient *))
 {
     NBDClient *client;
     client = g_malloc0(sizeof(NBDClient));
     client->refcount = 1;
     client->exp = exp;
+    client->tlscreds = tlscreds;
+    if (tlscreds) {
+        object_ref(OBJECT(client->tlscreds));
+    }
+    client->tlsaclname = g_strdup(tlsaclname);
     client->sioc = sioc;
     object_ref(OBJECT(client->sioc));
     client->ioc = QIO_CHANNEL(sioc);
@@ -1771,6 +2028,10 @@ NBDClient *nbd_client_new(NBDExport *exp,
     if (nbd_send_negotiate(client)) {
         object_unref(OBJECT(client->sioc));
         object_unref(OBJECT(client->ioc));
+        if (client->tlscreds) {
+            object_unref(OBJECT(client->tlscreds));
+        }
+        g_free(client->tlsaclname);
         g_free(client);
         return NULL;
     }
diff --git a/qemu-nbd.c b/qemu-nbd.c
index 7933ff5..53b22d7 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -254,6 +254,7 @@ static void *nbd_client_thread(void *arg)
     }
 
     ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), NULL, &nbdflags,
+                                NULL, NULL, NULL,
                                 &size, &local_error);
     if (ret < 0) {
         if (local_error) {
@@ -342,7 +343,8 @@ static gboolean nbd_accept(QIOChannel *ioc, GIOCondition cond, gpointer opaque)
         return TRUE;
     }
 
-    if (nbd_client_new(newproto ? NULL : exp, cioc, nbd_client_closed)) {
+    if (nbd_client_new(newproto ? NULL : exp, cioc,
+                       NULL, NULL, nbd_client_closed)) {
         nb_fds++;
         nbd_update_server_watch();
     }
-- 
2.5.0