bt: Implement application level acknowledgements

This updates the GATT transport to use application level
acknowledgements on top of unacknowledged GATT writes and
notifications. This significantly increases throughput and
matches the work done in the pouch repository.

Signed-off-by: Sam Friedman <sam@golioth.io>

Author: sam-golioth

Kconfig

@@ -39,6 +39,33 @@ config POUCH_GATEWAY_SERVER_CERT_BUILTIN
     bool
     default y if !POUCH_GATEWAY_CLOUD
 
+config POUCH_GATT_INFO_WINDOW_SIZE
+    int "Info GATT Window Size"
+    range 1 127
+    default 4
+    help
+        The number of unacknowledged packets that can be received by
+        the device on the info characteristic. Larger numbers can
+        increase throughput but will use more RAM.
+
+config POUCH_GATT_DEVICE_CERT_WINDOW_SIZE
+    int "Device Cert GATT Window Size"
+    range 1 127
+    default 4
+    help
+        The number of unacknowledged packets that can be received by
+        the device on the device certificate characteristic. Larger
+        numbers can increase throughput but will use more RAM.
+
+config POUCH_GATT_UPLINK_WINDOW_SIZE
+    int "Uplink GATT Window Size"
+    range 1 127
+    default 4
+    help
+        The number of unacknowledged packets that can be received by
+        the device on the uplink characteristic. Larger numbers can
+        increase throughput but will use more RAM.
+
 module = POUCH_GATEWAY
 module-str = Pouch Gateway Library
 source "subsys/logging/Kconfig.template.log_config"

include/pouch_gateway/types.h

@@ -23,6 +23,13 @@ enum pouch_gateway_gatt_attr
     POUCH_GATEWAY_GATT_ATTRS,
 };
 
+enum server_cert_next_state
+{
+    SERVER_CERT_NEXT_DEVICE_CERT,
+    SERVER_CERT_NEXT_SERVER_CERT,
+    SERVER_CERT_NEXT_END
+};
+
 struct pouch_gateway_attr_handle
 {
     uint16_t value;
@@ -32,37 +39,24 @@ struct pouch_gateway_attr_handle
 struct pouch_gateway_node_info
 {
     struct pouch_gateway_attr_handle attr_handles[POUCH_GATEWAY_GATT_ATTRS];
-    union
-    {
-        struct bt_gatt_discover_params discover_params;
-        struct bt_gatt_read_params read_params;
-        struct bt_gatt_subscribe_params subscribe_params;
-        struct bt_gatt_write_params write_params;
-    };
+    struct bt_gatt_discover_params discover_params;
+    struct bt_gatt_subscribe_params info_subscribe_params;
+    struct bt_gatt_subscribe_params server_cert_subscribe_params;
+    struct bt_gatt_subscribe_params device_cert_subscribe_params;
+    struct bt_gatt_subscribe_params uplink_subscribe_params;
+    struct bt_gatt_subscribe_params downlink_subscribe_params;
     struct pouch_gateway_downlink_context *downlink_ctx;
-    void *downlink_scratch;
-    void *server_cert_scratch;
+    struct pouch_gatt_receiver *info_receiver;
+    struct pouch_gatt_sender *server_cert_sender;
+    struct pouch_gatt_receiver *device_cert_receiver;
+    struct pouch_gatt_sender *downlink_sender;
+    struct pouch_gatt_receiver *uplink_receiver;
     struct pouch_gatt_packetizer *packetizer;
     struct pouch_gateway_uplink *uplink;
     struct pouch_gateway_info_context *info_ctx;
     struct pouch_gateway_device_cert_context *device_cert_ctx;
     struct pouch_gateway_server_cert_context *server_cert_ctx;
+    enum server_cert_next_state server_cert_next;
     bool server_cert_provisioned;
     bool device_cert_provisioned;
 };
-
-/**
- * Allocate a buffer the size of the current MTU for GATT operations.
- * @param conn Bluetooth connection
- * @return Pointer to allocated buffer or NULL on failure
- */
-static inline void *pouch_gateway_bt_gatt_mtu_malloc(struct bt_conn *conn)
-{
-    size_t mtu = bt_gatt_get_mtu(conn);
-    if (mtu < POUCH_GATEWAY_BT_ATT_OVERHEAD)
-    {
-        return NULL;
-    }
-
-    return malloc(mtu - POUCH_GATEWAY_BT_ATT_OVERHEAD);
-}

lib/bt/device_cert.c

@@ -11,6 +11,7 @@
 #include <zephyr/bluetooth/gatt.h>
 
 #include <pouch/transport/gatt/common/packetizer.h>
+#include <pouch/transport/gatt/common/receiver.h>
 
 #include <pouch_gateway/bt/connect.h>
 
@@ -22,7 +23,6 @@
 #include <zephyr/logging/log.h>
 LOG_MODULE_REGISTER(device_cert_gatt, CONFIG_POUCH_GATEWAY_GATT_LOG_LEVEL);
 
-
 static void device_cert_cleanup(struct bt_conn *conn)
 {
     struct pouch_gateway_node_info *node = pouch_gateway_get_node_info(conn);
@@ -32,80 +32,120 @@ static void device_cert_cleanup(struct bt_conn *conn)
         pouch_gateway_device_cert_abort(node->device_cert_ctx);
         node->device_cert_ctx = NULL;
     }
+
+    if (node->device_cert_receiver)
+    {
+        pouch_gatt_receiver_destroy(node->device_cert_receiver);
+        node->device_cert_receiver = NULL;
+    }
 }
 
-static uint8_t device_cert_read_cb(struct bt_conn *conn,
-                                   uint8_t err,
-                                   struct bt_gatt_read_params *params,
-                                   const void *data,
-                                   uint16_t length)
+static int device_cert_data_received_cb(void *conn,
+                                        const void *data,
+                                        size_t length,
+                                        bool is_first,
+                                        bool is_last)
 {
     struct pouch_gateway_node_info *node = pouch_gateway_get_node_info(conn);
 
+    int err = pouch_gateway_device_cert_push(node->device_cert_ctx, data, length);
     if (err)
     {
-        LOG_ERR("Failed to read BLE GATT %s (err %d)", "device cert", err);
-        device_cert_cleanup(conn);
-        pouch_gateway_bt_finished(conn);
-        return BT_GATT_ITER_STOP;
+        LOG_ERR("Failed to push device cert");
+        goto finish;
     }
 
-    if (length == 0)
+    if (is_last)
     {
-        LOG_ERR("No device cert");
-        device_cert_cleanup(conn);
-        pouch_gateway_bt_finished(conn);
-        return BT_GATT_ITER_STOP;
+        err = pouch_gateway_device_cert_finish(node->device_cert_ctx);
+        if (err)
+        {
+            LOG_ERR("Failed to finish device cert: %d", err);
+            goto finish;
+        }
+        node->device_cert_ctx = NULL;
     }
 
-    bool is_first = false;
-    bool is_last = false;
-    const void *payload = NULL;
-    ssize_t payload_len = pouch_gatt_packetizer_decode(data, length, &payload, &is_first, &is_last);
-    if (payload_len < 0)
+finish:
+    if (err)
     {
-        LOG_ERR("Failed to decode BLE GATT %s (err %d)", "device cert", (int) payload_len);
         device_cert_cleanup(conn);
         pouch_gateway_bt_finished(conn);
-        return BT_GATT_ITER_STOP;
     }
 
-    if (data)
+    return err;
+}
+
+static int send_ack_cb(void *conn, const void *data, size_t length)
+{
+    struct pouch_gateway_node_info *node = pouch_gateway_get_node_info(conn);
+    uint16_t handle = node->attr_handles[POUCH_GATEWAY_GATT_ATTR_DEVICE_CERT].value;
+
+    return bt_gatt_write_without_response(conn, handle, data, length, false);
+}
+
+static uint8_t device_cert_notify_cb(struct bt_conn *conn,
+                                     struct bt_gatt_subscribe_params *params,
+                                     const void *data,
+                                     uint16_t length)
+{
+    struct pouch_gateway_node_info *node = pouch_gateway_get_node_info(conn);
+
+    if (NULL == data)
     {
-        LOG_HEXDUMP_DBG(data, length, "[READ] BLE GATT device cert");
-    }
+        LOG_DBG("Subscription terminated");
+        device_cert_cleanup(conn);
 
-    pouch_gateway_device_cert_push(node->device_cert_ctx, payload, payload_len);
+        return BT_GATT_ITER_STOP;
+    }
 
-    if (is_last)
+    if (NULL == node->device_cert_receiver)
     {
-        int err = pouch_gateway_device_cert_finish(node->device_cert_ctx);
-        if (err)
+        enum pouch_gatt_ack_code code;
+        if (pouch_gatt_packetizer_is_fin(data, length, &code))
         {
-            LOG_ERR("Failed to finish device cert: %d", err);
-            device_cert_cleanup(conn);
-            pouch_gateway_bt_finished(conn);
-            return BT_GATT_ITER_STOP;
+            LOG_WRN("Received FIN while idle: %d", code);
+        }
+        else
+        {
+            LOG_ERR("Received packet while idle");
+
+            pouch_gatt_receiver_send_nack(send_ack_cb, conn, POUCH_GATT_NACK_IDLE);
         }
 
-        pouch_gateway_uplink_start(conn);
         return BT_GATT_ITER_STOP;
     }
 
-    err = bt_gatt_read(conn, params);
+    bool complete = false;
+    int err = pouch_gatt_receiver_receive_data(node->device_cert_receiver, data, length, &complete);
     if (err)
     {
-        LOG_ERR("BT (re)read request failed: %d", err);
+        LOG_ERR("Error receiving data: %d", err);
+
         device_cert_cleanup(conn);
+
         pouch_gateway_bt_finished(conn);
+
         return BT_GATT_ITER_STOP;
     }
 
-    return BT_GATT_ITER_STOP;
+    if (complete)
+    {
+        device_cert_cleanup(conn);
+
+        pouch_gateway_uplink_start(conn);
+
+        return BT_GATT_ITER_STOP;
+    }
+
+
+    return BT_GATT_ITER_CONTINUE;
 }
 
 void pouch_gateway_device_cert_read(struct bt_conn *conn)
 {
+    LOG_INF("Starting device cert read");
+
     struct pouch_gateway_node_info *node = pouch_gateway_get_node_info(conn);
 
     if (node->device_cert_provisioned)
@@ -114,8 +154,12 @@ void pouch_gateway_device_cert_read(struct bt_conn *conn)
         return;
     }
 
-    struct bt_gatt_read_params *read_params = &node->read_params;
-    memset(read_params, 0, sizeof(*read_params));
+    if (0 == node->attr_handles[POUCH_GATEWAY_GATT_ATTR_DEVICE_CERT].ccc)
+    {
+        LOG_ERR("Did not discover Device Cert CCC");
+        pouch_gateway_bt_finished(conn);
+        return;
+    }
 
     node->device_cert_ctx = pouch_gateway_device_cert_start();
     if (node->device_cert_ctx == NULL)
@@ -126,13 +170,32 @@ void pouch_gateway_device_cert_read(struct bt_conn *conn)
         return;
     }
 
-    read_params->func = device_cert_read_cb;
-    read_params->handle_count = 1;
-    read_params->single.handle = node->attr_handles[POUCH_GATEWAY_GATT_ATTR_DEVICE_CERT].value;
-    int err = bt_gatt_read(conn, read_params);
+    node->device_cert_receiver =
+        pouch_gatt_receiver_create(send_ack_cb,
+                                   conn,
+                                   device_cert_data_received_cb,
+                                   conn,
+                                   CONFIG_POUCH_GATT_DEVICE_CERT_WINDOW_SIZE);
+    if (NULL == node->device_cert_receiver)
+    {
+        LOG_ERR("Failed to create receiver");
+        device_cert_cleanup(conn);
+        pouch_gateway_bt_finished(conn);
+        return;
+    }
+
+    struct bt_gatt_subscribe_params *subscribe_params = &node->device_cert_subscribe_params;
+    memset(subscribe_params, 0, sizeof(*subscribe_params));
+
+    subscribe_params->notify = device_cert_notify_cb;
+    subscribe_params->value = BT_GATT_CCC_NOTIFY;
+    subscribe_params->value_handle = node->attr_handles[POUCH_GATEWAY_GATT_ATTR_DEVICE_CERT].value;
+    subscribe_params->ccc_handle = node->attr_handles[POUCH_GATEWAY_GATT_ATTR_DEVICE_CERT].ccc;
+    atomic_set_bit(subscribe_params->flags, BT_GATT_SUBSCRIBE_FLAG_VOLATILE);
+    int err = bt_gatt_subscribe(conn, subscribe_params);
     if (err)
     {
-        LOG_ERR("BT read request failed: %d", err);
+        LOG_ERR("BT subscribe request failed: %d", err);
         device_cert_cleanup(conn);
         pouch_gateway_bt_finished(conn);
     }