MQTT over QUIC support with openssl3.4

.github/workflows/build_linux.yml

@@ -4,37 +4,83 @@ on: [push, pull_request]
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # with/without openssl source
+        ssl: [true, false]
+        include:
+          - ssl: true
+            ssl_name: "with-openssl-src"
+            quic: true
+          - ssl: false
+            ssl_name: "without-openssl-src"
+            quic: false
+    name: Build ${{ matrix.ssl_name }}
     steps:
       - name: Check out code
         uses: actions/checkout@v2
+
       - name: Install dependencies
         run: |
           sudo apt-get update
           sudo apt-get install doxygen
+
+      - name: Install openssl3.4
+        if: matrix.ssl
+        run: |
+          pwd
+          mkdir -p openssl_root
+          wget https://github.com/openssl/openssl/releases/download/openssl-3.4.1/openssl-3.4.1.tar.gz &&
+            tar zxvf openssl-3.4.1.tar.gz --strip-components=1 -C openssl_root
+          cd openssl_root
+          ./Configure && make -j `nproc`
+
       - name: Build
         run: |
           rm -rf build.paho
           mkdir build.paho
           cd build.paho
           echo "pwd $PWD"
-          cmake -DPAHO_BUILD_STATIC=FALSE -DPAHO_BUILD_SHARED=TRUE -DCMAKE_BUILD_TYPE=Debug -DPAHO_WITH_SSL=TRUE -DOPENSSL_ROOT_DIR= -DPAHO_BUILD_DOCUMENTATION=FALSE -DPAHO_BUILD_SAMPLES=TRUE -DPAHO_HIGH_PERFORMANCE=TRUE ..
+          cmake -DPAHO_BUILD_STATIC=FALSE \
+                -DPAHO_BUILD_SHARED=TRUE \
+                -DCMAKE_BUILD_TYPE=Debug \
+                -DPAHO_WITH_SSL=TRUE \
+                ${{ matrix.ssl && '-DOPENSSL_ROOT_DIR=../openssl_root' || '' }} \
+                -DPAHO_BUILD_DOCUMENTATION=FALSE \
+                -DPAHO_BUILD_SAMPLES=TRUE \
+                -DPAHO_HIGH_PERFORMANCE=TRUE \
+                -DPAHO_WITH_QUIC=${{ matrix.quic }} ..
           cmake --build .
+
       - name: Start test broker
         run: |
           git clone https://github.com/eclipse/paho.mqtt.testing.git
           cd paho.mqtt.testing/interoperability
           python3 startbroker.py -c localhost_testing.conf &
+
+      - name: Start test broker for Quic
+        if: matrix.quic
+        run: |
+          docker run -d --name emqx \
+          -v `pwd`/test/ssl/emqx/etc/certs:/opt/emqx/etc/certs \
+          -v `pwd`/test/emqx.conf:/opt/emqx/etc/emqx.conf \
+          -p 8883:8883  -p 14567:14567/udp -p 18883:18883/udp -p 18886:18886/udp -p 18884:18884/udp -p 18885:18885/udp \
+          emqx/emqx:5.8.4
+
       - name: Start test proxy
         run: |
           python3 test/mqttsas.py &
+
       - name: run tests
         run: |
           cd build.paho
           ctest -VV --timeout 600
+
       - name: clean up
         run: |
           killall python3 || true
           sleep 3 # allow broker time to terminate and report
+
       - name: package
         run: |
           cd build.paho

CMakeLists.txt

@@ -54,6 +54,7 @@ endif()
 ## build options
 option(PAHO_WITH_SSL "Flag that defines whether to build ssl-enabled binaries too. " FALSE)
 option(PAHO_WITH_LIBRESSL "Flag that defines whether to build ssl-enabled binaries with LibreSSL instead of OpenSSL. " FALSE)
+option(PAHO_WITH_QUIC "Flag that defines whether to build ssl-enabled binaries with QUIC support. " FALSE)
 option(PAHO_WITH_LIBUUID "Flag that defines whether libuuid or a custom uuid implementation should be used" FALSE)
 option(PAHO_BUILD_SHARED "Build shared library" TRUE)
 option(PAHO_BUILD_STATIC "Build static library" FALSE)

src/CMakeLists.txt

@@ -236,6 +236,13 @@ if(PAHO_WITH_SSL OR PAHO_WITH_LIBRESSL)
     set(SSL_ROOT_DIR ${LIBRESSL_ROOT_DIR})
   else()
     find_package(OpenSSL REQUIRED)
+
+    if (PAHO_WITH_QUIC AND "${OPENSSL_VERSION}" MATCHES "3.4.*")
+      set(COMPILE_WITH_QUIC "WITH_OPENSSL_QUIC=1")
+    else()
+      set(COMPILE_WITH_QUIC "")
+    endif()
+
     set(SSL_LIBRARY_NAME OpenSSL)
     set(SSL_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR})
     set(SSL_ROOT_DIR ${OPENSSL_ROOT_DIR})
@@ -249,7 +256,7 @@ if(PAHO_WITH_SSL OR PAHO_WITH_LIBRESSL)
     target_include_directories(common_ssl_obj PUBLIC ${SSL_INCLUDE_DIR})
 
     set_property(TARGET common_ssl_obj PROPERTY POSITION_INDEPENDENT_CODE ON)
-    target_compile_definitions(common_ssl_obj PRIVATE OPENSSL=1 PAHO_MQTT_EXPORTS=1)
+    target_compile_definitions(common_ssl_obj PRIVATE OPENSSL=1 ${COMPILE_WITH_QUIC} PAHO_MQTT_EXPORTS=1)
 
     add_library(paho-mqtt3cs SHARED
       $<TARGET_OBJECTS:common_ssl_obj>
@@ -269,7 +276,7 @@ if(PAHO_WITH_SSL OR PAHO_WITH_LIBRESSL)
     set_target_properties(paho-mqtt3cs paho-mqtt3as PROPERTIES
       VERSION ${PROJECT_VERSION}
       SOVERSION ${PROJECT_VERSION_MAJOR}
-      COMPILE_DEFINITIONS "OPENSSL=1;PAHO_MQTT_EXPORTS=1"
+      COMPILE_DEFINITIONS "OPENSSL=1;${COMPILE_WITH_QUIC};PAHO_MQTT_EXPORTS=1"
     )
 
     if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
@@ -317,7 +324,7 @@ if(PAHO_WITH_SSL OR PAHO_WITH_LIBRESSL)
     target_include_directories(common_ssl_obj_static PUBLIC ${SSL_INCLUDE_DIR})
 
     set_property(TARGET common_ssl_obj_static PROPERTY POSITION_INDEPENDENT_CODE ON)
-    target_compile_definitions(common_ssl_obj_static PRIVATE OPENSSL=1 PAHO_MQTT_STATIC=1)
+    target_compile_definitions(common_ssl_obj_static PRIVATE OPENSSL=1 ${COMPILE_WITH_QUIC} PAHO_MQTT_STATIC=1)
 
     add_library(paho-mqtt3cs-static STATIC
       $<TARGET_OBJECTS:common_ssl_obj_static>
@@ -337,7 +344,7 @@ if(PAHO_WITH_SSL OR PAHO_WITH_LIBRESSL)
     set_target_properties(paho-mqtt3cs-static paho-mqtt3as-static PROPERTIES
       VERSION ${PROJECT_VERSION}
       SOVERSION ${PROJECT_VERSION_MAJOR}
-      COMPILE_DEFINITIONS "OPENSSL=1;PAHO_MQTT_STATIC=1"
+      COMPILE_DEFINITIONS "OPENSSL=1;${COMPILE_WITH_QUIC};PAHO_MQTT_STATIC=1"
     )
 
     if(NOT WIN32)

src/Clients.h

@@ -76,6 +76,18 @@ typedef struct
 	int qos;
 } willMessages;
 
+#if defined(OPENSSL)
+enum QUIC_MODE {
+    // QUIC_MODE is disabled
+	QUIC_MODE_NONE = 1,
+	// QUIC ONLY no fallback to TCP
+	QUIC_MODE_ONLY,
+	// FALLBACK to TCP if QUIC failed
+	QUIC_MODE_PREFERRED 
+};
+typedef enum QUIC_MODE QUIC_MODE;
+#endif
+
 typedef struct
 {
 	SOCKET socket;
@@ -87,6 +99,9 @@ typedef struct
 	SSL_CTX* ctx;
 	char *https_proxy;
 	char *https_proxy_auth;
+#endif
+#if defined(WITH_OPENSSL_QUIC)
+	QUIC_MODE quic_mode;
 #endif
 	char *http_proxy;
 	char *http_proxy_auth;

src/MQTTAsync.c

@@ -337,6 +337,9 @@ int MQTTAsync_createWithOptions(MQTTAsync* handle, const char* serverURI, const
 		 && strncmp(URI_SSL, serverURI, strlen(URI_SSL)) != 0
 		 && strncmp(URI_MQTTS, serverURI, strlen(URI_MQTTS)) != 0
 		 && strncmp(URI_WSS, serverURI, strlen(URI_WSS)) != 0
+#if defined(WITH_OPENSSL_QUIC)
+		 && strncmp(URI_QUIC, serverURI, strlen(URI_QUIC)) != 0
+#endif
 #endif
 			)
 		{
@@ -416,6 +419,14 @@ int MQTTAsync_createWithOptions(MQTTAsync* handle, const char* serverURI, const
 		m->ssl = 1;
 		m->websocket = 1;
 	}
+#if defined(WITH_OPENSSL_QUIC)
+	else if (strncmp(URI_QUIC, serverURI, strlen(URI_QUIC)) == 0)
+	{
+		serverURI += strlen(URI_QUIC);
+		m->ssl = 2;
+	}
+#endif
+
 #endif
 	if ((m->serverURI = MQTTStrdup(serverURI)) == NULL)
 	{

src/MQTTAsyncUtils.c

@@ -1359,6 +1359,14 @@ static int MQTTAsync_processCommand(void)
 						command->client->ssl = 1;
 						command->client->websocket = 1;
 					}
+#if defined(WITH_OPENSSL_QUIC)
+					else if (strncmp(URI_QUIC, serverURI, strlen(URI_QUIC)) == 0)
+					{
+						serverURI += strlen(URI_QUIC);
+						command->client->ssl = 2;
+					}
+#endif
+
 #endif
 				}
 			}
@@ -1392,8 +1400,10 @@ static int MQTTAsync_processCommand(void)
 #endif
 #endif
 
-			if (command->client->c->connect_state == NOT_IN_PROGRESS)
+			if (command->client->c->connect_state == NOT_IN_PROGRESS) {
+				Log(TRACE_MIN, -1, "Connect state is NOT_IN_PROGRESS for client %s", command->client->c->clientID);
 				rc = SOCKET_ERROR;
+			}
 
 			/* if the TCP connect is pending, then we must call select to determine when the connect has completed,
 			which is indicated by the socket being ready *either* for reading *or* writing.  The next couple of lines
@@ -2928,6 +2938,11 @@ static int MQTTAsync_connecting(MQTTAsyncs* m)
 			}
 
 			hostname_len = MQTTProtocol_addressPort(serverURI, &port, NULL, default_port);
+
+			if (m->ssl == 2)
+			{
+				m->c->sslopts->sslVersion = MQTT_SSL_VERSION_QUIC;
+			}
 			setSocketForSSLrc = SSLSocket_setSocketForSSL(&m->c->net, m->c->sslopts,
 					serverURI, hostname_len);
 

src/MQTTAsyncUtils.h

@@ -25,6 +25,7 @@
 #define URI_WS   "ws://"
 #define URI_WSS  "wss://"
 #define URI_UNIX "unix://"
+#define URI_QUIC "quic://"
 
 enum MQTTAsync_threadStates
 {

src/MQTTClient.h

@@ -661,6 +661,8 @@ typedef struct
 #define MQTT_SSL_VERSION_TLS_1_0 1
 #define MQTT_SSL_VERSION_TLS_1_1 2
 #define MQTT_SSL_VERSION_TLS_1_2 3
+#define MQTT_SSL_VERSION_TLS_1_3 4
+#define MQTT_SSL_VERSION_QUIC    5
 
 /**
 * MQTTClient_sslProperties defines the settings to establish an SSL/TLS connection using the

src/MQTTProtocolOut.c

@@ -266,6 +266,18 @@ int MQTTProtocol_connect(const char* address, Clients* aClient, int unixsock, in
 		rc = Socket_unix_new(address, addr_len, &(aClient->net.socket));
 	}
 #endif
+
+#if defined(OPENSSL) && defined(WITH_OPENSSL_QUIC)
+	else if (ssl == 2) {
+		addr_len = MQTTProtocol_addressPort(address, &port, NULL, MQTT_DEFAULT_PORT);
+		aClient->net.quic_mode = QUIC_MODE_PREFERRED;
+#if defined(__GNUC__) && defined(__linux__)
+		rc = Socket_dgram_new(address, addr_len, port, &(aClient->net.socket), timeout);
+#else
+		rc = Socket_dgram_new(address, addr_len, port, &(aClient->net.socket));
+#endif
+	}
+#endif
 	else {
 #if defined(OPENSSL)
 		addr_len = MQTTProtocol_addressPort(address, &port, NULL, ssl ?
@@ -283,10 +295,11 @@ int MQTTProtocol_connect(const char* address, Clients* aClient, int unixsock, in
 		rc = Socket_new(address, addr_len, port, &(aClient->net.socket));
 #endif
 	}
+
 	if (rc == EINPROGRESS || rc == EWOULDBLOCK)
 		aClient->connect_state = TCP_IN_PROGRESS; /* TCP connect called - wait for connect completion */
 	else if (rc == 0)
-	{	/* TCP connect completed. If SSL, send SSL connect */
+	{	/* TCP/UDP connect completed. If SSL, send SSL connect */
 #if defined(OPENSSL)
 		if (ssl)
 		{
@@ -301,8 +314,20 @@ int MQTTProtocol_connect(const char* address, Clients* aClient, int unixsock, in
 						aClient->sslopts->verify, aClient->sslopts->ssl_error_cb, aClient->sslopts->ssl_error_context) :
 					SSLSocket_connect(aClient->net.ssl, aClient->net.socket, address,
 						aClient->sslopts->verify, NULL, NULL);
-				if (rc == TCPSOCKET_INTERRUPTED)
+				// in this fun scope, 0 means success
+				if (rc == 1)
+				{
+#if defined(WITH_OPENSSL_QUIC)
+					SSL_set_blocking_mode(aClient->net.ssl, 0);
+#endif
+					rc = 0;
+				}
+				else if (rc == TCPSOCKET_INTERRUPTED)
 					aClient->connect_state = SSL_IN_PROGRESS; /* SSL connect called - wait for completion */
+				else if (!rc)
+				{
+					aClient->connect_state = NOT_IN_PROGRESS;
+				}
 			}
 			else
 				rc = SOCKET_ERROR;