Skip to content

Commit 841b545

Browse files
author
bneradt
committed
Add OpenSSL native QUIC backend
OpenSSL 3.5 can terminate QUIC connections directly, but ATS only had a quiche-backed HTTP/3 listener. Operators who want to use the system OpenSSL QUIC stack needed a separate downstream backend without changing the existing quiche path or origin HTTP/3 scope. This adds an optional ENABLE_OPENSSL_QUIC backend that uses OpenSSL's native QUIC listener and stream APIs for downstream HTTP/3. This keeps the backend mutually exclusive with quiche, exposes TS_HAS_OPENSSL_QUIC, and shares ATS's existing HTTP/3 stream handling above the transport. This also installs native-QUIC TLS callbacks for ALPN and SNI certificate selection before ATS has a QUIC NetVC to bind. OpenSSL native QUIC does not make a selected SSL_CTX certificate active via SSL_set_SSL_CTX alone, so this applies the selected cert, key, and chain to the connection SSL. This also broadens client-side HTTP/3 tests to run with either backend and hardens transaction cleanup when OpenSSL closes stream state before ATS finishes teardown. This caches stream identifiers, delivers terminal events through the UA VIOs, defers reentrant deletion, and adds focused H3 lifecycle coverage.
1 parent 9b12150 commit 841b545

54 files changed

Lines changed: 3054 additions & 130 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

CMakeLists.txt

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ set(ENABLE_TPROXY
212212
'X' where X is a number to use as the IP_TRANSPARENT sockopt,
213213
anything else to enable."
214214
)
215+
option(ENABLE_OPENSSL_QUIC "Use OpenSSL native QUIC (default OFF)")
215216
option(ENABLE_QUICHE "Use quiche (default OFF)")
216217

217218
option(ENABLE_EXAMPLE "Build example directory (default OFF)")
@@ -281,6 +282,7 @@ include(CheckOpenSSLIsBoringSSL)
281282
include(CheckOpenSSLIsQuictls)
282283
include(CheckOpenSSLIsAwsLc)
283284
include(CheckOpenSSLHasQuicTlsCbs)
285+
include(CheckOpenSSLHasNativeQuic)
284286
find_package(OpenSSL REQUIRED)
285287
check_openssl_is_boringssl(SSLLIB_IS_BORINGSSL BORINGSSL_VERSION "${OPENSSL_INCLUDE_DIR}")
286288
check_openssl_is_awslc(SSLLIB_IS_AWSLC AWSLC_VERSION "${OPENSSL_INCLUDE_DIR}")
@@ -320,6 +322,8 @@ if(SSLLIB_HAS_QUIC_TLS_CBS
320322
add_compile_definitions(TS_OPENSSL_QUIC_TLS_CBS_COMPAT)
321323
endif()
322324

325+
check_openssl_has_native_quic(SSLLIB_HAS_NATIVE_QUIC "${OPENSSL_INCLUDE_DIR}")
326+
323327
if(ENABLE_PROFILER)
324328
find_package(profiler REQUIRED)
325329
set(TS_HAS_PROFILER ${profiler_FOUND})
@@ -335,6 +339,25 @@ elseif(TS_HAS_MIMALLOC)
335339
link_libraries(mimalloc)
336340
endif()
337341

342+
if(ENABLE_OPENSSL_QUIC AND ENABLE_QUICHE)
343+
message(FATAL_ERROR "ENABLE_OPENSSL_QUIC and ENABLE_QUICHE are mutually exclusive QUIC backends")
344+
endif()
345+
346+
if(ENABLE_OPENSSL_QUIC)
347+
if(NOT SSLLIB_HAS_NATIVE_QUIC)
348+
message(FATAL_ERROR "OpenSSL native QUIC support requires OpenSSL 3.5 or newer with OSSL_QUIC_server_method")
349+
endif()
350+
if(SSLLIB_IS_BORINGSSL
351+
OR SSLLIB_IS_AWSLC
352+
OR SSLLIB_IS_QUICTLS
353+
)
354+
message(FATAL_ERROR "OpenSSL native QUIC support requires upstream OpenSSL 3.5 or newer")
355+
endif()
356+
set(TS_HAS_OPENSSL_QUIC TRUE)
357+
set(TS_USE_QUIC TRUE)
358+
message(STATUS "Using OpenSSL native QUIC")
359+
endif()
360+
338361
if(ENABLE_QUICHE)
339362
if(TS_OPENSSL_QUIC_TLS_CBS_COMPAT)
340363
set(quiche_USE_STATIC TRUE)

CMakePresets.json

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -183,11 +183,10 @@
183183
"inherits": ["ci"],
184184
"cacheVariables": {
185185
"ENABLE_PROBES": "ON",
186-
"OPENSSL_ROOT_DIR": "/opt/openssl-quic",
187186
"opentelemetry_ROOT": "/opt",
188-
"CURL_ROOT": "/opt",
189187
"wamr_ROOT": "/opt",
190-
"ENABLE_CRIPTS": "ON"
188+
"ENABLE_CRIPTS": "ON",
189+
"ENABLE_OPENSSL_QUIC": "ON"
191190
}
192191
},
193192
{
@@ -207,20 +206,19 @@
207206
"name": "ci-fedora-quiche",
208207
"displayName": "CI Fedora Quiche",
209208
"description": "CI Pipeline config for Fedora Linux (quiche build)",
210-
"inherits": ["ci"],
209+
"inherits": ["ci-fedora"],
211210
"cacheVariables": {
212211
"OPENSSL_ROOT_DIR": "/opt/h3-tools-boringssl/boringssl",
213212
"quiche_ROOT": "/opt/h3-tools-boringssl/quiche",
214-
"opentelemetry_ROOT": "/opt",
215213
"CURL_ROOT": "/opt",
216-
"wamr_ROOT": "/opt",
217214
"CMAKE_INSTALL_PREFIX": "/tmp/ats-quiche",
215+
"ENABLE_OPENSSL_QUIC": "OFF",
218216
"ENABLE_QUICHE": "ON"
219217
}
220218
},
221219
{
222220
"name": "ci-fedora-autest",
223-
"displayName": "CI Fedora Quiche Autest",
221+
"displayName": "CI Fedora Autest",
224222
"description": "CI Pipeline config for Fedora Linux (autest build)",
225223
"inherits": ["ci-fedora", "autest"]
226224
},
@@ -450,4 +448,3 @@
450448
}
451449
]
452450
}
453-
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#######################
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more contributor license
4+
# agreements. See the NOTICE file distributed with this work for additional information regarding
5+
# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with the License. You may obtain
7+
# a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software distributed under the License
12+
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
13+
# or implied. See the License for the specific language governing permissions and limitations under
14+
# the License.
15+
#
16+
#######################
17+
18+
function(CHECK_OPENSSL_HAS_NATIVE_QUIC OUT_VAR OPENSSL_INCLUDE_DIR)
19+
set(CHECK_PROGRAM
20+
"
21+
#include <openssl/ssl.h>
22+
#include <openssl/quic.h>
23+
#include <cstdint>
24+
25+
int main() {
26+
const SSL_METHOD *method = OSSL_QUIC_server_method();
27+
SSL *ssl = nullptr;
28+
uint64_t value = 0;
29+
return method == nullptr ||
30+
SSL_new_listener == nullptr ||
31+
SSL_listen == nullptr ||
32+
SSL_handle_events == nullptr ||
33+
SSL_accept_connection == nullptr ||
34+
SSL_accept_stream == nullptr ||
35+
SSL_new_stream == nullptr ||
36+
SSL_stream_conclude == nullptr ||
37+
SSL_get_stream_id == nullptr ||
38+
SSL_get_stream_type == nullptr ||
39+
SSL_get_stream_read_state == nullptr ||
40+
SSL_get_stream_write_buf_avail(ssl, &value) ||
41+
SSL_get_conn_close_info == nullptr ||
42+
SSL_shutdown_ex == nullptr ||
43+
SSL_set_default_stream_mode == nullptr ||
44+
SSL_set_blocking_mode == nullptr ||
45+
SSL_set_event_handling_mode(ssl, SSL_VALUE_EVENT_HANDLING_MODE_EXPLICIT) ||
46+
SSL_set_feature_request_uint(ssl, SSL_VALUE_QUIC_IDLE_TIMEOUT, value) ||
47+
SSL_set_incoming_stream_policy == nullptr;
48+
}
49+
"
50+
)
51+
set(CMAKE_REQUIRED_INCLUDES "${OPENSSL_INCLUDE_DIR}")
52+
set(CMAKE_REQUIRED_LIBRARIES OpenSSL::SSL OpenSSL::Crypto)
53+
include(CheckCXXSourceCompiles)
54+
check_cxx_source_compiles("${CHECK_PROGRAM}" ${OUT_VAR})
55+
set(${OUT_VAR}
56+
${${OUT_VAR}}
57+
PARENT_SCOPE
58+
)
59+
endfunction()

include/iocore/net/QUICMultiCertConfigLoader.h

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
#include "iocore/net/SSLMultiCertConfigLoader.h"
2727
#include "iocore/eventsystem/ConfigProcessor.h"
28+
#include "tscore/ink_config.h"
2829

2930
class QUICCertConfig
3031
{
@@ -48,7 +49,12 @@ class QUICMultiCertConfigLoader : public SSLMultiCertConfigLoader
4849
virtual SSL_CTX *default_server_ssl_ctx() override;
4950

5051
private:
51-
const char *_debug_tag() const override;
52+
const char *_debug_tag() const override;
53+
#if TS_HAS_OPENSSL_QUIC
54+
virtual void _set_handshake_callbacks(SSL_CTX *ctx) override;
55+
virtual bool _set_alpn_callback(SSL_CTX *ctx) override;
56+
virtual bool _set_curves(SSL_CTX *ctx) override;
57+
#endif
5258
virtual bool _setup_session_cache(SSL_CTX *ctx) override;
5359
virtual bool _set_cipher_suites_for_legacy_versions(SSL_CTX *ctx) override;
5460
virtual bool _set_info_callback(SSL_CTX *ctx) override;

include/iocore/net/quic/Mock.h

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
#include "iocore/net/quic/QUICStreamAdapter.h"
2929
#include "iocore/net/quic/QUICStream.h"
3030

31+
#include <algorithm>
32+
3133
class MockQUICContext;
3234

3335
using namespace std::literals;
@@ -519,13 +521,19 @@ class MockQUICStreamAdapter : public QUICStreamAdapter
519521
Ptr<IOBufferBlock>
520522
_read(size_t len) override
521523
{
522-
this->_sending_data_len -= len;
523-
Ptr<IOBufferBlock> block = make_ptr<IOBufferBlock>(new_IOBufferBlock());
524+
len = std::min(len, this->_sending_data_len);
525+
Ptr<IOBufferBlock> block = make_ptr<IOBufferBlock>(new_IOBufferBlock());
524526
block->alloc(iobuffer_size_to_index(len, BUFFER_SIZE_INDEX_32K));
525527
block->fill(len);
526528
return block;
527529
}
528530

531+
void
532+
_consume(size_t len) override
533+
{
534+
this->_sending_data_len -= std::min(len, this->_sending_data_len);
535+
}
536+
529537
private:
530538
size_t _sending_data_len = 0;
531539
size_t _total_sending_data_len = 0;

include/iocore/net/quic/QUICConfig.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,11 @@
2424
#pragma once
2525

2626
#include <openssl/ssl.h>
27+
#include "tscore/ink_config.h"
28+
29+
#if TS_HAS_QUICHE
2730
#include <quiche.h>
31+
#endif
2832

2933
#include "iocore/eventsystem/ConfigProcessor.h"
3034
#include "iocore/net/SSLTypes.h"
@@ -89,7 +93,9 @@ class QUICConfigParams : public ConfigInfo
8993

9094
bool disable_http_0_9() const;
9195

96+
#if TS_HAS_QUICHE
9297
quiche_cc_algorithm get_cc_algorithm() const;
98+
#endif
9399

94100
private:
95101
static int _connection_table_size;
@@ -163,3 +169,4 @@ class QUICConfig
163169
};
164170

165171
SSL_CTX *quic_new_ssl_ctx();
172+
SSL_CTX *quic_new_server_ssl_ctx();

include/iocore/net/quic/QUICConnection.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ class QUICConnectionInfoProvider
6060
virtual bool is_handshake_completed() const = 0;
6161
virtual QUICVersion negotiated_version() const = 0;
6262
virtual std::string_view negotiated_application_name() const = 0;
63+
virtual void
64+
on_stream_updated()
65+
{
66+
}
6367
};
6468

6569
class QUICConnection : public QUICConnectionInfoProvider

include/iocore/net/quic/QUICStream.h

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,30 @@
2626
#include "tscore/List.h"
2727

2828
#include "iocore/eventsystem/Event.h"
29+
#include "iocore/eventsystem/IOBuffer.h"
2930

3031
#include "iocore/net/quic/QUICConnection.h"
3132
#include "iocore/net/quic/QUICDebugNames.h"
3233

33-
#include <quiche.h>
34+
#include <cstddef>
35+
#include <cstdint>
3436

3537
class QUICStreamAdapter;
3638
class QUICStreamStateListener;
3739

40+
class QUICStreamIO
41+
{
42+
public:
43+
using ErrorCode = uint64_t;
44+
45+
virtual ~QUICStreamIO() = default;
46+
47+
virtual int64_t read_stream(QUICStreamId stream_id, uint8_t *buf, size_t len, bool &fin, ErrorCode &error_code) = 0;
48+
virtual bool stream_read_finished(QUICStreamId stream_id) = 0;
49+
virtual int64_t stream_write_capacity(QUICStreamId stream_id) = 0;
50+
virtual int64_t write_stream(QUICStreamId stream_id, uint8_t const *buf, size_t len, bool fin, ErrorCode &error_code) = 0;
51+
};
52+
3853
/**
3954
* @brief QUIC Stream
4055
* TODO: This is similar to Http2Stream. Need to think some integration.
@@ -53,19 +68,21 @@ class QUICStream
5368
QUICStreamDirection direction() const;
5469
bool is_bidirectional() const;
5570
bool has_no_more_data() const;
71+
bool has_data_to_send();
5672

5773
QUICOffset final_offset() const;
5874

5975
void stop_sending(QUICStreamErrorUPtr error);
6076
void reset(QUICStreamErrorUPtr error);
6177

62-
void receive_data(quiche_conn *quiche_con);
63-
void send_data(quiche_conn *quiche_con);
78+
void receive_data(QUICStreamIO &stream_io);
79+
int64_t send_data(QUICStreamIO &stream_io);
6480

6581
/*
6682
* QUICApplication need to call one of these functions when it process VC_EVENT_*
6783
*/
6884
void on_read();
85+
void on_write();
6986
void on_eos();
7087

7188
/**
@@ -85,6 +102,8 @@ class QUICStream
85102
uint64_t _received_bytes = 0;
86103
uint64_t _sent_bytes = 0;
87104
bool _has_no_more_data = false;
105+
Ptr<IOBufferBlock> _pending_send_block;
106+
bool _pending_send_fin = false;
88107
};
89108

90109
class QUICStreamStateListener

include/iocore/net/quic/QUICStreamAdapter.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ class QUICStreamAdapter
3939

4040
virtual int64_t write(QUICOffset offset, const uint8_t *data, uint64_t data_length, bool fin) = 0;
4141
Ptr<IOBufferBlock> read(size_t len);
42+
void consume(size_t len);
4243
virtual bool is_eos() = 0;
4344
virtual uint64_t unread_len() = 0;
4445
virtual uint64_t read_len() = 0;
@@ -60,6 +61,7 @@ class QUICStreamAdapter
6061
virtual void notify_eos() = 0;
6162

6263
protected:
63-
virtual Ptr<IOBufferBlock> _read(size_t len) = 0;
64+
virtual Ptr<IOBufferBlock> _read(size_t len) = 0;
65+
virtual void _consume(size_t len) = 0;
6466
QUICStream &_stream;
6567
};

include/iocore/net/quic/QUICStreamManager.h

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,10 @@ class QUICStreamManager : public QUICStreamStateListener
4848

4949
QUICStream *create_stream(QUICStreamId stream_id, QUICConnectionError &err);
5050

51-
QUICConnectionErrorUPtr create_uni_stream(QUICStreamId new_stream_id);
52-
QUICConnectionErrorUPtr create_bidi_stream(QUICStreamId new_stream_id);
53-
QUICConnectionErrorUPtr delete_stream(QUICStreamId new_stream_id);
54-
void reset_stream(QUICStreamId stream_id, QUICStreamErrorUPtr error);
51+
virtual QUICConnectionErrorUPtr create_uni_stream(QUICStreamId &new_stream_id);
52+
virtual QUICConnectionErrorUPtr create_bidi_stream(QUICStreamId &new_stream_id);
53+
QUICConnectionErrorUPtr delete_stream(QUICStreamId new_stream_id);
54+
void reset_stream(QUICStreamId stream_id, QUICStreamErrorUPtr error);
5555

5656
void set_default_application(QUICApplication *app);
5757

@@ -63,4 +63,6 @@ class QUICStreamManager : public QUICStreamStateListener
6363
protected:
6464
QUICContext *_context = nullptr;
6565
QUICApplicationMap *_app_map = nullptr;
66+
QUICStreamId _next_local_bidi_stream_id{0};
67+
QUICStreamId _next_local_uni_stream_id{0};
6668
};

0 commit comments

Comments
 (0)