Skip to content

Commit 445b7fa

Browse files
committed
Bugfix: RDMA client cannot fall back to TCP when RDMA is not enabled on the server
1 parent eb31fa5 commit 445b7fa

13 files changed

Lines changed: 567 additions & 233 deletions

src/brpc/global.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
#include "brpc/policy/nshead_mcpack_protocol.h"
8484
#include "brpc/policy/rtmp_protocol.h"
8585
#include "brpc/policy/esp_protocol.h"
86+
#include "brpc/policy/rdma_handshake_fallback_protocol.h"
8687
#ifdef ENABLE_THRIFT_FRAMED_PROTOCOL
8788
# include "brpc/policy/thrift_protocol.h"
8889
#endif
@@ -617,6 +618,15 @@ static void GlobalInitializeOrDieImpl() {
617618
exit(1);
618619
}
619620

621+
Protocol rdma_handshake_fallback_protocol = {
622+
ParseRdmaHandshake, NULL, NULL,
623+
ProcessRdmaHandshake, NULL,
624+
NULL, NULL, NULL,
625+
CONNECTION_TYPE_ALL, "rdma_handshake" };
626+
if (RegisterProtocol(PROTOCOL_RDMA_HANDSHAKE, rdma_handshake_fallback_protocol) != 0) {
627+
exit(1);
628+
}
629+
620630
std::vector<Protocol> protocols;
621631
ListProtocols(&protocols);
622632
for (size_t i = 0; i < protocols.size(); ++i) {

src/brpc/input_messenger.cpp

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ DECLARE_uint64(max_body_size);
7979
const size_t MSG_SIZE_WINDOW = 10; // Take last so many message into stat.
8080
const size_t MIN_ONCE_READ = 4096;
8181
const size_t MAX_ONCE_READ = 524288;
82-
const size_t PROTO_DUMMY_LEN = 4;
8382

8483
ParseResult InputMessenger::CutInputMessage(
8584
Socket* m, size_t* index, bool read_eof) {
@@ -107,16 +106,6 @@ ParseResult InputMessenger::CutInputMessage(
107106
<< " bytes, the connection will be closed."
108107
" Set max_body_size to allow bigger messages";
109108
return result;
110-
} else {
111-
if (m->_read_buf.size() >= 4) {
112-
// The length of `data' must be PROTO_DUMMY_LEN + 1 to store extra ending char '\0'
113-
char data[PROTO_DUMMY_LEN + 1];
114-
m->_read_buf.copy_to_cstr(data, PROTO_DUMMY_LEN);
115-
if (strncmp(data, "RDMA", PROTO_DUMMY_LEN) == 0) {
116-
// To avoid timeout when client uses RDMA but server uses TCP
117-
return MakeParseError(PARSE_ERROR_TRY_OTHERS);
118-
}
119-
}
120109
}
121110

122111
if (m->CreatedByConnect()) {

src/brpc/options.proto

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -43,28 +43,29 @@ enum ProtocolType {
4343
PROTOCOL_SOFA_PBRPC = 4;
4444
PROTOCOL_RTMP = 5;
4545
PROTOCOL_THRIFT = 6;
46-
PROTOCOL_HTTP = 7;
47-
PROTOCOL_PUBLIC_PBRPC = 8;
48-
PROTOCOL_NOVA_PBRPC = 9;
49-
PROTOCOL_REDIS = 10;
50-
PROTOCOL_NSHEAD_CLIENT = 11; // implemented in baidu-rpc-ub
51-
PROTOCOL_NSHEAD = 12;
52-
PROTOCOL_HADOOP_RPC = 13;
53-
PROTOCOL_HADOOP_SERVER_RPC = 14;
54-
PROTOCOL_MONGO = 15; // server side only
55-
PROTOCOL_UBRPC_COMPACK = 16;
56-
PROTOCOL_DIDX_CLIENT = 17; // Client side only
57-
PROTOCOL_MEMCACHE = 18; // Client side only
58-
PROTOCOL_ITP = 19;
59-
PROTOCOL_NSHEAD_MCPACK = 20;
60-
PROTOCOL_DISP_IDL = 21; // Client side only
61-
PROTOCOL_ERSDA_CLIENT = 22; // Client side only
62-
PROTOCOL_UBRPC_MCPACK2 = 23; // Client side only
46+
PROTOCOL_RDMA_HANDSHAKE = 7;
47+
PROTOCOL_HTTP = 8;
48+
PROTOCOL_PUBLIC_PBRPC = 9;
49+
PROTOCOL_NOVA_PBRPC = 10;
50+
PROTOCOL_REDIS = 11;
51+
PROTOCOL_NSHEAD_CLIENT = 12; // implemented in baidu-rpc-ub
52+
PROTOCOL_NSHEAD = 13;
53+
PROTOCOL_HADOOP_RPC = 14;
54+
PROTOCOL_HADOOP_SERVER_RPC = 15;
55+
PROTOCOL_MONGO = 16; // server side only
56+
PROTOCOL_UBRPC_COMPACK = 17;
57+
PROTOCOL_DIDX_CLIENT = 18; // Client side only
58+
PROTOCOL_MEMCACHE = 19; // Client side only
59+
PROTOCOL_ITP = 20;
60+
PROTOCOL_NSHEAD_MCPACK = 21;
61+
PROTOCOL_DISP_IDL = 22; // Client side only
62+
PROTOCOL_ERSDA_CLIENT = 23; // Client side only
63+
PROTOCOL_UBRPC_MCPACK2 = 24; // Client side only
6364
// Reserve special protocol for cds-agent, which depends on FIFO right now
64-
PROTOCOL_CDS_AGENT = 24; // Client side only
65-
PROTOCOL_ESP = 25; // Client side only
66-
PROTOCOL_H2 = 26;
67-
PROTOCOL_COUCHBASE = 27;
65+
PROTOCOL_CDS_AGENT = 25; // Client side only
66+
PROTOCOL_ESP = 26; // Client side only
67+
PROTOCOL_H2 = 27;
68+
PROTOCOL_COUCHBASE = 28;
6869
}
6970

7071
enum CompressType {

src/brpc/parse_result.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#ifndef BRPC_PARSE_RESULT_H
2020
#define BRPC_PARSE_RESULT_H
2121

22+
#include <stddef.h>
2223

2324
namespace brpc {
2425

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "brpc/policy/rdma_handshake_fallback_protocol.h"
19+
20+
#include <limits>
21+
#include <string>
22+
#include <string.h>
23+
#include "butil/object_pool.h"
24+
#include "butil/logging.h"
25+
#include "butil/raw_pack.h"
26+
#include "brpc/destroyable.h"
27+
#include "brpc/rdma/rdma_handshake.pb.h"
28+
#include "brpc/rdma/rdma_handshake_constants.h"
29+
30+
namespace brpc {
31+
namespace policy {
32+
33+
// Both handshake versions are handled:
34+
// - v2 ("RDMA"): [ "RDMA" 4B ][ msg_len 2B ][ body ], total >= 40B.
35+
// - v3 ("RDM3"): [ "RDM3" 4B ][ pb_size 4B (big-endian) ][ RdmaHello bytes ].
36+
37+
// v2 header: 4B magic + 2B msg_len.
38+
static constexpr size_t RDMA_V2_HEADER_LEN = rdma::HELLO_MAGIC_LEN + 2;
39+
// v3 header: 4B magic + 4B pb_size.
40+
static constexpr size_t RDMA_V3_HEADER_LEN =
41+
rdma::HELLO_MAGIC_LEN + rdma::HELLO_V3_PB_SIZE_LEN;
42+
// An intentionally-invalid v2 hello_ver written into the reply: the client's
43+
// ValidHelloMessage() requires hello_ver==2, so it rejects this and falls back.
44+
static constexpr uint16_t RDMA_HELLO_V2_VERSION_INVALID =
45+
std::numeric_limits<uint16_t>::max();
46+
47+
// Length of the gid field (== sizeof(ibv_gid)). Spelled as a literal so this
48+
// header stays free of <infiniband/verbs.h>.
49+
constexpr size_t RDMA_GID_LEN = 16;
50+
51+
namespace {
52+
struct RdmaFallbackContext : public Destroyable {
53+
static RdmaFallbackContext* create() { return butil::get_object<RdmaFallbackContext>(); }
54+
void Destroy() override { butil::return_object(this); }
55+
};
56+
57+
ParseResult HandleRdmaV2Hello(butil::IOBuf* source, Socket* socket) {
58+
char header_buf[RDMA_V2_HEADER_LEN];
59+
if (source->copy_to(header_buf, sizeof(header_buf)) < sizeof(header_buf)) {
60+
// Magic matched, but msg_len has not fully arrived yet.
61+
return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA);
62+
}
63+
64+
// msg_len (big-endian, right after the magic) is the total hello length.
65+
uint16_t hello_size = 0;
66+
butil::RawUnpacker(header_buf + rdma::HELLO_MAGIC_LEN).unpack16(hello_size);
67+
// A too-small value is malformed; a too-large one would make us wait
68+
// forever for bytes that never come (the real hello is 40B).
69+
if (hello_size < rdma::HELLO_V2_MSG_LEN_MIN || hello_size > rdma::HELLO_V2_MSG_LEN_MAX) {
70+
return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG);
71+
}
72+
// During the handshake the client sends exactly one hello and then blocks
73+
// waiting for the server hello: fewer bytes -> wait; more bytes -> the peer
74+
// is not a well-behaved RDMA client, bail out.
75+
if (source->size() < hello_size) {
76+
return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA);
77+
}
78+
if (source->size() > hello_size) {
79+
return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG);
80+
}
81+
source->pop_front(hello_size);
82+
83+
// Reply an incompatible hello so the client downgrades to TCP on THIS
84+
// connection: magic "RDMA" + msg_len(=40) + an invalid hello_ver. msg_len
85+
// must be valid, otherwise the client errors out at the IO layer instead of
86+
// negotiating. The rest of the 36B body stays zero (value-initialized);
87+
// the client's ValidHelloMessage() rejects hello_ver!=2 -> negotiated=false.
88+
char reply[rdma::HELLO_V2_MSG_LEN_MIN]{};
89+
fast_memcpy(reply, rdma::HELLO_MAGIC, rdma::HELLO_MAGIC_LEN);
90+
butil::RawPacker(reply + rdma::HELLO_MAGIC_LEN)
91+
.pack16(sizeof(reply)).pack16(RDMA_HELLO_V2_VERSION_INVALID);
92+
93+
butil::IOBuf out;
94+
out.append(reply, sizeof(reply));
95+
if (socket->Write(&out) != 0) {
96+
PLOG(WARNING) << "Fail to send RDMA v2 fallback hello to "
97+
<< socket->description();
98+
return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG,
99+
"fail to send RDMA v2 fallback hello");
100+
}
101+
102+
// Remember we are now draining the client's ACK across subsequent reads.
103+
socket->reset_parsing_context(RdmaFallbackContext::create());
104+
return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA);
105+
}
106+
107+
ParseResult HandleRdmaV3Hello(butil::IOBuf* source, Socket* socket) {
108+
char header_buf[RDMA_V3_HEADER_LEN];
109+
if (source->copy_to(header_buf, sizeof(header_buf)) < sizeof(header_buf)) {
110+
// Magic matched, but pb_size has not fully arrived yet.
111+
return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA);
112+
}
113+
uint32_t pb_size = 0;
114+
butil::RawUnpacker(header_buf + rdma::HELLO_MAGIC_LEN).unpack32(pb_size);
115+
// Zero is malformed; an over-large value would make us wait forever. The
116+
// upper bound also keeps the size_t addition below well within range.
117+
if (pb_size == 0 || pb_size > rdma::HELLO_V3_MAX_PB_SIZE) {
118+
return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG);
119+
}
120+
// size_t (>= 64-bit on supported platforms), so this addition cannot
121+
// overflow given the bound above.
122+
const size_t hello_size = RDMA_V3_HEADER_LEN + pb_size;
123+
// During the handshake the client sends exactly one hello and then blocks
124+
// waiting for the server hello: fewer bytes -> wait; more bytes -> the peer
125+
// is not a well-behaved RDMA client, bail out.
126+
if (source->size() < hello_size) {
127+
return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA);
128+
}
129+
if (source->size() > hello_size) {
130+
return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG);
131+
}
132+
source->pop_front(hello_size);
133+
134+
// Build a v3 hello the client will reject so it downgrades to TCP. All
135+
// `required` fields are populated so the client's ParseFromArray() succeeds
136+
// (a parse failure would surface as an IO error, not a clean fallback), but
137+
// block_size==0 (< MIN_BLOCK_SIZE) and qp_num==0 make its ValidRdmaHello()
138+
// return false -> negotiated=false regardless of g_skip_rdma_init.
139+
rdma::RdmaHello reply_msg;
140+
reply_msg.set_block_size(0);
141+
reply_msg.set_sq_size(0);
142+
reply_msg.set_rq_size(0);
143+
reply_msg.set_lid(0);
144+
reply_msg.set_gid(std::string(RDMA_GID_LEN, '\0'));
145+
reply_msg.set_qp_num(0);
146+
147+
// [ "RDM3" 4B ][ pb_size 4B (big-endian) ][ RdmaHello protobuf bytes ]
148+
butil::IOBuf packet;
149+
packet.append(rdma::HELLO_MAGIC_V3, rdma::HELLO_MAGIC_LEN);
150+
uint32_t pb_size_be = butil::HostToNet32(static_cast<uint32_t>(reply_msg.ByteSizeLong()));
151+
packet.append(&pb_size_be, sizeof(pb_size_be));
152+
153+
butil::IOBufAsZeroCopyOutputStream output(&packet);
154+
if (!reply_msg.SerializeToZeroCopyStream(&output)) {
155+
return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG,
156+
"Fail to serialize RDMA v3 fallback hello");
157+
}
158+
159+
if (socket->Write(&packet) != 0) {
160+
return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG,
161+
"Fail to write RDMA v3 fallback hello");
162+
}
163+
164+
// Remember we are now draining the client's ACK across subsequent reads.
165+
socket->reset_parsing_context(RdmaFallbackContext::create());
166+
return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA);
167+
}
168+
} // namespace
169+
170+
ParseResult ParseRdmaHandshake(butil::IOBuf* source, Socket* socket,
171+
bool /*read_eof*/, const void* /*arg*/) {
172+
// It is a small state machine kept across parse calls via Socket's
173+
// parsing_context() (same mechanism HTTP uses):
174+
// - no context yet: if the leading bytes are an RDMA magic ("RDMA" for v2 or
175+
// "RDM3" for v3), consume the client hello, reply a version-matched but
176+
// un-negotiable hello (so the client downgrades to TCP on this connection),
177+
// store a context and return NOT_ENOUGH_DATA to wait for the client's ACK;
178+
// - context present: drain the 4-byte ACK, drop the context and return
179+
// TRY_OTHERS so InputMessenger parses the following real request from the
180+
// first protocol.
181+
if (socket->parsing_context() != NULL) {
182+
if (source->size() < rdma::HELLO_ACK_LEN) {
183+
return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA);
184+
}
185+
source->pop_front(rdma::HELLO_ACK_LEN);
186+
socket->reset_parsing_context(NULL);
187+
return MakeParseError(PARSE_ERROR_TRY_OTHERS);
188+
}
189+
190+
char magic_buf[rdma::HELLO_MAGIC_LEN];
191+
const size_t mn = source->copy_to(magic_buf, sizeof(magic_buf));
192+
if (mn < rdma::HELLO_MAGIC_LEN) {
193+
return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA);
194+
}
195+
196+
if (*(const uint32_t*)magic_buf == *(const uint32_t*)rdma::HELLO_MAGIC) {
197+
return HandleRdmaV2Hello(source, socket);
198+
}
199+
if (*(const uint32_t*)magic_buf == *(const uint32_t*)rdma::HELLO_MAGIC_V3) {
200+
return HandleRdmaV3Hello(source, socket);
201+
}
202+
return MakeParseError(PARSE_ERROR_TRY_OTHERS);
203+
}
204+
205+
void ProcessRdmaHandshake(InputMessageBase* msg) {
206+
// ParseRdmaHandshake replies inline and only ever returns
207+
// NOT_ENOUGH_DATA / TRY_OTHERS / hard errors, never a real message, so this
208+
// must never run. Keep a placeholder (required for server registration).
209+
DestroyingPtr<InputMessageBase> destroying_msg(msg);
210+
CHECK(false) << "ProcessRdmaHandshake should never be called";
211+
}
212+
213+
} // namespace policy
214+
} // namespace brpc
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#ifndef BRPC_POLICY_RDMA_HANDSHAKE_FALLBACK_PROTOCOL_H
19+
#define BRPC_POLICY_RDMA_HANDSHAKE_FALLBACK_PROTOCOL_H
20+
21+
// NOTE: This file is intentionally INDEPENDENT of BRPC_WITH_RDMA. A server may
22+
// run in TCP mode either because it was built without RDMA, or because RDMA was
23+
// not enabled at runtime. In both cases an RDMA client that connects to it will
24+
// send an RDMA handshake magic ("RDMA" for v2, "RDM3" for v3) first. Without
25+
// special handling the server treats those bytes as an unknown protocol and
26+
// closes the connection, so the client (blocked reading the server hello) only
27+
// sees EOF and cannot fall back to TCP.
28+
//
29+
// To let the client fall back on the SAME connection, the server recognizes the
30+
// RDMA handshake as a "first-class" protocol (magic in the first 4 bytes,
31+
// PROTOCOL_RDMA_HANDSHAKE is ordered before PROTOCOL_HTTP), replies a hello with
32+
// an incompatible version so the client rejects it and downgrades to TCP, then
33+
// drains the client's subsequent ACK and lets normal RPC parsing continue.
34+
35+
#include "butil/iobuf.h"
36+
#include "brpc/input_message_base.h"
37+
#include "brpc/parse_result.h"
38+
#include "brpc/socket.h"
39+
40+
namespace brpc {
41+
namespace policy {
42+
43+
// Parse binary format of rdma handshake.
44+
ParseResult ParseRdmaHandshake(butil::IOBuf* source, Socket* socket,
45+
bool read_eof, const void* arg);
46+
47+
// Process callback placeholder. The parser never emits a real message (it
48+
// replies inline and only returns NOT_ENOUGH_DATA / TRY_OTHERS), so this must
49+
// never be invoked; it only exists to satisfy server-side protocol registration
50+
// which requires a non-NULL process_request.
51+
void ProcessRdmaHandshake(InputMessageBase* msg);
52+
53+
} // namespace policy
54+
} // namespace brpc
55+
56+
#endif // BRPC_POLICY_RDMA_HANDSHAKE_FALLBACK_PROTOCOL_H

0 commit comments

Comments
 (0)