api: add file descriptor passing over the UNIX API socket

The upcoming packet capture feature needs to share a memfd between
the grout daemon and its clients. Extend the API protocol to pass
a file descriptor alongside the response payload using SCM_RIGHTS
ancillary data on the UNIX socket.

On the server side, api_out_fd() sets an fd in struct api_out and
sendmsg() transmits it. On the client side, recvmsg() captures the
ancillary data. Out-of-order cached responses carry their fd so it
is preserved until the matching recv call picks it up.

Signed-off-by: Robin Jarry <rjarry@redhat.com>
Signed-off-by: Vincent Jardin <vjardin@free.fr>

Author: vjardin

api/gr_api.h

@@ -58,6 +58,14 @@ gr_api_client_send(struct gr_api_client *, uint32_t req_type, size_t tx_len, con
 // Returns -EMSGSIZE if payload is non-empty but smaller than min_resp_size.
 int gr_api_client_recv(struct gr_api_client *, uint32_t req_type, uint32_t for_id, void **rx_data);
 
+int gr_api_client_recv_fd(
+	struct gr_api_client *,
+	uint32_t req_type,
+	uint32_t for_id,
+	void **rx_data,
+	int *fd
+);
+
 // Send a request and receive the response.
 // Validates response payload size against GR_REQ-declared type.
 // Caller must free(*rx_data) after use.
@@ -78,6 +86,22 @@ static inline int gr_api_client_send_recv(
 // internal, called when interrupting gr_api_client_stream_foreach()
 int __gr_api_client_stream_drain(struct gr_api_client *, uint32_t req_type, uint32_t for_id);
 
+// Send a request and receive the response with an optional file descriptor.
+// If fd is non-NULL and the server sends an fd via SCM_RIGHTS, it is stored in *fd.
+static inline int gr_api_client_send_recv_fd(
+	struct gr_api_client *client,
+	uint32_t req_type,
+	size_t tx_len,
+	const void *tx_data,
+	void **rx_data,
+	int *fd
+) {
+	long int ret = gr_api_client_send(client, req_type, tx_len, tx_data);
+	if (ret < 0)
+		return ret;
+	return gr_api_client_recv_fd(client, req_type, ret, rx_data, fd);
+}
+
 // Send a request and iterate over the received stream of responses.
 //
 // @param obj Iterator variable (const pointer to response object type).

api/gr_api_client_impl.h

@@ -101,6 +101,7 @@ const char *gr_api_message_name(uint32_t type) {
 struct response {
 	struct gr_api_response header;
 	void *payload;
+	int fd; // received via SCM_RIGHTS, -1 if none
 	STAILQ_ENTRY(response) next;
 };
 
@@ -153,6 +154,8 @@ int gr_api_client_disconnect(struct gr_api_client *client) {
 	while (!STAILQ_EMPTY(&client->responses)) {
 		struct response *resp = STAILQ_FIRST(&client->responses);
 		STAILQ_REMOVE_HEAD(&client->responses, next);
+		if (resp->fd >= 0)
+			close(resp->fd);
 		free(resp->payload);
 		free(resp);
 	}
@@ -224,16 +227,61 @@ long int gr_api_client_send(
 	return req.id;
 }
 
-int gr_api_client_recv(
+// Receive a response header, potentially with an SCM_RIGHTS fd.
+// Uses recvmsg() so ancillary data is captured.
+static int
+recv_response_header(const struct gr_api_client *c, struct gr_api_response *resp, int *recv_fd) {
+	*recv_fd = -1;
+
+	union {
+		char buf[CMSG_SPACE(sizeof(int))];
+		struct cmsghdr align;
+	} cmsg_buf;
+	memset(&cmsg_buf, 0, sizeof(cmsg_buf));
+
+	struct iovec iov = {.iov_base = resp, .iov_len = sizeof(*resp)};
+	struct msghdr msg = {
+		.msg_iov = &iov,
+		.msg_iovlen = 1,
+		.msg_control = cmsg_buf.buf,
+		.msg_controllen = sizeof(cmsg_buf.buf),
+	};
+
+	ssize_t n = recvmsg(c->sock_fd, &msg, MSG_CMSG_CLOEXEC);
+
+	if (n == 0) {
+		errno = ECONNRESET;
+		return -1;
+	}
+	if (n < 0)
+		return -1;
+	if ((size_t)n < sizeof(*resp)) {
+		errno = EPROTO;
+		return -1;
+	}
+
+	struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+	if (cmsg != NULL && cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS)
+		memcpy(recv_fd, CMSG_DATA(cmsg), sizeof(int));
+
+	return 0;
+}
+
+int gr_api_client_recv_fd(
 	struct gr_api_client *client,
 	uint32_t req_type,
 	uint32_t for_id,
-	void **rx_data
+	void **rx_data,
+	int *fd
 ) {
 	struct response *cached = NULL;
 	const struct api_message *m;
 	struct gr_api_response resp;
 	void *payload = NULL;
+	int recv_fd = -1;
+
+	if (fd != NULL)
+		*fd = -1;
 
 	if (client == NULL)
 		return errno_set(EINVAL);
@@ -249,12 +297,13 @@ int gr_api_client_recv(
 		STAILQ_REMOVE(&client->responses, cached, response, next);
 		resp = cached->header;
 		payload = cached->payload;
+		recv_fd = cached->fd;
 		free(cached);
 		goto out;
 	}
 recv:
 	// No matching cached message, try to receive one from the socket.
-	if (recv_all(client, &resp, sizeof(resp)) != sizeof(resp))
+	if (recv_response_header(client, &resp, &recv_fd) < 0)
 		goto err;
 
 	if (resp.payload_len > GR_API_MAX_MSG_LEN) {
@@ -275,8 +324,10 @@ int gr_api_client_recv(
 			goto err;
 		cached->header = resp;
 		cached->payload = payload;
+		cached->fd = recv_fd;
 		STAILQ_INSERT_TAIL(&client->responses, cached, next);
 		payload = NULL;
+		recv_fd = -1;
 		// And try to receive the next message until we get the correct ID.
 		goto recv;
 	}
@@ -299,13 +350,28 @@ int gr_api_client_recv(
 		assert(rx_data != NULL);
 		*rx_data = payload;
 	}
+	if (fd != NULL)
+		*fd = recv_fd;
+	else if (recv_fd >= 0)
+		close(recv_fd);
 
 	return 0;
 err:
+	if (recv_fd >= 0)
+		close(recv_fd);
 	free(payload);
 	return -errno;
 }
 
+int gr_api_client_recv(
+	struct gr_api_client *client,
+	uint32_t req_type,
+	uint32_t for_id,
+	void **rx_data
+) {
+	return gr_api_client_recv_fd(client, req_type, for_id, rx_data, NULL);
+}
+
 int gr_api_client_event_recv(const struct gr_api_client *c, struct gr_api_event **event) {
 	const struct api_message *m;
 	struct gr_api_event header;

main/api.c

@@ -22,6 +22,7 @@
 #include <stdlib.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
+#include <sys/uio.h>
 #include <unistd.h>
 
 LOG_TYPE("api");
@@ -249,6 +250,50 @@ void api_send(struct api_ctx *ctx, uint32_t len, const void *payload) {
 		LOG(ERR, "pid=%d cannot write payload", ctx->pid);
 }
 
+// Send a response header + optional payload together with a file
+// descriptor via sendmsg(SCM_RIGHTS). Closes the fd after sending.
+static void
+send_response_with_fd(struct bufferevent *bev, struct gr_api_response *resp, struct api_out *out) {
+	bufferevent_flush(bev, EV_WRITE, BEV_FLUSH);
+
+	struct iovec iov[2];
+	int iovlen = 1;
+	iov[0].iov_base = resp;
+	iov[0].iov_len = sizeof(*resp);
+	if (out->len > 0 && out->payload != NULL) {
+		iov[1].iov_base = out->payload;
+		iov[1].iov_len = out->len;
+		iovlen = 2;
+	}
+
+	union {
+		char buf[CMSG_SPACE(sizeof(int))];
+		struct cmsghdr align;
+	} cmsg_buf;
+	memset(&cmsg_buf, 0, sizeof(cmsg_buf));
+
+	struct msghdr msg = {
+		.msg_iov = iov,
+		.msg_iovlen = iovlen,
+		.msg_control = cmsg_buf.buf,
+		.msg_controllen = sizeof(cmsg_buf.buf),
+	};
+	struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+	cmsg->cmsg_level = SOL_SOCKET;
+	cmsg->cmsg_type = SCM_RIGHTS;
+	cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+	memcpy(CMSG_DATA(cmsg), &out->fd, sizeof(int));
+
+	ssize_t ret;
+	do {
+		ret = sendmsg(bufferevent_getfd(bev), &msg, MSG_NOSIGNAL);
+	} while (ret < 0 && errno == EINTR);
+	if (ret < 0)
+		LOG(ERR, "sendmsg with fd: %s", strerror(errno));
+	close(out->fd);
+	out->fd = -1;
+}
+
 static void read_cb(struct bufferevent *bev, void *priv) {
 	struct evbuffer *input = bufferevent_get_input(bev);
 	struct api_ctx *ctx = priv;
@@ -294,7 +339,7 @@ static void read_cb(struct bufferevent *bev, void *priv) {
 	// Reset state for next request
 	ctx->header_complete = false;
 
-	struct api_out out;
+	struct api_out out = {.fd = -1};
 
 	// We have a complete request, process it
 	const struct api_handler *handler = lookup_api_handler(ctx->header.type);
@@ -333,16 +378,19 @@ static void read_cb(struct bufferevent *bev, void *priv) {
 		.payload_len = out.len,
 	};
 
-	if (bufferevent_write(bev, &resp, sizeof(resp)) < 0)
-		LOG(ERR, "failed to write header");
-	if (out.len > 0) {
-		assert(out.payload != NULL);
-		if (bufferevent_write(bev, out.payload, out.len) < 0)
-			LOG(ERR, "failed to write payload");
+	if (out.fd >= 0) {
+		send_response_with_fd(bev, &resp, &out);
+	} else {
+		if (bufferevent_write(bev, &resp, sizeof(resp)) < 0)
+			LOG(ERR, "failed to write header");
+		if (out.len > 0) {
+			assert(out.payload != NULL);
+			if (bufferevent_write(bev, out.payload, out.len) < 0)
+				LOG(ERR, "failed to write payload");
+		}
+		bufferevent_flush(bev, EV_WRITE, BEV_FLUSH);
 	}
 
-	bufferevent_flush(bev, EV_WRITE, BEV_FLUSH);
-
 	free(req_payload);
 	free(out.payload);
 

main/module.h

@@ -15,10 +15,16 @@ struct api_out {
 	uint32_t status;
 	uint32_t len;
 	void *payload;
+	int fd; // file descriptor to pass via SCM_RIGHTS, -1 = none
 };
 
 static inline struct api_out api_out(uint32_t status, uint32_t len, void *payload) {
-	struct api_out out = {status, len, payload};
+	struct api_out out = {status, len, payload, -1};
+	return out;
+}
+
+static inline struct api_out api_out_fd(uint32_t status, uint32_t len, void *payload, int fd) {
+	struct api_out out = {status, len, payload, fd};
 	return out;
 }