1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* RxRPC packet transmission
4 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10 #include <linux/net.h>
11 #include <linux/gfp.h>
12 #include <linux/skbuff.h>
13 #include <linux/export.h>
15 #include <net/af_rxrpc.h>
16 #include "ar-internal.h"
18 struct rxrpc_ack_buffer
{
19 struct rxrpc_wire_header whdr
;
20 struct rxrpc_ackpacket ack
;
23 struct rxrpc_ackinfo ackinfo
;
26 struct rxrpc_abort_buffer
{
27 struct rxrpc_wire_header whdr
;
31 static const char rxrpc_keepalive_string
[] = "";
34 * Increase Tx backoff on transmission failure and clear it on success.
36 static void rxrpc_tx_backoff(struct rxrpc_call
*call
, int ret
)
39 u16 tx_backoff
= READ_ONCE(call
->tx_backoff
);
42 WRITE_ONCE(call
->tx_backoff
, tx_backoff
+ 1);
44 WRITE_ONCE(call
->tx_backoff
, 0);
49 * Arrange for a keepalive ping a certain time after we last transmitted. This
50 * lets the far side know we're still interested in this call and helps keep
51 * the route through any intervening firewall open.
53 * Receiving a response to the ping will prevent the ->expect_rx_by timer from
56 static void rxrpc_set_keepalive(struct rxrpc_call
*call
)
58 unsigned long now
= jiffies
, keepalive_at
= call
->next_rx_timo
/ 6;
61 WRITE_ONCE(call
->keepalive_at
, keepalive_at
);
62 rxrpc_reduce_call_timer(call
, keepalive_at
, now
,
63 rxrpc_timer_set_for_keepalive
);
67 * Fill out an ACK packet.
69 static size_t rxrpc_fill_out_ack(struct rxrpc_connection
*conn
,
70 struct rxrpc_call
*call
,
71 struct rxrpc_ack_buffer
*pkt
,
72 rxrpc_seq_t
*_hard_ack
,
76 rxrpc_serial_t serial
;
77 rxrpc_seq_t hard_ack
, top
, seq
;
82 /* Barrier against rxrpc_input_data(). */
83 serial
= call
->ackr_serial
;
84 hard_ack
= READ_ONCE(call
->rx_hard_ack
);
85 top
= smp_load_acquire(&call
->rx_top
);
86 *_hard_ack
= hard_ack
;
89 pkt
->ack
.bufferSpace
= htons(8);
90 pkt
->ack
.maxSkew
= htons(call
->ackr_skew
);
91 pkt
->ack
.firstPacket
= htonl(hard_ack
+ 1);
92 pkt
->ack
.previousPacket
= htonl(call
->ackr_prev_seq
);
93 pkt
->ack
.serial
= htonl(serial
);
94 pkt
->ack
.reason
= reason
;
95 pkt
->ack
.nAcks
= top
- hard_ack
;
97 if (reason
== RXRPC_ACK_PING
)
98 pkt
->whdr
.flags
|= RXRPC_REQUEST_ACK
;
100 if (after(top
, hard_ack
)) {
103 ix
= seq
& RXRPC_RXTX_BUFF_MASK
;
104 if (call
->rxtx_buffer
[ix
])
105 *ackp
++ = RXRPC_ACK_TYPE_ACK
;
107 *ackp
++ = RXRPC_ACK_TYPE_NACK
;
109 } while (before_eq(seq
, top
));
112 mtu
= conn
->params
.peer
->if_mtu
;
113 mtu
-= conn
->params
.peer
->hdrsize
;
114 jmax
= (call
->nr_jumbo_bad
> 3) ? 1 : rxrpc_rx_jumbo_max
;
115 pkt
->ackinfo
.rxMTU
= htonl(rxrpc_rx_mtu
);
116 pkt
->ackinfo
.maxMTU
= htonl(mtu
);
117 pkt
->ackinfo
.rwind
= htonl(call
->rx_winsize
);
118 pkt
->ackinfo
.jumbo_max
= htonl(jmax
);
123 return top
- hard_ack
+ 3;
127 * Send an ACK call packet.
129 int rxrpc_send_ack_packet(struct rxrpc_call
*call
, bool ping
,
130 rxrpc_serial_t
*_serial
)
132 struct rxrpc_connection
*conn
= NULL
;
133 struct rxrpc_ack_buffer
*pkt
;
136 rxrpc_serial_t serial
;
137 rxrpc_seq_t hard_ack
, top
;
142 spin_lock_bh(&call
->lock
);
144 conn
= rxrpc_get_connection_maybe(call
->conn
);
145 spin_unlock_bh(&call
->lock
);
149 pkt
= kzalloc(sizeof(*pkt
), GFP_KERNEL
);
151 rxrpc_put_connection(conn
);
155 msg
.msg_name
= &call
->peer
->srx
.transport
;
156 msg
.msg_namelen
= call
->peer
->srx
.transport_len
;
157 msg
.msg_control
= NULL
;
158 msg
.msg_controllen
= 0;
161 pkt
->whdr
.epoch
= htonl(conn
->proto
.epoch
);
162 pkt
->whdr
.cid
= htonl(call
->cid
);
163 pkt
->whdr
.callNumber
= htonl(call
->call_id
);
165 pkt
->whdr
.type
= RXRPC_PACKET_TYPE_ACK
;
166 pkt
->whdr
.flags
= RXRPC_SLOW_START_OK
| conn
->out_clientflag
;
167 pkt
->whdr
.userStatus
= 0;
168 pkt
->whdr
.securityIndex
= call
->security_ix
;
170 pkt
->whdr
.serviceId
= htons(call
->service_id
);
172 spin_lock_bh(&call
->lock
);
174 reason
= RXRPC_ACK_PING
;
176 reason
= call
->ackr_reason
;
177 if (!call
->ackr_reason
) {
178 spin_unlock_bh(&call
->lock
);
182 call
->ackr_reason
= 0;
184 n
= rxrpc_fill_out_ack(conn
, call
, pkt
, &hard_ack
, &top
, reason
);
186 spin_unlock_bh(&call
->lock
);
188 iov
[0].iov_base
= pkt
;
189 iov
[0].iov_len
= sizeof(pkt
->whdr
) + sizeof(pkt
->ack
) + n
;
190 iov
[1].iov_base
= &pkt
->ackinfo
;
191 iov
[1].iov_len
= sizeof(pkt
->ackinfo
);
192 len
= iov
[0].iov_len
+ iov
[1].iov_len
;
194 serial
= atomic_inc_return(&conn
->serial
);
195 pkt
->whdr
.serial
= htonl(serial
);
196 trace_rxrpc_tx_ack(call
->debug_id
, serial
,
197 ntohl(pkt
->ack
.firstPacket
),
198 ntohl(pkt
->ack
.serial
),
199 pkt
->ack
.reason
, pkt
->ack
.nAcks
);
204 call
->ping_serial
= serial
;
206 /* We need to stick a time in before we send the packet in case
207 * the reply gets back before kernel_sendmsg() completes - but
208 * asking UDP to send the packet can take a relatively long
211 call
->ping_time
= ktime_get_real();
212 set_bit(RXRPC_CALL_PINGING
, &call
->flags
);
213 trace_rxrpc_rtt_tx(call
, rxrpc_rtt_tx_ping
, serial
);
216 ret
= kernel_sendmsg(conn
->params
.local
->socket
, &msg
, iov
, 2, len
);
217 conn
->params
.peer
->last_tx_at
= ktime_get_seconds();
219 trace_rxrpc_tx_fail(call
->debug_id
, serial
, ret
,
220 rxrpc_tx_point_call_ack
);
222 trace_rxrpc_tx_packet(call
->debug_id
, &pkt
->whdr
,
223 rxrpc_tx_point_call_ack
);
224 rxrpc_tx_backoff(call
, ret
);
226 if (call
->state
< RXRPC_CALL_COMPLETE
) {
229 clear_bit(RXRPC_CALL_PINGING
, &call
->flags
);
230 rxrpc_propose_ACK(call
, pkt
->ack
.reason
,
231 ntohs(pkt
->ack
.maxSkew
),
232 ntohl(pkt
->ack
.serial
),
234 rxrpc_propose_ack_retry_tx
);
236 spin_lock_bh(&call
->lock
);
237 if (after(hard_ack
, call
->ackr_consumed
))
238 call
->ackr_consumed
= hard_ack
;
239 if (after(top
, call
->ackr_seen
))
240 call
->ackr_seen
= top
;
241 spin_unlock_bh(&call
->lock
);
244 rxrpc_set_keepalive(call
);
248 rxrpc_put_connection(conn
);
254 * Send an ABORT call packet.
256 int rxrpc_send_abort_packet(struct rxrpc_call
*call
)
258 struct rxrpc_connection
*conn
= NULL
;
259 struct rxrpc_abort_buffer pkt
;
262 rxrpc_serial_t serial
;
265 /* Don't bother sending aborts for a client call once the server has
266 * hard-ACK'd all of its request data. After that point, we're not
267 * going to stop the operation proceeding, and whilst we might limit
268 * the reply, it's not worth it if we can send a new call on the same
269 * channel instead, thereby closing off this call.
271 if (rxrpc_is_client_call(call
) &&
272 test_bit(RXRPC_CALL_TX_LAST
, &call
->flags
))
275 spin_lock_bh(&call
->lock
);
277 conn
= rxrpc_get_connection_maybe(call
->conn
);
278 spin_unlock_bh(&call
->lock
);
282 msg
.msg_name
= &call
->peer
->srx
.transport
;
283 msg
.msg_namelen
= call
->peer
->srx
.transport_len
;
284 msg
.msg_control
= NULL
;
285 msg
.msg_controllen
= 0;
288 pkt
.whdr
.epoch
= htonl(conn
->proto
.epoch
);
289 pkt
.whdr
.cid
= htonl(call
->cid
);
290 pkt
.whdr
.callNumber
= htonl(call
->call_id
);
292 pkt
.whdr
.type
= RXRPC_PACKET_TYPE_ABORT
;
293 pkt
.whdr
.flags
= conn
->out_clientflag
;
294 pkt
.whdr
.userStatus
= 0;
295 pkt
.whdr
.securityIndex
= call
->security_ix
;
297 pkt
.whdr
.serviceId
= htons(call
->service_id
);
298 pkt
.abort_code
= htonl(call
->abort_code
);
300 iov
[0].iov_base
= &pkt
;
301 iov
[0].iov_len
= sizeof(pkt
);
303 serial
= atomic_inc_return(&conn
->serial
);
304 pkt
.whdr
.serial
= htonl(serial
);
306 ret
= kernel_sendmsg(conn
->params
.local
->socket
,
307 &msg
, iov
, 1, sizeof(pkt
));
308 conn
->params
.peer
->last_tx_at
= ktime_get_seconds();
310 trace_rxrpc_tx_fail(call
->debug_id
, serial
, ret
,
311 rxrpc_tx_point_call_abort
);
313 trace_rxrpc_tx_packet(call
->debug_id
, &pkt
.whdr
,
314 rxrpc_tx_point_call_abort
);
315 rxrpc_tx_backoff(call
, ret
);
317 rxrpc_put_connection(conn
);
322 * send a packet through the transport endpoint
324 int rxrpc_send_data_packet(struct rxrpc_call
*call
, struct sk_buff
*skb
,
327 struct rxrpc_connection
*conn
= call
->conn
;
328 struct rxrpc_wire_header whdr
;
329 struct rxrpc_skb_priv
*sp
= rxrpc_skb(skb
);
332 rxrpc_serial_t serial
;
336 _enter(",{%d}", skb
->len
);
338 /* Each transmission of a Tx packet needs a new serial number */
339 serial
= atomic_inc_return(&conn
->serial
);
341 whdr
.epoch
= htonl(conn
->proto
.epoch
);
342 whdr
.cid
= htonl(call
->cid
);
343 whdr
.callNumber
= htonl(call
->call_id
);
344 whdr
.seq
= htonl(sp
->hdr
.seq
);
345 whdr
.serial
= htonl(serial
);
346 whdr
.type
= RXRPC_PACKET_TYPE_DATA
;
347 whdr
.flags
= sp
->hdr
.flags
;
349 whdr
.securityIndex
= call
->security_ix
;
350 whdr
._rsvd
= htons(sp
->hdr
._rsvd
);
351 whdr
.serviceId
= htons(call
->service_id
);
353 if (test_bit(RXRPC_CONN_PROBING_FOR_UPGRADE
, &conn
->flags
) &&
355 whdr
.userStatus
= RXRPC_USERSTATUS_SERVICE_UPGRADE
;
357 iov
[0].iov_base
= &whdr
;
358 iov
[0].iov_len
= sizeof(whdr
);
359 iov
[1].iov_base
= skb
->head
;
360 iov
[1].iov_len
= skb
->len
;
361 len
= iov
[0].iov_len
+ iov
[1].iov_len
;
363 msg
.msg_name
= &call
->peer
->srx
.transport
;
364 msg
.msg_namelen
= call
->peer
->srx
.transport_len
;
365 msg
.msg_control
= NULL
;
366 msg
.msg_controllen
= 0;
369 /* If our RTT cache needs working on, request an ACK. Also request
370 * ACKs if a DATA packet appears to have been lost.
372 * However, we mustn't request an ACK on the last reply packet of a
373 * service call, lest OpenAFS incorrectly send us an ACK with some
374 * soft-ACKs in it and then never follow up with a proper hard ACK.
376 if ((!(sp
->hdr
.flags
& RXRPC_LAST_PACKET
) ||
379 (test_and_clear_bit(RXRPC_CALL_EV_ACK_LOST
, &call
->events
) ||
381 call
->cong_mode
== RXRPC_CALL_SLOW_START
||
382 (call
->peer
->rtt_usage
< 3 && sp
->hdr
.seq
& 1) ||
383 ktime_before(ktime_add_ms(call
->peer
->rtt_last_req
, 1000),
385 whdr
.flags
|= RXRPC_REQUEST_ACK
;
387 if (IS_ENABLED(CONFIG_AF_RXRPC_INJECT_LOSS
)) {
389 if ((lose
++ & 7) == 7) {
391 trace_rxrpc_tx_data(call
, sp
->hdr
.seq
, serial
,
392 whdr
.flags
, retrans
, true);
397 trace_rxrpc_tx_data(call
, sp
->hdr
.seq
, serial
, whdr
.flags
, retrans
,
400 /* send the packet with the don't fragment bit set if we currently
401 * think it's small enough */
402 if (iov
[1].iov_len
>= call
->peer
->maxdata
)
403 goto send_fragmentable
;
405 down_read(&conn
->params
.local
->defrag_sem
);
407 sp
->hdr
.serial
= serial
;
408 smp_wmb(); /* Set serial before timestamp */
409 skb
->tstamp
= ktime_get_real();
411 /* send the packet by UDP
412 * - returns -EMSGSIZE if UDP would have to fragment the packet
413 * to go out of the interface
414 * - in which case, we'll have processed the ICMP error
415 * message and update the peer record
417 ret
= kernel_sendmsg(conn
->params
.local
->socket
, &msg
, iov
, 2, len
);
418 conn
->params
.peer
->last_tx_at
= ktime_get_seconds();
420 up_read(&conn
->params
.local
->defrag_sem
);
422 trace_rxrpc_tx_fail(call
->debug_id
, serial
, ret
,
423 rxrpc_tx_point_call_data_nofrag
);
425 trace_rxrpc_tx_packet(call
->debug_id
, &whdr
,
426 rxrpc_tx_point_call_data_nofrag
);
427 rxrpc_tx_backoff(call
, ret
);
428 if (ret
== -EMSGSIZE
)
429 goto send_fragmentable
;
433 if (whdr
.flags
& RXRPC_REQUEST_ACK
) {
434 call
->peer
->rtt_last_req
= skb
->tstamp
;
435 trace_rxrpc_rtt_tx(call
, rxrpc_rtt_tx_data
, serial
);
436 if (call
->peer
->rtt_usage
> 1) {
437 unsigned long nowj
= jiffies
, ack_lost_at
;
439 ack_lost_at
= nsecs_to_jiffies(2 * call
->peer
->rtt
);
444 WRITE_ONCE(call
->ack_lost_at
, ack_lost_at
);
445 rxrpc_reduce_call_timer(call
, ack_lost_at
, nowj
,
446 rxrpc_timer_set_for_lost_ack
);
450 if (sp
->hdr
.seq
== 1 &&
451 !test_and_set_bit(RXRPC_CALL_BEGAN_RX_TIMER
,
453 unsigned long nowj
= jiffies
, expect_rx_by
;
455 expect_rx_by
= nowj
+ call
->next_rx_timo
;
456 WRITE_ONCE(call
->expect_rx_by
, expect_rx_by
);
457 rxrpc_reduce_call_timer(call
, expect_rx_by
, nowj
,
458 rxrpc_timer_set_for_normal
);
461 rxrpc_set_keepalive(call
);
463 /* Cancel the call if the initial transmission fails,
464 * particularly if that's due to network routing issues that
465 * aren't going away anytime soon. The layer above can arrange
466 * the retransmission.
468 if (!test_and_set_bit(RXRPC_CALL_BEGAN_RX_TIMER
, &call
->flags
))
469 rxrpc_set_call_completion(call
, RXRPC_CALL_LOCAL_ERROR
,
473 _leave(" = %d [%u]", ret
, call
->peer
->maxdata
);
477 /* attempt to send this message with fragmentation enabled */
478 _debug("send fragment");
480 down_write(&conn
->params
.local
->defrag_sem
);
482 sp
->hdr
.serial
= serial
;
483 smp_wmb(); /* Set serial before timestamp */
484 skb
->tstamp
= ktime_get_real();
486 switch (conn
->params
.local
->srx
.transport
.family
) {
488 opt
= IP_PMTUDISC_DONT
;
489 ret
= kernel_setsockopt(conn
->params
.local
->socket
,
490 SOL_IP
, IP_MTU_DISCOVER
,
491 (char *)&opt
, sizeof(opt
));
493 ret
= kernel_sendmsg(conn
->params
.local
->socket
, &msg
,
495 conn
->params
.peer
->last_tx_at
= ktime_get_seconds();
497 opt
= IP_PMTUDISC_DO
;
498 kernel_setsockopt(conn
->params
.local
->socket
, SOL_IP
,
500 (char *)&opt
, sizeof(opt
));
504 #ifdef CONFIG_AF_RXRPC_IPV6
506 opt
= IPV6_PMTUDISC_DONT
;
507 ret
= kernel_setsockopt(conn
->params
.local
->socket
,
508 SOL_IPV6
, IPV6_MTU_DISCOVER
,
509 (char *)&opt
, sizeof(opt
));
511 ret
= kernel_sendmsg(conn
->params
.local
->socket
, &msg
,
513 conn
->params
.peer
->last_tx_at
= ktime_get_seconds();
515 opt
= IPV6_PMTUDISC_DO
;
516 kernel_setsockopt(conn
->params
.local
->socket
,
517 SOL_IPV6
, IPV6_MTU_DISCOVER
,
518 (char *)&opt
, sizeof(opt
));
525 trace_rxrpc_tx_fail(call
->debug_id
, serial
, ret
,
526 rxrpc_tx_point_call_data_frag
);
528 trace_rxrpc_tx_packet(call
->debug_id
, &whdr
,
529 rxrpc_tx_point_call_data_frag
);
530 rxrpc_tx_backoff(call
, ret
);
532 up_write(&conn
->params
.local
->defrag_sem
);
537 * reject packets through the local endpoint
539 void rxrpc_reject_packets(struct rxrpc_local
*local
)
541 struct sockaddr_rxrpc srx
;
542 struct rxrpc_skb_priv
*sp
;
543 struct rxrpc_wire_header whdr
;
551 _enter("%d", local
->debug_id
);
553 iov
[0].iov_base
= &whdr
;
554 iov
[0].iov_len
= sizeof(whdr
);
555 iov
[1].iov_base
= &code
;
556 iov
[1].iov_len
= sizeof(code
);
558 msg
.msg_name
= &srx
.transport
;
559 msg
.msg_control
= NULL
;
560 msg
.msg_controllen
= 0;
563 memset(&whdr
, 0, sizeof(whdr
));
565 while ((skb
= skb_dequeue(&local
->reject_queue
))) {
566 rxrpc_see_skb(skb
, rxrpc_skb_rx_seen
);
570 case RXRPC_SKB_MARK_REJECT_BUSY
:
571 whdr
.type
= RXRPC_PACKET_TYPE_BUSY
;
575 case RXRPC_SKB_MARK_REJECT_ABORT
:
576 whdr
.type
= RXRPC_PACKET_TYPE_ABORT
;
577 code
= htonl(skb
->priority
);
578 size
= sizeof(whdr
) + sizeof(code
);
582 rxrpc_free_skb(skb
, rxrpc_skb_rx_freed
);
586 if (rxrpc_extract_addr_from_skb(&srx
, skb
) == 0) {
587 msg
.msg_namelen
= srx
.transport_len
;
589 whdr
.epoch
= htonl(sp
->hdr
.epoch
);
590 whdr
.cid
= htonl(sp
->hdr
.cid
);
591 whdr
.callNumber
= htonl(sp
->hdr
.callNumber
);
592 whdr
.serviceId
= htons(sp
->hdr
.serviceId
);
593 whdr
.flags
= sp
->hdr
.flags
;
594 whdr
.flags
^= RXRPC_CLIENT_INITIATED
;
595 whdr
.flags
&= RXRPC_CLIENT_INITIATED
;
597 ret
= kernel_sendmsg(local
->socket
, &msg
,
600 trace_rxrpc_tx_fail(local
->debug_id
, 0, ret
,
601 rxrpc_tx_point_reject
);
603 trace_rxrpc_tx_packet(local
->debug_id
, &whdr
,
604 rxrpc_tx_point_reject
);
607 rxrpc_free_skb(skb
, rxrpc_skb_rx_freed
);
614 * Send a VERSION reply to a peer as a keepalive.
616 void rxrpc_send_keepalive(struct rxrpc_peer
*peer
)
618 struct rxrpc_wire_header whdr
;
626 msg
.msg_name
= &peer
->srx
.transport
;
627 msg
.msg_namelen
= peer
->srx
.transport_len
;
628 msg
.msg_control
= NULL
;
629 msg
.msg_controllen
= 0;
632 whdr
.epoch
= htonl(peer
->local
->rxnet
->epoch
);
637 whdr
.type
= RXRPC_PACKET_TYPE_VERSION
; /* Not client-initiated */
638 whdr
.flags
= RXRPC_LAST_PACKET
;
640 whdr
.securityIndex
= 0;
644 iov
[0].iov_base
= &whdr
;
645 iov
[0].iov_len
= sizeof(whdr
);
646 iov
[1].iov_base
= (char *)rxrpc_keepalive_string
;
647 iov
[1].iov_len
= sizeof(rxrpc_keepalive_string
);
649 len
= iov
[0].iov_len
+ iov
[1].iov_len
;
651 _proto("Tx VERSION (keepalive)");
653 ret
= kernel_sendmsg(peer
->local
->socket
, &msg
, iov
, 2, len
);
655 trace_rxrpc_tx_fail(peer
->debug_id
, 0, ret
,
656 rxrpc_tx_point_version_keepalive
);
658 trace_rxrpc_tx_packet(peer
->debug_id
, &whdr
,
659 rxrpc_tx_point_version_keepalive
);
661 peer
->last_tx_at
= ktime_get_seconds();