1 /* RxRPC packet reception
3 * Copyright (C) 2007, 2016 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 #include <linux/module.h>
15 #include <linux/net.h>
16 #include <linux/skbuff.h>
17 #include <linux/errqueue.h>
18 #include <linux/udp.h>
20 #include <linux/in6.h>
21 #include <linux/icmp.h>
22 #include <linux/gfp.h>
24 #include <net/af_rxrpc.h>
27 #include <net/net_namespace.h>
28 #include "ar-internal.h"
30 static void rxrpc_proto_abort(const char *why
,
31 struct rxrpc_call
*call
, rxrpc_seq_t seq
)
33 if (rxrpc_abort_call(why
, call
, seq
, RX_PROTOCOL_ERROR
, -EBADMSG
)) {
34 set_bit(RXRPC_CALL_EV_ABORT
, &call
->events
);
35 rxrpc_queue_call(call
);
40 * Do TCP-style congestion management [RFC 5681].
42 static void rxrpc_congestion_management(struct rxrpc_call
*call
,
44 struct rxrpc_ack_summary
*summary
,
45 rxrpc_serial_t acked_serial
)
47 enum rxrpc_congest_change change
= rxrpc_cong_no_change
;
48 unsigned int cumulative_acks
= call
->cong_cumul_acks
;
49 unsigned int cwnd
= call
->cong_cwnd
;
52 summary
->flight_size
=
53 (call
->tx_top
- call
->tx_hard_ack
) - summary
->nr_acks
;
55 if (test_and_clear_bit(RXRPC_CALL_RETRANS_TIMEOUT
, &call
->flags
)) {
56 summary
->retrans_timeo
= true;
57 call
->cong_ssthresh
= max_t(unsigned int,
58 summary
->flight_size
/ 2, 2);
60 if (cwnd
>= call
->cong_ssthresh
&&
61 call
->cong_mode
== RXRPC_CALL_SLOW_START
) {
62 call
->cong_mode
= RXRPC_CALL_CONGEST_AVOIDANCE
;
63 call
->cong_tstamp
= skb
->tstamp
;
68 cumulative_acks
+= summary
->nr_new_acks
;
69 cumulative_acks
+= summary
->nr_rot_new_acks
;
70 if (cumulative_acks
> 255)
71 cumulative_acks
= 255;
73 summary
->mode
= call
->cong_mode
;
74 summary
->cwnd
= call
->cong_cwnd
;
75 summary
->ssthresh
= call
->cong_ssthresh
;
76 summary
->cumulative_acks
= cumulative_acks
;
77 summary
->dup_acks
= call
->cong_dup_acks
;
79 switch (call
->cong_mode
) {
80 case RXRPC_CALL_SLOW_START
:
81 if (summary
->nr_nacks
> 0)
82 goto packet_loss_detected
;
83 if (summary
->cumulative_acks
> 0)
85 if (cwnd
>= call
->cong_ssthresh
) {
86 call
->cong_mode
= RXRPC_CALL_CONGEST_AVOIDANCE
;
87 call
->cong_tstamp
= skb
->tstamp
;
91 case RXRPC_CALL_CONGEST_AVOIDANCE
:
92 if (summary
->nr_nacks
> 0)
93 goto packet_loss_detected
;
95 /* We analyse the number of packets that get ACK'd per RTT
96 * period and increase the window if we managed to fill it.
98 if (call
->peer
->rtt_usage
== 0)
100 if (ktime_before(skb
->tstamp
,
101 ktime_add_ns(call
->cong_tstamp
,
103 goto out_no_clear_ca
;
104 change
= rxrpc_cong_rtt_window_end
;
105 call
->cong_tstamp
= skb
->tstamp
;
106 if (cumulative_acks
>= cwnd
)
110 case RXRPC_CALL_PACKET_LOSS
:
111 if (summary
->nr_nacks
== 0)
112 goto resume_normality
;
114 if (summary
->new_low_nack
) {
115 change
= rxrpc_cong_new_low_nack
;
116 call
->cong_dup_acks
= 1;
117 if (call
->cong_extra
> 1)
118 call
->cong_extra
= 1;
119 goto send_extra_data
;
122 call
->cong_dup_acks
++;
123 if (call
->cong_dup_acks
< 3)
124 goto send_extra_data
;
126 change
= rxrpc_cong_begin_retransmission
;
127 call
->cong_mode
= RXRPC_CALL_FAST_RETRANSMIT
;
128 call
->cong_ssthresh
= max_t(unsigned int,
129 summary
->flight_size
/ 2, 2);
130 cwnd
= call
->cong_ssthresh
+ 3;
131 call
->cong_extra
= 0;
132 call
->cong_dup_acks
= 0;
136 case RXRPC_CALL_FAST_RETRANSMIT
:
137 if (!summary
->new_low_nack
) {
138 if (summary
->nr_new_acks
== 0)
140 call
->cong_dup_acks
++;
141 if (call
->cong_dup_acks
== 2) {
142 change
= rxrpc_cong_retransmit_again
;
143 call
->cong_dup_acks
= 0;
147 change
= rxrpc_cong_progress
;
148 cwnd
= call
->cong_ssthresh
;
149 if (summary
->nr_nacks
== 0)
150 goto resume_normality
;
160 change
= rxrpc_cong_cleared_nacks
;
161 call
->cong_dup_acks
= 0;
162 call
->cong_extra
= 0;
163 call
->cong_tstamp
= skb
->tstamp
;
164 if (cwnd
< call
->cong_ssthresh
)
165 call
->cong_mode
= RXRPC_CALL_SLOW_START
;
167 call
->cong_mode
= RXRPC_CALL_CONGEST_AVOIDANCE
;
171 if (cwnd
>= RXRPC_RXTX_BUFF_SIZE
- 1)
172 cwnd
= RXRPC_RXTX_BUFF_SIZE
- 1;
173 call
->cong_cwnd
= cwnd
;
174 call
->cong_cumul_acks
= cumulative_acks
;
175 trace_rxrpc_congest(call
, summary
, acked_serial
, change
);
176 if (resend
&& !test_and_set_bit(RXRPC_CALL_EV_RESEND
, &call
->events
))
177 rxrpc_queue_call(call
);
180 packet_loss_detected
:
181 change
= rxrpc_cong_saw_nack
;
182 call
->cong_mode
= RXRPC_CALL_PACKET_LOSS
;
183 call
->cong_dup_acks
= 0;
184 goto send_extra_data
;
187 /* Send some previously unsent DATA if we have some to advance the ACK
190 if (call
->rxtx_annotations
[call
->tx_top
& RXRPC_RXTX_BUFF_MASK
] &
191 RXRPC_TX_ANNO_LAST
||
192 summary
->nr_acks
!= call
->tx_top
- call
->tx_hard_ack
) {
194 wake_up(&call
->waitq
);
196 goto out_no_clear_ca
;
200 * Ping the other end to fill our RTT cache and to retrieve the rwind
201 * and MTU parameters.
203 static void rxrpc_send_ping(struct rxrpc_call
*call
, struct sk_buff
*skb
,
206 struct rxrpc_skb_priv
*sp
= rxrpc_skb(skb
);
207 ktime_t now
= skb
->tstamp
;
209 if (call
->peer
->rtt_usage
< 3 ||
210 ktime_before(ktime_add_ms(call
->peer
->rtt_last_req
, 1000), now
))
211 rxrpc_propose_ACK(call
, RXRPC_ACK_PING
, skew
, sp
->hdr
.serial
,
213 rxrpc_propose_ack_ping_for_params
);
217 * Apply a hard ACK by advancing the Tx window.
219 static bool rxrpc_rotate_tx_window(struct rxrpc_call
*call
, rxrpc_seq_t to
,
220 struct rxrpc_ack_summary
*summary
)
222 struct sk_buff
*skb
, *list
= NULL
;
223 bool rot_last
= false;
227 if (call
->acks_lowest_nak
== call
->tx_hard_ack
) {
228 call
->acks_lowest_nak
= to
;
229 } else if (before_eq(call
->acks_lowest_nak
, to
)) {
230 summary
->new_low_nack
= true;
231 call
->acks_lowest_nak
= to
;
234 spin_lock(&call
->lock
);
236 while (before(call
->tx_hard_ack
, to
)) {
238 ix
= call
->tx_hard_ack
& RXRPC_RXTX_BUFF_MASK
;
239 skb
= call
->rxtx_buffer
[ix
];
240 annotation
= call
->rxtx_annotations
[ix
];
241 rxrpc_see_skb(skb
, rxrpc_skb_tx_rotated
);
242 call
->rxtx_buffer
[ix
] = NULL
;
243 call
->rxtx_annotations
[ix
] = 0;
247 if (annotation
& RXRPC_TX_ANNO_LAST
) {
248 set_bit(RXRPC_CALL_TX_LAST
, &call
->flags
);
251 if ((annotation
& RXRPC_TX_ANNO_MASK
) != RXRPC_TX_ANNO_ACK
)
252 summary
->nr_rot_new_acks
++;
255 spin_unlock(&call
->lock
);
257 trace_rxrpc_transmit(call
, (rot_last
?
258 rxrpc_transmit_rotate_last
:
259 rxrpc_transmit_rotate
));
260 wake_up(&call
->waitq
);
266 rxrpc_free_skb(skb
, rxrpc_skb_tx_freed
);
273 * End the transmission phase of a call.
275 * This occurs when we get an ACKALL packet, the first DATA packet of a reply,
276 * or a final ACK packet.
278 static bool rxrpc_end_tx_phase(struct rxrpc_call
*call
, bool reply_begun
,
279 const char *abort_why
)
283 ASSERT(test_bit(RXRPC_CALL_TX_LAST
, &call
->flags
));
285 write_lock(&call
->state_lock
);
289 case RXRPC_CALL_CLIENT_SEND_REQUEST
:
290 case RXRPC_CALL_CLIENT_AWAIT_REPLY
:
292 call
->state
= state
= RXRPC_CALL_CLIENT_RECV_REPLY
;
294 call
->state
= state
= RXRPC_CALL_CLIENT_AWAIT_REPLY
;
297 case RXRPC_CALL_SERVER_AWAIT_ACK
:
298 __rxrpc_call_completed(call
);
299 rxrpc_notify_socket(call
);
307 write_unlock(&call
->state_lock
);
308 if (state
== RXRPC_CALL_CLIENT_AWAIT_REPLY
)
309 trace_rxrpc_transmit(call
, rxrpc_transmit_await_reply
);
311 trace_rxrpc_transmit(call
, rxrpc_transmit_end
);
316 write_unlock(&call
->state_lock
);
317 kdebug("end_tx %s", rxrpc_call_states
[call
->state
]);
318 rxrpc_proto_abort(abort_why
, call
, call
->tx_top
);
323 * Begin the reply reception phase of a call.
325 static bool rxrpc_receiving_reply(struct rxrpc_call
*call
)
327 struct rxrpc_ack_summary summary
= { 0 };
328 unsigned long now
, timo
;
329 rxrpc_seq_t top
= READ_ONCE(call
->tx_top
);
331 if (call
->ackr_reason
) {
332 spin_lock_bh(&call
->lock
);
333 call
->ackr_reason
= 0;
334 spin_unlock_bh(&call
->lock
);
336 timo
= now
+ MAX_JIFFY_OFFSET
;
337 WRITE_ONCE(call
->resend_at
, timo
);
338 WRITE_ONCE(call
->ack_at
, timo
);
339 trace_rxrpc_timer(call
, rxrpc_timer_init_for_reply
, now
);
342 if (!test_bit(RXRPC_CALL_TX_LAST
, &call
->flags
)) {
343 if (!rxrpc_rotate_tx_window(call
, top
, &summary
)) {
344 rxrpc_proto_abort("TXL", call
, top
);
348 if (!rxrpc_end_tx_phase(call
, true, "ETD"))
350 call
->tx_phase
= false;
355 * Scan a jumbo packet to validate its structure and to work out how many
356 * subpackets it contains.
358 * A jumbo packet is a collection of consecutive packets glued together with
359 * little headers between that indicate how to change the initial header for
362 * RXRPC_JUMBO_PACKET must be set on all but the last subpacket - and all but
363 * the last are RXRPC_JUMBO_DATALEN in size. The last subpacket may be of any
366 static bool rxrpc_validate_jumbo(struct sk_buff
*skb
)
368 struct rxrpc_skb_priv
*sp
= rxrpc_skb(skb
);
369 unsigned int offset
= sizeof(struct rxrpc_wire_header
);
370 unsigned int len
= skb
->len
;
372 u8 flags
= sp
->hdr
.flags
;
376 if (len
- offset
< RXRPC_JUMBO_SUBPKTLEN
)
378 if (flags
& RXRPC_LAST_PACKET
)
380 offset
+= RXRPC_JUMBO_DATALEN
;
381 if (skb_copy_bits(skb
, offset
, &flags
, 1) < 0)
383 offset
+= sizeof(struct rxrpc_jumbo_header
);
384 } while (flags
& RXRPC_JUMBO_PACKET
);
386 sp
->nr_jumbo
= nr_jumbo
;
394 * Handle reception of a duplicate packet.
396 * We have to take care to avoid an attack here whereby we're given a series of
397 * jumbograms, each with a sequence number one before the preceding one and
398 * filled up to maximum UDP size. If they never send us the first packet in
399 * the sequence, they can cause us to have to hold on to around 2MiB of kernel
400 * space until the call times out.
402 * We limit the space usage by only accepting three duplicate jumbo packets per
403 * call. After that, we tell the other side we're no longer accepting jumbos
404 * (that information is encoded in the ACK packet).
406 static void rxrpc_input_dup_data(struct rxrpc_call
*call
, rxrpc_seq_t seq
,
407 u8 annotation
, bool *_jumbo_bad
)
409 /* Discard normal packets that are duplicates. */
413 /* Skip jumbo subpackets that are duplicates. When we've had three or
414 * more partially duplicate jumbo packets, we refuse to take any more
415 * jumbos for this call.
418 call
->nr_jumbo_bad
++;
424 * Process a DATA packet, adding the packet to the Rx ring.
426 static void rxrpc_input_data(struct rxrpc_call
*call
, struct sk_buff
*skb
,
429 struct rxrpc_skb_priv
*sp
= rxrpc_skb(skb
);
430 enum rxrpc_call_state state
;
431 unsigned int offset
= sizeof(struct rxrpc_wire_header
);
433 rxrpc_serial_t serial
= sp
->hdr
.serial
, ack_serial
= 0;
434 rxrpc_seq_t seq
= sp
->hdr
.seq
, hard_ack
;
435 bool immediate_ack
= false, jumbo_bad
= false, queued
;
437 u8 ack
= 0, flags
, annotation
= 0;
439 _enter("{%u,%u},{%u,%u}",
440 call
->rx_hard_ack
, call
->rx_top
, skb
->len
, seq
);
442 _proto("Rx DATA %%%u { #%u f=%02x }",
443 sp
->hdr
.serial
, seq
, sp
->hdr
.flags
);
445 state
= READ_ONCE(call
->state
);
446 if (state
>= RXRPC_CALL_COMPLETE
)
449 if (call
->state
== RXRPC_CALL_SERVER_RECV_REQUEST
) {
450 unsigned long timo
= READ_ONCE(call
->next_req_timo
);
451 unsigned long now
, expect_req_by
;
455 expect_req_by
= now
+ timo
;
456 WRITE_ONCE(call
->expect_req_by
, expect_req_by
);
457 rxrpc_reduce_call_timer(call
, expect_req_by
, now
,
458 rxrpc_timer_set_for_idle
);
462 /* Received data implicitly ACKs all of the request packets we sent
463 * when we're acting as a client.
465 if ((state
== RXRPC_CALL_CLIENT_SEND_REQUEST
||
466 state
== RXRPC_CALL_CLIENT_AWAIT_REPLY
) &&
467 !rxrpc_receiving_reply(call
))
470 call
->ackr_prev_seq
= seq
;
472 hard_ack
= READ_ONCE(call
->rx_hard_ack
);
473 if (after(seq
, hard_ack
+ call
->rx_winsize
)) {
474 ack
= RXRPC_ACK_EXCEEDS_WINDOW
;
479 flags
= sp
->hdr
.flags
;
480 if (flags
& RXRPC_JUMBO_PACKET
) {
481 if (call
->nr_jumbo_bad
> 3) {
482 ack
= RXRPC_ACK_NOSPACE
;
491 ix
= seq
& RXRPC_RXTX_BUFF_MASK
;
493 if (flags
& RXRPC_JUMBO_PACKET
)
494 len
= RXRPC_JUMBO_DATALEN
;
496 if (flags
& RXRPC_LAST_PACKET
) {
497 if (test_bit(RXRPC_CALL_RX_LAST
, &call
->flags
) &&
499 return rxrpc_proto_abort("LSN", call
, seq
);
501 if (test_bit(RXRPC_CALL_RX_LAST
, &call
->flags
) &&
502 after_eq(seq
, call
->rx_top
))
503 return rxrpc_proto_abort("LSA", call
, seq
);
506 trace_rxrpc_rx_data(call
, seq
, serial
, flags
, annotation
);
507 if (before_eq(seq
, hard_ack
)) {
508 ack
= RXRPC_ACK_DUPLICATE
;
513 if (flags
& RXRPC_REQUEST_ACK
&& !ack
) {
514 ack
= RXRPC_ACK_REQUESTED
;
518 if (call
->rxtx_buffer
[ix
]) {
519 rxrpc_input_dup_data(call
, seq
, annotation
, &jumbo_bad
);
520 if (ack
!= RXRPC_ACK_DUPLICATE
) {
521 ack
= RXRPC_ACK_DUPLICATE
;
524 immediate_ack
= true;
528 /* Queue the packet. We use a couple of memory barriers here as need
529 * to make sure that rx_top is perceived to be set after the buffer
530 * pointer and that the buffer pointer is set after the annotation and
533 * Barriers against rxrpc_recvmsg_data() and rxrpc_rotate_rx_window()
534 * and also rxrpc_fill_out_ack().
536 rxrpc_get_skb(skb
, rxrpc_skb_rx_got
);
537 call
->rxtx_annotations
[ix
] = annotation
;
539 call
->rxtx_buffer
[ix
] = skb
;
540 if (after(seq
, call
->rx_top
)) {
541 smp_store_release(&call
->rx_top
, seq
);
542 } else if (before(seq
, call
->rx_top
)) {
543 /* Send an immediate ACK if we fill in a hole */
545 ack
= RXRPC_ACK_DELAY
;
548 immediate_ack
= true;
550 if (flags
& RXRPC_LAST_PACKET
) {
551 set_bit(RXRPC_CALL_RX_LAST
, &call
->flags
);
552 trace_rxrpc_receive(call
, rxrpc_receive_queue_last
, serial
, seq
);
554 trace_rxrpc_receive(call
, rxrpc_receive_queue
, serial
, seq
);
558 if (after_eq(seq
, call
->rx_expect_next
)) {
559 if (after(seq
, call
->rx_expect_next
)) {
560 _net("OOS %u > %u", seq
, call
->rx_expect_next
);
561 ack
= RXRPC_ACK_OUT_OF_SEQUENCE
;
564 call
->rx_expect_next
= seq
+ 1;
569 if (flags
& RXRPC_JUMBO_PACKET
) {
570 if (skb_copy_bits(skb
, offset
, &flags
, 1) < 0)
571 return rxrpc_proto_abort("XJF", call
, seq
);
572 offset
+= sizeof(struct rxrpc_jumbo_header
);
576 if (flags
& RXRPC_JUMBO_PACKET
)
577 annotation
|= RXRPC_RX_ANNO_JLAST
;
578 if (after(seq
, hard_ack
+ call
->rx_winsize
)) {
579 ack
= RXRPC_ACK_EXCEEDS_WINDOW
;
582 call
->nr_jumbo_bad
++;
588 _proto("Rx DATA Jumbo %%%u", serial
);
592 if (queued
&& flags
& RXRPC_LAST_PACKET
&& !ack
) {
593 ack
= RXRPC_ACK_DELAY
;
599 rxrpc_propose_ACK(call
, ack
, skew
, ack_serial
,
601 rxrpc_propose_ack_input_data
);
603 if (sp
->hdr
.seq
== READ_ONCE(call
->rx_hard_ack
) + 1)
604 rxrpc_notify_socket(call
);
609 * Process a requested ACK.
611 static void rxrpc_input_requested_ack(struct rxrpc_call
*call
,
613 rxrpc_serial_t orig_serial
,
614 rxrpc_serial_t ack_serial
)
616 struct rxrpc_skb_priv
*sp
;
621 for (ix
= 0; ix
< RXRPC_RXTX_BUFF_SIZE
; ix
++) {
622 skb
= call
->rxtx_buffer
[ix
];
627 if (sp
->hdr
.serial
!= orig_serial
)
630 sent_at
= skb
->tstamp
;
636 rxrpc_peer_add_rtt(call
, rxrpc_rtt_rx_requested_ack
,
637 orig_serial
, ack_serial
, sent_at
, resp_time
);
641 * Process the response to a ping that we sent to find out if we lost an ACK.
643 * If we got back a ping response that indicates a lower tx_top than what we
644 * had at the time of the ping transmission, we adjudge all the DATA packets
645 * sent between the response tx_top and the ping-time tx_top to have been lost.
647 static void rxrpc_input_check_for_lost_ack(struct rxrpc_call
*call
)
649 rxrpc_seq_t top
, bottom
, seq
;
652 spin_lock_bh(&call
->lock
);
654 bottom
= call
->tx_hard_ack
+ 1;
655 top
= call
->acks_lost_top
;
656 if (before(bottom
, top
)) {
657 for (seq
= bottom
; before_eq(seq
, top
); seq
++) {
658 int ix
= seq
& RXRPC_RXTX_BUFF_MASK
;
659 u8 annotation
= call
->rxtx_annotations
[ix
];
660 u8 anno_type
= annotation
& RXRPC_TX_ANNO_MASK
;
662 if (anno_type
!= RXRPC_TX_ANNO_UNACK
)
664 annotation
&= ~RXRPC_TX_ANNO_MASK
;
665 annotation
|= RXRPC_TX_ANNO_RETRANS
;
666 call
->rxtx_annotations
[ix
] = annotation
;
671 spin_unlock_bh(&call
->lock
);
673 if (resend
&& !test_and_set_bit(RXRPC_CALL_EV_RESEND
, &call
->events
))
674 rxrpc_queue_call(call
);
678 * Process a ping response.
680 static void rxrpc_input_ping_response(struct rxrpc_call
*call
,
682 rxrpc_serial_t orig_serial
,
683 rxrpc_serial_t ack_serial
)
685 rxrpc_serial_t ping_serial
;
688 ping_time
= call
->ping_time
;
690 ping_serial
= call
->ping_serial
;
692 if (orig_serial
== call
->acks_lost_ping
)
693 rxrpc_input_check_for_lost_ack(call
);
695 if (!test_bit(RXRPC_CALL_PINGING
, &call
->flags
) ||
696 before(orig_serial
, ping_serial
))
698 clear_bit(RXRPC_CALL_PINGING
, &call
->flags
);
699 if (after(orig_serial
, ping_serial
))
702 rxrpc_peer_add_rtt(call
, rxrpc_rtt_rx_ping_response
,
703 orig_serial
, ack_serial
, ping_time
, resp_time
);
707 * Process the extra information that may be appended to an ACK packet
709 static void rxrpc_input_ackinfo(struct rxrpc_call
*call
, struct sk_buff
*skb
,
710 struct rxrpc_ackinfo
*ackinfo
)
712 struct rxrpc_skb_priv
*sp
= rxrpc_skb(skb
);
713 struct rxrpc_peer
*peer
;
716 u32 rwind
= ntohl(ackinfo
->rwind
);
718 _proto("Rx ACK %%%u Info { rx=%u max=%u rwin=%u jm=%u }",
720 ntohl(ackinfo
->rxMTU
), ntohl(ackinfo
->maxMTU
),
721 rwind
, ntohl(ackinfo
->jumbo_max
));
723 if (call
->tx_winsize
!= rwind
) {
724 if (rwind
> RXRPC_RXTX_BUFF_SIZE
- 1)
725 rwind
= RXRPC_RXTX_BUFF_SIZE
- 1;
726 if (rwind
> call
->tx_winsize
)
728 trace_rxrpc_rx_rwind_change(call
, sp
->hdr
.serial
,
729 ntohl(ackinfo
->rwind
), wake
);
730 call
->tx_winsize
= rwind
;
733 if (call
->cong_ssthresh
> rwind
)
734 call
->cong_ssthresh
= rwind
;
736 mtu
= min(ntohl(ackinfo
->rxMTU
), ntohl(ackinfo
->maxMTU
));
739 if (mtu
< peer
->maxdata
) {
740 spin_lock_bh(&peer
->lock
);
742 peer
->mtu
= mtu
+ peer
->hdrsize
;
743 spin_unlock_bh(&peer
->lock
);
744 _net("Net MTU %u (maxdata %u)", peer
->mtu
, peer
->maxdata
);
748 wake_up(&call
->waitq
);
752 * Process individual soft ACKs.
754 * Each ACK in the array corresponds to one packet and can be either an ACK or
755 * a NAK. If we get find an explicitly NAK'd packet we resend immediately;
756 * packets that lie beyond the end of the ACK list are scheduled for resend by
757 * the timer on the basis that the peer might just not have processed them at
758 * the time the ACK was sent.
760 static void rxrpc_input_soft_acks(struct rxrpc_call
*call
, u8
*acks
,
761 rxrpc_seq_t seq
, int nr_acks
,
762 struct rxrpc_ack_summary
*summary
)
765 u8 annotation
, anno_type
;
767 for (; nr_acks
> 0; nr_acks
--, seq
++) {
768 ix
= seq
& RXRPC_RXTX_BUFF_MASK
;
769 annotation
= call
->rxtx_annotations
[ix
];
770 anno_type
= annotation
& RXRPC_TX_ANNO_MASK
;
771 annotation
&= ~RXRPC_TX_ANNO_MASK
;
773 case RXRPC_ACK_TYPE_ACK
:
775 if (anno_type
== RXRPC_TX_ANNO_ACK
)
777 summary
->nr_new_acks
++;
778 call
->rxtx_annotations
[ix
] =
779 RXRPC_TX_ANNO_ACK
| annotation
;
781 case RXRPC_ACK_TYPE_NACK
:
782 if (!summary
->nr_nacks
&&
783 call
->acks_lowest_nak
!= seq
) {
784 call
->acks_lowest_nak
= seq
;
785 summary
->new_low_nack
= true;
788 if (anno_type
== RXRPC_TX_ANNO_NAK
)
790 summary
->nr_new_nacks
++;
791 if (anno_type
== RXRPC_TX_ANNO_RETRANS
)
793 call
->rxtx_annotations
[ix
] =
794 RXRPC_TX_ANNO_NAK
| annotation
;
797 return rxrpc_proto_abort("SFT", call
, 0);
803 * Process an ACK packet.
805 * ack.firstPacket is the sequence number of the first soft-ACK'd/NAK'd packet
806 * in the ACK array. Anything before that is hard-ACK'd and may be discarded.
808 * A hard-ACK means that a packet has been processed and may be discarded; a
809 * soft-ACK means that the packet may be discarded and retransmission
810 * requested. A phase is complete when all packets are hard-ACK'd.
812 static void rxrpc_input_ack(struct rxrpc_call
*call
, struct sk_buff
*skb
,
815 struct rxrpc_ack_summary summary
= { 0 };
816 struct rxrpc_skb_priv
*sp
= rxrpc_skb(skb
);
818 struct rxrpc_ackpacket ack
;
819 struct rxrpc_ackinfo info
;
820 u8 acks
[RXRPC_MAXACKS
];
822 rxrpc_serial_t acked_serial
;
823 rxrpc_seq_t first_soft_ack
, hard_ack
;
824 int nr_acks
, offset
, ioffset
;
828 offset
= sizeof(struct rxrpc_wire_header
);
829 if (skb_copy_bits(skb
, offset
, &buf
.ack
, sizeof(buf
.ack
)) < 0) {
830 _debug("extraction failure");
831 return rxrpc_proto_abort("XAK", call
, 0);
833 offset
+= sizeof(buf
.ack
);
835 acked_serial
= ntohl(buf
.ack
.serial
);
836 first_soft_ack
= ntohl(buf
.ack
.firstPacket
);
837 hard_ack
= first_soft_ack
- 1;
838 nr_acks
= buf
.ack
.nAcks
;
839 summary
.ack_reason
= (buf
.ack
.reason
< RXRPC_ACK__INVALID
?
840 buf
.ack
.reason
: RXRPC_ACK__INVALID
);
842 trace_rxrpc_rx_ack(call
, sp
->hdr
.serial
, acked_serial
,
843 first_soft_ack
, ntohl(buf
.ack
.previousPacket
),
844 summary
.ack_reason
, nr_acks
);
846 if (buf
.ack
.reason
== RXRPC_ACK_PING_RESPONSE
)
847 rxrpc_input_ping_response(call
, skb
->tstamp
, acked_serial
,
849 if (buf
.ack
.reason
== RXRPC_ACK_REQUESTED
)
850 rxrpc_input_requested_ack(call
, skb
->tstamp
, acked_serial
,
853 if (buf
.ack
.reason
== RXRPC_ACK_PING
) {
854 _proto("Rx ACK %%%u PING Request", sp
->hdr
.serial
);
855 rxrpc_propose_ACK(call
, RXRPC_ACK_PING_RESPONSE
,
856 skew
, sp
->hdr
.serial
, true, true,
857 rxrpc_propose_ack_respond_to_ping
);
858 } else if (sp
->hdr
.flags
& RXRPC_REQUEST_ACK
) {
859 rxrpc_propose_ACK(call
, RXRPC_ACK_REQUESTED
,
860 skew
, sp
->hdr
.serial
, true, true,
861 rxrpc_propose_ack_respond_to_ack
);
864 /* Discard any out-of-order or duplicate ACKs. */
865 if (before_eq(sp
->hdr
.serial
, call
->acks_latest
)) {
866 _debug("discard ACK %d <= %d",
867 sp
->hdr
.serial
, call
->acks_latest
);
870 call
->acks_latest_ts
= skb
->tstamp
;
871 call
->acks_latest
= sp
->hdr
.serial
;
873 /* Parse rwind and mtu sizes if provided. */
874 ioffset
= offset
+ nr_acks
+ 3;
875 if (skb
->len
>= ioffset
+ sizeof(buf
.info
)) {
876 if (skb_copy_bits(skb
, ioffset
, &buf
.info
, sizeof(buf
.info
)) < 0)
877 return rxrpc_proto_abort("XAI", call
, 0);
878 rxrpc_input_ackinfo(call
, skb
, &buf
.info
);
881 if (first_soft_ack
== 0)
882 return rxrpc_proto_abort("AK0", call
, 0);
884 /* Ignore ACKs unless we are or have just been transmitting. */
885 switch (READ_ONCE(call
->state
)) {
886 case RXRPC_CALL_CLIENT_SEND_REQUEST
:
887 case RXRPC_CALL_CLIENT_AWAIT_REPLY
:
888 case RXRPC_CALL_SERVER_SEND_REPLY
:
889 case RXRPC_CALL_SERVER_AWAIT_ACK
:
895 if (before(hard_ack
, call
->tx_hard_ack
) ||
896 after(hard_ack
, call
->tx_top
))
897 return rxrpc_proto_abort("AKW", call
, 0);
898 if (nr_acks
> call
->tx_top
- hard_ack
)
899 return rxrpc_proto_abort("AKN", call
, 0);
901 if (after(hard_ack
, call
->tx_hard_ack
)) {
902 if (rxrpc_rotate_tx_window(call
, hard_ack
, &summary
)) {
903 rxrpc_end_tx_phase(call
, false, "ETA");
909 if (skb_copy_bits(skb
, offset
, buf
.acks
, nr_acks
) < 0)
910 return rxrpc_proto_abort("XSA", call
, 0);
911 rxrpc_input_soft_acks(call
, buf
.acks
, first_soft_ack
, nr_acks
,
915 if (call
->rxtx_annotations
[call
->tx_top
& RXRPC_RXTX_BUFF_MASK
] &
916 RXRPC_TX_ANNO_LAST
&&
917 summary
.nr_acks
== call
->tx_top
- hard_ack
&&
918 rxrpc_is_client_call(call
))
919 rxrpc_propose_ACK(call
, RXRPC_ACK_PING
, skew
, sp
->hdr
.serial
,
921 rxrpc_propose_ack_ping_for_lost_reply
);
923 return rxrpc_congestion_management(call
, skb
, &summary
, acked_serial
);
927 * Process an ACKALL packet.
929 static void rxrpc_input_ackall(struct rxrpc_call
*call
, struct sk_buff
*skb
)
931 struct rxrpc_ack_summary summary
= { 0 };
932 struct rxrpc_skb_priv
*sp
= rxrpc_skb(skb
);
934 _proto("Rx ACKALL %%%u", sp
->hdr
.serial
);
936 if (rxrpc_rotate_tx_window(call
, call
->tx_top
, &summary
))
937 rxrpc_end_tx_phase(call
, false, "ETL");
941 * Process an ABORT packet directed at a call.
943 static void rxrpc_input_abort(struct rxrpc_call
*call
, struct sk_buff
*skb
)
945 struct rxrpc_skb_priv
*sp
= rxrpc_skb(skb
);
947 u32 abort_code
= RX_CALL_DEAD
;
952 skb_copy_bits(skb
, sizeof(struct rxrpc_wire_header
),
953 &wtmp
, sizeof(wtmp
)) >= 0)
954 abort_code
= ntohl(wtmp
);
956 trace_rxrpc_rx_abort(call
, sp
->hdr
.serial
, abort_code
);
958 _proto("Rx ABORT %%%u { %x }", sp
->hdr
.serial
, abort_code
);
960 if (rxrpc_set_call_completion(call
, RXRPC_CALL_REMOTELY_ABORTED
,
961 abort_code
, -ECONNABORTED
))
962 rxrpc_notify_socket(call
);
966 * Process an incoming call packet.
968 static void rxrpc_input_call_packet(struct rxrpc_call
*call
,
969 struct sk_buff
*skb
, u16 skew
)
971 struct rxrpc_skb_priv
*sp
= rxrpc_skb(skb
);
974 _enter("%p,%p", call
, skb
);
976 timo
= READ_ONCE(call
->next_rx_timo
);
978 unsigned long now
= jiffies
, expect_rx_by
;
980 expect_rx_by
= now
+ timo
;
981 WRITE_ONCE(call
->expect_rx_by
, expect_rx_by
);
982 rxrpc_reduce_call_timer(call
, expect_rx_by
, now
,
983 rxrpc_timer_set_for_normal
);
986 switch (sp
->hdr
.type
) {
987 case RXRPC_PACKET_TYPE_DATA
:
988 rxrpc_input_data(call
, skb
, skew
);
991 case RXRPC_PACKET_TYPE_ACK
:
992 rxrpc_input_ack(call
, skb
, skew
);
995 case RXRPC_PACKET_TYPE_BUSY
:
996 _proto("Rx BUSY %%%u", sp
->hdr
.serial
);
998 /* Just ignore BUSY packets from the server; the retry and
999 * lifespan timers will take care of business. BUSY packets
1000 * from the client don't make sense.
1004 case RXRPC_PACKET_TYPE_ABORT
:
1005 rxrpc_input_abort(call
, skb
);
1008 case RXRPC_PACKET_TYPE_ACKALL
:
1009 rxrpc_input_ackall(call
, skb
);
1020 * Handle a new call on a channel implicitly completing the preceding call on
1023 * TODO: If callNumber > call_id + 1, renegotiate security.
1025 static void rxrpc_input_implicit_end_call(struct rxrpc_connection
*conn
,
1026 struct rxrpc_call
*call
)
1028 switch (READ_ONCE(call
->state
)) {
1029 case RXRPC_CALL_SERVER_AWAIT_ACK
:
1030 rxrpc_call_completed(call
);
1032 case RXRPC_CALL_COMPLETE
:
1035 if (rxrpc_abort_call("IMP", call
, 0, RX_CALL_DEAD
, -ESHUTDOWN
)) {
1036 set_bit(RXRPC_CALL_EV_ABORT
, &call
->events
);
1037 rxrpc_queue_call(call
);
1042 trace_rxrpc_improper_term(call
);
1043 __rxrpc_disconnect_call(conn
, call
);
1044 rxrpc_notify_socket(call
);
1048 * post connection-level events to the connection
1049 * - this includes challenges, responses, some aborts and call terminal packet
1052 static void rxrpc_post_packet_to_conn(struct rxrpc_connection
*conn
,
1053 struct sk_buff
*skb
)
1055 _enter("%p,%p", conn
, skb
);
1057 skb_queue_tail(&conn
->rx_queue
, skb
);
1058 rxrpc_queue_conn(conn
);
1062 * post endpoint-level events to the local endpoint
1063 * - this includes debug and version messages
1065 static void rxrpc_post_packet_to_local(struct rxrpc_local
*local
,
1066 struct sk_buff
*skb
)
1068 _enter("%p,%p", local
, skb
);
1070 skb_queue_tail(&local
->event_queue
, skb
);
1071 rxrpc_queue_local(local
);
1075 * put a packet up for transport-level abort
1077 static void rxrpc_reject_packet(struct rxrpc_local
*local
, struct sk_buff
*skb
)
1079 CHECK_SLAB_OKAY(&local
->usage
);
1081 skb_queue_tail(&local
->reject_queue
, skb
);
1082 rxrpc_queue_local(local
);
1086 * Extract the wire header from a packet and translate the byte order.
1089 int rxrpc_extract_header(struct rxrpc_skb_priv
*sp
, struct sk_buff
*skb
)
1091 struct rxrpc_wire_header whdr
;
1093 /* dig out the RxRPC connection details */
1094 if (skb_copy_bits(skb
, 0, &whdr
, sizeof(whdr
)) < 0) {
1095 trace_rxrpc_rx_eproto(NULL
, sp
->hdr
.serial
,
1096 tracepoint_string("bad_hdr"));
1100 memset(sp
, 0, sizeof(*sp
));
1101 sp
->hdr
.epoch
= ntohl(whdr
.epoch
);
1102 sp
->hdr
.cid
= ntohl(whdr
.cid
);
1103 sp
->hdr
.callNumber
= ntohl(whdr
.callNumber
);
1104 sp
->hdr
.seq
= ntohl(whdr
.seq
);
1105 sp
->hdr
.serial
= ntohl(whdr
.serial
);
1106 sp
->hdr
.flags
= whdr
.flags
;
1107 sp
->hdr
.type
= whdr
.type
;
1108 sp
->hdr
.userStatus
= whdr
.userStatus
;
1109 sp
->hdr
.securityIndex
= whdr
.securityIndex
;
1110 sp
->hdr
._rsvd
= ntohs(whdr
._rsvd
);
1111 sp
->hdr
.serviceId
= ntohs(whdr
.serviceId
);
1116 * handle data received on the local endpoint
1117 * - may be called in interrupt context
1119 * The socket is locked by the caller and this prevents the socket from being
1120 * shut down and the local endpoint from going away, thus sk_user_data will not
1121 * be cleared until this function returns.
1123 void rxrpc_data_ready(struct sock
*udp_sk
)
1125 struct rxrpc_connection
*conn
;
1126 struct rxrpc_channel
*chan
;
1127 struct rxrpc_call
*call
;
1128 struct rxrpc_skb_priv
*sp
;
1129 struct rxrpc_local
*local
= udp_sk
->sk_user_data
;
1130 struct sk_buff
*skb
;
1131 unsigned int channel
;
1134 _enter("%p", udp_sk
);
1136 ASSERT(!irqs_disabled());
1138 skb
= skb_recv_udp(udp_sk
, 0, 1, &ret
);
1142 _debug("UDP socket error %d", ret
);
1146 rxrpc_new_skb(skb
, rxrpc_skb_rx_received
);
1148 _net("recv skb %p", skb
);
1150 /* we'll probably need to checksum it (didn't call sock_recvmsg) */
1151 if (skb_checksum_complete(skb
)) {
1152 rxrpc_free_skb(skb
, rxrpc_skb_rx_freed
);
1153 __UDP_INC_STATS(&init_net
, UDP_MIB_INERRORS
, 0);
1154 _leave(" [CSUM failed]");
1158 __UDP_INC_STATS(&init_net
, UDP_MIB_INDATAGRAMS
, 0);
1160 /* The UDP protocol already released all skb resources;
1161 * we are free to add our own data there.
1163 sp
= rxrpc_skb(skb
);
1165 /* dig out the RxRPC connection details */
1166 if (rxrpc_extract_header(sp
, skb
) < 0)
1169 if (IS_ENABLED(CONFIG_AF_RXRPC_INJECT_LOSS
)) {
1171 if ((lose
++ & 7) == 7) {
1172 trace_rxrpc_rx_lose(sp
);
1173 rxrpc_lose_skb(skb
, rxrpc_skb_rx_lost
);
1178 trace_rxrpc_rx_packet(sp
);
1180 if (sp
->hdr
.type
>= RXRPC_N_PACKET_TYPES
||
1181 !((RXRPC_SUPPORTED_PACKET_TYPES
>> sp
->hdr
.type
) & 1)) {
1182 _proto("Rx Bad Packet Type %u", sp
->hdr
.type
);
1186 switch (sp
->hdr
.type
) {
1187 case RXRPC_PACKET_TYPE_VERSION
:
1188 if (rxrpc_to_client(sp
))
1190 rxrpc_post_packet_to_local(local
, skb
);
1193 case RXRPC_PACKET_TYPE_BUSY
:
1194 if (rxrpc_to_server(sp
))
1198 case RXRPC_PACKET_TYPE_DATA
:
1199 if (sp
->hdr
.callNumber
== 0)
1201 if (sp
->hdr
.flags
& RXRPC_JUMBO_PACKET
&&
1202 !rxrpc_validate_jumbo(skb
))
1209 conn
= rxrpc_find_connection_rcu(local
, skb
);
1211 if (sp
->hdr
.securityIndex
!= conn
->security_ix
)
1212 goto wrong_security
;
1214 if (sp
->hdr
.serviceId
!= conn
->service_id
) {
1215 if (!test_bit(RXRPC_CONN_PROBING_FOR_UPGRADE
, &conn
->flags
) ||
1216 conn
->service_id
!= conn
->params
.service_id
)
1218 conn
->service_id
= sp
->hdr
.serviceId
;
1221 if (sp
->hdr
.callNumber
== 0) {
1222 /* Connection-level packet */
1223 _debug("CONN %p {%d}", conn
, conn
->debug_id
);
1224 rxrpc_post_packet_to_conn(conn
, skb
);
1228 /* Note the serial number skew here */
1229 skew
= (int)sp
->hdr
.serial
- (int)conn
->hi_serial
;
1232 conn
->hi_serial
= sp
->hdr
.serial
;
1235 skew
= min(skew
, 65535);
1238 /* Call-bound packets are routed by connection channel. */
1239 channel
= sp
->hdr
.cid
& RXRPC_CHANNELMASK
;
1240 chan
= &conn
->channels
[channel
];
1242 /* Ignore really old calls */
1243 if (sp
->hdr
.callNumber
< chan
->last_call
)
1244 goto discard_unlock
;
1246 if (sp
->hdr
.callNumber
== chan
->last_call
) {
1248 sp
->hdr
.type
== RXRPC_PACKET_TYPE_ABORT
)
1249 goto discard_unlock
;
1251 /* For the previous service call, if completed
1252 * successfully, we discard all further packets.
1254 if (rxrpc_conn_is_service(conn
) &&
1255 chan
->last_type
== RXRPC_PACKET_TYPE_ACK
)
1256 goto discard_unlock
;
1258 /* But otherwise we need to retransmit the final packet
1259 * from data cached in the connection record.
1261 rxrpc_post_packet_to_conn(conn
, skb
);
1265 call
= rcu_dereference(chan
->call
);
1267 if (sp
->hdr
.callNumber
> chan
->call_id
) {
1268 if (rxrpc_to_client(sp
)) {
1273 rxrpc_input_implicit_end_call(conn
, call
);
1277 if (call
&& sp
->hdr
.serviceId
!= call
->service_id
)
1278 call
->service_id
= sp
->hdr
.serviceId
;
1284 if (!call
|| atomic_read(&call
->usage
) == 0) {
1285 if (rxrpc_to_client(sp
) ||
1286 sp
->hdr
.callNumber
== 0 ||
1287 sp
->hdr
.type
!= RXRPC_PACKET_TYPE_DATA
)
1288 goto bad_message_unlock
;
1289 if (sp
->hdr
.seq
!= 1)
1290 goto discard_unlock
;
1291 call
= rxrpc_new_incoming_call(local
, conn
, skb
);
1296 rxrpc_send_ping(call
, skb
, skew
);
1297 mutex_unlock(&call
->user_mutex
);
1300 rxrpc_input_call_packet(call
, skb
, skew
);
1301 goto discard_unlock
;
1306 rxrpc_free_skb(skb
, rxrpc_skb_rx_freed
);
1308 trace_rxrpc_rx_done(0, 0);
1317 trace_rxrpc_abort("SEC", sp
->hdr
.cid
, sp
->hdr
.callNumber
, sp
->hdr
.seq
,
1318 RXKADINCONSISTENCY
, EBADMSG
);
1319 skb
->priority
= RXKADINCONSISTENCY
;
1324 trace_rxrpc_abort("UPG", sp
->hdr
.cid
, sp
->hdr
.callNumber
, sp
->hdr
.seq
,
1325 RX_PROTOCOL_ERROR
, EBADMSG
);
1326 goto protocol_error
;
1331 trace_rxrpc_abort("BAD", sp
->hdr
.cid
, sp
->hdr
.callNumber
, sp
->hdr
.seq
,
1332 RX_PROTOCOL_ERROR
, EBADMSG
);
1334 skb
->priority
= RX_PROTOCOL_ERROR
;
1336 skb
->mark
= RXRPC_SKB_MARK_LOCAL_ABORT
;
1338 trace_rxrpc_rx_done(skb
->mark
, skb
->priority
);
1339 rxrpc_reject_packet(local
, skb
);
1340 _leave(" [badmsg]");