net/rxrpc/call_event.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* Management of Tx window, Tx resend, ACKs and out-of-sequence reception
   3  *
   4  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  */
   7
   8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9
  10 #include <linux/module.h>
  11 #include <linux/circ_buf.h>
  12 #include <linux/net.h>
  13 #include <linux/skbuff.h>
  14 #include <linux/slab.h>
  15 #include <linux/udp.h>
  16 #include <net/sock.h>
  17 #include <net/af_rxrpc.h>
  18 #include "ar-internal.h"
  19
  20 /*
  21  * Propose a PING ACK be sent.
  22  */
  23 static void rxrpc_propose_ping(struct rxrpc_call *call,
  24                                bool immediate, bool background)
  25 {
  26         if (immediate) {
  27                 if (background &&
  28                     !test_and_set_bit(RXRPC_CALL_EV_PING, &call->events))
  29                         rxrpc_queue_call(call);
  30         } else {
  31                 unsigned long now = jiffies;
  32                 unsigned long ping_at = now + rxrpc_idle_ack_delay;
  33
  34                 if (time_before(ping_at, call->ping_at)) {
  35                         WRITE_ONCE(call->ping_at, ping_at);
  36                         rxrpc_reduce_call_timer(call, ping_at, now,
  37                                                 rxrpc_timer_set_for_ping);
  38                 }
  39         }
  40 }
  41
  42 /*
  43  * propose an ACK be sent
  44  */
  45 static void __rxrpc_propose_ACK(struct rxrpc_call *call, u8 ack_reason,
  46                                 u16 skew, u32 serial, bool immediate,
  47                                 bool background,
  48                                 enum rxrpc_propose_ack_trace why)
  49 {
  50         enum rxrpc_propose_ack_outcome outcome = rxrpc_propose_ack_use;
  51         unsigned long expiry = rxrpc_soft_ack_delay;
  52         s8 prior = rxrpc_ack_priority[ack_reason];
  53
  54         /* Pings are handled specially because we don't want to accidentally
  55          * lose a ping response by subsuming it into a ping.
  56          */
  57         if (ack_reason == RXRPC_ACK_PING) {
  58                 rxrpc_propose_ping(call, immediate, background);
  59                 goto trace;
  60         }
  61
  62         /* Update DELAY, IDLE, REQUESTED and PING_RESPONSE ACK serial
  63          * numbers, but we don't alter the timeout.
  64          */
  65         _debug("prior %u %u vs %u %u",
  66                ack_reason, prior,
  67                call->ackr_reason, rxrpc_ack_priority[call->ackr_reason]);
  68         if (ack_reason == call->ackr_reason) {
  69                 if (RXRPC_ACK_UPDATEABLE & (1 << ack_reason)) {
  70                         outcome = rxrpc_propose_ack_update;
  71                         call->ackr_serial = serial;
  72                         call->ackr_skew = skew;
  73                 }
  74                 if (!immediate)
  75                         goto trace;
  76         } else if (prior > rxrpc_ack_priority[call->ackr_reason]) {
  77                 call->ackr_reason = ack_reason;
  78                 call->ackr_serial = serial;
  79                 call->ackr_skew = skew;
  80         } else {
  81                 outcome = rxrpc_propose_ack_subsume;
  82         }
  83
  84         switch (ack_reason) {
  85         case RXRPC_ACK_REQUESTED:
  86                 if (rxrpc_requested_ack_delay < expiry)
  87                         expiry = rxrpc_requested_ack_delay;
  88                 if (serial == 1)
  89                         immediate = false;
  90                 break;
  91
  92         case RXRPC_ACK_DELAY:
  93                 if (rxrpc_soft_ack_delay < expiry)
  94                         expiry = rxrpc_soft_ack_delay;
  95                 break;
  96
  97         case RXRPC_ACK_IDLE:
  98                 if (rxrpc_idle_ack_delay < expiry)
  99                         expiry = rxrpc_idle_ack_delay;
 100                 break;
 101
 102         default:
 103                 immediate = true;
 104                 break;
 105         }
 106
 107         if (test_bit(RXRPC_CALL_EV_ACK, &call->events)) {
 108                 _debug("already scheduled");
 109         } else if (immediate || expiry == 0) {
 110                 _debug("immediate ACK %lx", call->events);
 111                 if (!test_and_set_bit(RXRPC_CALL_EV_ACK, &call->events) &&
 112                     background)
 113                         rxrpc_queue_call(call);
 114         } else {
 115                 unsigned long now = jiffies, ack_at;
 116
 117                 if (call->peer->rtt_usage > 0)
 118                         ack_at = nsecs_to_jiffies(call->peer->rtt);
 119                 else
 120                         ack_at = expiry;
 121
 122                 ack_at += READ_ONCE(call->tx_backoff);
 123                 ack_at += now;
 124                 if (time_before(ack_at, call->ack_at)) {
 125                         WRITE_ONCE(call->ack_at, ack_at);
 126                         rxrpc_reduce_call_timer(call, ack_at, now,
 127                                                 rxrpc_timer_set_for_ack);
 128                 }
 129         }
 130
 131 trace:
 132         trace_rxrpc_propose_ack(call, why, ack_reason, serial, immediate,
 133                                 background, outcome);
 134 }
 135
 136 /*
 137  * propose an ACK be sent, locking the call structure
 138  */
 139 void rxrpc_propose_ACK(struct rxrpc_call *call, u8 ack_reason,
 140                        u16 skew, u32 serial, bool immediate, bool background,
 141                        enum rxrpc_propose_ack_trace why)
 142 {
 143         spin_lock_bh(&call->lock);
 144         __rxrpc_propose_ACK(call, ack_reason, skew, serial,
 145                             immediate, background, why);
 146         spin_unlock_bh(&call->lock);
 147 }
 148
 149 /*
 150  * Handle congestion being detected by the retransmit timeout.
 151  */
 152 static void rxrpc_congestion_timeout(struct rxrpc_call *call)
 153 {
 154         set_bit(RXRPC_CALL_RETRANS_TIMEOUT, &call->flags);
 155 }
 156
 157 /*
 158  * Perform retransmission of NAK'd and unack'd packets.
 159  */
 160 static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j)
 161 {
 162         struct sk_buff *skb;
 163         unsigned long resend_at;
 164         rxrpc_seq_t cursor, seq, top;
 165         ktime_t now, max_age, oldest, ack_ts, timeout, min_timeo;
 166         int ix;
 167         u8 annotation, anno_type, retrans = 0, unacked = 0;
 168
 169         _enter("{%d,%d}", call->tx_hard_ack, call->tx_top);
 170
 171         if (call->peer->rtt_usage > 1)
 172                 timeout = ns_to_ktime(call->peer->rtt * 3 / 2);
 173         else
 174                 timeout = ms_to_ktime(rxrpc_resend_timeout);
 175         min_timeo = ns_to_ktime((1000000000 / HZ) * 4);
 176         if (ktime_before(timeout, min_timeo))
 177                 timeout = min_timeo;
 178
 179         now = ktime_get_real();
 180         max_age = ktime_sub(now, timeout);
 181
 182         spin_lock_bh(&call->lock);
 183
 184         cursor = call->tx_hard_ack;
 185         top = call->tx_top;
 186         ASSERT(before_eq(cursor, top));
 187         if (cursor == top)
 188                 goto out_unlock;
 189
 190         /* Scan the packet list without dropping the lock and decide which of
 191          * the packets in the Tx buffer we're going to resend and what the new
 192          * resend timeout will be.
 193          */
 194         trace_rxrpc_resend(call, (cursor + 1) & RXRPC_RXTX_BUFF_MASK);
 195         oldest = now;
 196         for (seq = cursor + 1; before_eq(seq, top); seq++) {
 197                 ix = seq & RXRPC_RXTX_BUFF_MASK;
 198                 annotation = call->rxtx_annotations[ix];
 199                 anno_type = annotation & RXRPC_TX_ANNO_MASK;
 200                 annotation &= ~RXRPC_TX_ANNO_MASK;
 201                 if (anno_type == RXRPC_TX_ANNO_ACK)
 202                         continue;
 203
 204                 skb = call->rxtx_buffer[ix];
 205                 rxrpc_see_skb(skb, rxrpc_skb_tx_seen);
 206
 207                 if (anno_type == RXRPC_TX_ANNO_UNACK) {
 208                         if (ktime_after(skb->tstamp, max_age)) {
 209                                 if (ktime_before(skb->tstamp, oldest))
 210                                         oldest = skb->tstamp;
 211                                 continue;
 212                         }
 213                         if (!(annotation & RXRPC_TX_ANNO_RESENT))
 214                                 unacked++;
 215                 }
 216
 217                 /* Okay, we need to retransmit a packet. */
 218                 call->rxtx_annotations[ix] = RXRPC_TX_ANNO_RETRANS | annotation;
 219                 retrans++;
 220                 trace_rxrpc_retransmit(call, seq, annotation | anno_type,
 221                                        ktime_to_ns(ktime_sub(skb->tstamp, max_age)));
 222         }
 223
 224         resend_at = nsecs_to_jiffies(ktime_to_ns(ktime_sub(now, oldest)));
 225         resend_at += jiffies + rxrpc_resend_timeout;
 226         WRITE_ONCE(call->resend_at, resend_at);
 227
 228         if (unacked)
 229                 rxrpc_congestion_timeout(call);
 230
 231         /* If there was nothing that needed retransmission then it's likely
 232          * that an ACK got lost somewhere.  Send a ping to find out instead of
 233          * retransmitting data.
 234          */
 235         if (!retrans) {
 236                 rxrpc_reduce_call_timer(call, resend_at, now_j,
 237                                         rxrpc_timer_set_for_resend);
 238                 spin_unlock_bh(&call->lock);
 239                 ack_ts = ktime_sub(now, call->acks_latest_ts);
 240                 if (ktime_to_ns(ack_ts) < call->peer->rtt)
 241                         goto out;
 242                 rxrpc_propose_ACK(call, RXRPC_ACK_PING, 0, 0, true, false,
 243                                   rxrpc_propose_ack_ping_for_lost_ack);
 244                 rxrpc_send_ack_packet(call, true, NULL);
 245                 goto out;
 246         }
 247
 248         /* Now go through the Tx window and perform the retransmissions.  We
 249          * have to drop the lock for each send.  If an ACK comes in whilst the
 250          * lock is dropped, it may clear some of the retransmission markers for
 251          * packets that it soft-ACKs.
 252          */
 253         for (seq = cursor + 1; before_eq(seq, top); seq++) {
 254                 ix = seq & RXRPC_RXTX_BUFF_MASK;
 255                 annotation = call->rxtx_annotations[ix];
 256                 anno_type = annotation & RXRPC_TX_ANNO_MASK;
 257                 if (anno_type != RXRPC_TX_ANNO_RETRANS)
 258                         continue;
 259
 260                 skb = call->rxtx_buffer[ix];
 261                 rxrpc_get_skb(skb, rxrpc_skb_tx_got);
 262                 spin_unlock_bh(&call->lock);
 263
 264                 if (rxrpc_send_data_packet(call, skb, true) < 0) {
 265                         rxrpc_free_skb(skb, rxrpc_skb_tx_freed);
 266                         return;
 267                 }
 268
 269                 if (rxrpc_is_client_call(call))
 270                         rxrpc_expose_client_call(call);
 271
 272                 rxrpc_free_skb(skb, rxrpc_skb_tx_freed);
 273                 spin_lock_bh(&call->lock);
 274
 275                 /* We need to clear the retransmit state, but there are two
 276                  * things we need to be aware of: A new ACK/NAK might have been
 277                  * received and the packet might have been hard-ACK'd (in which
 278                  * case it will no longer be in the buffer).
 279                  */
 280                 if (after(seq, call->tx_hard_ack)) {
 281                         annotation = call->rxtx_annotations[ix];
 282                         anno_type = annotation & RXRPC_TX_ANNO_MASK;
 283                         if (anno_type == RXRPC_TX_ANNO_RETRANS ||
 284                             anno_type == RXRPC_TX_ANNO_NAK) {
 285                                 annotation &= ~RXRPC_TX_ANNO_MASK;
 286                                 annotation |= RXRPC_TX_ANNO_UNACK;
 287                         }
 288                         annotation |= RXRPC_TX_ANNO_RESENT;
 289                         call->rxtx_annotations[ix] = annotation;
 290                 }
 291
 292                 if (after(call->tx_hard_ack, seq))
 293                         seq = call->tx_hard_ack;
 294         }
 295
 296 out_unlock:
 297         spin_unlock_bh(&call->lock);
 298 out:
 299         _leave("");
 300 }
 301
 302 /*
 303  * Handle retransmission and deferred ACK/abort generation.
 304  */
 305 void rxrpc_process_call(struct work_struct *work)
 306 {
 307         struct rxrpc_call *call =
 308                 container_of(work, struct rxrpc_call, processor);
 309         rxrpc_serial_t *send_ack;
 310         unsigned long now, next, t;
 311         unsigned int iterations = 0;
 312
 313         rxrpc_see_call(call);
 314
 315         //printk("\n--------------------\n");
 316         _enter("{%d,%s,%lx}",
 317                call->debug_id, rxrpc_call_states[call->state], call->events);
 318
 319 recheck_state:
 320         /* Limit the number of times we do this before returning to the manager */
 321         iterations++;
 322         if (iterations > 5)
 323                 goto requeue;
 324
 325         if (test_and_clear_bit(RXRPC_CALL_EV_ABORT, &call->events)) {
 326                 rxrpc_send_abort_packet(call);
 327                 goto recheck_state;
 328         }
 329
 330         if (call->state == RXRPC_CALL_COMPLETE) {
 331                 del_timer_sync(&call->timer);
 332                 rxrpc_notify_socket(call);
 333                 goto out_put;
 334         }
 335
 336         /* Work out if any timeouts tripped */
 337         now = jiffies;
 338         t = READ_ONCE(call->expect_rx_by);
 339         if (time_after_eq(now, t)) {
 340                 trace_rxrpc_timer(call, rxrpc_timer_exp_normal, now);
 341                 set_bit(RXRPC_CALL_EV_EXPIRED, &call->events);
 342         }
 343
 344         t = READ_ONCE(call->expect_req_by);
 345         if (call->state == RXRPC_CALL_SERVER_RECV_REQUEST &&
 346             time_after_eq(now, t)) {
 347                 trace_rxrpc_timer(call, rxrpc_timer_exp_idle, now);
 348                 set_bit(RXRPC_CALL_EV_EXPIRED, &call->events);
 349         }
 350
 351         t = READ_ONCE(call->expect_term_by);
 352         if (time_after_eq(now, t)) {
 353                 trace_rxrpc_timer(call, rxrpc_timer_exp_hard, now);
 354                 set_bit(RXRPC_CALL_EV_EXPIRED, &call->events);
 355         }
 356
 357         t = READ_ONCE(call->ack_at);
 358         if (time_after_eq(now, t)) {
 359                 trace_rxrpc_timer(call, rxrpc_timer_exp_ack, now);
 360                 cmpxchg(&call->ack_at, t, now + MAX_JIFFY_OFFSET);
 361                 set_bit(RXRPC_CALL_EV_ACK, &call->events);
 362         }
 363
 364         t = READ_ONCE(call->ack_lost_at);
 365         if (time_after_eq(now, t)) {
 366                 trace_rxrpc_timer(call, rxrpc_timer_exp_lost_ack, now);
 367                 cmpxchg(&call->ack_lost_at, t, now + MAX_JIFFY_OFFSET);
 368                 set_bit(RXRPC_CALL_EV_ACK_LOST, &call->events);
 369         }
 370
 371         t = READ_ONCE(call->keepalive_at);
 372         if (time_after_eq(now, t)) {
 373                 trace_rxrpc_timer(call, rxrpc_timer_exp_keepalive, now);
 374                 cmpxchg(&call->keepalive_at, t, now + MAX_JIFFY_OFFSET);
 375                 rxrpc_propose_ACK(call, RXRPC_ACK_PING, 0, 0, true, true,
 376                                   rxrpc_propose_ack_ping_for_keepalive);
 377                 set_bit(RXRPC_CALL_EV_PING, &call->events);
 378         }
 379
 380         t = READ_ONCE(call->ping_at);
 381         if (time_after_eq(now, t)) {
 382                 trace_rxrpc_timer(call, rxrpc_timer_exp_ping, now);
 383                 cmpxchg(&call->ping_at, t, now + MAX_JIFFY_OFFSET);
 384                 set_bit(RXRPC_CALL_EV_PING, &call->events);
 385         }
 386
 387         t = READ_ONCE(call->resend_at);
 388         if (time_after_eq(now, t)) {
 389                 trace_rxrpc_timer(call, rxrpc_timer_exp_resend, now);
 390                 cmpxchg(&call->resend_at, t, now + MAX_JIFFY_OFFSET);
 391                 set_bit(RXRPC_CALL_EV_RESEND, &call->events);
 392         }
 393
 394         /* Process events */
 395         if (test_and_clear_bit(RXRPC_CALL_EV_EXPIRED, &call->events)) {
 396                 if (test_bit(RXRPC_CALL_RX_HEARD, &call->flags) &&
 397                     (int)call->conn->hi_serial - (int)call->rx_serial > 0) {
 398                         trace_rxrpc_call_reset(call);
 399                         rxrpc_abort_call("EXP", call, 0, RX_USER_ABORT, -ECONNRESET);
 400                 } else {
 401                         rxrpc_abort_call("EXP", call, 0, RX_USER_ABORT, -ETIME);
 402                 }
 403                 set_bit(RXRPC_CALL_EV_ABORT, &call->events);
 404                 goto recheck_state;
 405         }
 406
 407         send_ack = NULL;
 408         if (test_and_clear_bit(RXRPC_CALL_EV_ACK_LOST, &call->events)) {
 409                 call->acks_lost_top = call->tx_top;
 410                 rxrpc_propose_ACK(call, RXRPC_ACK_PING, 0, 0, true, false,
 411                                   rxrpc_propose_ack_ping_for_lost_ack);
 412                 send_ack = &call->acks_lost_ping;
 413         }
 414
 415         if (test_and_clear_bit(RXRPC_CALL_EV_ACK, &call->events) ||
 416             send_ack) {
 417                 if (call->ackr_reason) {
 418                         rxrpc_send_ack_packet(call, false, send_ack);
 419                         goto recheck_state;
 420                 }
 421         }
 422
 423         if (test_and_clear_bit(RXRPC_CALL_EV_PING, &call->events)) {
 424                 rxrpc_send_ack_packet(call, true, NULL);
 425                 goto recheck_state;
 426         }
 427
 428         if (test_and_clear_bit(RXRPC_CALL_EV_RESEND, &call->events)) {
 429                 rxrpc_resend(call, now);
 430                 goto recheck_state;
 431         }
 432
 433         /* Make sure the timer is restarted */
 434         next = call->expect_rx_by;
 435
 436 #define set(T) { t = READ_ONCE(T); if (time_before(t, next)) next = t; }
 437
 438         set(call->expect_req_by);
 439         set(call->expect_term_by);
 440         set(call->ack_at);
 441         set(call->ack_lost_at);
 442         set(call->resend_at);
 443         set(call->keepalive_at);
 444         set(call->ping_at);
 445
 446         now = jiffies;
 447         if (time_after_eq(now, next))
 448                 goto recheck_state;
 449
 450         rxrpc_reduce_call_timer(call, next, now, rxrpc_timer_restart);
 451
 452         /* other events may have been raised since we started checking */
 453         if (call->events && call->state < RXRPC_CALL_COMPLETE)
 454                 goto requeue;
 455
 456 out_put:
 457         rxrpc_put_call(call, rxrpc_call_put);
 458 out:
 459         _leave("");
 460         return;
 461
 462 requeue:
 463         __rxrpc_queue_call(call);
 464         goto out;
 465 }