net/rxrpc/output.c

   1 /* RxRPC packet transmission
   2  *
   3  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  13
  14 #include <linux/net.h>
  15 #include <linux/gfp.h>
  16 #include <linux/skbuff.h>
  17 #include <linux/circ_buf.h>
  18 #include <linux/export.h>
  19 #include <net/sock.h>
  20 #include <net/af_rxrpc.h>
  21 #include "ar-internal.h"
  22
  23 /*
  24  * Time till packet resend (in jiffies).
  25  */
  26 unsigned int rxrpc_resend_timeout = 4 * HZ;
  27
  28 static int rxrpc_send_data(struct rxrpc_sock *rx,
  29                            struct rxrpc_call *call,
  30                            struct msghdr *msg, size_t len);
  31
  32 /*
  33  * extract control messages from the sendmsg() control buffer
  34  */
  35 static int rxrpc_sendmsg_cmsg(struct msghdr *msg,
  36                               unsigned long *user_call_ID,
  37                               enum rxrpc_command *command,
  38                               u32 *abort_code,
  39                               bool *_exclusive)
  40 {
  41         struct cmsghdr *cmsg;
  42         bool got_user_ID = false;
  43         int len;
  44
  45         *command = RXRPC_CMD_SEND_DATA;
  46
  47         if (msg->msg_controllen == 0)
  48                 return -EINVAL;
  49
  50         for_each_cmsghdr(cmsg, msg) {
  51                 if (!CMSG_OK(msg, cmsg))
  52                         return -EINVAL;
  53
  54                 len = cmsg->cmsg_len - CMSG_ALIGN(sizeof(struct cmsghdr));
  55                 _debug("CMSG %d, %d, %d",
  56                        cmsg->cmsg_level, cmsg->cmsg_type, len);
  57
  58                 if (cmsg->cmsg_level != SOL_RXRPC)
  59                         continue;
  60
  61                 switch (cmsg->cmsg_type) {
  62                 case RXRPC_USER_CALL_ID:
  63                         if (msg->msg_flags & MSG_CMSG_COMPAT) {
  64                                 if (len != sizeof(u32))
  65                                         return -EINVAL;
  66                                 *user_call_ID = *(u32 *) CMSG_DATA(cmsg);
  67                         } else {
  68                                 if (len != sizeof(unsigned long))
  69                                         return -EINVAL;
  70                                 *user_call_ID = *(unsigned long *)
  71                                         CMSG_DATA(cmsg);
  72                         }
  73                         _debug("User Call ID %lx", *user_call_ID);
  74                         got_user_ID = true;
  75                         break;
  76
  77                 case RXRPC_ABORT:
  78                         if (*command != RXRPC_CMD_SEND_DATA)
  79                                 return -EINVAL;
  80                         *command = RXRPC_CMD_SEND_ABORT;
  81                         if (len != sizeof(*abort_code))
  82                                 return -EINVAL;
  83                         *abort_code = *(unsigned int *) CMSG_DATA(cmsg);
  84                         _debug("Abort %x", *abort_code);
  85                         if (*abort_code == 0)
  86                                 return -EINVAL;
  87                         break;
  88
  89                 case RXRPC_ACCEPT:
  90                         if (*command != RXRPC_CMD_SEND_DATA)
  91                                 return -EINVAL;
  92                         *command = RXRPC_CMD_ACCEPT;
  93                         if (len != 0)
  94                                 return -EINVAL;
  95                         break;
  96
  97                 case RXRPC_EXCLUSIVE_CALL:
  98                         *_exclusive = true;
  99                         if (len != 0)
 100                                 return -EINVAL;
 101                         break;
 102                 default:
 103                         return -EINVAL;
 104                 }
 105         }
 106
 107         if (!got_user_ID)
 108                 return -EINVAL;
 109         _leave(" = 0");
 110         return 0;
 111 }
 112
 113 /*
 114  * abort a call, sending an ABORT packet to the peer
 115  */
 116 static void rxrpc_send_abort(struct rxrpc_call *call, u32 abort_code)
 117 {
 118         write_lock_bh(&call->state_lock);
 119
 120         if (call->state <= RXRPC_CALL_COMPLETE) {
 121                 call->state = RXRPC_CALL_LOCALLY_ABORTED;
 122                 call->local_abort = abort_code;
 123                 set_bit(RXRPC_CALL_EV_ABORT, &call->events);
 124                 del_timer_sync(&call->resend_timer);
 125                 del_timer_sync(&call->ack_timer);
 126                 clear_bit(RXRPC_CALL_EV_RESEND_TIMER, &call->events);
 127                 clear_bit(RXRPC_CALL_EV_ACK, &call->events);
 128                 clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
 129                 rxrpc_queue_call(call);
 130         }
 131
 132         write_unlock_bh(&call->state_lock);
 133 }
 134
 135 /*
 136  * Create a new client call for sendmsg().
 137  */
 138 static struct rxrpc_call *
 139 rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg,
 140                                   unsigned long user_call_ID, bool exclusive)
 141 {
 142         struct rxrpc_conn_parameters cp;
 143         struct rxrpc_call *call;
 144         struct key *key;
 145
 146         DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx, msg->msg_name);
 147
 148         _enter("");
 149
 150         if (!msg->msg_name)
 151                 return ERR_PTR(-EDESTADDRREQ);
 152
 153         key = rx->key;
 154         if (key && !rx->key->payload.data[0])
 155                 key = NULL;
 156
 157         memset(&cp, 0, sizeof(cp));
 158         cp.local                = rx->local;
 159         cp.key                  = rx->key;
 160         cp.security_level       = rx->min_sec_level;
 161         cp.exclusive            = rx->exclusive | exclusive;
 162         cp.service_id           = srx->srx_service;
 163         call = rxrpc_new_client_call(rx, &cp, srx, user_call_ID, GFP_KERNEL);
 164
 165         _leave(" = %p\n", call);
 166         return call;
 167 }
 168
 169 /*
 170  * send a message forming part of a client call through an RxRPC socket
 171  * - caller holds the socket locked
 172  * - the socket may be either a client socket or a server socket
 173  */
 174 int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len)
 175 {
 176         enum rxrpc_command cmd;
 177         struct rxrpc_call *call;
 178         unsigned long user_call_ID = 0;
 179         bool exclusive = false;
 180         u32 abort_code = 0;
 181         int ret;
 182
 183         _enter("");
 184
 185         ret = rxrpc_sendmsg_cmsg(msg, &user_call_ID, &cmd, &abort_code,
 186                                  &exclusive);
 187         if (ret < 0)
 188                 return ret;
 189
 190         if (cmd == RXRPC_CMD_ACCEPT) {
 191                 if (rx->sk.sk_state != RXRPC_SERVER_LISTENING)
 192                         return -EINVAL;
 193                 call = rxrpc_accept_call(rx, user_call_ID);
 194                 if (IS_ERR(call))
 195                         return PTR_ERR(call);
 196                 rxrpc_put_call(call);
 197                 return 0;
 198         }
 199
 200         call = rxrpc_find_call_by_user_ID(rx, user_call_ID);
 201         if (!call) {
 202                 if (cmd != RXRPC_CMD_SEND_DATA)
 203                         return -EBADSLT;
 204                 call = rxrpc_new_client_call_for_sendmsg(rx, msg, user_call_ID,
 205                                                          exclusive);
 206                 if (IS_ERR(call))
 207                         return PTR_ERR(call);
 208         }
 209
 210         _debug("CALL %d USR %lx ST %d on CONN %p",
 211                call->debug_id, call->user_call_ID, call->state, call->conn);
 212
 213         if (call->state >= RXRPC_CALL_COMPLETE) {
 214                 /* it's too late for this call */
 215                 ret = -ECONNRESET;
 216         } else if (cmd == RXRPC_CMD_SEND_ABORT) {
 217                 rxrpc_send_abort(call, abort_code);
 218                 ret = 0;
 219         } else if (cmd != RXRPC_CMD_SEND_DATA) {
 220                 ret = -EINVAL;
 221         } else if (!call->in_clientflag &&
 222                    call->state != RXRPC_CALL_CLIENT_SEND_REQUEST) {
 223                 /* request phase complete for this client call */
 224                 ret = -EPROTO;
 225         } else if (call->in_clientflag &&
 226                    call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
 227                    call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
 228                 /* Reply phase not begun or not complete for service call. */
 229                 ret = -EPROTO;
 230         } else {
 231                 ret = rxrpc_send_data(rx, call, msg, len);
 232         }
 233
 234         rxrpc_put_call(call);
 235         _leave(" = %d", ret);
 236         return ret;
 237 }
 238
 239 /**
 240  * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
 241  * @call: The call to send data through
 242  * @msg: The data to send
 243  * @len: The amount of data to send
 244  *
 245  * Allow a kernel service to send data on a call.  The call must be in an state
 246  * appropriate to sending data.  No control data should be supplied in @msg,
 247  * nor should an address be supplied.  MSG_MORE should be flagged if there's
 248  * more data to come, otherwise this data will end the transmission phase.
 249  */
 250 int rxrpc_kernel_send_data(struct rxrpc_call *call, struct msghdr *msg,
 251                            size_t len)
 252 {
 253         int ret;
 254
 255         _enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]);
 256
 257         ASSERTCMP(msg->msg_name, ==, NULL);
 258         ASSERTCMP(msg->msg_control, ==, NULL);
 259
 260         lock_sock(&call->socket->sk);
 261
 262         _debug("CALL %d USR %lx ST %d on CONN %p",
 263                call->debug_id, call->user_call_ID, call->state, call->conn);
 264
 265         if (call->state >= RXRPC_CALL_COMPLETE) {
 266                 ret = -ESHUTDOWN; /* it's too late for this call */
 267         } else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
 268                    call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
 269                    call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
 270                 ret = -EPROTO; /* request phase complete for this client call */
 271         } else {
 272                 ret = rxrpc_send_data(call->socket, call, msg, len);
 273         }
 274
 275         release_sock(&call->socket->sk);
 276         _leave(" = %d", ret);
 277         return ret;
 278 }
 279
 280 EXPORT_SYMBOL(rxrpc_kernel_send_data);
 281
 282 /**
 283  * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
 284  * @call: The call to be aborted
 285  * @abort_code: The abort code to stick into the ABORT packet
 286  *
 287  * Allow a kernel service to abort a call, if it's still in an abortable state.
 288  */
 289 void rxrpc_kernel_abort_call(struct rxrpc_call *call, u32 abort_code)
 290 {
 291         _enter("{%d},%d", call->debug_id, abort_code);
 292
 293         lock_sock(&call->socket->sk);
 294
 295         _debug("CALL %d USR %lx ST %d on CONN %p",
 296                call->debug_id, call->user_call_ID, call->state, call->conn);
 297
 298         if (call->state < RXRPC_CALL_COMPLETE)
 299                 rxrpc_send_abort(call, abort_code);
 300
 301         release_sock(&call->socket->sk);
 302         _leave("");
 303 }
 304
 305 EXPORT_SYMBOL(rxrpc_kernel_abort_call);
 306
 307 /*
 308  * send a packet through the transport endpoint
 309  */
 310 int rxrpc_send_data_packet(struct rxrpc_connection *conn, struct sk_buff *skb)
 311 {
 312         struct kvec iov[1];
 313         struct msghdr msg;
 314         int ret, opt;
 315
 316         _enter(",{%d}", skb->len);
 317
 318         iov[0].iov_base = skb->head;
 319         iov[0].iov_len = skb->len;
 320
 321         msg.msg_name = &conn->params.peer->srx.transport;
 322         msg.msg_namelen = conn->params.peer->srx.transport_len;
 323         msg.msg_control = NULL;
 324         msg.msg_controllen = 0;
 325         msg.msg_flags = 0;
 326
 327         /* send the packet with the don't fragment bit set if we currently
 328          * think it's small enough */
 329         if (skb->len - sizeof(struct rxrpc_wire_header) < conn->params.peer->maxdata) {
 330                 down_read(&conn->params.local->defrag_sem);
 331                 /* send the packet by UDP
 332                  * - returns -EMSGSIZE if UDP would have to fragment the packet
 333                  *   to go out of the interface
 334                  *   - in which case, we'll have processed the ICMP error
 335                  *     message and update the peer record
 336                  */
 337                 ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 1,
 338                                      iov[0].iov_len);
 339
 340                 up_read(&conn->params.local->defrag_sem);
 341                 if (ret == -EMSGSIZE)
 342                         goto send_fragmentable;
 343
 344                 _leave(" = %d [%u]", ret, conn->params.peer->maxdata);
 345                 return ret;
 346         }
 347
 348 send_fragmentable:
 349         /* attempt to send this message with fragmentation enabled */
 350         _debug("send fragment");
 351
 352         down_write(&conn->params.local->defrag_sem);
 353
 354         switch (conn->params.local->srx.transport.family) {
 355         case AF_INET:
 356                 opt = IP_PMTUDISC_DONT;
 357                 ret = kernel_setsockopt(conn->params.local->socket,
 358                                         SOL_IP, IP_MTU_DISCOVER,
 359                                         (char *)&opt, sizeof(opt));
 360                 if (ret == 0) {
 361                         ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 1,
 362                                              iov[0].iov_len);
 363
 364                         opt = IP_PMTUDISC_DO;
 365                         kernel_setsockopt(conn->params.local->socket, SOL_IP,
 366                                           IP_MTU_DISCOVER,
 367                                           (char *)&opt, sizeof(opt));
 368                 }
 369                 break;
 370         }
 371
 372         up_write(&conn->params.local->defrag_sem);
 373         _leave(" = %d [frag %u]", ret, conn->params.peer->maxdata);
 374         return ret;
 375 }
 376
 377 /*
 378  * wait for space to appear in the transmit/ACK window
 379  * - caller holds the socket locked
 380  */
 381 static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx,
 382                                     struct rxrpc_call *call,
 383                                     long *timeo)
 384 {
 385         DECLARE_WAITQUEUE(myself, current);
 386         int ret;
 387
 388         _enter(",{%d},%ld",
 389                CIRC_SPACE(call->acks_head, ACCESS_ONCE(call->acks_tail),
 390                           call->acks_winsz),
 391                *timeo);
 392
 393         add_wait_queue(&call->tx_waitq, &myself);
 394
 395         for (;;) {
 396                 set_current_state(TASK_INTERRUPTIBLE);
 397                 ret = 0;
 398                 if (CIRC_SPACE(call->acks_head, ACCESS_ONCE(call->acks_tail),
 399                                call->acks_winsz) > 0)
 400                         break;
 401                 if (signal_pending(current)) {
 402                         ret = sock_intr_errno(*timeo);
 403                         break;
 404                 }
 405
 406                 release_sock(&rx->sk);
 407                 *timeo = schedule_timeout(*timeo);
 408                 lock_sock(&rx->sk);
 409         }
 410
 411         remove_wait_queue(&call->tx_waitq, &myself);
 412         set_current_state(TASK_RUNNING);
 413         _leave(" = %d", ret);
 414         return ret;
 415 }
 416
 417 /*
 418  * attempt to schedule an instant Tx resend
 419  */
 420 static inline void rxrpc_instant_resend(struct rxrpc_call *call)
 421 {
 422         read_lock_bh(&call->state_lock);
 423         if (try_to_del_timer_sync(&call->resend_timer) >= 0) {
 424                 clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
 425                 if (call->state < RXRPC_CALL_COMPLETE &&
 426                     !test_and_set_bit(RXRPC_CALL_EV_RESEND_TIMER, &call->events))
 427                         rxrpc_queue_call(call);
 428         }
 429         read_unlock_bh(&call->state_lock);
 430 }
 431
 432 /*
 433  * queue a packet for transmission, set the resend timer and attempt
 434  * to send the packet immediately
 435  */
 436 static void rxrpc_queue_packet(struct rxrpc_call *call, struct sk_buff *skb,
 437                                bool last)
 438 {
 439         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 440         int ret;
 441
 442         _net("queue skb %p [%d]", skb, call->acks_head);
 443
 444         ASSERT(call->acks_window != NULL);
 445         call->acks_window[call->acks_head] = (unsigned long) skb;
 446         smp_wmb();
 447         call->acks_head = (call->acks_head + 1) & (call->acks_winsz - 1);
 448
 449         if (last || call->state == RXRPC_CALL_SERVER_ACK_REQUEST) {
 450                 _debug("________awaiting reply/ACK__________");
 451                 write_lock_bh(&call->state_lock);
 452                 switch (call->state) {
 453                 case RXRPC_CALL_CLIENT_SEND_REQUEST:
 454                         call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY;
 455                         break;
 456                 case RXRPC_CALL_SERVER_ACK_REQUEST:
 457                         call->state = RXRPC_CALL_SERVER_SEND_REPLY;
 458                         if (!last)
 459                                 break;
 460                 case RXRPC_CALL_SERVER_SEND_REPLY:
 461                         call->state = RXRPC_CALL_SERVER_AWAIT_ACK;
 462                         break;
 463                 default:
 464                         break;
 465                 }
 466                 write_unlock_bh(&call->state_lock);
 467         }
 468
 469         _proto("Tx DATA %%%u { #%u }", sp->hdr.serial, sp->hdr.seq);
 470
 471         sp->need_resend = false;
 472         sp->resend_at = jiffies + rxrpc_resend_timeout;
 473         if (!test_and_set_bit(RXRPC_CALL_RUN_RTIMER, &call->flags)) {
 474                 _debug("run timer");
 475                 call->resend_timer.expires = sp->resend_at;
 476                 add_timer(&call->resend_timer);
 477         }
 478
 479         /* attempt to cancel the rx-ACK timer, deferring reply transmission if
 480          * we're ACK'ing the request phase of an incoming call */
 481         ret = -EAGAIN;
 482         if (try_to_del_timer_sync(&call->ack_timer) >= 0) {
 483                 /* the packet may be freed by rxrpc_process_call() before this
 484                  * returns */
 485                 ret = rxrpc_send_data_packet(call->conn, skb);
 486                 _net("sent skb %p", skb);
 487         } else {
 488                 _debug("failed to delete ACK timer");
 489         }
 490
 491         if (ret < 0) {
 492                 _debug("need instant resend %d", ret);
 493                 sp->need_resend = true;
 494                 rxrpc_instant_resend(call);
 495         }
 496
 497         _leave("");
 498 }
 499
 500 /*
 501  * Convert a host-endian header into a network-endian header.
 502  */
 503 static void rxrpc_insert_header(struct sk_buff *skb)
 504 {
 505         struct rxrpc_wire_header whdr;
 506         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 507
 508         whdr.epoch      = htonl(sp->hdr.epoch);
 509         whdr.cid        = htonl(sp->hdr.cid);
 510         whdr.callNumber = htonl(sp->hdr.callNumber);
 511         whdr.seq        = htonl(sp->hdr.seq);
 512         whdr.serial     = htonl(sp->hdr.serial);
 513         whdr.type       = sp->hdr.type;
 514         whdr.flags      = sp->hdr.flags;
 515         whdr.userStatus = sp->hdr.userStatus;
 516         whdr.securityIndex = sp->hdr.securityIndex;
 517         whdr._rsvd      = htons(sp->hdr._rsvd);
 518         whdr.serviceId  = htons(sp->hdr.serviceId);
 519
 520         memcpy(skb->head, &whdr, sizeof(whdr));
 521 }
 522
 523 /*
 524  * send data through a socket
 525  * - must be called in process context
 526  * - caller holds the socket locked
 527  */
 528 static int rxrpc_send_data(struct rxrpc_sock *rx,
 529                            struct rxrpc_call *call,
 530                            struct msghdr *msg, size_t len)
 531 {
 532         struct rxrpc_skb_priv *sp;
 533         struct sk_buff *skb;
 534         struct sock *sk = &rx->sk;
 535         long timeo;
 536         bool more;
 537         int ret, copied;
 538
 539         timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
 540
 541         /* this should be in poll */
 542         sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 543
 544         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 545                 return -EPIPE;
 546
 547         more = msg->msg_flags & MSG_MORE;
 548
 549         skb = call->tx_pending;
 550         call->tx_pending = NULL;
 551
 552         copied = 0;
 553         do {
 554                 if (!skb) {
 555                         size_t size, chunk, max, space;
 556
 557                         _debug("alloc");
 558
 559                         if (CIRC_SPACE(call->acks_head,
 560                                        ACCESS_ONCE(call->acks_tail),
 561                                        call->acks_winsz) <= 0) {
 562                                 ret = -EAGAIN;
 563                                 if (msg->msg_flags & MSG_DONTWAIT)
 564                                         goto maybe_error;
 565                                 ret = rxrpc_wait_for_tx_window(rx, call,
 566                                                                &timeo);
 567                                 if (ret < 0)
 568                                         goto maybe_error;
 569                         }
 570
 571                         max = call->conn->params.peer->maxdata;
 572                         max -= call->conn->security_size;
 573                         max &= ~(call->conn->size_align - 1UL);
 574
 575                         chunk = max;
 576                         if (chunk > msg_data_left(msg) && !more)
 577                                 chunk = msg_data_left(msg);
 578
 579                         space = chunk + call->conn->size_align;
 580                         space &= ~(call->conn->size_align - 1UL);
 581
 582                         size = space + call->conn->header_size;
 583
 584                         _debug("SIZE: %zu/%zu/%zu", chunk, space, size);
 585
 586                         /* create a buffer that we can retain until it's ACK'd */
 587                         skb = sock_alloc_send_skb(
 588                                 sk, size, msg->msg_flags & MSG_DONTWAIT, &ret);
 589                         if (!skb)
 590                                 goto maybe_error;
 591
 592                         rxrpc_new_skb(skb);
 593
 594                         _debug("ALLOC SEND %p", skb);
 595
 596                         ASSERTCMP(skb->mark, ==, 0);
 597
 598                         _debug("HS: %u", call->conn->header_size);
 599                         skb_reserve(skb, call->conn->header_size);
 600                         skb->len += call->conn->header_size;
 601
 602                         sp = rxrpc_skb(skb);
 603                         sp->remain = chunk;
 604                         if (sp->remain > skb_tailroom(skb))
 605                                 sp->remain = skb_tailroom(skb);
 606
 607                         _net("skb: hr %d, tr %d, hl %d, rm %d",
 608                                skb_headroom(skb),
 609                                skb_tailroom(skb),
 610                                skb_headlen(skb),
 611                                sp->remain);
 612
 613                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 614                 }
 615
 616                 _debug("append");
 617                 sp = rxrpc_skb(skb);
 618
 619                 /* append next segment of data to the current buffer */
 620                 if (msg_data_left(msg) > 0) {
 621                         int copy = skb_tailroom(skb);
 622                         ASSERTCMP(copy, >, 0);
 623                         if (copy > msg_data_left(msg))
 624                                 copy = msg_data_left(msg);
 625                         if (copy > sp->remain)
 626                                 copy = sp->remain;
 627
 628                         _debug("add");
 629                         ret = skb_add_data(skb, &msg->msg_iter, copy);
 630                         _debug("added");
 631                         if (ret < 0)
 632                                 goto efault;
 633                         sp->remain -= copy;
 634                         skb->mark += copy;
 635                         copied += copy;
 636                 }
 637
 638                 /* check for the far side aborting the call or a network error
 639                  * occurring */
 640                 if (call->state > RXRPC_CALL_COMPLETE)
 641                         goto call_aborted;
 642
 643                 /* add the packet to the send queue if it's now full */
 644                 if (sp->remain <= 0 ||
 645                     (msg_data_left(msg) == 0 && !more)) {
 646                         struct rxrpc_connection *conn = call->conn;
 647                         uint32_t seq;
 648                         size_t pad;
 649
 650                         /* pad out if we're using security */
 651                         if (conn->security_ix) {
 652                                 pad = conn->security_size + skb->mark;
 653                                 pad = conn->size_align - pad;
 654                                 pad &= conn->size_align - 1;
 655                                 _debug("pad %zu", pad);
 656                                 if (pad)
 657                                         memset(skb_put(skb, pad), 0, pad);
 658                         }
 659
 660                         seq = atomic_inc_return(&call->sequence);
 661
 662                         sp->hdr.epoch   = conn->proto.epoch;
 663                         sp->hdr.cid     = call->cid;
 664                         sp->hdr.callNumber = call->call_id;
 665                         sp->hdr.seq     = seq;
 666                         sp->hdr.serial  = atomic_inc_return(&conn->serial);
 667                         sp->hdr.type    = RXRPC_PACKET_TYPE_DATA;
 668                         sp->hdr.userStatus = 0;
 669                         sp->hdr.securityIndex = conn->security_ix;
 670                         sp->hdr._rsvd   = 0;
 671                         sp->hdr.serviceId = call->service_id;
 672
 673                         sp->hdr.flags = conn->out_clientflag;
 674                         if (msg_data_left(msg) == 0 && !more)
 675                                 sp->hdr.flags |= RXRPC_LAST_PACKET;
 676                         else if (CIRC_SPACE(call->acks_head,
 677                                             ACCESS_ONCE(call->acks_tail),
 678                                             call->acks_winsz) > 1)
 679                                 sp->hdr.flags |= RXRPC_MORE_PACKETS;
 680                         if (more && seq & 1)
 681                                 sp->hdr.flags |= RXRPC_REQUEST_ACK;
 682
 683                         ret = conn->security->secure_packet(
 684                                 call, skb, skb->mark,
 685                                 skb->head + sizeof(struct rxrpc_wire_header));
 686                         if (ret < 0)
 687                                 goto out;
 688
 689                         rxrpc_insert_header(skb);
 690                         rxrpc_queue_packet(call, skb, !msg_data_left(msg) && !more);
 691                         skb = NULL;
 692                 }
 693         } while (msg_data_left(msg) > 0);
 694
 695 success:
 696         ret = copied;
 697 out:
 698         call->tx_pending = skb;
 699         _leave(" = %d", ret);
 700         return ret;
 701
 702 call_aborted:
 703         rxrpc_free_skb(skb);
 704         if (call->state == RXRPC_CALL_NETWORK_ERROR)
 705                 ret = call->error_report < RXRPC_LOCAL_ERROR_OFFSET ?
 706                         call->error_report :
 707                         call->error_report - RXRPC_LOCAL_ERROR_OFFSET;
 708         else
 709                 ret = -ECONNABORTED;
 710         _leave(" = %d", ret);
 711         return ret;
 712
 713 maybe_error:
 714         if (copied)
 715                 goto success;
 716         goto out;
 717
 718 efault:
 719         ret = -EFAULT;
 720         goto out;
 721 }