include/linux/skmsg.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
   3
   4 #ifndef _LINUX_SKMSG_H
   5 #define _LINUX_SKMSG_H
   6
   7 #include <linux/bpf.h>
   8 #include <linux/filter.h>
   9 #include <linux/scatterlist.h>
  10 #include <linux/skbuff.h>
  11
  12 #include <net/sock.h>
  13 #include <net/tcp.h>
  14 #include <net/strparser.h>
  15
  16 #define MAX_MSG_FRAGS                   MAX_SKB_FRAGS
  17
  18 enum __sk_action {
  19         __SK_DROP = 0,
  20         __SK_PASS,
  21         __SK_REDIRECT,
  22         __SK_NONE,
  23 };
  24
  25 struct sk_msg_sg {
  26         u32                             start;
  27         u32                             curr;
  28         u32                             end;
  29         u32                             size;
  30         u32                             copybreak;
  31         bool                            copy[MAX_MSG_FRAGS];
  32         /* The extra element is used for chaining the front and sections when
  33          * the list becomes partitioned (e.g. end < start). The crypto APIs
  34          * require the chaining.
  35          */
  36         struct scatterlist              data[MAX_MSG_FRAGS + 1];
  37 };
  38
  39 struct sk_msg {
  40         struct sk_msg_sg                sg;
  41         void                            *data;
  42         void                            *data_end;
  43         u32                             apply_bytes;
  44         u32                             cork_bytes;
  45         u32                             flags;
  46         struct sk_buff                  *skb;
  47         struct sock                     *sk_redir;
  48         struct sock                     *sk;
  49         struct list_head                list;
  50 };
  51
  52 struct sk_psock_progs {
  53         struct bpf_prog                 *msg_parser;
  54         struct bpf_prog                 *skb_parser;
  55         struct bpf_prog                 *skb_verdict;
  56 };
  57
  58 enum sk_psock_state_bits {
  59         SK_PSOCK_TX_ENABLED,
  60 };
  61
  62 struct sk_psock_link {
  63         struct list_head                list;
  64         struct bpf_map                  *map;
  65         void                            *link_raw;
  66 };
  67
  68 struct sk_psock_parser {
  69         struct strparser                strp;
  70         bool                            enabled;
  71         void (*saved_data_ready)(struct sock *sk);
  72 };
  73
  74 struct sk_psock_work_state {
  75         struct sk_buff                  *skb;
  76         u32                             len;
  77         u32                             off;
  78 };
  79
  80 struct sk_psock {
  81         struct sock                     *sk;
  82         struct sock                     *sk_redir;
  83         u32                             apply_bytes;
  84         u32                             cork_bytes;
  85         u32                             eval;
  86         struct sk_msg                   *cork;
  87         struct sk_psock_progs           progs;
  88         struct sk_psock_parser          parser;
  89         struct sk_buff_head             ingress_skb;
  90         struct list_head                ingress_msg;
  91         unsigned long                   state;
  92         struct list_head                link;
  93         spinlock_t                      link_lock;
  94         refcount_t                      refcnt;
  95         void (*saved_unhash)(struct sock *sk);
  96         void (*saved_close)(struct sock *sk, long timeout);
  97         void (*saved_write_space)(struct sock *sk);
  98         struct proto                    *sk_proto;
  99         struct sk_psock_work_state      work_state;
 100         struct work_struct              work;
 101         union {
 102                 struct rcu_head         rcu;
 103                 struct work_struct      gc;
 104         };
 105 };
 106
 107 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
 108                  int elem_first_coalesce);
 109 int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
 110                  u32 off, u32 len);
 111 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
 112 int sk_msg_free(struct sock *sk, struct sk_msg *msg);
 113 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
 114 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes);
 115 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
 116                                   u32 bytes);
 117
 118 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes);
 119 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes);
 120
 121 int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
 122                               struct sk_msg *msg, u32 bytes);
 123 int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
 124                              struct sk_msg *msg, u32 bytes);
 125
 126 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)
 127 {
 128         WARN_ON(i == msg->sg.end && bytes);
 129 }
 130
 131 static inline void sk_msg_apply_bytes(struct sk_psock *psock, u32 bytes)
 132 {
 133         if (psock->apply_bytes) {
 134                 if (psock->apply_bytes < bytes)
 135                         psock->apply_bytes = 0;
 136                 else
 137                         psock->apply_bytes -= bytes;
 138         }
 139 }
 140
 141 #define sk_msg_iter_var_prev(var)                       \
 142         do {                                            \
 143                 if (var == 0)                           \
 144                         var = MAX_MSG_FRAGS - 1;        \
 145                 else                                    \
 146                         var--;                          \
 147         } while (0)
 148
 149 #define sk_msg_iter_var_next(var)                       \
 150         do {                                            \
 151                 var++;                                  \
 152                 if (var == MAX_MSG_FRAGS)               \
 153                         var = 0;                        \
 154         } while (0)
 155
 156 #define sk_msg_iter_prev(msg, which)                    \
 157         sk_msg_iter_var_prev(msg->sg.which)
 158
 159 #define sk_msg_iter_next(msg, which)                    \
 160         sk_msg_iter_var_next(msg->sg.which)
 161
 162 static inline void sk_msg_clear_meta(struct sk_msg *msg)
 163 {
 164         memset(&msg->sg, 0, offsetofend(struct sk_msg_sg, copy));
 165 }
 166
 167 static inline void sk_msg_init(struct sk_msg *msg)
 168 {
 169         BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != MAX_MSG_FRAGS);
 170         memset(msg, 0, sizeof(*msg));
 171         sg_init_marker(msg->sg.data, MAX_MSG_FRAGS);
 172 }
 173
 174 static inline void sk_msg_xfer(struct sk_msg *dst, struct sk_msg *src,
 175                                int which, u32 size)
 176 {
 177         dst->sg.data[which] = src->sg.data[which];
 178         dst->sg.data[which].length  = size;
 179         dst->sg.size               += size;
 180         src->sg.data[which].length -= size;
 181         src->sg.data[which].offset += size;
 182 }
 183
 184 static inline void sk_msg_xfer_full(struct sk_msg *dst, struct sk_msg *src)
 185 {
 186         memcpy(dst, src, sizeof(*src));
 187         sk_msg_init(src);
 188 }
 189
 190 static inline bool sk_msg_full(const struct sk_msg *msg)
 191 {
 192         return (msg->sg.end == msg->sg.start) && msg->sg.size;
 193 }
 194
 195 static inline u32 sk_msg_elem_used(const struct sk_msg *msg)
 196 {
 197         if (sk_msg_full(msg))
 198                 return MAX_MSG_FRAGS;
 199
 200         return msg->sg.end >= msg->sg.start ?
 201                 msg->sg.end - msg->sg.start :
 202                 msg->sg.end + (MAX_MSG_FRAGS - msg->sg.start);
 203 }
 204
 205 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which)
 206 {
 207         return &msg->sg.data[which];
 208 }
 209
 210 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which)
 211 {
 212         return msg->sg.data[which];
 213 }
 214
 215 static inline struct page *sk_msg_page(struct sk_msg *msg, int which)
 216 {
 217         return sg_page(sk_msg_elem(msg, which));
 218 }
 219
 220 static inline bool sk_msg_to_ingress(const struct sk_msg *msg)
 221 {
 222         return msg->flags & BPF_F_INGRESS;
 223 }
 224
 225 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg)
 226 {
 227         struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start);
 228
 229         if (msg->sg.copy[msg->sg.start]) {
 230                 msg->data = NULL;
 231                 msg->data_end = NULL;
 232         } else {
 233                 msg->data = sg_virt(sge);
 234                 msg->data_end = msg->data + sge->length;
 235         }
 236 }
 237
 238 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page,
 239                                    u32 len, u32 offset)
 240 {
 241         struct scatterlist *sge;
 242
 243         get_page(page);
 244         sge = sk_msg_elem(msg, msg->sg.end);
 245         sg_set_page(sge, page, len, offset);
 246         sg_unmark_end(sge);
 247
 248         msg->sg.copy[msg->sg.end] = true;
 249         msg->sg.size += len;
 250         sk_msg_iter_next(msg, end);
 251 }
 252
 253 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state)
 254 {
 255         do {
 256                 msg->sg.copy[i] = copy_state;
 257                 sk_msg_iter_var_next(i);
 258                 if (i == msg->sg.end)
 259                         break;
 260         } while (1);
 261 }
 262
 263 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start)
 264 {
 265         sk_msg_sg_copy(msg, start, true);
 266 }
 267
 268 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start)
 269 {
 270         sk_msg_sg_copy(msg, start, false);
 271 }
 272
 273 static inline struct sk_psock *sk_psock(const struct sock *sk)
 274 {
 275         return rcu_dereference_sk_user_data(sk);
 276 }
 277
 278 static inline void sk_psock_queue_msg(struct sk_psock *psock,
 279                                       struct sk_msg *msg)
 280 {
 281         list_add_tail(&msg->list, &psock->ingress_msg);
 282 }
 283
 284 static inline bool sk_psock_queue_empty(const struct sk_psock *psock)
 285 {
 286         return psock ? list_empty(&psock->ingress_msg) : true;
 287 }
 288
 289 static inline void sk_psock_report_error(struct sk_psock *psock, int err)
 290 {
 291         struct sock *sk = psock->sk;
 292
 293         sk->sk_err = err;
 294         sk->sk_error_report(sk);
 295 }
 296
 297 struct sk_psock *sk_psock_init(struct sock *sk, int node);
 298
 299 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock);
 300 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock);
 301 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock);
 302
 303 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
 304                          struct sk_msg *msg);
 305
 306 static inline struct sk_psock_link *sk_psock_init_link(void)
 307 {
 308         return kzalloc(sizeof(struct sk_psock_link),
 309                        GFP_ATOMIC | __GFP_NOWARN);
 310 }
 311
 312 static inline void sk_psock_free_link(struct sk_psock_link *link)
 313 {
 314         kfree(link);
 315 }
 316
 317 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock);
 318 #if defined(CONFIG_BPF_STREAM_PARSER)
 319 void sk_psock_unlink(struct sock *sk, struct sk_psock_link *link);
 320 #else
 321 static inline void sk_psock_unlink(struct sock *sk,
 322                                    struct sk_psock_link *link)
 323 {
 324 }
 325 #endif
 326
 327 void __sk_psock_purge_ingress_msg(struct sk_psock *psock);
 328
 329 static inline void sk_psock_cork_free(struct sk_psock *psock)
 330 {
 331         if (psock->cork) {
 332                 sk_msg_free(psock->sk, psock->cork);
 333                 kfree(psock->cork);
 334                 psock->cork = NULL;
 335         }
 336 }
 337
 338 static inline void sk_psock_update_proto(struct sock *sk,
 339                                          struct sk_psock *psock,
 340                                          struct proto *ops)
 341 {
 342         psock->saved_unhash = sk->sk_prot->unhash;
 343         psock->saved_close = sk->sk_prot->close;
 344         psock->saved_write_space = sk->sk_write_space;
 345
 346         psock->sk_proto = sk->sk_prot;
 347         sk->sk_prot = ops;
 348 }
 349
 350 static inline void sk_psock_restore_proto(struct sock *sk,
 351                                           struct sk_psock *psock)
 352 {
 353         if (psock->sk_proto) {
 354                 sk->sk_prot = psock->sk_proto;
 355                 psock->sk_proto = NULL;
 356         }
 357 }
 358
 359 static inline void sk_psock_set_state(struct sk_psock *psock,
 360                                       enum sk_psock_state_bits bit)
 361 {
 362         set_bit(bit, &psock->state);
 363 }
 364
 365 static inline void sk_psock_clear_state(struct sk_psock *psock,
 366                                         enum sk_psock_state_bits bit)
 367 {
 368         clear_bit(bit, &psock->state);
 369 }
 370
 371 static inline bool sk_psock_test_state(const struct sk_psock *psock,
 372                                        enum sk_psock_state_bits bit)
 373 {
 374         return test_bit(bit, &psock->state);
 375 }
 376
 377 static inline struct sk_psock *sk_psock_get_checked(struct sock *sk)
 378 {
 379         struct sk_psock *psock;
 380
 381         rcu_read_lock();
 382         psock = sk_psock(sk);
 383         if (psock) {
 384                 if (sk->sk_prot->recvmsg != tcp_bpf_recvmsg) {
 385                         psock = ERR_PTR(-EBUSY);
 386                         goto out;
 387                 }
 388
 389                 if (!refcount_inc_not_zero(&psock->refcnt))
 390                         psock = ERR_PTR(-EBUSY);
 391         }
 392 out:
 393         rcu_read_unlock();
 394         return psock;
 395 }
 396
 397 static inline struct sk_psock *sk_psock_get(struct sock *sk)
 398 {
 399         struct sk_psock *psock;
 400
 401         rcu_read_lock();
 402         psock = sk_psock(sk);
 403         if (psock && !refcount_inc_not_zero(&psock->refcnt))
 404                 psock = NULL;
 405         rcu_read_unlock();
 406         return psock;
 407 }
 408
 409 void sk_psock_stop(struct sock *sk, struct sk_psock *psock);
 410 void sk_psock_destroy(struct rcu_head *rcu);
 411 void sk_psock_drop(struct sock *sk, struct sk_psock *psock);
 412
 413 static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock)
 414 {
 415         if (refcount_dec_and_test(&psock->refcnt))
 416                 sk_psock_drop(sk, psock);
 417 }
 418
 419 static inline void psock_set_prog(struct bpf_prog **pprog,
 420                                   struct bpf_prog *prog)
 421 {
 422         prog = xchg(pprog, prog);
 423         if (prog)
 424                 bpf_prog_put(prog);
 425 }
 426
 427 static inline void psock_progs_drop(struct sk_psock_progs *progs)
 428 {
 429         psock_set_prog(&progs->msg_parser, NULL);
 430         psock_set_prog(&progs->skb_parser, NULL);
 431         psock_set_prog(&progs->skb_verdict, NULL);
 432 }
 433
 434 #endif /* _LINUX_SKMSG_H */