ceph/src/spdk/dpdk/lib/librte_gro/gro_tcp4.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mbuf.h>
   7 #include <rte_cycles.h>
   8 #include <rte_ethdev.h>
   9
  10 #include "gro_tcp4.h"
  11
  12 void *
  13 gro_tcp4_tbl_create(uint16_t socket_id,
  14                 uint16_t max_flow_num,
  15                 uint16_t max_item_per_flow)
  16 {
  17         struct gro_tcp4_tbl *tbl;
  18         size_t size;
  19         uint32_t entries_num, i;
  20
  21         entries_num = max_flow_num * max_item_per_flow;
  22         entries_num = RTE_MIN(entries_num, GRO_TCP4_TBL_MAX_ITEM_NUM);
  23
  24         if (entries_num == 0)
  25                 return NULL;
  26
  27         tbl = rte_zmalloc_socket(__func__,
  28                         sizeof(struct gro_tcp4_tbl),
  29                         RTE_CACHE_LINE_SIZE,
  30                         socket_id);
  31         if (tbl == NULL)
  32                 return NULL;
  33
  34         size = sizeof(struct gro_tcp4_item) * entries_num;
  35         tbl->items = rte_zmalloc_socket(__func__,
  36                         size,
  37                         RTE_CACHE_LINE_SIZE,
  38                         socket_id);
  39         if (tbl->items == NULL) {
  40                 rte_free(tbl);
  41                 return NULL;
  42         }
  43         tbl->max_item_num = entries_num;
  44
  45         size = sizeof(struct gro_tcp4_flow) * entries_num;
  46         tbl->flows = rte_zmalloc_socket(__func__,
  47                         size,
  48                         RTE_CACHE_LINE_SIZE,
  49                         socket_id);
  50         if (tbl->flows == NULL) {
  51                 rte_free(tbl->items);
  52                 rte_free(tbl);
  53                 return NULL;
  54         }
  55         /* INVALID_ARRAY_INDEX indicates an empty flow */
  56         for (i = 0; i < entries_num; i++)
  57                 tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
  58         tbl->max_flow_num = entries_num;
  59
  60         return tbl;
  61 }
  62
  63 void
  64 gro_tcp4_tbl_destroy(void *tbl)
  65 {
  66         struct gro_tcp4_tbl *tcp_tbl = tbl;
  67
  68         if (tcp_tbl) {
  69                 rte_free(tcp_tbl->items);
  70                 rte_free(tcp_tbl->flows);
  71         }
  72         rte_free(tcp_tbl);
  73 }
  74
  75 static inline uint32_t
  76 find_an_empty_item(struct gro_tcp4_tbl *tbl)
  77 {
  78         uint32_t i;
  79         uint32_t max_item_num = tbl->max_item_num;
  80
  81         for (i = 0; i < max_item_num; i++)
  82                 if (tbl->items[i].firstseg == NULL)
  83                         return i;
  84         return INVALID_ARRAY_INDEX;
  85 }
  86
  87 static inline uint32_t
  88 find_an_empty_flow(struct gro_tcp4_tbl *tbl)
  89 {
  90         uint32_t i;
  91         uint32_t max_flow_num = tbl->max_flow_num;
  92
  93         for (i = 0; i < max_flow_num; i++)
  94                 if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
  95                         return i;
  96         return INVALID_ARRAY_INDEX;
  97 }
  98
  99 static inline uint32_t
 100 insert_new_item(struct gro_tcp4_tbl *tbl,
 101                 struct rte_mbuf *pkt,
 102                 uint64_t start_time,
 103                 uint32_t prev_idx,
 104                 uint32_t sent_seq,
 105                 uint16_t ip_id,
 106                 uint8_t is_atomic)
 107 {
 108         uint32_t item_idx;
 109
 110         item_idx = find_an_empty_item(tbl);
 111         if (item_idx == INVALID_ARRAY_INDEX)
 112                 return INVALID_ARRAY_INDEX;
 113
 114         tbl->items[item_idx].firstseg = pkt;
 115         tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
 116         tbl->items[item_idx].start_time = start_time;
 117         tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
 118         tbl->items[item_idx].sent_seq = sent_seq;
 119         tbl->items[item_idx].ip_id = ip_id;
 120         tbl->items[item_idx].nb_merged = 1;
 121         tbl->items[item_idx].is_atomic = is_atomic;
 122         tbl->item_num++;
 123
 124         /* if the previous packet exists, chain them together. */
 125         if (prev_idx != INVALID_ARRAY_INDEX) {
 126                 tbl->items[item_idx].next_pkt_idx =
 127                         tbl->items[prev_idx].next_pkt_idx;
 128                 tbl->items[prev_idx].next_pkt_idx = item_idx;
 129         }
 130
 131         return item_idx;
 132 }
 133
 134 static inline uint32_t
 135 delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
 136                 uint32_t prev_item_idx)
 137 {
 138         uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
 139
 140         /* NULL indicates an empty item */
 141         tbl->items[item_idx].firstseg = NULL;
 142         tbl->item_num--;
 143         if (prev_item_idx != INVALID_ARRAY_INDEX)
 144                 tbl->items[prev_item_idx].next_pkt_idx = next_idx;
 145
 146         return next_idx;
 147 }
 148
 149 static inline uint32_t
 150 insert_new_flow(struct gro_tcp4_tbl *tbl,
 151                 struct tcp4_flow_key *src,
 152                 uint32_t item_idx)
 153 {
 154         struct tcp4_flow_key *dst;
 155         uint32_t flow_idx;
 156
 157         flow_idx = find_an_empty_flow(tbl);
 158         if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
 159                 return INVALID_ARRAY_INDEX;
 160
 161         dst = &(tbl->flows[flow_idx].key);
 162
 163         ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
 164         ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
 165         dst->ip_src_addr = src->ip_src_addr;
 166         dst->ip_dst_addr = src->ip_dst_addr;
 167         dst->recv_ack = src->recv_ack;
 168         dst->src_port = src->src_port;
 169         dst->dst_port = src->dst_port;
 170
 171         tbl->flows[flow_idx].start_index = item_idx;
 172         tbl->flow_num++;
 173
 174         return flow_idx;
 175 }
 176
 177 /*
 178  * update the packet length for the flushed packet.
 179  */
 180 static inline void
 181 update_header(struct gro_tcp4_item *item)
 182 {
 183         struct ipv4_hdr *ipv4_hdr;
 184         struct rte_mbuf *pkt = item->firstseg;
 185
 186         ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
 187                         pkt->l2_len);
 188         ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
 189                         pkt->l2_len);
 190 }
 191
 192 int32_t
 193 gro_tcp4_reassemble(struct rte_mbuf *pkt,
 194                 struct gro_tcp4_tbl *tbl,
 195                 uint64_t start_time)
 196 {
 197         struct ether_hdr *eth_hdr;
 198         struct ipv4_hdr *ipv4_hdr;
 199         struct tcp_hdr *tcp_hdr;
 200         uint32_t sent_seq;
 201         uint16_t tcp_dl, ip_id, hdr_len, frag_off;
 202         uint8_t is_atomic;
 203
 204         struct tcp4_flow_key key;
 205         uint32_t cur_idx, prev_idx, item_idx;
 206         uint32_t i, max_flow_num, remaining_flow_num;
 207         int cmp;
 208         uint8_t find;
 209
 210         eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
 211         ipv4_hdr = (struct ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
 212         tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
 213         hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
 214
 215         /*
 216          * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
 217          * or CWR set.
 218          */
 219         if (tcp_hdr->tcp_flags != TCP_ACK_FLAG)
 220                 return -1;
 221         /*
 222          * Don't process the packet whose payload length is less than or
 223          * equal to 0.
 224          */
 225         tcp_dl = pkt->pkt_len - hdr_len;
 226         if (tcp_dl <= 0)
 227                 return -1;
 228
 229         /*
 230          * Save IPv4 ID for the packet whose DF bit is 0. For the packet
 231          * whose DF bit is 1, IPv4 ID is ignored.
 232          */
 233         frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
 234         is_atomic = (frag_off & IPV4_HDR_DF_FLAG) == IPV4_HDR_DF_FLAG;
 235         ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
 236         sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
 237
 238         ether_addr_copy(&(eth_hdr->s_addr), &(key.eth_saddr));
 239         ether_addr_copy(&(eth_hdr->d_addr), &(key.eth_daddr));
 240         key.ip_src_addr = ipv4_hdr->src_addr;
 241         key.ip_dst_addr = ipv4_hdr->dst_addr;
 242         key.src_port = tcp_hdr->src_port;
 243         key.dst_port = tcp_hdr->dst_port;
 244         key.recv_ack = tcp_hdr->recv_ack;
 245
 246         /* Search for a matched flow. */
 247         max_flow_num = tbl->max_flow_num;
 248         remaining_flow_num = tbl->flow_num;
 249         find = 0;
 250         for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
 251                 if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
 252                         if (is_same_tcp4_flow(tbl->flows[i].key, key)) {
 253                                 find = 1;
 254                                 break;
 255                         }
 256                         remaining_flow_num--;
 257                 }
 258         }
 259
 260         /*
 261          * Fail to find a matched flow. Insert a new flow and store the
 262          * packet into the flow.
 263          */
 264         if (find == 0) {
 265                 item_idx = insert_new_item(tbl, pkt, start_time,
 266                                 INVALID_ARRAY_INDEX, sent_seq, ip_id,
 267                                 is_atomic);
 268                 if (item_idx == INVALID_ARRAY_INDEX)
 269                         return -1;
 270                 if (insert_new_flow(tbl, &key, item_idx) ==
 271                                 INVALID_ARRAY_INDEX) {
 272                         /*
 273                          * Fail to insert a new flow, so delete the
 274                          * stored packet.
 275                          */
 276                         delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
 277                         return -1;
 278                 }
 279                 return 0;
 280         }
 281
 282         /*
 283          * Check all packets in the flow and try to find a neighbor for
 284          * the input packet.
 285          */
 286         cur_idx = tbl->flows[i].start_index;
 287         prev_idx = cur_idx;
 288         do {
 289                 cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
 290                                 sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
 291                                 is_atomic);
 292                 if (cmp) {
 293                         if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
 294                                                 pkt, cmp, sent_seq, ip_id, 0))
 295                                 return 1;
 296                         /*
 297                          * Fail to merge the two packets, as the packet
 298                          * length is greater than the max value. Store
 299                          * the packet into the flow.
 300                          */
 301                         if (insert_new_item(tbl, pkt, start_time, prev_idx,
 302                                                 sent_seq, ip_id, is_atomic) ==
 303                                         INVALID_ARRAY_INDEX)
 304                                 return -1;
 305                         return 0;
 306                 }
 307                 prev_idx = cur_idx;
 308                 cur_idx = tbl->items[cur_idx].next_pkt_idx;
 309         } while (cur_idx != INVALID_ARRAY_INDEX);
 310
 311         /* Fail to find a neighbor, so store the packet into the flow. */
 312         if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
 313                                 ip_id, is_atomic) == INVALID_ARRAY_INDEX)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 uint16_t
 320 gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
 321                 uint64_t flush_timestamp,
 322                 struct rte_mbuf **out,
 323                 uint16_t nb_out)
 324 {
 325         uint16_t k = 0;
 326         uint32_t i, j;
 327         uint32_t max_flow_num = tbl->max_flow_num;
 328
 329         for (i = 0; i < max_flow_num; i++) {
 330                 if (unlikely(tbl->flow_num == 0))
 331                         return k;
 332
 333                 j = tbl->flows[i].start_index;
 334                 while (j != INVALID_ARRAY_INDEX) {
 335                         if (tbl->items[j].start_time <= flush_timestamp) {
 336                                 out[k++] = tbl->items[j].firstseg;
 337                                 if (tbl->items[j].nb_merged > 1)
 338                                         update_header(&(tbl->items[j]));
 339                                 /*
 340                                  * Delete the packet and get the next
 341                                  * packet in the flow.
 342                                  */
 343                                 j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
 344                                 tbl->flows[i].start_index = j;
 345                                 if (j == INVALID_ARRAY_INDEX)
 346                                         tbl->flow_num--;
 347
 348                                 if (unlikely(k == nb_out))
 349                                         return k;
 350                         } else
 351                                 /*
 352                                  * The left packets in this flow won't be
 353                                  * timeout. Go to check other flows.
 354                                  */
 355                                 break;
 356                 }
 357         }
 358         return k;
 359 }
 360
 361 uint32_t
 362 gro_tcp4_tbl_pkt_count(void *tbl)
 363 {
 364         struct gro_tcp4_tbl *gro_tbl = tbl;
 365
 366         if (gro_tbl)
 367                 return gro_tbl->item_num;
 368
 369         return 0;
 370 }