1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
5 #include <sys/socket.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
21 #include <sys/resource.h>
22 #include <sys/types.h>
23 #include <sys/sendfile.h>
25 #include <linux/netlink.h>
26 #include <linux/socket.h>
27 #include <linux/sock_diag.h>
28 #include <linux/bpf.h>
29 #include <linux/if_link.h>
30 #include <linux/tls.h>
37 #include <bpf/libbpf.h>
40 #include "bpf_rlimit.h"
41 #include "cgroup_helpers.h"
44 static void running_handler(int a
);
53 /* randomly selected ports for testing on lo */
57 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
58 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
59 #define CG_PATH "/sockmap"
62 int s1
, s2
, c1
, c2
, p1
, p2
;
67 struct bpf_map
*maps
[8];
73 int txmsg_redir_noisy
;
88 static const struct option long_options
[] = {
89 {"help", no_argument
, NULL
, 'h' },
90 {"cgroup", required_argument
, NULL
, 'c' },
91 {"rate", required_argument
, NULL
, 'r' },
92 {"verbose", no_argument
, NULL
, 'v' },
93 {"iov_count", required_argument
, NULL
, 'i' },
94 {"length", required_argument
, NULL
, 'l' },
95 {"test", required_argument
, NULL
, 't' },
96 {"data_test", no_argument
, NULL
, 'd' },
97 {"txmsg", no_argument
, &txmsg_pass
, 1 },
98 {"txmsg_noisy", no_argument
, &txmsg_noisy
, 1 },
99 {"txmsg_redir", no_argument
, &txmsg_redir
, 1 },
100 {"txmsg_redir_noisy", no_argument
, &txmsg_redir_noisy
, 1},
101 {"txmsg_drop", no_argument
, &txmsg_drop
, 1 },
102 {"txmsg_apply", required_argument
, NULL
, 'a'},
103 {"txmsg_cork", required_argument
, NULL
, 'k'},
104 {"txmsg_start", required_argument
, NULL
, 's'},
105 {"txmsg_end", required_argument
, NULL
, 'e'},
106 {"txmsg_start_push", required_argument
, NULL
, 'p'},
107 {"txmsg_end_push", required_argument
, NULL
, 'q'},
108 {"txmsg_start_pop", required_argument
, NULL
, 'w'},
109 {"txmsg_pop", required_argument
, NULL
, 'x'},
110 {"txmsg_ingress", no_argument
, &txmsg_ingress
, 1 },
111 {"txmsg_skb", no_argument
, &txmsg_skb
, 1 },
112 {"ktls", no_argument
, &ktls
, 1 },
113 {"peek", no_argument
, &peek_flag
, 1 },
117 static void usage(char *argv
[])
121 printf(" Usage: %s --cgroup <cgroup_path>\n", argv
[0]);
122 printf(" options:\n");
123 for (i
= 0; long_options
[i
].name
!= 0; i
++) {
124 printf(" --%-12s", long_options
[i
].name
);
125 if (long_options
[i
].flag
!= NULL
)
126 printf(" flag (internal value:%d)\n",
127 *long_options
[i
].flag
);
129 printf(" -%c\n", long_options
[i
].val
);
134 char *sock_to_string(int s
)
152 static int sockmap_init_ktls(int verbose
, int s
)
154 struct tls12_crypto_info_aes_gcm_128 tls_tx
= {
156 .version
= TLS_1_2_VERSION
,
157 .cipher_type
= TLS_CIPHER_AES_GCM_128
,
160 struct tls12_crypto_info_aes_gcm_128 tls_rx
= {
162 .version
= TLS_1_2_VERSION
,
163 .cipher_type
= TLS_CIPHER_AES_GCM_128
,
166 int so_buf
= 6553500;
169 err
= setsockopt(s
, 6, TCP_ULP
, "tls", sizeof("tls"));
171 fprintf(stderr
, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s
), err
);
174 err
= setsockopt(s
, SOL_TLS
, TLS_TX
, (void *)&tls_tx
, sizeof(tls_tx
));
176 fprintf(stderr
, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s
), err
);
179 err
= setsockopt(s
, SOL_TLS
, TLS_RX
, (void *)&tls_rx
, sizeof(tls_rx
));
181 fprintf(stderr
, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s
), err
);
184 err
= setsockopt(s
, SOL_SOCKET
, SO_SNDBUF
, &so_buf
, sizeof(so_buf
));
186 fprintf(stderr
, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s
), err
);
189 err
= setsockopt(s
, SOL_SOCKET
, SO_RCVBUF
, &so_buf
, sizeof(so_buf
));
191 fprintf(stderr
, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s
), err
);
196 fprintf(stdout
, "socket(%s) kTLS enabled\n", sock_to_string(s
));
199 static int sockmap_init_sockets(int verbose
)
202 struct sockaddr_in addr
;
203 int *fds
[4] = {&s1
, &s2
, &c1
, &c2
};
205 s1
= s2
= p1
= p2
= c1
= c2
= 0;
208 for (i
= 0; i
< 4; i
++) {
209 *fds
[i
] = socket(AF_INET
, SOCK_STREAM
, 0);
211 perror("socket s1 failed()");
217 for (i
= 0; i
< 2; i
++) {
218 err
= setsockopt(*fds
[i
], SOL_SOCKET
, SO_REUSEADDR
,
219 (char *)&one
, sizeof(one
));
221 perror("setsockopt failed()");
226 /* Non-blocking sockets */
227 for (i
= 0; i
< 2; i
++) {
228 err
= ioctl(*fds
[i
], FIONBIO
, (char *)&one
);
230 perror("ioctl s1 failed()");
235 /* Bind server sockets */
236 memset(&addr
, 0, sizeof(struct sockaddr_in
));
237 addr
.sin_family
= AF_INET
;
238 addr
.sin_addr
.s_addr
= inet_addr("127.0.0.1");
240 addr
.sin_port
= htons(S1_PORT
);
241 err
= bind(s1
, (struct sockaddr
*)&addr
, sizeof(addr
));
243 perror("bind s1 failed()");
247 addr
.sin_port
= htons(S2_PORT
);
248 err
= bind(s2
, (struct sockaddr
*)&addr
, sizeof(addr
));
250 perror("bind s2 failed()");
254 /* Listen server sockets */
255 addr
.sin_port
= htons(S1_PORT
);
256 err
= listen(s1
, 32);
258 perror("listen s1 failed()");
262 addr
.sin_port
= htons(S2_PORT
);
263 err
= listen(s2
, 32);
265 perror("listen s1 failed()");
269 /* Initiate Connect */
270 addr
.sin_port
= htons(S1_PORT
);
271 err
= connect(c1
, (struct sockaddr
*)&addr
, sizeof(addr
));
272 if (err
< 0 && errno
!= EINPROGRESS
) {
273 perror("connect c1 failed()");
277 addr
.sin_port
= htons(S2_PORT
);
278 err
= connect(c2
, (struct sockaddr
*)&addr
, sizeof(addr
));
279 if (err
< 0 && errno
!= EINPROGRESS
) {
280 perror("connect c2 failed()");
282 } else if (err
< 0) {
286 /* Accept Connecrtions */
287 p1
= accept(s1
, NULL
, NULL
);
289 perror("accept s1 failed()");
293 p2
= accept(s2
, NULL
, NULL
);
295 perror("accept s1 failed()");
300 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
301 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
310 struct timespec start
;
314 struct sockmap_options
{
325 static int msg_loop_sendpage(int fd
, int iov_length
, int cnt
,
327 struct sockmap_options
*opt
)
329 bool drop
= opt
->drop_expected
;
334 file
= fopen(".sendpage_tst.tmp", "w+");
336 perror("create file for sendpage");
339 for (i
= 0; i
< iov_length
* cnt
; i
++, k
++)
340 fwrite(&k
, sizeof(char), 1, file
);
342 fseek(file
, 0, SEEK_SET
);
345 fp
= open(".sendpage_tst.tmp", O_RDONLY
);
347 perror("reopen file for sendpage");
351 clock_gettime(CLOCK_MONOTONIC
, &s
->start
);
352 for (i
= 0; i
< cnt
; i
++) {
353 int sent
= sendfile(fd
, fp
, NULL
, iov_length
);
355 if (!drop
&& sent
< 0) {
356 perror("send loop error");
359 } else if (drop
&& sent
>= 0) {
360 printf("sendpage loop error expected: %i\n", sent
);
366 s
->bytes_sent
+= sent
;
368 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
373 static void msg_free_iov(struct msghdr
*msg
)
377 for (i
= 0; i
< msg
->msg_iovlen
; i
++)
378 free(msg
->msg_iov
[i
].iov_base
);
384 static int msg_alloc_iov(struct msghdr
*msg
,
385 int iov_count
, int iov_length
,
386 bool data
, bool xmit
)
392 iov
= calloc(iov_count
, sizeof(struct iovec
));
396 for (i
= 0; i
< iov_count
; i
++) {
397 unsigned char *d
= calloc(iov_length
, sizeof(char));
400 fprintf(stderr
, "iov_count %i/%i OOM\n", i
, iov_count
);
404 iov
[i
].iov_len
= iov_length
;
409 for (j
= 0; j
< iov_length
; j
++)
415 msg
->msg_iovlen
= iov_count
;
419 for (i
--; i
>= 0 ; i
--)
420 free(msg
->msg_iov
[i
].iov_base
);
424 static int msg_verify_data(struct msghdr
*msg
, int size
, int chunk_sz
)
426 int i
, j
, bytes_cnt
= 0;
429 for (i
= 0; i
< msg
->msg_iovlen
; i
++) {
430 unsigned char *d
= msg
->msg_iov
[i
].iov_base
;
433 j
< msg
->msg_iov
[i
].iov_len
&& size
; j
++) {
436 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
437 i
, j
, d
[j
], k
- 1, d
[j
+1], k
);
441 if (bytes_cnt
== chunk_sz
) {
451 static int msg_loop(int fd
, int iov_count
, int iov_length
, int cnt
,
452 struct msg_stats
*s
, bool tx
,
453 struct sockmap_options
*opt
)
455 struct msghdr msg
= {0}, msg_peek
= {0};
456 int err
, i
, flags
= MSG_NOSIGNAL
;
457 bool drop
= opt
->drop_expected
;
458 bool data
= opt
->data_test
;
460 err
= msg_alloc_iov(&msg
, iov_count
, iov_length
, data
, tx
);
464 err
= msg_alloc_iov(&msg_peek
, iov_count
, iov_length
, data
, tx
);
470 clock_gettime(CLOCK_MONOTONIC
, &s
->start
);
471 for (i
= 0; i
< cnt
; i
++) {
472 int sent
= sendmsg(fd
, &msg
, flags
);
474 if (!drop
&& sent
< 0) {
475 perror("send loop error");
477 } else if (drop
&& sent
>= 0) {
478 printf("send loop error expected: %i\n", sent
);
483 s
->bytes_sent
+= sent
;
485 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
487 int slct
, recvp
= 0, recv
, max_fd
= fd
;
488 float total_bytes
, txmsg_pop_total
;
489 int fd_flags
= O_NONBLOCK
;
490 struct timeval timeout
;
494 /* Account for pop bytes noting each iteration of apply will
495 * call msg_pop_data helper so we need to account for this
496 * by calculating the number of apply iterations. Note user
497 * of the tool can create cases where no data is sent by
498 * manipulating pop/push/pull/etc. For example txmsg_apply 1
499 * with txmsg_pop 1 will try to apply 1B at a time but each
500 * iteration will then pop 1B so no data will ever be sent.
501 * This is really only useful for testing edge cases in code
504 total_bytes
= (float)iov_count
* (float)iov_length
* (float)cnt
;
505 txmsg_pop_total
= txmsg_pop
;
507 txmsg_pop_total
*= (total_bytes
/ txmsg_apply
);
508 total_bytes
-= txmsg_pop_total
;
509 err
= clock_gettime(CLOCK_MONOTONIC
, &s
->start
);
511 perror("recv start time");
512 while (s
->bytes_recvd
< total_bytes
) {
515 timeout
.tv_usec
= 300000;
525 slct
= select(max_fd
+ 1, &w
, NULL
, NULL
, &timeout
);
528 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
532 fprintf(stderr
, "unexpected timeout: recved %zu/%f pop_total %f\n", s
->bytes_recvd
, total_bytes
, txmsg_pop_total
);
534 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
541 recvp
= recvmsg(fd
, &msg_peek
, flags
);
543 if (errno
!= EWOULDBLOCK
) {
544 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
551 recv
= recvmsg(fd
, &msg
, flags
);
553 if (errno
!= EWOULDBLOCK
) {
554 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
555 perror("recv failed()");
560 s
->bytes_recvd
+= recv
;
563 int chunk_sz
= opt
->sendpage
?
565 iov_length
* iov_count
;
567 errno
= msg_verify_data(&msg
, recv
, chunk_sz
);
569 perror("data verify msg failed");
573 errno
= msg_verify_data(&msg_peek
,
577 perror("data verify msg_peek failed");
583 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
587 msg_free_iov(&msg_peek
);
591 msg_free_iov(&msg_peek
);
595 static float giga
= 1000000000;
597 static inline float sentBps(struct msg_stats s
)
599 return s
.bytes_sent
/ (s
.end
.tv_sec
- s
.start
.tv_sec
);
602 static inline float recvdBps(struct msg_stats s
)
604 return s
.bytes_recvd
/ (s
.end
.tv_sec
- s
.start
.tv_sec
);
607 static int sendmsg_test(struct sockmap_options
*opt
)
609 float sent_Bps
= 0, recvd_Bps
= 0;
610 int rx_fd
, txpid
, rxpid
, err
= 0;
611 struct msg_stats s
= {0};
612 int iov_count
= opt
->iov_count
;
613 int iov_buf
= opt
->iov_length
;
614 int rx_status
, tx_status
;
625 /* Redirecting into non-TLS socket which sends into a TLS
626 * socket is not a valid test. So in this case lets not
627 * enable kTLS but still run the test.
629 if (!txmsg_redir
|| (txmsg_redir
&& txmsg_ingress
)) {
630 err
= sockmap_init_ktls(opt
->verbose
, rx_fd
);
634 err
= sockmap_init_ktls(opt
->verbose
, c1
);
641 if (opt
->drop_expected
)
646 err
= msg_loop(rx_fd
, iov_count
, iov_buf
,
647 cnt
, &s
, false, opt
);
650 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
651 iov_count
, iov_buf
, cnt
, err
);
652 if (s
.end
.tv_sec
- s
.start
.tv_sec
) {
653 sent_Bps
= sentBps(s
);
654 recvd_Bps
= recvdBps(s
);
658 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
659 s
.bytes_sent
, sent_Bps
, sent_Bps
/giga
,
660 s
.bytes_recvd
, recvd_Bps
, recvd_Bps
/giga
,
661 peek_flag
? "(peek_msg)" : "");
662 if (err
&& txmsg_cork
)
665 } else if (rxpid
== -1) {
666 perror("msg_loop_rx");
673 err
= msg_loop_sendpage(c1
, iov_buf
, cnt
, &s
, opt
);
675 err
= msg_loop(c1
, iov_count
, iov_buf
,
680 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
681 iov_count
, iov_buf
, cnt
, err
);
682 if (s
.end
.tv_sec
- s
.start
.tv_sec
) {
683 sent_Bps
= sentBps(s
);
684 recvd_Bps
= recvdBps(s
);
688 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
689 s
.bytes_sent
, sent_Bps
, sent_Bps
/giga
,
690 s
.bytes_recvd
, recvd_Bps
, recvd_Bps
/giga
);
692 } else if (txpid
== -1) {
693 perror("msg_loop_tx");
697 assert(waitpid(rxpid
, &rx_status
, 0) == rxpid
);
698 assert(waitpid(txpid
, &tx_status
, 0) == txpid
);
699 if (WIFEXITED(rx_status
)) {
700 err
= WEXITSTATUS(rx_status
);
702 fprintf(stderr
, "rx thread exited with err %d. ", err
);
706 if (WIFEXITED(tx_status
)) {
707 err
= WEXITSTATUS(tx_status
);
709 fprintf(stderr
, "tx thread exited with err %d. ", err
);
715 static int forever_ping_pong(int rate
, struct sockmap_options
*opt
)
717 struct timeval timeout
;
718 char buf
[1024] = {0};
724 /* Ping/Pong data from client to server */
725 sc
= send(c1
, buf
, sizeof(buf
), 0);
727 perror("send failed()");
732 int s
, rc
, i
, max_fd
= p2
;
742 s
= select(max_fd
+ 1, &w
, NULL
, NULL
, &timeout
);
747 fprintf(stderr
, "unexpected timeout\n");
751 for (i
= 0; i
<= max_fd
&& s
> 0; ++i
) {
752 if (!FD_ISSET(i
, &w
))
757 rc
= recv(i
, buf
, sizeof(buf
), 0);
759 if (errno
!= EWOULDBLOCK
) {
760 perror("recv failed()");
770 sc
= send(i
, buf
, rc
, 0);
772 perror("send failed()");
798 static int run_options(struct sockmap_options
*options
, int cg_fd
, int test
)
800 int i
, key
, next_key
, err
, tx_prog_fd
= -1, zero
= 0;
802 /* If base test skip BPF setup */
803 if (test
== BASE
|| test
== BASE_SENDPAGE
)
806 /* Attach programs to sockmap */
807 err
= bpf_prog_attach(prog_fd
[0], map_fd
[0],
808 BPF_SK_SKB_STREAM_PARSER
, 0);
811 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
812 prog_fd
[0], map_fd
[0], err
, strerror(errno
));
816 err
= bpf_prog_attach(prog_fd
[1], map_fd
[0],
817 BPF_SK_SKB_STREAM_VERDICT
, 0);
819 fprintf(stderr
, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
820 err
, strerror(errno
));
824 /* Attach to cgroups */
825 err
= bpf_prog_attach(prog_fd
[2], cg_fd
, BPF_CGROUP_SOCK_OPS
, 0);
827 fprintf(stderr
, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
828 err
, strerror(errno
));
833 err
= sockmap_init_sockets(options
->verbose
);
835 fprintf(stderr
, "ERROR: test socket failed: %d\n", err
);
839 /* Attach txmsg program to sockmap */
841 tx_prog_fd
= prog_fd
[3];
842 else if (txmsg_noisy
)
843 tx_prog_fd
= prog_fd
[4];
844 else if (txmsg_redir
)
845 tx_prog_fd
= prog_fd
[5];
846 else if (txmsg_redir_noisy
)
847 tx_prog_fd
= prog_fd
[6];
849 tx_prog_fd
= prog_fd
[9];
850 /* apply and cork must be last */
851 else if (txmsg_apply
)
852 tx_prog_fd
= prog_fd
[7];
854 tx_prog_fd
= prog_fd
[8];
861 err
= bpf_prog_attach(tx_prog_fd
,
862 map_fd
[1], BPF_SK_MSG_VERDICT
, 0);
865 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
866 err
, strerror(errno
));
870 err
= bpf_map_update_elem(map_fd
[1], &i
, &c1
, BPF_ANY
);
873 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
874 err
, strerror(errno
));
878 if (txmsg_redir
|| txmsg_redir_noisy
)
883 err
= bpf_map_update_elem(map_fd
[2], &i
, &redir_fd
, BPF_ANY
);
886 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
887 err
, strerror(errno
));
892 err
= bpf_map_update_elem(map_fd
[3],
893 &i
, &txmsg_apply
, BPF_ANY
);
896 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
897 err
, strerror(errno
));
903 err
= bpf_map_update_elem(map_fd
[4],
904 &i
, &txmsg_cork
, BPF_ANY
);
907 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
908 err
, strerror(errno
));
914 err
= bpf_map_update_elem(map_fd
[5],
915 &i
, &txmsg_start
, BPF_ANY
);
918 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
919 err
, strerror(errno
));
926 err
= bpf_map_update_elem(map_fd
[5],
927 &i
, &txmsg_end
, BPF_ANY
);
930 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
931 err
, strerror(errno
));
936 if (txmsg_start_push
) {
938 err
= bpf_map_update_elem(map_fd
[5],
939 &i
, &txmsg_start_push
, BPF_ANY
);
942 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
943 err
, strerror(errno
));
948 if (txmsg_end_push
) {
950 err
= bpf_map_update_elem(map_fd
[5],
951 &i
, &txmsg_end_push
, BPF_ANY
);
954 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
955 txmsg_end_push
, i
, err
, strerror(errno
));
960 if (txmsg_start_pop
) {
962 err
= bpf_map_update_elem(map_fd
[5],
963 &i
, &txmsg_start_pop
, BPF_ANY
);
966 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
967 txmsg_start_pop
, i
, err
, strerror(errno
));
972 bpf_map_update_elem(map_fd
[5],
973 &i
, &txmsg_start_pop
, BPF_ANY
);
978 err
= bpf_map_update_elem(map_fd
[5],
979 &i
, &txmsg_pop
, BPF_ANY
);
982 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
983 txmsg_pop
, i
, err
, strerror(errno
));
988 bpf_map_update_elem(map_fd
[5],
989 &i
, &txmsg_pop
, BPF_ANY
);
994 int in
= BPF_F_INGRESS
;
997 err
= bpf_map_update_elem(map_fd
[6], &i
, &in
, BPF_ANY
);
1000 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1001 err
, strerror(errno
));
1004 err
= bpf_map_update_elem(map_fd
[1], &i
, &p1
, BPF_ANY
);
1007 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1008 err
, strerror(errno
));
1010 err
= bpf_map_update_elem(map_fd
[2], &i
, &p1
, BPF_ANY
);
1013 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1014 err
, strerror(errno
));
1018 err
= bpf_map_update_elem(map_fd
[2], &i
, &p2
, BPF_ANY
);
1021 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1022 err
, strerror(errno
));
1027 int skb_fd
= (test
== SENDMSG
|| test
== SENDPAGE
) ?
1029 int ingress
= BPF_F_INGRESS
;
1032 err
= bpf_map_update_elem(map_fd
[7],
1033 &i
, &ingress
, BPF_ANY
);
1036 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1037 err
, strerror(errno
));
1041 err
= bpf_map_update_elem(map_fd
[0],
1042 &i
, &skb_fd
, BPF_ANY
);
1045 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1046 err
, strerror(errno
));
1052 options
->drop_expected
= true;
1054 if (test
== PING_PONG
)
1055 err
= forever_ping_pong(options
->rate
, options
);
1056 else if (test
== SENDMSG
) {
1057 options
->base
= false;
1058 options
->sendpage
= false;
1059 err
= sendmsg_test(options
);
1060 } else if (test
== SENDPAGE
) {
1061 options
->base
= false;
1062 options
->sendpage
= true;
1063 err
= sendmsg_test(options
);
1064 } else if (test
== BASE
) {
1065 options
->base
= true;
1066 options
->sendpage
= false;
1067 err
= sendmsg_test(options
);
1068 } else if (test
== BASE_SENDPAGE
) {
1069 options
->base
= true;
1070 options
->sendpage
= true;
1071 err
= sendmsg_test(options
);
1073 fprintf(stderr
, "unknown test\n");
1075 /* Detatch and zero all the maps */
1076 bpf_prog_detach2(prog_fd
[2], cg_fd
, BPF_CGROUP_SOCK_OPS
);
1077 bpf_prog_detach2(prog_fd
[0], map_fd
[0], BPF_SK_SKB_STREAM_PARSER
);
1078 bpf_prog_detach2(prog_fd
[1], map_fd
[0], BPF_SK_SKB_STREAM_VERDICT
);
1079 if (tx_prog_fd
>= 0)
1080 bpf_prog_detach2(tx_prog_fd
, map_fd
[1], BPF_SK_MSG_VERDICT
);
1082 for (i
= 0; i
< 8; i
++) {
1084 bpf_map_update_elem(map_fd
[i
], &key
, &zero
, BPF_ANY
);
1085 while (bpf_map_get_next_key(map_fd
[i
], &key
, &next_key
) == 0) {
1086 bpf_map_update_elem(map_fd
[i
], &key
, &zero
, BPF_ANY
);
1100 static char *test_to_str(int test
)
1111 #define OPTSTRING 60
1112 static void test_options(char *options
)
1114 char tstr
[OPTSTRING
];
1116 memset(options
, 0, OPTSTRING
);
1119 strncat(options
, "pass,", OPTSTRING
);
1121 strncat(options
, "pass_noisy,", OPTSTRING
);
1123 strncat(options
, "redir,", OPTSTRING
);
1124 if (txmsg_redir_noisy
)
1125 strncat(options
, "redir_noisy,", OPTSTRING
);
1127 strncat(options
, "drop,", OPTSTRING
);
1129 snprintf(tstr
, OPTSTRING
, "apply %d,", txmsg_apply
);
1130 strncat(options
, tstr
, OPTSTRING
);
1133 snprintf(tstr
, OPTSTRING
, "cork %d,", txmsg_cork
);
1134 strncat(options
, tstr
, OPTSTRING
);
1137 snprintf(tstr
, OPTSTRING
, "start %d,", txmsg_start
);
1138 strncat(options
, tstr
, OPTSTRING
);
1141 snprintf(tstr
, OPTSTRING
, "end %d,", txmsg_end
);
1142 strncat(options
, tstr
, OPTSTRING
);
1144 if (txmsg_start_pop
) {
1145 snprintf(tstr
, OPTSTRING
, "pop (%d,%d),",
1146 txmsg_start_pop
, txmsg_start_pop
+ txmsg_pop
);
1147 strncat(options
, tstr
, OPTSTRING
);
1150 strncat(options
, "ingress,", OPTSTRING
);
1152 strncat(options
, "skb,", OPTSTRING
);
1154 strncat(options
, "ktls,", OPTSTRING
);
1156 strncat(options
, "peek,", OPTSTRING
);
1159 static int __test_exec(int cgrp
, int test
, struct sockmap_options
*opt
)
1161 char *options
= calloc(OPTSTRING
, sizeof(char));
1164 if (test
== SENDPAGE
)
1165 opt
->sendpage
= true;
1167 opt
->sendpage
= false;
1170 opt
->drop_expected
= true;
1172 opt
->drop_expected
= false;
1174 test_options(options
);
1177 "[TEST %i]: (%i, %i, %i, %s, %s): ",
1178 test_cnt
, opt
->rate
, opt
->iov_count
, opt
->iov_length
,
1179 test_to_str(test
), options
);
1181 err
= run_options(opt
, cgrp
, test
);
1182 fprintf(stdout
, "%s\n", !err
? "PASS" : "FAILED");
1184 !err
? passed
++ : failed
++;
1189 static int test_exec(int cgrp
, struct sockmap_options
*opt
)
1191 int err
= __test_exec(cgrp
, SENDMSG
, opt
);
1196 err
= __test_exec(cgrp
, SENDPAGE
, opt
);
1201 static int test_loop(int cgrp
)
1203 struct sockmap_options opt
;
1209 opt
.sendpage
= false;
1210 opt
.data_test
= false;
1211 opt
.drop_expected
= false;
1217 for (i
= 1; i
< 100; i
+= 33) {
1218 for (l
= 1; l
< 100; l
+= 33) {
1222 err
= test_exec(cgrp
, &opt
);
1232 static int test_txmsg(int cgrp
)
1236 txmsg_pass
= txmsg_noisy
= txmsg_redir_noisy
= txmsg_drop
= 0;
1237 txmsg_apply
= txmsg_cork
= 0;
1238 txmsg_ingress
= txmsg_skb
= 0;
1241 err
= test_loop(cgrp
);
1247 err
= test_loop(cgrp
);
1253 err
= test_loop(cgrp
);
1260 err
= test_loop(cgrp
);
1272 static int test_send(struct sockmap_options
*opt
, int cgrp
)
1276 opt
->iov_length
= 1;
1279 err
= test_exec(cgrp
, opt
);
1283 opt
->iov_length
= 1;
1284 opt
->iov_count
= 1024;
1286 err
= test_exec(cgrp
, opt
);
1290 opt
->iov_length
= 1024;
1293 err
= test_exec(cgrp
, opt
);
1297 opt
->iov_length
= 1;
1300 err
= test_exec(cgrp
, opt
);
1304 opt
->iov_length
= 256;
1305 opt
->iov_count
= 1024;
1307 err
= test_exec(cgrp
, opt
);
1313 opt
->iov_length
= 5;
1314 err
= test_exec(cgrp
, opt
);
1322 static int test_mixed(int cgrp
)
1324 struct sockmap_options opt
= {0};
1327 txmsg_pass
= txmsg_noisy
= txmsg_redir_noisy
= txmsg_drop
= 0;
1328 txmsg_apply
= txmsg_cork
= 0;
1329 txmsg_start
= txmsg_end
= 0;
1330 txmsg_start_push
= txmsg_end_push
= 0;
1331 txmsg_start_pop
= txmsg_pop
= 0;
1333 /* Test small and large iov_count values with pass/redir/apply/cork */
1338 err
= test_send(&opt
, cgrp
);
1346 err
= test_send(&opt
, cgrp
);
1354 err
= test_send(&opt
, cgrp
);
1362 err
= test_send(&opt
, cgrp
);
1370 err
= test_send(&opt
, cgrp
);
1378 err
= test_send(&opt
, cgrp
);
1386 err
= test_send(&opt
, cgrp
);
1394 err
= test_send(&opt
, cgrp
);
1402 err
= test_send(&opt
, cgrp
);
1410 err
= test_send(&opt
, cgrp
);
1418 err
= test_send(&opt
, cgrp
);
1426 err
= test_send(&opt
, cgrp
);
1434 err
= test_send(&opt
, cgrp
);
1441 static int test_start_end(int cgrp
)
1443 struct sockmap_options opt
= {0};
1446 /* Test basic start/end with lots of iov_count and iov_lengths */
1449 txmsg_start_push
= 1;
1451 txmsg_start_pop
= 1;
1453 err
= test_txmsg(cgrp
);
1457 /* Cut a byte of pushed data but leave reamining in place */
1460 txmsg_start_push
= 1;
1462 txmsg_start_pop
= 1;
1464 err
= test_txmsg(cgrp
);
1468 /* Test start/end with cork */
1471 opt
.iov_length
= 100;
1474 txmsg_start_pop
= 0;
1477 for (i
= 99; i
<= 1600; i
+= 500) {
1480 txmsg_start_push
= 0;
1482 err
= test_exec(cgrp
, &opt
);
1487 /* Test pop data in middle of cork */
1488 for (i
= 99; i
<= 1600; i
+= 500) {
1489 txmsg_start_pop
= 10;
1491 err
= test_exec(cgrp
, &opt
);
1495 txmsg_start_pop
= 0;
1498 /* Test start/end with cork but pull data in middle */
1499 for (i
= 199; i
<= 1600; i
+= 500) {
1502 txmsg_start_push
= 100;
1504 err
= test_exec(cgrp
, &opt
);
1509 /* Test start/end with cork pulling last sg entry */
1512 txmsg_start_push
= 1500;
1513 txmsg_end_push
= 1600;
1514 err
= test_exec(cgrp
, &opt
);
1518 /* Test pop with cork pulling last sg entry */
1519 txmsg_start_pop
= 1500;
1521 err
= test_exec(cgrp
, &opt
);
1524 txmsg_start_pop
= 0;
1527 /* Test start/end pull of single byte in last page */
1530 txmsg_start_push
= 1111;
1531 txmsg_end_push
= 1112;
1532 err
= test_exec(cgrp
, &opt
);
1536 /* Test pop of single byte in last page */
1537 txmsg_start_pop
= 1111;
1539 err
= test_exec(cgrp
, &opt
);
1543 /* Test start/end with end < start */
1546 txmsg_start_push
= 1111;
1548 err
= test_exec(cgrp
, &opt
);
1552 /* Test start/end with end > data */
1555 txmsg_start_push
= 0;
1556 txmsg_end_push
= 1601;
1557 err
= test_exec(cgrp
, &opt
);
1561 /* Test start/end with start > data */
1564 txmsg_start_push
= 1601;
1565 txmsg_end_push
= 1600;
1566 err
= test_exec(cgrp
, &opt
);
1570 /* Test pop with start > data */
1571 txmsg_start_pop
= 1601;
1573 err
= test_exec(cgrp
, &opt
);
1577 /* Test pop with pop range > data */
1578 txmsg_start_pop
= 1599;
1580 err
= test_exec(cgrp
, &opt
);
1588 char *map_names
[] = {
1599 int prog_attach_type
[] = {
1600 BPF_SK_SKB_STREAM_PARSER
,
1601 BPF_SK_SKB_STREAM_VERDICT
,
1602 BPF_CGROUP_SOCK_OPS
,
1613 BPF_PROG_TYPE_SK_SKB
,
1614 BPF_PROG_TYPE_SK_SKB
,
1615 BPF_PROG_TYPE_SOCK_OPS
,
1616 BPF_PROG_TYPE_SK_MSG
,
1617 BPF_PROG_TYPE_SK_MSG
,
1618 BPF_PROG_TYPE_SK_MSG
,
1619 BPF_PROG_TYPE_SK_MSG
,
1620 BPF_PROG_TYPE_SK_MSG
,
1621 BPF_PROG_TYPE_SK_MSG
,
1622 BPF_PROG_TYPE_SK_MSG
,
1625 static int populate_progs(char *bpf_file
)
1627 struct bpf_program
*prog
;
1628 struct bpf_object
*obj
;
1632 obj
= bpf_object__open(bpf_file
);
1633 err
= libbpf_get_error(obj
);
1637 libbpf_strerror(err
, err_buf
, sizeof(err_buf
));
1638 printf("Unable to load eBPF objects in file '%s' : %s\n",
1643 bpf_object__for_each_program(prog
, obj
) {
1644 bpf_program__set_type(prog
, prog_type
[i
]);
1645 bpf_program__set_expected_attach_type(prog
,
1646 prog_attach_type
[i
]);
1650 i
= bpf_object__load(obj
);
1652 bpf_object__for_each_program(prog
, obj
) {
1653 prog_fd
[i
] = bpf_program__fd(prog
);
1657 for (i
= 0; i
< sizeof(map_fd
)/sizeof(int); i
++) {
1658 maps
[i
] = bpf_object__find_map_by_name(obj
, map_names
[i
]);
1659 map_fd
[i
] = bpf_map__fd(maps
[i
]);
1660 if (map_fd
[i
] < 0) {
1661 fprintf(stderr
, "load_bpf_file: (%i) %s\n",
1662 map_fd
[i
], strerror(errno
));
1670 static int __test_suite(int cg_fd
, char *bpf_file
)
1672 int err
, cleanup
= cg_fd
;
1674 err
= populate_progs(bpf_file
);
1676 fprintf(stderr
, "ERROR: (%i) load bpf failed\n", err
);
1681 if (setup_cgroup_environment()) {
1682 fprintf(stderr
, "ERROR: cgroup env failed\n");
1686 cg_fd
= create_and_get_cgroup(CG_PATH
);
1689 "ERROR: (%i) open cg path failed: %s\n",
1694 if (join_cgroup(CG_PATH
)) {
1695 fprintf(stderr
, "ERROR: failed to join cgroup\n");
1700 /* Tests basic commands and APIs with range of iov values */
1701 txmsg_start
= txmsg_end
= txmsg_start_push
= txmsg_end_push
= 0;
1702 err
= test_txmsg(cg_fd
);
1706 /* Tests interesting combinations of APIs used together */
1707 err
= test_mixed(cg_fd
);
1711 /* Tests pull_data API using start/end API */
1712 err
= test_start_end(cg_fd
);
1717 printf("Summary: %i PASSED %i FAILED\n", passed
, failed
);
1719 cleanup_cgroup_environment();
1725 static int test_suite(int cg_fd
)
1729 err
= __test_suite(cg_fd
, BPF_SOCKMAP_FILENAME
);
1732 err
= __test_suite(cg_fd
, BPF_SOCKHASH_FILENAME
);
1739 int main(int argc
, char **argv
)
1741 int iov_count
= 1, length
= 1024, rate
= 1;
1742 struct sockmap_options options
= {0};
1743 int opt
, longindex
, err
, cg_fd
= 0;
1744 char *bpf_file
= BPF_SOCKMAP_FILENAME
;
1745 int test
= PING_PONG
;
1748 return test_suite(-1);
1750 while ((opt
= getopt_long(argc
, argv
, ":dhvc:r:i:l:t:p:q:",
1751 long_options
, &longindex
)) != -1) {
1754 txmsg_start
= atoi(optarg
);
1757 txmsg_end
= atoi(optarg
);
1760 txmsg_start_push
= atoi(optarg
);
1763 txmsg_end_push
= atoi(optarg
);
1766 txmsg_start_pop
= atoi(optarg
);
1769 txmsg_pop
= atoi(optarg
);
1772 txmsg_apply
= atoi(optarg
);
1775 txmsg_cork
= atoi(optarg
);
1778 cg_fd
= open(optarg
, O_DIRECTORY
, O_RDONLY
);
1781 "ERROR: (%i) open cg path failed: %s\n",
1787 rate
= atoi(optarg
);
1790 options
.verbose
= 1;
1793 iov_count
= atoi(optarg
);
1796 length
= atoi(optarg
);
1799 options
.data_test
= true;
1802 if (strcmp(optarg
, "ping") == 0) {
1804 } else if (strcmp(optarg
, "sendmsg") == 0) {
1806 } else if (strcmp(optarg
, "base") == 0) {
1808 } else if (strcmp(optarg
, "base_sendpage") == 0) {
1809 test
= BASE_SENDPAGE
;
1810 } else if (strcmp(optarg
, "sendpage") == 0) {
1826 if (argc
<= 3 && cg_fd
)
1827 return test_suite(cg_fd
);
1830 fprintf(stderr
, "%s requires cgroup option: --cgroup <path>\n",
1835 err
= populate_progs(bpf_file
);
1837 fprintf(stderr
, "populate program: (%s) %s\n",
1838 bpf_file
, strerror(errno
));
1844 signal(SIGINT
, running_handler
);
1846 options
.iov_count
= iov_count
;
1847 options
.iov_length
= length
;
1848 options
.rate
= rate
;
1850 err
= run_options(&options
, cg_fd
, test
);
1855 void running_handler(int a
)