]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - tools/testing/selftests/bpf/test_sockmap.c
projects / mirror_ubuntu-hirsute-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'locking-urgent-2020-11-01' of git://git.kernel.org/pub/scm/linux/kernel...
[mirror_ubuntu-hirsute-kernel.git] / tools / testing / selftests / bpf / test_sockmap.c
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <stdbool.h>
14 #include <signal.h>
15 #include <fcntl.h>
16 #include <sys/wait.h>
17 #include <time.h>
18 #include <sched.h>
19
20 #include <sys/time.h>
21 #include <sys/resource.h>
22 #include <sys/types.h>
23 #include <sys/sendfile.h>
24
25 #include <linux/netlink.h>
26 #include <linux/socket.h>
27 #include <linux/sock_diag.h>
28 #include <linux/bpf.h>
29 #include <linux/if_link.h>
30 #include <linux/tls.h>
31 #include <assert.h>
32 #include <libgen.h>
33
34 #include <getopt.h>
35
36 #include <bpf/bpf.h>
37 #include <bpf/libbpf.h>
38
39 #include "bpf_util.h"
40 #include "bpf_rlimit.h"
41 #include "cgroup_helpers.h"
42
43 int running;
44 static void running_handler(int a);
45
46 #ifndef TCP_ULP
47 # define TCP_ULP 31
48 #endif
49 #ifndef SOL_TLS
50 # define SOL_TLS 282
51 #endif
52
53 /* randomly selected ports for testing on lo */
54 #define S1_PORT 10000
55 #define S2_PORT 10001
56
57 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
58 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
59 #define CG_PATH "/sockmap"
60
61 /* global sockets */
62 int s1, s2, c1, c2, p1, p2;
63 int test_cnt;
64 int passed;
65 int failed;
66 int map_fd[9];
67 struct bpf_map *maps[9];
68 int prog_fd[11];
69
70 int txmsg_pass;
71 int txmsg_redir;
72 int txmsg_drop;
73 int txmsg_apply;
74 int txmsg_cork;
75 int txmsg_start;
76 int txmsg_end;
77 int txmsg_start_push;
78 int txmsg_end_push;
79 int txmsg_start_pop;
80 int txmsg_pop;
81 int txmsg_ingress;
82 int txmsg_redir_skb;
83 int txmsg_ktls_skb;
84 int txmsg_ktls_skb_drop;
85 int txmsg_ktls_skb_redir;
86 int ktls;
87 int peek_flag;
88 int skb_use_parser;
89 int txmsg_omit_skb_parser;
90
91 static const struct option long_options[] = {
92 {"help", no_argument, NULL, 'h' },
93 {"cgroup", required_argument, NULL, 'c' },
94 {"rate", required_argument, NULL, 'r' },
95 {"verbose", optional_argument, NULL, 'v' },
96 {"iov_count", required_argument, NULL, 'i' },
97 {"length", required_argument, NULL, 'l' },
98 {"test", required_argument, NULL, 't' },
99 {"data_test", no_argument, NULL, 'd' },
100 {"txmsg", no_argument, &txmsg_pass, 1 },
101 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
102 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
103 {"txmsg_apply", required_argument, NULL, 'a'},
104 {"txmsg_cork", required_argument, NULL, 'k'},
105 {"txmsg_start", required_argument, NULL, 's'},
106 {"txmsg_end", required_argument, NULL, 'e'},
107 {"txmsg_start_push", required_argument, NULL, 'p'},
108 {"txmsg_end_push", required_argument, NULL, 'q'},
109 {"txmsg_start_pop", required_argument, NULL, 'w'},
110 {"txmsg_pop", required_argument, NULL, 'x'},
111 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
112 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
113 {"ktls", no_argument, &ktls, 1 },
114 {"peek", no_argument, &peek_flag, 1 },
115 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
116 {"whitelist", required_argument, NULL, 'n' },
117 {"blacklist", required_argument, NULL, 'b' },
118 {0, 0, NULL, 0 }
119 };
120
121 struct test_env {
122 const char *type;
123 const char *subtest;
124 const char *prepend;
125
126 int test_num;
127 int subtest_num;
128
129 int succ_cnt;
130 int fail_cnt;
131 int fail_last;
132 };
133
134 struct test_env env;
135
136 struct sockmap_options {
137 int verbose;
138 bool base;
139 bool sendpage;
140 bool data_test;
141 bool drop_expected;
142 int iov_count;
143 int iov_length;
144 int rate;
145 char *map;
146 char *whitelist;
147 char *blacklist;
148 char *prepend;
149 };
150
151 struct _test {
152 char *title;
153 void (*tester)(int cg_fd, struct sockmap_options *opt);
154 };
155
156 static void test_start(void)
157 {
158 env.subtest_num++;
159 }
160
161 static void test_fail(void)
162 {
163 env.fail_cnt++;
164 }
165
166 static void test_pass(void)
167 {
168 env.succ_cnt++;
169 }
170
171 static void test_reset(void)
172 {
173 txmsg_start = txmsg_end = 0;
174 txmsg_start_pop = txmsg_pop = 0;
175 txmsg_start_push = txmsg_end_push = 0;
176 txmsg_pass = txmsg_drop = txmsg_redir = 0;
177 txmsg_apply = txmsg_cork = 0;
178 txmsg_ingress = txmsg_redir_skb = 0;
179 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
180 txmsg_omit_skb_parser = 0;
181 skb_use_parser = 0;
182 }
183
184 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
185 {
186 env.type = o->map;
187 env.subtest = t->title;
188 env.prepend = o->prepend;
189 env.test_num++;
190 env.subtest_num = 0;
191 env.fail_last = env.fail_cnt;
192 test_reset();
193 return 0;
194 }
195
196 static void test_end_subtest(void)
197 {
198 int error = env.fail_cnt - env.fail_last;
199 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
200
201 if (!error)
202 test_pass();
203
204 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
205 env.test_num, env.subtest_num,
206 !type ? "sockmap" : "sockhash",
207 env.prepend ? : "",
208 env.subtest, error ? "FAIL" : "OK");
209 }
210
211 static void test_print_results(void)
212 {
213 fprintf(stdout, "Pass: %d Fail: %d\n",
214 env.succ_cnt, env.fail_cnt);
215 }
216
217 static void usage(char *argv[])
218 {
219 int i;
220
221 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
222 printf(" options:\n");
223 for (i = 0; long_options[i].name != 0; i++) {
224 printf(" --%-12s", long_options[i].name);
225 if (long_options[i].flag != NULL)
226 printf(" flag (internal value:%d)\n",
227 *long_options[i].flag);
228 else
229 printf(" -%c\n", long_options[i].val);
230 }
231 printf("\n");
232 }
233
234 char *sock_to_string(int s)
235 {
236 if (s == c1)
237 return "client1";
238 else if (s == c2)
239 return "client2";
240 else if (s == s1)
241 return "server1";
242 else if (s == s2)
243 return "server2";
244 else if (s == p1)
245 return "peer1";
246 else if (s == p2)
247 return "peer2";
248 else
249 return "unknown";
250 }
251
252 static int sockmap_init_ktls(int verbose, int s)
253 {
254 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
255 .info = {
256 .version = TLS_1_2_VERSION,
257 .cipher_type = TLS_CIPHER_AES_GCM_128,
258 },
259 };
260 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
261 .info = {
262 .version = TLS_1_2_VERSION,
263 .cipher_type = TLS_CIPHER_AES_GCM_128,
264 },
265 };
266 int so_buf = 6553500;
267 int err;
268
269 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
270 if (err) {
271 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
272 return -EINVAL;
273 }
274 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
275 if (err) {
276 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
277 return -EINVAL;
278 }
279 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
280 if (err) {
281 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
282 return -EINVAL;
283 }
284 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
285 if (err) {
286 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
287 return -EINVAL;
288 }
289 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
290 if (err) {
291 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
292 return -EINVAL;
293 }
294
295 if (verbose)
296 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
297 return 0;
298 }
299 static int sockmap_init_sockets(int verbose)
300 {
301 int i, err, one = 1;
302 struct sockaddr_in addr;
303 int *fds[4] = {&s1, &s2, &c1, &c2};
304
305 s1 = s2 = p1 = p2 = c1 = c2 = 0;
306
307 /* Init sockets */
308 for (i = 0; i < 4; i++) {
309 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
310 if (*fds[i] < 0) {
311 perror("socket s1 failed()");
312 return errno;
313 }
314 }
315
316 /* Allow reuse */
317 for (i = 0; i < 2; i++) {
318 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
319 (char *)&one, sizeof(one));
320 if (err) {
321 perror("setsockopt failed()");
322 return errno;
323 }
324 }
325
326 /* Non-blocking sockets */
327 for (i = 0; i < 2; i++) {
328 err = ioctl(*fds[i], FIONBIO, (char *)&one);
329 if (err < 0) {
330 perror("ioctl s1 failed()");
331 return errno;
332 }
333 }
334
335 /* Bind server sockets */
336 memset(&addr, 0, sizeof(struct sockaddr_in));
337 addr.sin_family = AF_INET;
338 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
339
340 addr.sin_port = htons(S1_PORT);
341 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
342 if (err < 0) {
343 perror("bind s1 failed()");
344 return errno;
345 }
346
347 addr.sin_port = htons(S2_PORT);
348 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
349 if (err < 0) {
350 perror("bind s2 failed()");
351 return errno;
352 }
353
354 /* Listen server sockets */
355 addr.sin_port = htons(S1_PORT);
356 err = listen(s1, 32);
357 if (err < 0) {
358 perror("listen s1 failed()");
359 return errno;
360 }
361
362 addr.sin_port = htons(S2_PORT);
363 err = listen(s2, 32);
364 if (err < 0) {
365 perror("listen s1 failed()");
366 return errno;
367 }
368
369 /* Initiate Connect */
370 addr.sin_port = htons(S1_PORT);
371 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
372 if (err < 0 && errno != EINPROGRESS) {
373 perror("connect c1 failed()");
374 return errno;
375 }
376
377 addr.sin_port = htons(S2_PORT);
378 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
379 if (err < 0 && errno != EINPROGRESS) {
380 perror("connect c2 failed()");
381 return errno;
382 } else if (err < 0) {
383 err = 0;
384 }
385
386 /* Accept Connecrtions */
387 p1 = accept(s1, NULL, NULL);
388 if (p1 < 0) {
389 perror("accept s1 failed()");
390 return errno;
391 }
392
393 p2 = accept(s2, NULL, NULL);
394 if (p2 < 0) {
395 perror("accept s1 failed()");
396 return errno;
397 }
398
399 if (verbose > 1) {
400 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
401 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
402 c1, s1, c2, s2);
403 }
404 return 0;
405 }
406
407 struct msg_stats {
408 size_t bytes_sent;
409 size_t bytes_recvd;
410 struct timespec start;
411 struct timespec end;
412 };
413
414 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
415 struct msg_stats *s,
416 struct sockmap_options *opt)
417 {
418 bool drop = opt->drop_expected;
419 unsigned char k = 0;
420 FILE *file;
421 int i, fp;
422
423 file = tmpfile();
424 if (!file) {
425 perror("create file for sendpage");
426 return 1;
427 }
428 for (i = 0; i < iov_length * cnt; i++, k++)
429 fwrite(&k, sizeof(char), 1, file);
430 fflush(file);
431 fseek(file, 0, SEEK_SET);
432
433 fp = fileno(file);
434
435 clock_gettime(CLOCK_MONOTONIC, &s->start);
436 for (i = 0; i < cnt; i++) {
437 int sent;
438
439 errno = 0;
440 sent = sendfile(fd, fp, NULL, iov_length);
441
442 if (!drop && sent < 0) {
443 perror("sendpage loop error");
444 fclose(file);
445 return sent;
446 } else if (drop && sent >= 0) {
447 printf("sendpage loop error expected: %i errno %i\n",
448 sent, errno);
449 fclose(file);
450 return -EIO;
451 }
452
453 if (sent > 0)
454 s->bytes_sent += sent;
455 }
456 clock_gettime(CLOCK_MONOTONIC, &s->end);
457 fclose(file);
458 return 0;
459 }
460
461 static void msg_free_iov(struct msghdr *msg)
462 {
463 int i;
464
465 for (i = 0; i < msg->msg_iovlen; i++)
466 free(msg->msg_iov[i].iov_base);
467 free(msg->msg_iov);
468 msg->msg_iov = NULL;
469 msg->msg_iovlen = 0;
470 }
471
472 static int msg_alloc_iov(struct msghdr *msg,
473 int iov_count, int iov_length,
474 bool data, bool xmit)
475 {
476 unsigned char k = 0;
477 struct iovec *iov;
478 int i;
479
480 iov = calloc(iov_count, sizeof(struct iovec));
481 if (!iov)
482 return errno;
483
484 for (i = 0; i < iov_count; i++) {
485 unsigned char *d = calloc(iov_length, sizeof(char));
486
487 if (!d) {
488 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
489 goto unwind_iov;
490 }
491 iov[i].iov_base = d;
492 iov[i].iov_len = iov_length;
493
494 if (data && xmit) {
495 int j;
496
497 for (j = 0; j < iov_length; j++)
498 d[j] = k++;
499 }
500 }
501
502 msg->msg_iov = iov;
503 msg->msg_iovlen = iov_count;
504
505 return 0;
506 unwind_iov:
507 for (i--; i >= 0 ; i--)
508 free(msg->msg_iov[i].iov_base);
509 return -ENOMEM;
510 }
511
512 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
513 {
514 int i, j = 0, bytes_cnt = 0;
515 unsigned char k = 0;
516
517 for (i = 0; i < msg->msg_iovlen; i++) {
518 unsigned char *d = msg->msg_iov[i].iov_base;
519
520 /* Special case test for skb ingress + ktls */
521 if (i == 0 && txmsg_ktls_skb) {
522 if (msg->msg_iov[i].iov_len < 4)
523 return -EIO;
524 if (memcmp(d, "PASS", 4) != 0) {
525 fprintf(stderr,
526 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
527 i, 0, d[0], d[1], d[2], d[3]);
528 return -EIO;
529 }
530 j = 4; /* advance index past PASS header */
531 }
532
533 for (; j < msg->msg_iov[i].iov_len && size; j++) {
534 if (d[j] != k++) {
535 fprintf(stderr,
536 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
537 i, j, d[j], k - 1, d[j+1], k);
538 return -EIO;
539 }
540 bytes_cnt++;
541 if (bytes_cnt == chunk_sz) {
542 k = 0;
543 bytes_cnt = 0;
544 }
545 size--;
546 }
547 }
548 return 0;
549 }
550
551 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
552 struct msg_stats *s, bool tx,
553 struct sockmap_options *opt)
554 {
555 struct msghdr msg = {0}, msg_peek = {0};
556 int err, i, flags = MSG_NOSIGNAL;
557 bool drop = opt->drop_expected;
558 bool data = opt->data_test;
559
560 err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx);
561 if (err)
562 goto out_errno;
563 if (peek_flag) {
564 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
565 if (err)
566 goto out_errno;
567 }
568
569 if (tx) {
570 clock_gettime(CLOCK_MONOTONIC, &s->start);
571 for (i = 0; i < cnt; i++) {
572 int sent;
573
574 errno = 0;
575 sent = sendmsg(fd, &msg, flags);
576
577 if (!drop && sent < 0) {
578 perror("sendmsg loop error");
579 goto out_errno;
580 } else if (drop && sent >= 0) {
581 fprintf(stderr,
582 "sendmsg loop error expected: %i errno %i\n",
583 sent, errno);
584 errno = -EIO;
585 goto out_errno;
586 }
587 if (sent > 0)
588 s->bytes_sent += sent;
589 }
590 clock_gettime(CLOCK_MONOTONIC, &s->end);
591 } else {
592 int slct, recvp = 0, recv, max_fd = fd;
593 float total_bytes, txmsg_pop_total;
594 int fd_flags = O_NONBLOCK;
595 struct timeval timeout;
596 fd_set w;
597
598 fcntl(fd, fd_flags);
599 /* Account for pop bytes noting each iteration of apply will
600 * call msg_pop_data helper so we need to account for this
601 * by calculating the number of apply iterations. Note user
602 * of the tool can create cases where no data is sent by
603 * manipulating pop/push/pull/etc. For example txmsg_apply 1
604 * with txmsg_pop 1 will try to apply 1B at a time but each
605 * iteration will then pop 1B so no data will ever be sent.
606 * This is really only useful for testing edge cases in code
607 * paths.
608 */
609 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
610 if (txmsg_apply)
611 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
612 else
613 txmsg_pop_total = txmsg_pop * cnt;
614 total_bytes -= txmsg_pop_total;
615 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
616 if (err < 0)
617 perror("recv start time");
618 while (s->bytes_recvd < total_bytes) {
619 if (txmsg_cork) {
620 timeout.tv_sec = 0;
621 timeout.tv_usec = 300000;
622 } else {
623 timeout.tv_sec = 3;
624 timeout.tv_usec = 0;
625 }
626
627 /* FD sets */
628 FD_ZERO(&w);
629 FD_SET(fd, &w);
630
631 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
632 if (slct == -1) {
633 perror("select()");
634 clock_gettime(CLOCK_MONOTONIC, &s->end);
635 goto out_errno;
636 } else if (!slct) {
637 if (opt->verbose)
638 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
639 errno = -EIO;
640 clock_gettime(CLOCK_MONOTONIC, &s->end);
641 goto out_errno;
642 }
643
644 errno = 0;
645 if (peek_flag) {
646 flags |= MSG_PEEK;
647 recvp = recvmsg(fd, &msg_peek, flags);
648 if (recvp < 0) {
649 if (errno != EWOULDBLOCK) {
650 clock_gettime(CLOCK_MONOTONIC, &s->end);
651 goto out_errno;
652 }
653 }
654 flags = 0;
655 }
656
657 recv = recvmsg(fd, &msg, flags);
658 if (recv < 0) {
659 if (errno != EWOULDBLOCK) {
660 clock_gettime(CLOCK_MONOTONIC, &s->end);
661 perror("recv failed()");
662 goto out_errno;
663 }
664 }
665
666 s->bytes_recvd += recv;
667
668 if (data) {
669 int chunk_sz = opt->sendpage ?
670 iov_length * cnt :
671 iov_length * iov_count;
672
673 errno = msg_verify_data(&msg, recv, chunk_sz);
674 if (errno) {
675 perror("data verify msg failed");
676 goto out_errno;
677 }
678 if (recvp) {
679 errno = msg_verify_data(&msg_peek,
680 recvp,
681 chunk_sz);
682 if (errno) {
683 perror("data verify msg_peek failed");
684 goto out_errno;
685 }
686 }
687 }
688 }
689 clock_gettime(CLOCK_MONOTONIC, &s->end);
690 }
691
692 msg_free_iov(&msg);
693 msg_free_iov(&msg_peek);
694 return err;
695 out_errno:
696 msg_free_iov(&msg);
697 msg_free_iov(&msg_peek);
698 return errno;
699 }
700
701 static float giga = 1000000000;
702
703 static inline float sentBps(struct msg_stats s)
704 {
705 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
706 }
707
708 static inline float recvdBps(struct msg_stats s)
709 {
710 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
711 }
712
713 static int sendmsg_test(struct sockmap_options *opt)
714 {
715 float sent_Bps = 0, recvd_Bps = 0;
716 int rx_fd, txpid, rxpid, err = 0;
717 struct msg_stats s = {0};
718 int iov_count = opt->iov_count;
719 int iov_buf = opt->iov_length;
720 int rx_status, tx_status;
721 int cnt = opt->rate;
722
723 errno = 0;
724
725 if (opt->base)
726 rx_fd = p1;
727 else
728 rx_fd = p2;
729
730 if (ktls) {
731 /* Redirecting into non-TLS socket which sends into a TLS
732 * socket is not a valid test. So in this case lets not
733 * enable kTLS but still run the test.
734 */
735 if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) {
736 err = sockmap_init_ktls(opt->verbose, rx_fd);
737 if (err)
738 return err;
739 }
740 err = sockmap_init_ktls(opt->verbose, c1);
741 if (err)
742 return err;
743 }
744
745 rxpid = fork();
746 if (rxpid == 0) {
747 iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
748 if (opt->drop_expected || txmsg_ktls_skb_drop)
749 _exit(0);
750
751 if (!iov_buf) /* zero bytes sent case */
752 _exit(0);
753
754 if (opt->sendpage)
755 iov_count = 1;
756 err = msg_loop(rx_fd, iov_count, iov_buf,
757 cnt, &s, false, opt);
758 if (opt->verbose > 1)
759 fprintf(stderr,
760 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
761 iov_count, iov_buf, cnt, err);
762 if (s.end.tv_sec - s.start.tv_sec) {
763 sent_Bps = sentBps(s);
764 recvd_Bps = recvdBps(s);
765 }
766 if (opt->verbose > 1)
767 fprintf(stdout,
768 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
769 s.bytes_sent, sent_Bps, sent_Bps/giga,
770 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
771 peek_flag ? "(peek_msg)" : "");
772 if (err && txmsg_cork)
773 err = 0;
774 exit(err ? 1 : 0);
775 } else if (rxpid == -1) {
776 perror("msg_loop_rx");
777 return errno;
778 }
779
780 txpid = fork();
781 if (txpid == 0) {
782 if (opt->sendpage)
783 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
784 else
785 err = msg_loop(c1, iov_count, iov_buf,
786 cnt, &s, true, opt);
787
788 if (err)
789 fprintf(stderr,
790 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
791 iov_count, iov_buf, cnt, err);
792 if (s.end.tv_sec - s.start.tv_sec) {
793 sent_Bps = sentBps(s);
794 recvd_Bps = recvdBps(s);
795 }
796 if (opt->verbose > 1)
797 fprintf(stdout,
798 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
799 s.bytes_sent, sent_Bps, sent_Bps/giga,
800 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
801 exit(err ? 1 : 0);
802 } else if (txpid == -1) {
803 perror("msg_loop_tx");
804 return errno;
805 }
806
807 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
808 assert(waitpid(txpid, &tx_status, 0) == txpid);
809 if (WIFEXITED(rx_status)) {
810 err = WEXITSTATUS(rx_status);
811 if (err) {
812 fprintf(stderr, "rx thread exited with err %d.\n", err);
813 goto out;
814 }
815 }
816 if (WIFEXITED(tx_status)) {
817 err = WEXITSTATUS(tx_status);
818 if (err)
819 fprintf(stderr, "tx thread exited with err %d.\n", err);
820 }
821 out:
822 return err;
823 }
824
825 static int forever_ping_pong(int rate, struct sockmap_options *opt)
826 {
827 struct timeval timeout;
828 char buf[1024] = {0};
829 int sc;
830
831 timeout.tv_sec = 10;
832 timeout.tv_usec = 0;
833
834 /* Ping/Pong data from client to server */
835 sc = send(c1, buf, sizeof(buf), 0);
836 if (sc < 0) {
837 perror("send failed()");
838 return sc;
839 }
840
841 do {
842 int s, rc, i, max_fd = p2;
843 fd_set w;
844
845 /* FD sets */
846 FD_ZERO(&w);
847 FD_SET(c1, &w);
848 FD_SET(c2, &w);
849 FD_SET(p1, &w);
850 FD_SET(p2, &w);
851
852 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
853 if (s == -1) {
854 perror("select()");
855 break;
856 } else if (!s) {
857 fprintf(stderr, "unexpected timeout\n");
858 break;
859 }
860
861 for (i = 0; i <= max_fd && s > 0; ++i) {
862 if (!FD_ISSET(i, &w))
863 continue;
864
865 s--;
866
867 rc = recv(i, buf, sizeof(buf), 0);
868 if (rc < 0) {
869 if (errno != EWOULDBLOCK) {
870 perror("recv failed()");
871 return rc;
872 }
873 }
874
875 if (rc == 0) {
876 close(i);
877 break;
878 }
879
880 sc = send(i, buf, rc, 0);
881 if (sc < 0) {
882 perror("send failed()");
883 return sc;
884 }
885 }
886
887 if (rate)
888 sleep(rate);
889
890 if (opt->verbose) {
891 printf(".");
892 fflush(stdout);
893
894 }
895 } while (running);
896
897 return 0;
898 }
899
900 enum {
901 SELFTESTS,
902 PING_PONG,
903 SENDMSG,
904 BASE,
905 BASE_SENDPAGE,
906 SENDPAGE,
907 };
908
909 static int run_options(struct sockmap_options *options, int cg_fd, int test)
910 {
911 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
912
913 /* If base test skip BPF setup */
914 if (test == BASE || test == BASE_SENDPAGE)
915 goto run;
916
917 /* Attach programs to sockmap */
918 if (!txmsg_omit_skb_parser) {
919 err = bpf_prog_attach(prog_fd[0], map_fd[0],
920 BPF_SK_SKB_STREAM_PARSER, 0);
921 if (err) {
922 fprintf(stderr,
923 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
924 prog_fd[0], map_fd[0], err, strerror(errno));
925 return err;
926 }
927 }
928
929 err = bpf_prog_attach(prog_fd[1], map_fd[0],
930 BPF_SK_SKB_STREAM_VERDICT, 0);
931 if (err) {
932 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
933 err, strerror(errno));
934 return err;
935 }
936
937 /* Attach programs to TLS sockmap */
938 if (txmsg_ktls_skb) {
939 if (!txmsg_omit_skb_parser) {
940 err = bpf_prog_attach(prog_fd[0], map_fd[8],
941 BPF_SK_SKB_STREAM_PARSER, 0);
942 if (err) {
943 fprintf(stderr,
944 "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
945 prog_fd[0], map_fd[8], err, strerror(errno));
946 return err;
947 }
948 }
949
950 err = bpf_prog_attach(prog_fd[2], map_fd[8],
951 BPF_SK_SKB_STREAM_VERDICT, 0);
952 if (err) {
953 fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
954 err, strerror(errno));
955 return err;
956 }
957 }
958
959 /* Attach to cgroups */
960 err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
961 if (err) {
962 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
963 err, strerror(errno));
964 return err;
965 }
966
967 run:
968 err = sockmap_init_sockets(options->verbose);
969 if (err) {
970 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
971 goto out;
972 }
973
974 /* Attach txmsg program to sockmap */
975 if (txmsg_pass)
976 tx_prog_fd = prog_fd[4];
977 else if (txmsg_redir)
978 tx_prog_fd = prog_fd[5];
979 else if (txmsg_apply)
980 tx_prog_fd = prog_fd[6];
981 else if (txmsg_cork)
982 tx_prog_fd = prog_fd[7];
983 else if (txmsg_drop)
984 tx_prog_fd = prog_fd[8];
985 else
986 tx_prog_fd = 0;
987
988 if (tx_prog_fd) {
989 int redir_fd, i = 0;
990
991 err = bpf_prog_attach(tx_prog_fd,
992 map_fd[1], BPF_SK_MSG_VERDICT, 0);
993 if (err) {
994 fprintf(stderr,
995 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
996 err, strerror(errno));
997 goto out;
998 }
999
1000 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1001 if (err) {
1002 fprintf(stderr,
1003 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1004 err, strerror(errno));
1005 goto out;
1006 }
1007
1008 if (txmsg_redir)
1009 redir_fd = c2;
1010 else
1011 redir_fd = c1;
1012
1013 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1014 if (err) {
1015 fprintf(stderr,
1016 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1017 err, strerror(errno));
1018 goto out;
1019 }
1020
1021 if (txmsg_apply) {
1022 err = bpf_map_update_elem(map_fd[3],
1023 &i, &txmsg_apply, BPF_ANY);
1024 if (err) {
1025 fprintf(stderr,
1026 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1027 err, strerror(errno));
1028 goto out;
1029 }
1030 }
1031
1032 if (txmsg_cork) {
1033 err = bpf_map_update_elem(map_fd[4],
1034 &i, &txmsg_cork, BPF_ANY);
1035 if (err) {
1036 fprintf(stderr,
1037 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1038 err, strerror(errno));
1039 goto out;
1040 }
1041 }
1042
1043 if (txmsg_start) {
1044 err = bpf_map_update_elem(map_fd[5],
1045 &i, &txmsg_start, BPF_ANY);
1046 if (err) {
1047 fprintf(stderr,
1048 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1049 err, strerror(errno));
1050 goto out;
1051 }
1052 }
1053
1054 if (txmsg_end) {
1055 i = 1;
1056 err = bpf_map_update_elem(map_fd[5],
1057 &i, &txmsg_end, BPF_ANY);
1058 if (err) {
1059 fprintf(stderr,
1060 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1061 err, strerror(errno));
1062 goto out;
1063 }
1064 }
1065
1066 if (txmsg_start_push) {
1067 i = 2;
1068 err = bpf_map_update_elem(map_fd[5],
1069 &i, &txmsg_start_push, BPF_ANY);
1070 if (err) {
1071 fprintf(stderr,
1072 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1073 err, strerror(errno));
1074 goto out;
1075 }
1076 }
1077
1078 if (txmsg_end_push) {
1079 i = 3;
1080 err = bpf_map_update_elem(map_fd[5],
1081 &i, &txmsg_end_push, BPF_ANY);
1082 if (err) {
1083 fprintf(stderr,
1084 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1085 txmsg_end_push, i, err, strerror(errno));
1086 goto out;
1087 }
1088 }
1089
1090 if (txmsg_start_pop) {
1091 i = 4;
1092 err = bpf_map_update_elem(map_fd[5],
1093 &i, &txmsg_start_pop, BPF_ANY);
1094 if (err) {
1095 fprintf(stderr,
1096 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1097 txmsg_start_pop, i, err, strerror(errno));
1098 goto out;
1099 }
1100 } else {
1101 i = 4;
1102 bpf_map_update_elem(map_fd[5],
1103 &i, &txmsg_start_pop, BPF_ANY);
1104 }
1105
1106 if (txmsg_pop) {
1107 i = 5;
1108 err = bpf_map_update_elem(map_fd[5],
1109 &i, &txmsg_pop, BPF_ANY);
1110 if (err) {
1111 fprintf(stderr,
1112 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1113 txmsg_pop, i, err, strerror(errno));
1114 goto out;
1115 }
1116 } else {
1117 i = 5;
1118 bpf_map_update_elem(map_fd[5],
1119 &i, &txmsg_pop, BPF_ANY);
1120
1121 }
1122
1123 if (txmsg_ingress) {
1124 int in = BPF_F_INGRESS;
1125
1126 i = 0;
1127 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1128 if (err) {
1129 fprintf(stderr,
1130 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1131 err, strerror(errno));
1132 }
1133 i = 1;
1134 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1135 if (err) {
1136 fprintf(stderr,
1137 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1138 err, strerror(errno));
1139 }
1140 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1141 if (err) {
1142 fprintf(stderr,
1143 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1144 err, strerror(errno));
1145 }
1146
1147 i = 2;
1148 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1149 if (err) {
1150 fprintf(stderr,
1151 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1152 err, strerror(errno));
1153 }
1154 }
1155
1156 if (txmsg_ktls_skb) {
1157 int ingress = BPF_F_INGRESS;
1158
1159 i = 0;
1160 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1161 if (err) {
1162 fprintf(stderr,
1163 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1164 err, strerror(errno));
1165 }
1166
1167 if (txmsg_ktls_skb_redir) {
1168 i = 1;
1169 err = bpf_map_update_elem(map_fd[7],
1170 &i, &ingress, BPF_ANY);
1171 if (err) {
1172 fprintf(stderr,
1173 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1174 err, strerror(errno));
1175 }
1176 }
1177
1178 if (txmsg_ktls_skb_drop) {
1179 i = 1;
1180 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1181 }
1182 }
1183
1184 if (txmsg_redir_skb) {
1185 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1186 p2 : p1;
1187 int ingress = BPF_F_INGRESS;
1188
1189 i = 0;
1190 err = bpf_map_update_elem(map_fd[7],
1191 &i, &ingress, BPF_ANY);
1192 if (err) {
1193 fprintf(stderr,
1194 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1195 err, strerror(errno));
1196 }
1197
1198 i = 3;
1199 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1200 if (err) {
1201 fprintf(stderr,
1202 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1203 err, strerror(errno));
1204 }
1205 }
1206 }
1207
1208 if (skb_use_parser) {
1209 i = 2;
1210 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1211 }
1212
1213 if (txmsg_drop)
1214 options->drop_expected = true;
1215
1216 if (test == PING_PONG)
1217 err = forever_ping_pong(options->rate, options);
1218 else if (test == SENDMSG) {
1219 options->base = false;
1220 options->sendpage = false;
1221 err = sendmsg_test(options);
1222 } else if (test == SENDPAGE) {
1223 options->base = false;
1224 options->sendpage = true;
1225 err = sendmsg_test(options);
1226 } else if (test == BASE) {
1227 options->base = true;
1228 options->sendpage = false;
1229 err = sendmsg_test(options);
1230 } else if (test == BASE_SENDPAGE) {
1231 options->base = true;
1232 options->sendpage = true;
1233 err = sendmsg_test(options);
1234 } else
1235 fprintf(stderr, "unknown test\n");
1236 out:
1237 /* Detatch and zero all the maps */
1238 bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1239 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1240 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1241 bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1242 bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1243
1244 if (tx_prog_fd >= 0)
1245 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1246
1247 for (i = 0; i < 8; i++) {
1248 key = next_key = 0;
1249 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1250 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1251 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1252 key = next_key;
1253 }
1254 }
1255
1256 close(s1);
1257 close(s2);
1258 close(p1);
1259 close(p2);
1260 close(c1);
1261 close(c2);
1262 return err;
1263 }
1264
1265 static char *test_to_str(int test)
1266 {
1267 switch (test) {
1268 case SENDMSG:
1269 return "sendmsg";
1270 case SENDPAGE:
1271 return "sendpage";
1272 }
1273 return "unknown";
1274 }
1275
1276 #define OPTSTRING 60
1277 static void test_options(char *options)
1278 {
1279 char tstr[OPTSTRING];
1280
1281 memset(options, 0, OPTSTRING);
1282
1283 if (txmsg_pass)
1284 strncat(options, "pass,", OPTSTRING);
1285 if (txmsg_redir)
1286 strncat(options, "redir,", OPTSTRING);
1287 if (txmsg_drop)
1288 strncat(options, "drop,", OPTSTRING);
1289 if (txmsg_apply) {
1290 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1291 strncat(options, tstr, OPTSTRING);
1292 }
1293 if (txmsg_cork) {
1294 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1295 strncat(options, tstr, OPTSTRING);
1296 }
1297 if (txmsg_start) {
1298 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1299 strncat(options, tstr, OPTSTRING);
1300 }
1301 if (txmsg_end) {
1302 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1303 strncat(options, tstr, OPTSTRING);
1304 }
1305 if (txmsg_start_pop) {
1306 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1307 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1308 strncat(options, tstr, OPTSTRING);
1309 }
1310 if (txmsg_ingress)
1311 strncat(options, "ingress,", OPTSTRING);
1312 if (txmsg_redir_skb)
1313 strncat(options, "redir_skb,", OPTSTRING);
1314 if (txmsg_ktls_skb)
1315 strncat(options, "ktls_skb,", OPTSTRING);
1316 if (ktls)
1317 strncat(options, "ktls,", OPTSTRING);
1318 if (peek_flag)
1319 strncat(options, "peek,", OPTSTRING);
1320 }
1321
1322 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1323 {
1324 char *options = calloc(OPTSTRING, sizeof(char));
1325 int err;
1326
1327 if (test == SENDPAGE)
1328 opt->sendpage = true;
1329 else
1330 opt->sendpage = false;
1331
1332 if (txmsg_drop)
1333 opt->drop_expected = true;
1334 else
1335 opt->drop_expected = false;
1336
1337 test_options(options);
1338
1339 if (opt->verbose) {
1340 fprintf(stdout,
1341 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1342 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1343 test_to_str(test), options);
1344 fflush(stdout);
1345 }
1346 err = run_options(opt, cgrp, test);
1347 if (opt->verbose)
1348 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1349 test_cnt++;
1350 !err ? passed++ : failed++;
1351 free(options);
1352 return err;
1353 }
1354
1355 static void test_exec(int cgrp, struct sockmap_options *opt)
1356 {
1357 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1358 int err;
1359
1360 if (type == 0) {
1361 test_start();
1362 err = __test_exec(cgrp, SENDMSG, opt);
1363 if (err)
1364 test_fail();
1365 } else {
1366 test_start();
1367 err = __test_exec(cgrp, SENDPAGE, opt);
1368 if (err)
1369 test_fail();
1370 }
1371 }
1372
1373 static void test_send_one(struct sockmap_options *opt, int cgrp)
1374 {
1375 opt->iov_length = 1;
1376 opt->iov_count = 1;
1377 opt->rate = 1;
1378 test_exec(cgrp, opt);
1379
1380 opt->iov_length = 1;
1381 opt->iov_count = 1024;
1382 opt->rate = 1;
1383 test_exec(cgrp, opt);
1384
1385 opt->iov_length = 1024;
1386 opt->iov_count = 1;
1387 opt->rate = 1;
1388 test_exec(cgrp, opt);
1389
1390 }
1391
1392 static void test_send_many(struct sockmap_options *opt, int cgrp)
1393 {
1394 opt->iov_length = 3;
1395 opt->iov_count = 1;
1396 opt->rate = 512;
1397 test_exec(cgrp, opt);
1398
1399 opt->rate = 100;
1400 opt->iov_count = 1;
1401 opt->iov_length = 5;
1402 test_exec(cgrp, opt);
1403 }
1404
1405 static void test_send_large(struct sockmap_options *opt, int cgrp)
1406 {
1407 opt->iov_length = 256;
1408 opt->iov_count = 1024;
1409 opt->rate = 2;
1410 test_exec(cgrp, opt);
1411 }
1412
1413 static void test_send(struct sockmap_options *opt, int cgrp)
1414 {
1415 test_send_one(opt, cgrp);
1416 test_send_many(opt, cgrp);
1417 test_send_large(opt, cgrp);
1418 sched_yield();
1419 }
1420
1421 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1422 {
1423 /* Test small and large iov_count values with pass/redir/apply/cork */
1424 txmsg_pass = 1;
1425 test_send(opt, cgrp);
1426 }
1427
1428 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1429 {
1430 txmsg_redir = 1;
1431 test_send(opt, cgrp);
1432 }
1433
1434 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1435 {
1436 txmsg_drop = 1;
1437 test_send(opt, cgrp);
1438 }
1439
1440 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1441 {
1442 txmsg_pass = txmsg_drop = 0;
1443 txmsg_ingress = txmsg_redir = 1;
1444 test_send(opt, cgrp);
1445 }
1446
1447 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1448 {
1449 bool data = opt->data_test;
1450 int k = ktls;
1451
1452 opt->data_test = true;
1453 ktls = 1;
1454
1455 txmsg_pass = txmsg_drop = 0;
1456 txmsg_ingress = txmsg_redir = 0;
1457 txmsg_ktls_skb = 1;
1458 txmsg_pass = 1;
1459
1460 /* Using data verification so ensure iov layout is
1461 * expected from test receiver side. e.g. has enough
1462 * bytes to write test code.
1463 */
1464 opt->iov_length = 100;
1465 opt->iov_count = 1;
1466 opt->rate = 1;
1467 test_exec(cgrp, opt);
1468
1469 txmsg_ktls_skb_drop = 1;
1470 test_exec(cgrp, opt);
1471
1472 txmsg_ktls_skb_drop = 0;
1473 txmsg_ktls_skb_redir = 1;
1474 test_exec(cgrp, opt);
1475 txmsg_ktls_skb_redir = 0;
1476
1477 /* Tests that omit skb_parser */
1478 txmsg_omit_skb_parser = 1;
1479 ktls = 0;
1480 txmsg_ktls_skb = 0;
1481 test_exec(cgrp, opt);
1482
1483 txmsg_ktls_skb_drop = 1;
1484 test_exec(cgrp, opt);
1485 txmsg_ktls_skb_drop = 0;
1486
1487 txmsg_ktls_skb_redir = 1;
1488 test_exec(cgrp, opt);
1489
1490 ktls = 1;
1491 test_exec(cgrp, opt);
1492 txmsg_omit_skb_parser = 0;
1493
1494 opt->data_test = data;
1495 ktls = k;
1496 }
1497
1498 /* Test cork with hung data. This tests poor usage patterns where
1499 * cork can leave data on the ring if user program is buggy and
1500 * doesn't flush them somehow. They do take some time however
1501 * because they wait for a timeout. Test pass, redir and cork with
1502 * apply logic. Use cork size of 4097 with send_large to avoid
1503 * aligning cork size with send size.
1504 */
1505 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1506 {
1507 txmsg_pass = 1;
1508 txmsg_redir = 0;
1509 txmsg_cork = 4097;
1510 txmsg_apply = 4097;
1511 test_send_large(opt, cgrp);
1512
1513 txmsg_pass = 0;
1514 txmsg_redir = 1;
1515 txmsg_apply = 0;
1516 txmsg_cork = 4097;
1517 test_send_large(opt, cgrp);
1518
1519 txmsg_pass = 0;
1520 txmsg_redir = 1;
1521 txmsg_apply = 4097;
1522 txmsg_cork = 4097;
1523 test_send_large(opt, cgrp);
1524 }
1525
1526 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1527 {
1528 /* Test basic start/end */
1529 txmsg_start = 1;
1530 txmsg_end = 2;
1531 test_send(opt, cgrp);
1532
1533 /* Test >4k pull */
1534 txmsg_start = 4096;
1535 txmsg_end = 9182;
1536 test_send_large(opt, cgrp);
1537
1538 /* Test pull + redirect */
1539 txmsg_redir = 0;
1540 txmsg_start = 1;
1541 txmsg_end = 2;
1542 test_send(opt, cgrp);
1543
1544 /* Test pull + cork */
1545 txmsg_redir = 0;
1546 txmsg_cork = 512;
1547 txmsg_start = 1;
1548 txmsg_end = 2;
1549 test_send_many(opt, cgrp);
1550
1551 /* Test pull + cork + redirect */
1552 txmsg_redir = 1;
1553 txmsg_cork = 512;
1554 txmsg_start = 1;
1555 txmsg_end = 2;
1556 test_send_many(opt, cgrp);
1557 }
1558
1559 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1560 {
1561 /* Test basic pop */
1562 txmsg_start_pop = 1;
1563 txmsg_pop = 2;
1564 test_send_many(opt, cgrp);
1565
1566 /* Test pop with >4k */
1567 txmsg_start_pop = 4096;
1568 txmsg_pop = 4096;
1569 test_send_large(opt, cgrp);
1570
1571 /* Test pop + redirect */
1572 txmsg_redir = 1;
1573 txmsg_start_pop = 1;
1574 txmsg_pop = 2;
1575 test_send_many(opt, cgrp);
1576
1577 /* Test pop + cork */
1578 txmsg_redir = 0;
1579 txmsg_cork = 512;
1580 txmsg_start_pop = 1;
1581 txmsg_pop = 2;
1582 test_send_many(opt, cgrp);
1583
1584 /* Test pop + redirect + cork */
1585 txmsg_redir = 1;
1586 txmsg_cork = 4;
1587 txmsg_start_pop = 1;
1588 txmsg_pop = 2;
1589 test_send_many(opt, cgrp);
1590 }
1591
1592 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1593 {
1594 /* Test basic push */
1595 txmsg_start_push = 1;
1596 txmsg_end_push = 1;
1597 test_send(opt, cgrp);
1598
1599 /* Test push 4kB >4k */
1600 txmsg_start_push = 4096;
1601 txmsg_end_push = 4096;
1602 test_send_large(opt, cgrp);
1603
1604 /* Test push + redirect */
1605 txmsg_redir = 1;
1606 txmsg_start_push = 1;
1607 txmsg_end_push = 2;
1608 test_send_many(opt, cgrp);
1609
1610 /* Test push + cork */
1611 txmsg_redir = 0;
1612 txmsg_cork = 512;
1613 txmsg_start_push = 1;
1614 txmsg_end_push = 2;
1615 test_send_many(opt, cgrp);
1616 }
1617
1618 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1619 {
1620 txmsg_start_push = 1;
1621 txmsg_end_push = 10;
1622 txmsg_start_pop = 5;
1623 txmsg_pop = 4;
1624 test_send_large(opt, cgrp);
1625 }
1626
1627 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1628 {
1629 txmsg_pass = 1;
1630 txmsg_redir = 0;
1631 txmsg_apply = 1;
1632 txmsg_cork = 0;
1633 test_send_one(opt, cgrp);
1634
1635 txmsg_pass = 0;
1636 txmsg_redir = 1;
1637 txmsg_apply = 1;
1638 txmsg_cork = 0;
1639 test_send_one(opt, cgrp);
1640
1641 txmsg_pass = 1;
1642 txmsg_redir = 0;
1643 txmsg_apply = 1024;
1644 txmsg_cork = 0;
1645 test_send_large(opt, cgrp);
1646
1647 txmsg_pass = 0;
1648 txmsg_redir = 1;
1649 txmsg_apply = 1024;
1650 txmsg_cork = 0;
1651 test_send_large(opt, cgrp);
1652 }
1653
1654 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1655 {
1656 txmsg_pass = 1;
1657 txmsg_redir = 0;
1658 txmsg_apply = 0;
1659 txmsg_cork = 1;
1660 test_send(opt, cgrp);
1661
1662 txmsg_pass = 1;
1663 txmsg_redir = 0;
1664 txmsg_apply = 1;
1665 txmsg_cork = 1;
1666 test_send(opt, cgrp);
1667 }
1668
1669 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1670 {
1671 txmsg_pass = 1;
1672 skb_use_parser = 512;
1673 opt->iov_length = 256;
1674 opt->iov_count = 1;
1675 opt->rate = 2;
1676 test_exec(cgrp, opt);
1677 }
1678
1679 char *map_names[] = {
1680 "sock_map",
1681 "sock_map_txmsg",
1682 "sock_map_redir",
1683 "sock_apply_bytes",
1684 "sock_cork_bytes",
1685 "sock_bytes",
1686 "sock_redir_flags",
1687 "sock_skb_opts",
1688 "tls_sock_map",
1689 };
1690
1691 int prog_attach_type[] = {
1692 BPF_SK_SKB_STREAM_PARSER,
1693 BPF_SK_SKB_STREAM_VERDICT,
1694 BPF_SK_SKB_STREAM_VERDICT,
1695 BPF_CGROUP_SOCK_OPS,
1696 BPF_SK_MSG_VERDICT,
1697 BPF_SK_MSG_VERDICT,
1698 BPF_SK_MSG_VERDICT,
1699 BPF_SK_MSG_VERDICT,
1700 BPF_SK_MSG_VERDICT,
1701 BPF_SK_MSG_VERDICT,
1702 BPF_SK_MSG_VERDICT,
1703 };
1704
1705 int prog_type[] = {
1706 BPF_PROG_TYPE_SK_SKB,
1707 BPF_PROG_TYPE_SK_SKB,
1708 BPF_PROG_TYPE_SK_SKB,
1709 BPF_PROG_TYPE_SOCK_OPS,
1710 BPF_PROG_TYPE_SK_MSG,
1711 BPF_PROG_TYPE_SK_MSG,
1712 BPF_PROG_TYPE_SK_MSG,
1713 BPF_PROG_TYPE_SK_MSG,
1714 BPF_PROG_TYPE_SK_MSG,
1715 BPF_PROG_TYPE_SK_MSG,
1716 BPF_PROG_TYPE_SK_MSG,
1717 };
1718
1719 static int populate_progs(char *bpf_file)
1720 {
1721 struct bpf_program *prog;
1722 struct bpf_object *obj;
1723 int i = 0;
1724 long err;
1725
1726 obj = bpf_object__open(bpf_file);
1727 err = libbpf_get_error(obj);
1728 if (err) {
1729 char err_buf[256];
1730
1731 libbpf_strerror(err, err_buf, sizeof(err_buf));
1732 printf("Unable to load eBPF objects in file '%s' : %s\n",
1733 bpf_file, err_buf);
1734 return -1;
1735 }
1736
1737 bpf_object__for_each_program(prog, obj) {
1738 bpf_program__set_type(prog, prog_type[i]);
1739 bpf_program__set_expected_attach_type(prog,
1740 prog_attach_type[i]);
1741 i++;
1742 }
1743
1744 i = bpf_object__load(obj);
1745 i = 0;
1746 bpf_object__for_each_program(prog, obj) {
1747 prog_fd[i] = bpf_program__fd(prog);
1748 i++;
1749 }
1750
1751 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1752 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1753 map_fd[i] = bpf_map__fd(maps[i]);
1754 if (map_fd[i] < 0) {
1755 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1756 map_fd[i], strerror(errno));
1757 return -1;
1758 }
1759 }
1760
1761 return 0;
1762 }
1763
1764 struct _test test[] = {
1765 {"txmsg test passthrough", test_txmsg_pass},
1766 {"txmsg test redirect", test_txmsg_redir},
1767 {"txmsg test drop", test_txmsg_drop},
1768 {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1769 {"txmsg test skb", test_txmsg_skb},
1770 {"txmsg test apply", test_txmsg_apply},
1771 {"txmsg test cork", test_txmsg_cork},
1772 {"txmsg test hanging corks", test_txmsg_cork_hangs},
1773 {"txmsg test push_data", test_txmsg_push},
1774 {"txmsg test pull-data", test_txmsg_pull},
1775 {"txmsg test pop-data", test_txmsg_pop},
1776 {"txmsg test push/pop data", test_txmsg_push_pop},
1777 {"txmsg text ingress parser", test_txmsg_ingress_parser},
1778 };
1779
1780 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1781 {
1782 char *entry, *ptr;
1783
1784 if (!opt->whitelist)
1785 return 0;
1786 ptr = strdup(opt->whitelist);
1787 if (!ptr)
1788 return -ENOMEM;
1789 entry = strtok(ptr, ",");
1790 while (entry) {
1791 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1792 strstr(opt->map, entry) != 0 ||
1793 strstr(t->title, entry) != 0)
1794 return 0;
1795 entry = strtok(NULL, ",");
1796 }
1797 return -EINVAL;
1798 }
1799
1800 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1801 {
1802 char *entry, *ptr;
1803
1804 if (!opt->blacklist)
1805 return -EINVAL;
1806 ptr = strdup(opt->blacklist);
1807 if (!ptr)
1808 return -ENOMEM;
1809 entry = strtok(ptr, ",");
1810 while (entry) {
1811 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1812 strstr(opt->map, entry) != 0 ||
1813 strstr(t->title, entry) != 0)
1814 return 0;
1815 entry = strtok(NULL, ",");
1816 }
1817 return -EINVAL;
1818 }
1819
1820 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1821 {
1822 int i, err;
1823
1824 err = populate_progs(opt->map);
1825 if (err < 0) {
1826 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1827 return err;
1828 }
1829
1830 /* Tests basic commands and APIs */
1831 for (i = 0; i < sizeof(test)/sizeof(struct _test); i++) {
1832 struct _test t = test[i];
1833
1834 if (check_whitelist(&t, opt) != 0)
1835 continue;
1836 if (check_blacklist(&t, opt) == 0)
1837 continue;
1838
1839 test_start_subtest(&t, opt);
1840 t.tester(cg_fd, opt);
1841 test_end_subtest();
1842 }
1843
1844 return err;
1845 }
1846
1847 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1848 {
1849 opt->map = BPF_SOCKMAP_FILENAME;
1850 __test_selftests(cg_fd, opt);
1851 }
1852
1853 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1854 {
1855 opt->map = BPF_SOCKHASH_FILENAME;
1856 __test_selftests(cg_fd, opt);
1857 }
1858
1859 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1860 {
1861 opt->map = BPF_SOCKHASH_FILENAME;
1862 opt->prepend = "ktls";
1863 ktls = 1;
1864 __test_selftests(cg_fd, opt);
1865 ktls = 0;
1866 }
1867
1868 static int test_selftest(int cg_fd, struct sockmap_options *opt)
1869 {
1870
1871 test_selftests_sockmap(cg_fd, opt);
1872 test_selftests_sockhash(cg_fd, opt);
1873 test_selftests_ktls(cg_fd, opt);
1874 test_print_results();
1875 return 0;
1876 }
1877
1878 int main(int argc, char **argv)
1879 {
1880 int iov_count = 1, length = 1024, rate = 1;
1881 struct sockmap_options options = {0};
1882 int opt, longindex, err, cg_fd = 0;
1883 char *bpf_file = BPF_SOCKMAP_FILENAME;
1884 int test = SELFTESTS;
1885 bool cg_created = 0;
1886
1887 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1888 long_options, &longindex)) != -1) {
1889 switch (opt) {
1890 case 's':
1891 txmsg_start = atoi(optarg);
1892 break;
1893 case 'e':
1894 txmsg_end = atoi(optarg);
1895 break;
1896 case 'p':
1897 txmsg_start_push = atoi(optarg);
1898 break;
1899 case 'q':
1900 txmsg_end_push = atoi(optarg);
1901 break;
1902 case 'w':
1903 txmsg_start_pop = atoi(optarg);
1904 break;
1905 case 'x':
1906 txmsg_pop = atoi(optarg);
1907 break;
1908 case 'a':
1909 txmsg_apply = atoi(optarg);
1910 break;
1911 case 'k':
1912 txmsg_cork = atoi(optarg);
1913 break;
1914 case 'c':
1915 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1916 if (cg_fd < 0) {
1917 fprintf(stderr,
1918 "ERROR: (%i) open cg path failed: %s\n",
1919 cg_fd, optarg);
1920 return cg_fd;
1921 }
1922 break;
1923 case 'r':
1924 rate = atoi(optarg);
1925 break;
1926 case 'v':
1927 options.verbose = 1;
1928 if (optarg)
1929 options.verbose = atoi(optarg);
1930 break;
1931 case 'i':
1932 iov_count = atoi(optarg);
1933 break;
1934 case 'l':
1935 length = atoi(optarg);
1936 break;
1937 case 'd':
1938 options.data_test = true;
1939 break;
1940 case 't':
1941 if (strcmp(optarg, "ping") == 0) {
1942 test = PING_PONG;
1943 } else if (strcmp(optarg, "sendmsg") == 0) {
1944 test = SENDMSG;
1945 } else if (strcmp(optarg, "base") == 0) {
1946 test = BASE;
1947 } else if (strcmp(optarg, "base_sendpage") == 0) {
1948 test = BASE_SENDPAGE;
1949 } else if (strcmp(optarg, "sendpage") == 0) {
1950 test = SENDPAGE;
1951 } else {
1952 usage(argv);
1953 return -1;
1954 }
1955 break;
1956 case 'n':
1957 options.whitelist = strdup(optarg);
1958 if (!options.whitelist)
1959 return -ENOMEM;
1960 break;
1961 case 'b':
1962 options.blacklist = strdup(optarg);
1963 if (!options.blacklist)
1964 return -ENOMEM;
1965 case 0:
1966 break;
1967 case 'h':
1968 default:
1969 usage(argv);
1970 return -1;
1971 }
1972 }
1973
1974 if (!cg_fd) {
1975 cg_fd = cgroup_setup_and_join(CG_PATH);
1976 if (cg_fd < 0)
1977 return cg_fd;
1978 cg_created = 1;
1979 }
1980
1981 if (test == SELFTESTS) {
1982 err = test_selftest(cg_fd, &options);
1983 goto out;
1984 }
1985
1986 err = populate_progs(bpf_file);
1987 if (err) {
1988 fprintf(stderr, "populate program: (%s) %s\n",
1989 bpf_file, strerror(errno));
1990 return 1;
1991 }
1992 running = 1;
1993
1994 /* catch SIGINT */
1995 signal(SIGINT, running_handler);
1996
1997 options.iov_count = iov_count;
1998 options.iov_length = length;
1999 options.rate = rate;
2000
2001 err = run_options(&options, cg_fd, test);
2002 out:
2003 if (options.whitelist)
2004 free(options.whitelist);
2005 if (options.blacklist)
2006 free(options.blacklist);
2007 if (cg_created)
2008 cleanup_cgroup_environment();
2009 close(cg_fd);
2010 return err;
2011 }
2012
2013 void running_handler(int a)
2014 {
2015 running = 0;
2016 }
Ubuntu Hirsute Linux kernel mirror