1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
5 #include <sys/socket.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
21 #include <sys/resource.h>
22 #include <sys/types.h>
23 #include <sys/sendfile.h>
25 #include <linux/netlink.h>
26 #include <linux/socket.h>
27 #include <linux/sock_diag.h>
28 #include <linux/bpf.h>
29 #include <linux/if_link.h>
30 #include <linux/tls.h>
37 #include <bpf/libbpf.h>
40 #include "bpf_rlimit.h"
41 #include "cgroup_helpers.h"
44 static void running_handler(int a
);
53 /* randomly selected ports for testing on lo */
57 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
58 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
59 #define CG_PATH "/sockmap"
62 int s1
, s2
, c1
, c2
, p1
, p2
;
67 struct bpf_map
*maps
[9];
84 int txmsg_ktls_skb_drop
;
85 int txmsg_ktls_skb_redir
;
89 int txmsg_omit_skb_parser
;
91 static const struct option long_options
[] = {
92 {"help", no_argument
, NULL
, 'h' },
93 {"cgroup", required_argument
, NULL
, 'c' },
94 {"rate", required_argument
, NULL
, 'r' },
95 {"verbose", optional_argument
, NULL
, 'v' },
96 {"iov_count", required_argument
, NULL
, 'i' },
97 {"length", required_argument
, NULL
, 'l' },
98 {"test", required_argument
, NULL
, 't' },
99 {"data_test", no_argument
, NULL
, 'd' },
100 {"txmsg", no_argument
, &txmsg_pass
, 1 },
101 {"txmsg_redir", no_argument
, &txmsg_redir
, 1 },
102 {"txmsg_drop", no_argument
, &txmsg_drop
, 1 },
103 {"txmsg_apply", required_argument
, NULL
, 'a'},
104 {"txmsg_cork", required_argument
, NULL
, 'k'},
105 {"txmsg_start", required_argument
, NULL
, 's'},
106 {"txmsg_end", required_argument
, NULL
, 'e'},
107 {"txmsg_start_push", required_argument
, NULL
, 'p'},
108 {"txmsg_end_push", required_argument
, NULL
, 'q'},
109 {"txmsg_start_pop", required_argument
, NULL
, 'w'},
110 {"txmsg_pop", required_argument
, NULL
, 'x'},
111 {"txmsg_ingress", no_argument
, &txmsg_ingress
, 1 },
112 {"txmsg_redir_skb", no_argument
, &txmsg_redir_skb
, 1 },
113 {"ktls", no_argument
, &ktls
, 1 },
114 {"peek", no_argument
, &peek_flag
, 1 },
115 {"txmsg_omit_skb_parser", no_argument
, &txmsg_omit_skb_parser
, 1},
116 {"whitelist", required_argument
, NULL
, 'n' },
117 {"blacklist", required_argument
, NULL
, 'b' },
136 struct sockmap_options
{
153 void (*tester
)(int cg_fd
, struct sockmap_options
*opt
);
156 static void test_start(void)
161 static void test_fail(void)
166 static void test_pass(void)
171 static void test_reset(void)
173 txmsg_start
= txmsg_end
= 0;
174 txmsg_start_pop
= txmsg_pop
= 0;
175 txmsg_start_push
= txmsg_end_push
= 0;
176 txmsg_pass
= txmsg_drop
= txmsg_redir
= 0;
177 txmsg_apply
= txmsg_cork
= 0;
178 txmsg_ingress
= txmsg_redir_skb
= 0;
179 txmsg_ktls_skb
= txmsg_ktls_skb_drop
= txmsg_ktls_skb_redir
= 0;
180 txmsg_omit_skb_parser
= 0;
184 static int test_start_subtest(const struct _test
*t
, struct sockmap_options
*o
)
187 env
.subtest
= t
->title
;
188 env
.prepend
= o
->prepend
;
191 env
.fail_last
= env
.fail_cnt
;
196 static void test_end_subtest(void)
198 int error
= env
.fail_cnt
- env
.fail_last
;
199 int type
= strcmp(env
.type
, BPF_SOCKMAP_FILENAME
);
204 fprintf(stdout
, "#%2d/%2d %8s:%s:%s:%s\n",
205 env
.test_num
, env
.subtest_num
,
206 !type
? "sockmap" : "sockhash",
208 env
.subtest
, error
? "FAIL" : "OK");
211 static void test_print_results(void)
213 fprintf(stdout
, "Pass: %d Fail: %d\n",
214 env
.succ_cnt
, env
.fail_cnt
);
217 static void usage(char *argv
[])
221 printf(" Usage: %s --cgroup <cgroup_path>\n", argv
[0]);
222 printf(" options:\n");
223 for (i
= 0; long_options
[i
].name
!= 0; i
++) {
224 printf(" --%-12s", long_options
[i
].name
);
225 if (long_options
[i
].flag
!= NULL
)
226 printf(" flag (internal value:%d)\n",
227 *long_options
[i
].flag
);
229 printf(" -%c\n", long_options
[i
].val
);
234 char *sock_to_string(int s
)
252 static int sockmap_init_ktls(int verbose
, int s
)
254 struct tls12_crypto_info_aes_gcm_128 tls_tx
= {
256 .version
= TLS_1_2_VERSION
,
257 .cipher_type
= TLS_CIPHER_AES_GCM_128
,
260 struct tls12_crypto_info_aes_gcm_128 tls_rx
= {
262 .version
= TLS_1_2_VERSION
,
263 .cipher_type
= TLS_CIPHER_AES_GCM_128
,
266 int so_buf
= 6553500;
269 err
= setsockopt(s
, 6, TCP_ULP
, "tls", sizeof("tls"));
271 fprintf(stderr
, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s
), err
);
274 err
= setsockopt(s
, SOL_TLS
, TLS_TX
, (void *)&tls_tx
, sizeof(tls_tx
));
276 fprintf(stderr
, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s
), err
);
279 err
= setsockopt(s
, SOL_TLS
, TLS_RX
, (void *)&tls_rx
, sizeof(tls_rx
));
281 fprintf(stderr
, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s
), err
);
284 err
= setsockopt(s
, SOL_SOCKET
, SO_SNDBUF
, &so_buf
, sizeof(so_buf
));
286 fprintf(stderr
, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s
), err
);
289 err
= setsockopt(s
, SOL_SOCKET
, SO_RCVBUF
, &so_buf
, sizeof(so_buf
));
291 fprintf(stderr
, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s
), err
);
296 fprintf(stdout
, "socket(%s) kTLS enabled\n", sock_to_string(s
));
299 static int sockmap_init_sockets(int verbose
)
302 struct sockaddr_in addr
;
303 int *fds
[4] = {&s1
, &s2
, &c1
, &c2
};
305 s1
= s2
= p1
= p2
= c1
= c2
= 0;
308 for (i
= 0; i
< 4; i
++) {
309 *fds
[i
] = socket(AF_INET
, SOCK_STREAM
, 0);
311 perror("socket s1 failed()");
317 for (i
= 0; i
< 2; i
++) {
318 err
= setsockopt(*fds
[i
], SOL_SOCKET
, SO_REUSEADDR
,
319 (char *)&one
, sizeof(one
));
321 perror("setsockopt failed()");
326 /* Non-blocking sockets */
327 for (i
= 0; i
< 2; i
++) {
328 err
= ioctl(*fds
[i
], FIONBIO
, (char *)&one
);
330 perror("ioctl s1 failed()");
335 /* Bind server sockets */
336 memset(&addr
, 0, sizeof(struct sockaddr_in
));
337 addr
.sin_family
= AF_INET
;
338 addr
.sin_addr
.s_addr
= inet_addr("127.0.0.1");
340 addr
.sin_port
= htons(S1_PORT
);
341 err
= bind(s1
, (struct sockaddr
*)&addr
, sizeof(addr
));
343 perror("bind s1 failed()");
347 addr
.sin_port
= htons(S2_PORT
);
348 err
= bind(s2
, (struct sockaddr
*)&addr
, sizeof(addr
));
350 perror("bind s2 failed()");
354 /* Listen server sockets */
355 addr
.sin_port
= htons(S1_PORT
);
356 err
= listen(s1
, 32);
358 perror("listen s1 failed()");
362 addr
.sin_port
= htons(S2_PORT
);
363 err
= listen(s2
, 32);
365 perror("listen s1 failed()");
369 /* Initiate Connect */
370 addr
.sin_port
= htons(S1_PORT
);
371 err
= connect(c1
, (struct sockaddr
*)&addr
, sizeof(addr
));
372 if (err
< 0 && errno
!= EINPROGRESS
) {
373 perror("connect c1 failed()");
377 addr
.sin_port
= htons(S2_PORT
);
378 err
= connect(c2
, (struct sockaddr
*)&addr
, sizeof(addr
));
379 if (err
< 0 && errno
!= EINPROGRESS
) {
380 perror("connect c2 failed()");
382 } else if (err
< 0) {
386 /* Accept Connecrtions */
387 p1
= accept(s1
, NULL
, NULL
);
389 perror("accept s1 failed()");
393 p2
= accept(s2
, NULL
, NULL
);
395 perror("accept s1 failed()");
400 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
401 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
410 struct timespec start
;
414 static int msg_loop_sendpage(int fd
, int iov_length
, int cnt
,
416 struct sockmap_options
*opt
)
418 bool drop
= opt
->drop_expected
;
425 perror("create file for sendpage");
428 for (i
= 0; i
< iov_length
* cnt
; i
++, k
++)
429 fwrite(&k
, sizeof(char), 1, file
);
431 fseek(file
, 0, SEEK_SET
);
435 clock_gettime(CLOCK_MONOTONIC
, &s
->start
);
436 for (i
= 0; i
< cnt
; i
++) {
440 sent
= sendfile(fd
, fp
, NULL
, iov_length
);
442 if (!drop
&& sent
< 0) {
443 perror("sendpage loop error");
446 } else if (drop
&& sent
>= 0) {
447 printf("sendpage loop error expected: %i errno %i\n",
454 s
->bytes_sent
+= sent
;
456 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
461 static void msg_free_iov(struct msghdr
*msg
)
465 for (i
= 0; i
< msg
->msg_iovlen
; i
++)
466 free(msg
->msg_iov
[i
].iov_base
);
472 static int msg_alloc_iov(struct msghdr
*msg
,
473 int iov_count
, int iov_length
,
474 bool data
, bool xmit
)
480 iov
= calloc(iov_count
, sizeof(struct iovec
));
484 for (i
= 0; i
< iov_count
; i
++) {
485 unsigned char *d
= calloc(iov_length
, sizeof(char));
488 fprintf(stderr
, "iov_count %i/%i OOM\n", i
, iov_count
);
492 iov
[i
].iov_len
= iov_length
;
497 for (j
= 0; j
< iov_length
; j
++)
503 msg
->msg_iovlen
= iov_count
;
507 for (i
--; i
>= 0 ; i
--)
508 free(msg
->msg_iov
[i
].iov_base
);
512 static int msg_verify_data(struct msghdr
*msg
, int size
, int chunk_sz
)
514 int i
, j
= 0, bytes_cnt
= 0;
517 for (i
= 0; i
< msg
->msg_iovlen
; i
++) {
518 unsigned char *d
= msg
->msg_iov
[i
].iov_base
;
520 /* Special case test for skb ingress + ktls */
521 if (i
== 0 && txmsg_ktls_skb
) {
522 if (msg
->msg_iov
[i
].iov_len
< 4)
524 if (memcmp(d
, "PASS", 4) != 0) {
526 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
527 i
, 0, d
[0], d
[1], d
[2], d
[3]);
530 j
= 4; /* advance index past PASS header */
533 for (; j
< msg
->msg_iov
[i
].iov_len
&& size
; j
++) {
536 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
537 i
, j
, d
[j
], k
- 1, d
[j
+1], k
);
541 if (bytes_cnt
== chunk_sz
) {
551 static int msg_loop(int fd
, int iov_count
, int iov_length
, int cnt
,
552 struct msg_stats
*s
, bool tx
,
553 struct sockmap_options
*opt
)
555 struct msghdr msg
= {0}, msg_peek
= {0};
556 int err
, i
, flags
= MSG_NOSIGNAL
;
557 bool drop
= opt
->drop_expected
;
558 bool data
= opt
->data_test
;
560 err
= msg_alloc_iov(&msg
, iov_count
, iov_length
, data
, tx
);
564 err
= msg_alloc_iov(&msg_peek
, iov_count
, iov_length
, data
, tx
);
570 clock_gettime(CLOCK_MONOTONIC
, &s
->start
);
571 for (i
= 0; i
< cnt
; i
++) {
575 sent
= sendmsg(fd
, &msg
, flags
);
577 if (!drop
&& sent
< 0) {
578 perror("sendmsg loop error");
580 } else if (drop
&& sent
>= 0) {
582 "sendmsg loop error expected: %i errno %i\n",
588 s
->bytes_sent
+= sent
;
590 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
592 int slct
, recvp
= 0, recv
, max_fd
= fd
;
593 float total_bytes
, txmsg_pop_total
;
594 int fd_flags
= O_NONBLOCK
;
595 struct timeval timeout
;
599 /* Account for pop bytes noting each iteration of apply will
600 * call msg_pop_data helper so we need to account for this
601 * by calculating the number of apply iterations. Note user
602 * of the tool can create cases where no data is sent by
603 * manipulating pop/push/pull/etc. For example txmsg_apply 1
604 * with txmsg_pop 1 will try to apply 1B at a time but each
605 * iteration will then pop 1B so no data will ever be sent.
606 * This is really only useful for testing edge cases in code
609 total_bytes
= (float)iov_count
* (float)iov_length
* (float)cnt
;
611 txmsg_pop_total
= txmsg_pop
* (total_bytes
/ txmsg_apply
);
613 txmsg_pop_total
= txmsg_pop
* cnt
;
614 total_bytes
-= txmsg_pop_total
;
615 err
= clock_gettime(CLOCK_MONOTONIC
, &s
->start
);
617 perror("recv start time");
618 while (s
->bytes_recvd
< total_bytes
) {
621 timeout
.tv_usec
= 300000;
631 slct
= select(max_fd
+ 1, &w
, NULL
, NULL
, &timeout
);
634 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
638 fprintf(stderr
, "unexpected timeout: recved %zu/%f pop_total %f\n", s
->bytes_recvd
, total_bytes
, txmsg_pop_total
);
640 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
647 recvp
= recvmsg(fd
, &msg_peek
, flags
);
649 if (errno
!= EWOULDBLOCK
) {
650 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
657 recv
= recvmsg(fd
, &msg
, flags
);
659 if (errno
!= EWOULDBLOCK
) {
660 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
661 perror("recv failed()");
666 s
->bytes_recvd
+= recv
;
669 int chunk_sz
= opt
->sendpage
?
671 iov_length
* iov_count
;
673 errno
= msg_verify_data(&msg
, recv
, chunk_sz
);
675 perror("data verify msg failed");
679 errno
= msg_verify_data(&msg_peek
,
683 perror("data verify msg_peek failed");
689 clock_gettime(CLOCK_MONOTONIC
, &s
->end
);
693 msg_free_iov(&msg_peek
);
697 msg_free_iov(&msg_peek
);
701 static float giga
= 1000000000;
703 static inline float sentBps(struct msg_stats s
)
705 return s
.bytes_sent
/ (s
.end
.tv_sec
- s
.start
.tv_sec
);
708 static inline float recvdBps(struct msg_stats s
)
710 return s
.bytes_recvd
/ (s
.end
.tv_sec
- s
.start
.tv_sec
);
713 static int sendmsg_test(struct sockmap_options
*opt
)
715 float sent_Bps
= 0, recvd_Bps
= 0;
716 int rx_fd
, txpid
, rxpid
, err
= 0;
717 struct msg_stats s
= {0};
718 int iov_count
= opt
->iov_count
;
719 int iov_buf
= opt
->iov_length
;
720 int rx_status
, tx_status
;
731 /* Redirecting into non-TLS socket which sends into a TLS
732 * socket is not a valid test. So in this case lets not
733 * enable kTLS but still run the test.
735 if (!txmsg_redir
|| txmsg_ingress
) {
736 err
= sockmap_init_ktls(opt
->verbose
, rx_fd
);
740 err
= sockmap_init_ktls(opt
->verbose
, c1
);
747 iov_buf
-= (txmsg_pop
- txmsg_start_pop
+ 1);
748 if (opt
->drop_expected
|| txmsg_ktls_skb_drop
)
751 if (!iov_buf
) /* zero bytes sent case */
756 err
= msg_loop(rx_fd
, iov_count
, iov_buf
,
757 cnt
, &s
, false, opt
);
758 if (opt
->verbose
> 1)
760 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
761 iov_count
, iov_buf
, cnt
, err
);
762 if (s
.end
.tv_sec
- s
.start
.tv_sec
) {
763 sent_Bps
= sentBps(s
);
764 recvd_Bps
= recvdBps(s
);
766 if (opt
->verbose
> 1)
768 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
769 s
.bytes_sent
, sent_Bps
, sent_Bps
/giga
,
770 s
.bytes_recvd
, recvd_Bps
, recvd_Bps
/giga
,
771 peek_flag
? "(peek_msg)" : "");
772 if (err
&& txmsg_cork
)
775 } else if (rxpid
== -1) {
776 perror("msg_loop_rx");
783 err
= msg_loop_sendpage(c1
, iov_buf
, cnt
, &s
, opt
);
785 err
= msg_loop(c1
, iov_count
, iov_buf
,
790 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
791 iov_count
, iov_buf
, cnt
, err
);
792 if (s
.end
.tv_sec
- s
.start
.tv_sec
) {
793 sent_Bps
= sentBps(s
);
794 recvd_Bps
= recvdBps(s
);
796 if (opt
->verbose
> 1)
798 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
799 s
.bytes_sent
, sent_Bps
, sent_Bps
/giga
,
800 s
.bytes_recvd
, recvd_Bps
, recvd_Bps
/giga
);
802 } else if (txpid
== -1) {
803 perror("msg_loop_tx");
807 assert(waitpid(rxpid
, &rx_status
, 0) == rxpid
);
808 assert(waitpid(txpid
, &tx_status
, 0) == txpid
);
809 if (WIFEXITED(rx_status
)) {
810 err
= WEXITSTATUS(rx_status
);
812 fprintf(stderr
, "rx thread exited with err %d.\n", err
);
816 if (WIFEXITED(tx_status
)) {
817 err
= WEXITSTATUS(tx_status
);
819 fprintf(stderr
, "tx thread exited with err %d.\n", err
);
825 static int forever_ping_pong(int rate
, struct sockmap_options
*opt
)
827 struct timeval timeout
;
828 char buf
[1024] = {0};
834 /* Ping/Pong data from client to server */
835 sc
= send(c1
, buf
, sizeof(buf
), 0);
837 perror("send failed()");
842 int s
, rc
, i
, max_fd
= p2
;
852 s
= select(max_fd
+ 1, &w
, NULL
, NULL
, &timeout
);
857 fprintf(stderr
, "unexpected timeout\n");
861 for (i
= 0; i
<= max_fd
&& s
> 0; ++i
) {
862 if (!FD_ISSET(i
, &w
))
867 rc
= recv(i
, buf
, sizeof(buf
), 0);
869 if (errno
!= EWOULDBLOCK
) {
870 perror("recv failed()");
880 sc
= send(i
, buf
, rc
, 0);
882 perror("send failed()");
909 static int run_options(struct sockmap_options
*options
, int cg_fd
, int test
)
911 int i
, key
, next_key
, err
, tx_prog_fd
= -1, zero
= 0;
913 /* If base test skip BPF setup */
914 if (test
== BASE
|| test
== BASE_SENDPAGE
)
917 /* Attach programs to sockmap */
918 if (!txmsg_omit_skb_parser
) {
919 err
= bpf_prog_attach(prog_fd
[0], map_fd
[0],
920 BPF_SK_SKB_STREAM_PARSER
, 0);
923 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
924 prog_fd
[0], map_fd
[0], err
, strerror(errno
));
929 err
= bpf_prog_attach(prog_fd
[1], map_fd
[0],
930 BPF_SK_SKB_STREAM_VERDICT
, 0);
932 fprintf(stderr
, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
933 err
, strerror(errno
));
937 /* Attach programs to TLS sockmap */
938 if (txmsg_ktls_skb
) {
939 if (!txmsg_omit_skb_parser
) {
940 err
= bpf_prog_attach(prog_fd
[0], map_fd
[8],
941 BPF_SK_SKB_STREAM_PARSER
, 0);
944 "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
945 prog_fd
[0], map_fd
[8], err
, strerror(errno
));
950 err
= bpf_prog_attach(prog_fd
[2], map_fd
[8],
951 BPF_SK_SKB_STREAM_VERDICT
, 0);
953 fprintf(stderr
, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
954 err
, strerror(errno
));
959 /* Attach to cgroups */
960 err
= bpf_prog_attach(prog_fd
[3], cg_fd
, BPF_CGROUP_SOCK_OPS
, 0);
962 fprintf(stderr
, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
963 err
, strerror(errno
));
968 err
= sockmap_init_sockets(options
->verbose
);
970 fprintf(stderr
, "ERROR: test socket failed: %d\n", err
);
974 /* Attach txmsg program to sockmap */
976 tx_prog_fd
= prog_fd
[4];
977 else if (txmsg_redir
)
978 tx_prog_fd
= prog_fd
[5];
979 else if (txmsg_apply
)
980 tx_prog_fd
= prog_fd
[6];
982 tx_prog_fd
= prog_fd
[7];
984 tx_prog_fd
= prog_fd
[8];
991 err
= bpf_prog_attach(tx_prog_fd
,
992 map_fd
[1], BPF_SK_MSG_VERDICT
, 0);
995 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
996 err
, strerror(errno
));
1000 err
= bpf_map_update_elem(map_fd
[1], &i
, &c1
, BPF_ANY
);
1003 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1004 err
, strerror(errno
));
1013 err
= bpf_map_update_elem(map_fd
[2], &i
, &redir_fd
, BPF_ANY
);
1016 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1017 err
, strerror(errno
));
1022 err
= bpf_map_update_elem(map_fd
[3],
1023 &i
, &txmsg_apply
, BPF_ANY
);
1026 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1027 err
, strerror(errno
));
1033 err
= bpf_map_update_elem(map_fd
[4],
1034 &i
, &txmsg_cork
, BPF_ANY
);
1037 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1038 err
, strerror(errno
));
1044 err
= bpf_map_update_elem(map_fd
[5],
1045 &i
, &txmsg_start
, BPF_ANY
);
1048 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1049 err
, strerror(errno
));
1056 err
= bpf_map_update_elem(map_fd
[5],
1057 &i
, &txmsg_end
, BPF_ANY
);
1060 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1061 err
, strerror(errno
));
1066 if (txmsg_start_push
) {
1068 err
= bpf_map_update_elem(map_fd
[5],
1069 &i
, &txmsg_start_push
, BPF_ANY
);
1072 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1073 err
, strerror(errno
));
1078 if (txmsg_end_push
) {
1080 err
= bpf_map_update_elem(map_fd
[5],
1081 &i
, &txmsg_end_push
, BPF_ANY
);
1084 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1085 txmsg_end_push
, i
, err
, strerror(errno
));
1090 if (txmsg_start_pop
) {
1092 err
= bpf_map_update_elem(map_fd
[5],
1093 &i
, &txmsg_start_pop
, BPF_ANY
);
1096 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1097 txmsg_start_pop
, i
, err
, strerror(errno
));
1102 bpf_map_update_elem(map_fd
[5],
1103 &i
, &txmsg_start_pop
, BPF_ANY
);
1108 err
= bpf_map_update_elem(map_fd
[5],
1109 &i
, &txmsg_pop
, BPF_ANY
);
1112 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1113 txmsg_pop
, i
, err
, strerror(errno
));
1118 bpf_map_update_elem(map_fd
[5],
1119 &i
, &txmsg_pop
, BPF_ANY
);
1123 if (txmsg_ingress
) {
1124 int in
= BPF_F_INGRESS
;
1127 err
= bpf_map_update_elem(map_fd
[6], &i
, &in
, BPF_ANY
);
1130 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1131 err
, strerror(errno
));
1134 err
= bpf_map_update_elem(map_fd
[1], &i
, &p1
, BPF_ANY
);
1137 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1138 err
, strerror(errno
));
1140 err
= bpf_map_update_elem(map_fd
[2], &i
, &p1
, BPF_ANY
);
1143 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1144 err
, strerror(errno
));
1148 err
= bpf_map_update_elem(map_fd
[2], &i
, &p2
, BPF_ANY
);
1151 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1152 err
, strerror(errno
));
1156 if (txmsg_ktls_skb
) {
1157 int ingress
= BPF_F_INGRESS
;
1160 err
= bpf_map_update_elem(map_fd
[8], &i
, &p2
, BPF_ANY
);
1163 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1164 err
, strerror(errno
));
1167 if (txmsg_ktls_skb_redir
) {
1169 err
= bpf_map_update_elem(map_fd
[7],
1170 &i
, &ingress
, BPF_ANY
);
1173 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1174 err
, strerror(errno
));
1178 if (txmsg_ktls_skb_drop
) {
1180 err
= bpf_map_update_elem(map_fd
[7], &i
, &i
, BPF_ANY
);
1184 if (txmsg_redir_skb
) {
1185 int skb_fd
= (test
== SENDMSG
|| test
== SENDPAGE
) ?
1187 int ingress
= BPF_F_INGRESS
;
1190 err
= bpf_map_update_elem(map_fd
[7],
1191 &i
, &ingress
, BPF_ANY
);
1194 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1195 err
, strerror(errno
));
1199 err
= bpf_map_update_elem(map_fd
[0], &i
, &skb_fd
, BPF_ANY
);
1202 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1203 err
, strerror(errno
));
1208 if (skb_use_parser
) {
1210 err
= bpf_map_update_elem(map_fd
[7], &i
, &skb_use_parser
, BPF_ANY
);
1214 options
->drop_expected
= true;
1216 if (test
== PING_PONG
)
1217 err
= forever_ping_pong(options
->rate
, options
);
1218 else if (test
== SENDMSG
) {
1219 options
->base
= false;
1220 options
->sendpage
= false;
1221 err
= sendmsg_test(options
);
1222 } else if (test
== SENDPAGE
) {
1223 options
->base
= false;
1224 options
->sendpage
= true;
1225 err
= sendmsg_test(options
);
1226 } else if (test
== BASE
) {
1227 options
->base
= true;
1228 options
->sendpage
= false;
1229 err
= sendmsg_test(options
);
1230 } else if (test
== BASE_SENDPAGE
) {
1231 options
->base
= true;
1232 options
->sendpage
= true;
1233 err
= sendmsg_test(options
);
1235 fprintf(stderr
, "unknown test\n");
1237 /* Detatch and zero all the maps */
1238 bpf_prog_detach2(prog_fd
[3], cg_fd
, BPF_CGROUP_SOCK_OPS
);
1239 bpf_prog_detach2(prog_fd
[0], map_fd
[0], BPF_SK_SKB_STREAM_PARSER
);
1240 bpf_prog_detach2(prog_fd
[1], map_fd
[0], BPF_SK_SKB_STREAM_VERDICT
);
1241 bpf_prog_detach2(prog_fd
[0], map_fd
[8], BPF_SK_SKB_STREAM_PARSER
);
1242 bpf_prog_detach2(prog_fd
[2], map_fd
[8], BPF_SK_SKB_STREAM_VERDICT
);
1244 if (tx_prog_fd
>= 0)
1245 bpf_prog_detach2(tx_prog_fd
, map_fd
[1], BPF_SK_MSG_VERDICT
);
1247 for (i
= 0; i
< 8; i
++) {
1249 bpf_map_update_elem(map_fd
[i
], &key
, &zero
, BPF_ANY
);
1250 while (bpf_map_get_next_key(map_fd
[i
], &key
, &next_key
) == 0) {
1251 bpf_map_update_elem(map_fd
[i
], &key
, &zero
, BPF_ANY
);
1265 static char *test_to_str(int test
)
1276 static void append_str(char *dst
, const char *src
, size_t dst_cap
)
1278 size_t avail
= dst_cap
- strlen(dst
);
1280 if (avail
<= 1) /* just zero byte could be written */
1283 strncat(dst
, src
, avail
- 1); /* strncat() adds + 1 for zero byte */
1286 #define OPTSTRING 60
1287 static void test_options(char *options
)
1289 char tstr
[OPTSTRING
];
1291 memset(options
, 0, OPTSTRING
);
1294 append_str(options
, "pass,", OPTSTRING
);
1296 append_str(options
, "redir,", OPTSTRING
);
1298 append_str(options
, "drop,", OPTSTRING
);
1300 snprintf(tstr
, OPTSTRING
, "apply %d,", txmsg_apply
);
1301 append_str(options
, tstr
, OPTSTRING
);
1304 snprintf(tstr
, OPTSTRING
, "cork %d,", txmsg_cork
);
1305 append_str(options
, tstr
, OPTSTRING
);
1308 snprintf(tstr
, OPTSTRING
, "start %d,", txmsg_start
);
1309 append_str(options
, tstr
, OPTSTRING
);
1312 snprintf(tstr
, OPTSTRING
, "end %d,", txmsg_end
);
1313 append_str(options
, tstr
, OPTSTRING
);
1315 if (txmsg_start_pop
) {
1316 snprintf(tstr
, OPTSTRING
, "pop (%d,%d),",
1317 txmsg_start_pop
, txmsg_start_pop
+ txmsg_pop
);
1318 append_str(options
, tstr
, OPTSTRING
);
1321 append_str(options
, "ingress,", OPTSTRING
);
1322 if (txmsg_redir_skb
)
1323 append_str(options
, "redir_skb,", OPTSTRING
);
1325 append_str(options
, "ktls_skb,", OPTSTRING
);
1327 append_str(options
, "ktls,", OPTSTRING
);
1329 append_str(options
, "peek,", OPTSTRING
);
1332 static int __test_exec(int cgrp
, int test
, struct sockmap_options
*opt
)
1334 char *options
= calloc(OPTSTRING
, sizeof(char));
1337 if (test
== SENDPAGE
)
1338 opt
->sendpage
= true;
1340 opt
->sendpage
= false;
1343 opt
->drop_expected
= true;
1345 opt
->drop_expected
= false;
1347 test_options(options
);
1351 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1352 test_cnt
, opt
->rate
, opt
->iov_count
, opt
->iov_length
,
1353 test_to_str(test
), options
);
1356 err
= run_options(opt
, cgrp
, test
);
1358 fprintf(stdout
, " %s\n", !err
? "PASS" : "FAILED");
1360 !err
? passed
++ : failed
++;
1365 static void test_exec(int cgrp
, struct sockmap_options
*opt
)
1367 int type
= strcmp(opt
->map
, BPF_SOCKMAP_FILENAME
);
1372 err
= __test_exec(cgrp
, SENDMSG
, opt
);
1377 err
= __test_exec(cgrp
, SENDPAGE
, opt
);
1383 static void test_send_one(struct sockmap_options
*opt
, int cgrp
)
1385 opt
->iov_length
= 1;
1388 test_exec(cgrp
, opt
);
1390 opt
->iov_length
= 1;
1391 opt
->iov_count
= 1024;
1393 test_exec(cgrp
, opt
);
1395 opt
->iov_length
= 1024;
1398 test_exec(cgrp
, opt
);
1402 static void test_send_many(struct sockmap_options
*opt
, int cgrp
)
1404 opt
->iov_length
= 3;
1407 test_exec(cgrp
, opt
);
1411 opt
->iov_length
= 5;
1412 test_exec(cgrp
, opt
);
1415 static void test_send_large(struct sockmap_options
*opt
, int cgrp
)
1417 opt
->iov_length
= 256;
1418 opt
->iov_count
= 1024;
1420 test_exec(cgrp
, opt
);
1423 static void test_send(struct sockmap_options
*opt
, int cgrp
)
1425 test_send_one(opt
, cgrp
);
1426 test_send_many(opt
, cgrp
);
1427 test_send_large(opt
, cgrp
);
1431 static void test_txmsg_pass(int cgrp
, struct sockmap_options
*opt
)
1433 /* Test small and large iov_count values with pass/redir/apply/cork */
1435 test_send(opt
, cgrp
);
1438 static void test_txmsg_redir(int cgrp
, struct sockmap_options
*opt
)
1441 test_send(opt
, cgrp
);
1444 static void test_txmsg_drop(int cgrp
, struct sockmap_options
*opt
)
1447 test_send(opt
, cgrp
);
1450 static void test_txmsg_ingress_redir(int cgrp
, struct sockmap_options
*opt
)
1452 txmsg_pass
= txmsg_drop
= 0;
1453 txmsg_ingress
= txmsg_redir
= 1;
1454 test_send(opt
, cgrp
);
1457 static void test_txmsg_skb(int cgrp
, struct sockmap_options
*opt
)
1459 bool data
= opt
->data_test
;
1462 opt
->data_test
= true;
1465 txmsg_pass
= txmsg_drop
= 0;
1466 txmsg_ingress
= txmsg_redir
= 0;
1470 /* Using data verification so ensure iov layout is
1471 * expected from test receiver side. e.g. has enough
1472 * bytes to write test code.
1474 opt
->iov_length
= 100;
1477 test_exec(cgrp
, opt
);
1479 txmsg_ktls_skb_drop
= 1;
1480 test_exec(cgrp
, opt
);
1482 txmsg_ktls_skb_drop
= 0;
1483 txmsg_ktls_skb_redir
= 1;
1484 test_exec(cgrp
, opt
);
1485 txmsg_ktls_skb_redir
= 0;
1487 /* Tests that omit skb_parser */
1488 txmsg_omit_skb_parser
= 1;
1491 test_exec(cgrp
, opt
);
1493 txmsg_ktls_skb_drop
= 1;
1494 test_exec(cgrp
, opt
);
1495 txmsg_ktls_skb_drop
= 0;
1497 txmsg_ktls_skb_redir
= 1;
1498 test_exec(cgrp
, opt
);
1501 test_exec(cgrp
, opt
);
1502 txmsg_omit_skb_parser
= 0;
1504 opt
->data_test
= data
;
1508 /* Test cork with hung data. This tests poor usage patterns where
1509 * cork can leave data on the ring if user program is buggy and
1510 * doesn't flush them somehow. They do take some time however
1511 * because they wait for a timeout. Test pass, redir and cork with
1512 * apply logic. Use cork size of 4097 with send_large to avoid
1513 * aligning cork size with send size.
1515 static void test_txmsg_cork_hangs(int cgrp
, struct sockmap_options
*opt
)
1521 test_send_large(opt
, cgrp
);
1527 test_send_large(opt
, cgrp
);
1533 test_send_large(opt
, cgrp
);
1536 static void test_txmsg_pull(int cgrp
, struct sockmap_options
*opt
)
1538 /* Test basic start/end */
1541 test_send(opt
, cgrp
);
1546 test_send_large(opt
, cgrp
);
1548 /* Test pull + redirect */
1552 test_send(opt
, cgrp
);
1554 /* Test pull + cork */
1559 test_send_many(opt
, cgrp
);
1561 /* Test pull + cork + redirect */
1566 test_send_many(opt
, cgrp
);
1569 static void test_txmsg_pop(int cgrp
, struct sockmap_options
*opt
)
1571 /* Test basic pop */
1572 txmsg_start_pop
= 1;
1574 test_send_many(opt
, cgrp
);
1576 /* Test pop with >4k */
1577 txmsg_start_pop
= 4096;
1579 test_send_large(opt
, cgrp
);
1581 /* Test pop + redirect */
1583 txmsg_start_pop
= 1;
1585 test_send_many(opt
, cgrp
);
1587 /* Test pop + cork */
1590 txmsg_start_pop
= 1;
1592 test_send_many(opt
, cgrp
);
1594 /* Test pop + redirect + cork */
1597 txmsg_start_pop
= 1;
1599 test_send_many(opt
, cgrp
);
1602 static void test_txmsg_push(int cgrp
, struct sockmap_options
*opt
)
1604 /* Test basic push */
1605 txmsg_start_push
= 1;
1607 test_send(opt
, cgrp
);
1609 /* Test push 4kB >4k */
1610 txmsg_start_push
= 4096;
1611 txmsg_end_push
= 4096;
1612 test_send_large(opt
, cgrp
);
1614 /* Test push + redirect */
1616 txmsg_start_push
= 1;
1618 test_send_many(opt
, cgrp
);
1620 /* Test push + cork */
1623 txmsg_start_push
= 1;
1625 test_send_many(opt
, cgrp
);
1628 static void test_txmsg_push_pop(int cgrp
, struct sockmap_options
*opt
)
1630 txmsg_start_push
= 1;
1631 txmsg_end_push
= 10;
1632 txmsg_start_pop
= 5;
1634 test_send_large(opt
, cgrp
);
1637 static void test_txmsg_apply(int cgrp
, struct sockmap_options
*opt
)
1643 test_send_one(opt
, cgrp
);
1649 test_send_one(opt
, cgrp
);
1655 test_send_large(opt
, cgrp
);
1661 test_send_large(opt
, cgrp
);
1664 static void test_txmsg_cork(int cgrp
, struct sockmap_options
*opt
)
1670 test_send(opt
, cgrp
);
1676 test_send(opt
, cgrp
);
1679 static void test_txmsg_ingress_parser(int cgrp
, struct sockmap_options
*opt
)
1682 skb_use_parser
= 512;
1684 skb_use_parser
= 570;
1685 opt
->iov_length
= 256;
1688 test_exec(cgrp
, opt
);
1691 char *map_names
[] = {
1703 int prog_attach_type
[] = {
1704 BPF_SK_SKB_STREAM_PARSER
,
1705 BPF_SK_SKB_STREAM_VERDICT
,
1706 BPF_SK_SKB_STREAM_VERDICT
,
1707 BPF_CGROUP_SOCK_OPS
,
1718 BPF_PROG_TYPE_SK_SKB
,
1719 BPF_PROG_TYPE_SK_SKB
,
1720 BPF_PROG_TYPE_SK_SKB
,
1721 BPF_PROG_TYPE_SOCK_OPS
,
1722 BPF_PROG_TYPE_SK_MSG
,
1723 BPF_PROG_TYPE_SK_MSG
,
1724 BPF_PROG_TYPE_SK_MSG
,
1725 BPF_PROG_TYPE_SK_MSG
,
1726 BPF_PROG_TYPE_SK_MSG
,
1727 BPF_PROG_TYPE_SK_MSG
,
1728 BPF_PROG_TYPE_SK_MSG
,
1731 static int populate_progs(char *bpf_file
)
1733 struct bpf_program
*prog
;
1734 struct bpf_object
*obj
;
1738 obj
= bpf_object__open(bpf_file
);
1739 err
= libbpf_get_error(obj
);
1743 libbpf_strerror(err
, err_buf
, sizeof(err_buf
));
1744 printf("Unable to load eBPF objects in file '%s' : %s\n",
1749 bpf_object__for_each_program(prog
, obj
) {
1750 bpf_program__set_type(prog
, prog_type
[i
]);
1751 bpf_program__set_expected_attach_type(prog
,
1752 prog_attach_type
[i
]);
1756 i
= bpf_object__load(obj
);
1758 bpf_object__for_each_program(prog
, obj
) {
1759 prog_fd
[i
] = bpf_program__fd(prog
);
1763 for (i
= 0; i
< sizeof(map_fd
)/sizeof(int); i
++) {
1764 maps
[i
] = bpf_object__find_map_by_name(obj
, map_names
[i
]);
1765 map_fd
[i
] = bpf_map__fd(maps
[i
]);
1766 if (map_fd
[i
] < 0) {
1767 fprintf(stderr
, "load_bpf_file: (%i) %s\n",
1768 map_fd
[i
], strerror(errno
));
1776 struct _test test
[] = {
1777 {"txmsg test passthrough", test_txmsg_pass
},
1778 {"txmsg test redirect", test_txmsg_redir
},
1779 {"txmsg test drop", test_txmsg_drop
},
1780 {"txmsg test ingress redirect", test_txmsg_ingress_redir
},
1781 {"txmsg test skb", test_txmsg_skb
},
1782 {"txmsg test apply", test_txmsg_apply
},
1783 {"txmsg test cork", test_txmsg_cork
},
1784 {"txmsg test hanging corks", test_txmsg_cork_hangs
},
1785 {"txmsg test push_data", test_txmsg_push
},
1786 {"txmsg test pull-data", test_txmsg_pull
},
1787 {"txmsg test pop-data", test_txmsg_pop
},
1788 {"txmsg test push/pop data", test_txmsg_push_pop
},
1789 {"txmsg text ingress parser", test_txmsg_ingress_parser
},
1792 static int check_whitelist(struct _test
*t
, struct sockmap_options
*opt
)
1796 if (!opt
->whitelist
)
1798 ptr
= strdup(opt
->whitelist
);
1801 entry
= strtok(ptr
, ",");
1803 if ((opt
->prepend
&& strstr(opt
->prepend
, entry
) != 0) ||
1804 strstr(opt
->map
, entry
) != 0 ||
1805 strstr(t
->title
, entry
) != 0)
1807 entry
= strtok(NULL
, ",");
1812 static int check_blacklist(struct _test
*t
, struct sockmap_options
*opt
)
1816 if (!opt
->blacklist
)
1818 ptr
= strdup(opt
->blacklist
);
1821 entry
= strtok(ptr
, ",");
1823 if ((opt
->prepend
&& strstr(opt
->prepend
, entry
) != 0) ||
1824 strstr(opt
->map
, entry
) != 0 ||
1825 strstr(t
->title
, entry
) != 0)
1827 entry
= strtok(NULL
, ",");
1832 static int __test_selftests(int cg_fd
, struct sockmap_options
*opt
)
1836 err
= populate_progs(opt
->map
);
1838 fprintf(stderr
, "ERROR: (%i) load bpf failed\n", err
);
1842 /* Tests basic commands and APIs */
1843 for (i
= 0; i
< sizeof(test
)/sizeof(struct _test
); i
++) {
1844 struct _test t
= test
[i
];
1846 if (check_whitelist(&t
, opt
) != 0)
1848 if (check_blacklist(&t
, opt
) == 0)
1851 test_start_subtest(&t
, opt
);
1852 t
.tester(cg_fd
, opt
);
1859 static void test_selftests_sockmap(int cg_fd
, struct sockmap_options
*opt
)
1861 opt
->map
= BPF_SOCKMAP_FILENAME
;
1862 __test_selftests(cg_fd
, opt
);
1865 static void test_selftests_sockhash(int cg_fd
, struct sockmap_options
*opt
)
1867 opt
->map
= BPF_SOCKHASH_FILENAME
;
1868 __test_selftests(cg_fd
, opt
);
1871 static void test_selftests_ktls(int cg_fd
, struct sockmap_options
*opt
)
1873 opt
->map
= BPF_SOCKHASH_FILENAME
;
1874 opt
->prepend
= "ktls";
1876 __test_selftests(cg_fd
, opt
);
1880 static int test_selftest(int cg_fd
, struct sockmap_options
*opt
)
1883 test_selftests_sockmap(cg_fd
, opt
);
1884 test_selftests_sockhash(cg_fd
, opt
);
1885 test_selftests_ktls(cg_fd
, opt
);
1886 test_print_results();
1890 int main(int argc
, char **argv
)
1892 int iov_count
= 1, length
= 1024, rate
= 1;
1893 struct sockmap_options options
= {0};
1894 int opt
, longindex
, err
, cg_fd
= 0;
1895 char *bpf_file
= BPF_SOCKMAP_FILENAME
;
1896 int test
= SELFTESTS
;
1897 bool cg_created
= 0;
1899 while ((opt
= getopt_long(argc
, argv
, ":dhv:c:r:i:l:t:p:q:n:b:",
1900 long_options
, &longindex
)) != -1) {
1903 txmsg_start
= atoi(optarg
);
1906 txmsg_end
= atoi(optarg
);
1909 txmsg_start_push
= atoi(optarg
);
1912 txmsg_end_push
= atoi(optarg
);
1915 txmsg_start_pop
= atoi(optarg
);
1918 txmsg_pop
= atoi(optarg
);
1921 txmsg_apply
= atoi(optarg
);
1924 txmsg_cork
= atoi(optarg
);
1927 cg_fd
= open(optarg
, O_DIRECTORY
, O_RDONLY
);
1930 "ERROR: (%i) open cg path failed: %s\n",
1936 rate
= atoi(optarg
);
1939 options
.verbose
= 1;
1941 options
.verbose
= atoi(optarg
);
1944 iov_count
= atoi(optarg
);
1947 length
= atoi(optarg
);
1950 options
.data_test
= true;
1953 if (strcmp(optarg
, "ping") == 0) {
1955 } else if (strcmp(optarg
, "sendmsg") == 0) {
1957 } else if (strcmp(optarg
, "base") == 0) {
1959 } else if (strcmp(optarg
, "base_sendpage") == 0) {
1960 test
= BASE_SENDPAGE
;
1961 } else if (strcmp(optarg
, "sendpage") == 0) {
1969 options
.whitelist
= strdup(optarg
);
1970 if (!options
.whitelist
)
1974 options
.blacklist
= strdup(optarg
);
1975 if (!options
.blacklist
)
1987 cg_fd
= cgroup_setup_and_join(CG_PATH
);
1993 if (test
== SELFTESTS
) {
1994 err
= test_selftest(cg_fd
, &options
);
1998 err
= populate_progs(bpf_file
);
2000 fprintf(stderr
, "populate program: (%s) %s\n",
2001 bpf_file
, strerror(errno
));
2007 signal(SIGINT
, running_handler
);
2009 options
.iov_count
= iov_count
;
2010 options
.iov_length
= length
;
2011 options
.rate
= rate
;
2013 err
= run_options(&options
, cg_fd
, test
);
2015 if (options
.whitelist
)
2016 free(options
.whitelist
);
2017 if (options
.blacklist
)
2018 free(options
.blacklist
);
2020 cleanup_cgroup_environment();
2025 void running_handler(int a
)