4 * Copyright(c) 2015 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 * This application is a simple Layer 2 PTP v2 client. It shows delta values
36 * which are used to synchronize the PHC clock. if the "-T 1" parameter is
37 * passed to the application the Linux kernel clock is also synchronized.
43 #include <rte_ethdev.h>
44 #include <rte_cycles.h>
45 #include <rte_lcore.h>
52 #define RX_RING_SIZE 128
53 #define TX_RING_SIZE 512
55 #define NUM_MBUFS 8191
56 #define MBUF_CACHE_SIZE 250
58 /* Values for the PTP messageType field. */
61 #define PDELAY_REQ 0x2
62 #define PDELAY_RESP 0x3
64 #define DELAY_RESP 0x9
65 #define PDELAY_RESP_FOLLOW_UP 0xA
68 #define MANAGEMENT 0xD
70 #define NSEC_PER_SEC 1000000000L
71 #define KERNEL_TIME_ADJUST_LIMIT 20000
72 #define PTP_PROTOCOL 0x88F7
74 struct rte_mempool
*mbuf_pool
;
75 uint32_t ptp_enabled_port_mask
;
76 uint8_t ptp_enabled_port_nb
;
77 static uint8_t ptp_enabled_ports
[RTE_MAX_ETHPORTS
];
79 static const struct rte_eth_conf port_conf_default
= {
80 .rxmode
= { .max_rx_pkt_len
= ETHER_MAX_LEN
}
83 static const struct ether_addr ether_multicast
= {
84 .addr_bytes
= {0x01, 0x1b, 0x19, 0x0, 0x0, 0x0}
87 /* Structs used for PTP handling. */
92 } __attribute__((packed
));
99 struct clock_id clock_id
;
100 uint16_t port_number
;
101 } __attribute__((packed
));
106 uint16_t message_length
;
107 uint8_t domain_number
;
109 uint8_t flag_field
[2];
112 struct port_id source_port_id
;
115 int8_t log_message_interval
;
116 } __attribute__((packed
));
119 struct ptp_header hdr
;
120 struct tstamp origin_tstamp
;
121 } __attribute__((packed
));
123 struct follow_up_msg
{
124 struct ptp_header hdr
;
125 struct tstamp precise_origin_tstamp
;
127 } __attribute__((packed
));
129 struct delay_req_msg
{
130 struct ptp_header hdr
;
131 struct tstamp origin_tstamp
;
132 } __attribute__((packed
));
134 struct delay_resp_msg
{
135 struct ptp_header hdr
;
136 struct tstamp rx_tstamp
;
137 struct port_id req_port_id
;
139 } __attribute__((packed
));
143 struct ptp_header header
;
144 struct sync_msg sync
;
145 struct delay_req_msg delay_req
;
146 struct follow_up_msg follow_up
;
147 struct delay_resp_msg delay_resp
;
148 } __attribute__((packed
));
151 struct ptpv2_data_slave_ordinary
{
153 struct timespec tstamp1
;
154 struct timespec tstamp2
;
155 struct timespec tstamp3
;
156 struct timespec tstamp4
;
157 struct clock_id client_clock_id
;
158 struct clock_id master_clock_id
;
159 struct timeval new_adj
;
163 uint16_t seqID_FOLLOWUP
;
165 uint8_t kernel_time_set
;
166 uint8_t current_ptp_port
;
169 static struct ptpv2_data_slave_ordinary ptp_data
;
171 static inline uint64_t timespec64_to_ns(const struct timespec
*ts
)
173 return ((uint64_t) ts
->tv_sec
* NSEC_PER_SEC
) + ts
->tv_nsec
;
176 static struct timeval
177 ns_to_timeval(int64_t nsec
)
179 struct timespec t_spec
= {0, 0};
180 struct timeval t_eval
= {0, 0};
185 rem
= nsec
% NSEC_PER_SEC
;
186 t_spec
.tv_sec
= nsec
/ NSEC_PER_SEC
;
193 t_spec
.tv_nsec
= rem
;
194 t_eval
.tv_sec
= t_spec
.tv_sec
;
195 t_eval
.tv_usec
= t_spec
.tv_nsec
/ 1000;
201 * Initializes a given port using global settings and with the RX buffers
202 * coming from the mbuf_pool passed as a parameter.
205 port_init(uint8_t port
, struct rte_mempool
*mbuf_pool
)
207 struct rte_eth_dev_info dev_info
;
208 struct rte_eth_conf port_conf
= port_conf_default
;
209 const uint16_t rx_rings
= 1;
210 const uint16_t tx_rings
= 1;
214 if (port
>= rte_eth_dev_count())
217 /* Configure the Ethernet device. */
218 retval
= rte_eth_dev_configure(port
, rx_rings
, tx_rings
, &port_conf
);
222 /* Allocate and set up 1 RX queue per Ethernet port. */
223 for (q
= 0; q
< rx_rings
; q
++) {
224 retval
= rte_eth_rx_queue_setup(port
, q
, RX_RING_SIZE
,
225 rte_eth_dev_socket_id(port
), NULL
, mbuf_pool
);
231 /* Allocate and set up 1 TX queue per Ethernet port. */
232 for (q
= 0; q
< tx_rings
; q
++) {
233 /* Setup txq_flags */
234 struct rte_eth_txconf
*txconf
;
236 rte_eth_dev_info_get(q
, &dev_info
);
237 txconf
= &dev_info
.default_txconf
;
238 txconf
->txq_flags
= 0;
240 retval
= rte_eth_tx_queue_setup(port
, q
, TX_RING_SIZE
,
241 rte_eth_dev_socket_id(port
), txconf
);
246 /* Start the Ethernet port. */
247 retval
= rte_eth_dev_start(port
);
251 /* Enable timesync timestamping for the Ethernet device */
252 rte_eth_timesync_enable(port
);
254 /* Enable RX in promiscuous mode for the Ethernet device. */
255 rte_eth_promiscuous_enable(port
);
261 print_clock_info(struct ptpv2_data_slave_ordinary
*ptp_data
)
264 struct timespec net_time
, sys_time
;
266 printf("Master Clock id: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
267 ptp_data
->master_clock_id
.id
[0],
268 ptp_data
->master_clock_id
.id
[1],
269 ptp_data
->master_clock_id
.id
[2],
270 ptp_data
->master_clock_id
.id
[3],
271 ptp_data
->master_clock_id
.id
[4],
272 ptp_data
->master_clock_id
.id
[5],
273 ptp_data
->master_clock_id
.id
[6],
274 ptp_data
->master_clock_id
.id
[7]);
276 printf("\nT2 - Slave Clock. %lds %ldns",
277 (ptp_data
->tstamp2
.tv_sec
),
278 (ptp_data
->tstamp2
.tv_nsec
));
280 printf("\nT1 - Master Clock. %lds %ldns ",
281 ptp_data
->tstamp1
.tv_sec
,
282 (ptp_data
->tstamp1
.tv_nsec
));
284 printf("\nT3 - Slave Clock. %lds %ldns",
285 ptp_data
->tstamp3
.tv_sec
,
286 (ptp_data
->tstamp3
.tv_nsec
));
288 printf("\nT4 - Master Clock. %lds %ldns ",
289 ptp_data
->tstamp4
.tv_sec
,
290 (ptp_data
->tstamp4
.tv_nsec
));
292 printf("\nDelta between master and slave clocks:%"PRId64
"ns\n",
295 clock_gettime(CLOCK_REALTIME
, &sys_time
);
296 rte_eth_timesync_read_time(ptp_data
->current_ptp_port
, &net_time
);
298 time_t ts
= net_time
.tv_sec
;
300 printf("\n\nComparison between Linux kernel Time and PTP:");
302 printf("\nCurrent PTP Time: %.24s %.9ld ns",
303 ctime(&ts
), net_time
.tv_nsec
);
305 nsec
= (int64_t)timespec64_to_ns(&net_time
) -
306 (int64_t)timespec64_to_ns(&sys_time
);
307 ptp_data
->new_adj
= ns_to_timeval(nsec
);
309 gettimeofday(&ptp_data
->new_adj
, NULL
);
311 time_t tp
= ptp_data
->new_adj
.tv_sec
;
313 printf("\nCurrent SYS Time: %.24s %.6ld ns",
314 ctime(&tp
), ptp_data
->new_adj
.tv_usec
);
316 printf("\nDelta between PTP and Linux Kernel time:%"PRId64
"ns\n",
319 printf("[Ctrl+C to quit]\n");
321 /* Clear screen and put cursor in column 1, row 1 */
322 printf("\033[2J\033[1;1H");
326 delta_eval(struct ptpv2_data_slave_ordinary
*ptp_data
)
334 t1
= timespec64_to_ns(&ptp_data
->tstamp1
);
335 t2
= timespec64_to_ns(&ptp_data
->tstamp2
);
336 t3
= timespec64_to_ns(&ptp_data
->tstamp3
);
337 t4
= timespec64_to_ns(&ptp_data
->tstamp4
);
339 delta
= -((int64_t)((t2
- t1
) - (t4
- t3
))) / 2;
345 * Parse the PTP SYNC message.
348 parse_sync(struct ptpv2_data_slave_ordinary
*ptp_data
, uint16_t rx_tstamp_idx
)
350 struct ptp_header
*ptp_hdr
;
352 ptp_hdr
= (struct ptp_header
*)(rte_pktmbuf_mtod(ptp_data
->m
, char *)
353 + sizeof(struct ether_hdr
));
354 ptp_data
->seqID_SYNC
= rte_be_to_cpu_16(ptp_hdr
->seq_id
);
356 if (ptp_data
->ptpset
== 0) {
357 rte_memcpy(&ptp_data
->master_clock_id
,
358 &ptp_hdr
->source_port_id
.clock_id
,
359 sizeof(struct clock_id
));
360 ptp_data
->ptpset
= 1;
363 if (memcmp(&ptp_hdr
->source_port_id
.clock_id
,
364 &ptp_hdr
->source_port_id
.clock_id
,
365 sizeof(struct clock_id
)) == 0) {
367 if (ptp_data
->ptpset
== 1)
368 rte_eth_timesync_read_rx_timestamp(ptp_data
->portid
,
369 &ptp_data
->tstamp2
, rx_tstamp_idx
);
375 * Parse the PTP FOLLOWUP message and send DELAY_REQ to the master clock.
378 parse_fup(struct ptpv2_data_slave_ordinary
*ptp_data
)
380 struct ether_hdr
*eth_hdr
;
381 struct ptp_header
*ptp_hdr
;
382 struct clock_id
*client_clkid
;
383 struct ptp_message
*ptp_msg
;
384 struct rte_mbuf
*created_pkt
;
385 struct tstamp
*origin_tstamp
;
386 struct ether_addr eth_multicast
= ether_multicast
;
389 struct rte_mbuf
*m
= ptp_data
->m
;
391 eth_hdr
= rte_pktmbuf_mtod(m
, struct ether_hdr
*);
392 ptp_hdr
= (struct ptp_header
*)(rte_pktmbuf_mtod(m
, char *)
393 + sizeof(struct ether_hdr
));
394 if (memcmp(&ptp_data
->master_clock_id
,
395 &ptp_hdr
->source_port_id
.clock_id
,
396 sizeof(struct clock_id
)) != 0)
399 ptp_data
->seqID_FOLLOWUP
= rte_be_to_cpu_16(ptp_hdr
->seq_id
);
400 ptp_msg
= (struct ptp_message
*) (rte_pktmbuf_mtod(m
, char *) +
401 sizeof(struct ether_hdr
));
403 origin_tstamp
= &ptp_msg
->follow_up
.precise_origin_tstamp
;
404 ptp_data
->tstamp1
.tv_nsec
= ntohl(origin_tstamp
->ns
);
405 ptp_data
->tstamp1
.tv_sec
=
406 ((uint64_t)ntohl(origin_tstamp
->sec_lsb
)) |
407 (((uint64_t)ntohs(origin_tstamp
->sec_msb
)) << 32);
409 if (ptp_data
->seqID_FOLLOWUP
== ptp_data
->seqID_SYNC
) {
411 created_pkt
= rte_pktmbuf_alloc(mbuf_pool
);
412 pkt_size
= sizeof(struct ether_hdr
) +
413 sizeof(struct ptp_message
);
414 created_pkt
->data_len
= pkt_size
;
415 created_pkt
->pkt_len
= pkt_size
;
416 eth_hdr
= rte_pktmbuf_mtod(created_pkt
, struct ether_hdr
*);
417 rte_eth_macaddr_get(ptp_data
->portid
, ð_hdr
->s_addr
);
419 /* Set multicast address 01-1B-19-00-00-00. */
420 ether_addr_copy(ð_multicast
, ð_hdr
->d_addr
);
422 eth_hdr
->ether_type
= htons(PTP_PROTOCOL
);
423 ptp_msg
= (struct ptp_message
*)
424 (rte_pktmbuf_mtod(created_pkt
, char *) +
425 sizeof(struct ether_hdr
));
427 ptp_msg
->delay_req
.hdr
.seq_id
= htons(ptp_data
->seqID_SYNC
);
428 ptp_msg
->delay_req
.hdr
.msg_type
= DELAY_REQ
;
429 ptp_msg
->delay_req
.hdr
.ver
= 2;
430 ptp_msg
->delay_req
.hdr
.control
= 1;
431 ptp_msg
->delay_req
.hdr
.log_message_interval
= 127;
433 /* Set up clock id. */
435 &ptp_msg
->delay_req
.hdr
.source_port_id
.clock_id
;
437 client_clkid
->id
[0] = eth_hdr
->s_addr
.addr_bytes
[0];
438 client_clkid
->id
[1] = eth_hdr
->s_addr
.addr_bytes
[1];
439 client_clkid
->id
[2] = eth_hdr
->s_addr
.addr_bytes
[2];
440 client_clkid
->id
[3] = 0xFF;
441 client_clkid
->id
[4] = 0xFE;
442 client_clkid
->id
[5] = eth_hdr
->s_addr
.addr_bytes
[3];
443 client_clkid
->id
[6] = eth_hdr
->s_addr
.addr_bytes
[4];
444 client_clkid
->id
[7] = eth_hdr
->s_addr
.addr_bytes
[5];
446 rte_memcpy(&ptp_data
->client_clock_id
,
448 sizeof(struct clock_id
));
450 /* Enable flag for hardware timestamping. */
451 created_pkt
->ol_flags
|= PKT_TX_IEEE1588_TMST
;
453 /*Read value from NIC to prevent latching with old value. */
454 rte_eth_timesync_read_tx_timestamp(ptp_data
->portid
,
457 /* Transmit the packet. */
458 rte_eth_tx_burst(ptp_data
->portid
, 0, &created_pkt
, 1);
461 ptp_data
->tstamp3
.tv_nsec
= 0;
462 ptp_data
->tstamp3
.tv_sec
= 0;
464 /* Wait at least 1 us to read TX timestamp. */
465 while ((rte_eth_timesync_read_tx_timestamp(ptp_data
->portid
,
466 &ptp_data
->tstamp3
) < 0) && (wait_us
< 1000)) {
474 * Update the kernel time with the difference between it and the current NIC
478 update_kernel_time(void)
481 struct timespec net_time
, sys_time
;
483 clock_gettime(CLOCK_REALTIME
, &sys_time
);
484 rte_eth_timesync_read_time(ptp_data
.current_ptp_port
, &net_time
);
486 nsec
= (int64_t)timespec64_to_ns(&net_time
) -
487 (int64_t)timespec64_to_ns(&sys_time
);
489 ptp_data
.new_adj
= ns_to_timeval(nsec
);
492 * If difference between kernel time and system time in NIC is too big
493 * (more than +/- 20 microseconds), use clock_settime to set directly
494 * the kernel time, as adjtime is better for small adjustments (takes
495 * longer to adjust the time).
498 if (nsec
> KERNEL_TIME_ADJUST_LIMIT
|| nsec
< -KERNEL_TIME_ADJUST_LIMIT
)
499 clock_settime(CLOCK_REALTIME
, &net_time
);
501 adjtime(&ptp_data
.new_adj
, 0);
507 * Parse the DELAY_RESP message.
510 parse_drsp(struct ptpv2_data_slave_ordinary
*ptp_data
)
512 struct rte_mbuf
*m
= ptp_data
->m
;
513 struct ptp_message
*ptp_msg
;
514 struct tstamp
*rx_tstamp
;
517 ptp_msg
= (struct ptp_message
*) (rte_pktmbuf_mtod(m
, char *) +
518 sizeof(struct ether_hdr
));
519 seq_id
= rte_be_to_cpu_16(ptp_msg
->delay_resp
.hdr
.seq_id
);
520 if (memcmp(&ptp_data
->client_clock_id
,
521 &ptp_msg
->delay_resp
.req_port_id
.clock_id
,
522 sizeof(struct clock_id
)) == 0) {
523 if (seq_id
== ptp_data
->seqID_FOLLOWUP
) {
524 rx_tstamp
= &ptp_msg
->delay_resp
.rx_tstamp
;
525 ptp_data
->tstamp4
.tv_nsec
= ntohl(rx_tstamp
->ns
);
526 ptp_data
->tstamp4
.tv_sec
=
527 ((uint64_t)ntohl(rx_tstamp
->sec_lsb
)) |
528 (((uint64_t)ntohs(rx_tstamp
->sec_msb
)) << 32);
530 /* Evaluate the delta for adjustment. */
531 ptp_data
->delta
= delta_eval(ptp_data
);
533 rte_eth_timesync_adjust_time(ptp_data
->portid
,
536 ptp_data
->current_ptp_port
= ptp_data
->portid
;
538 /* Update kernel time if enabled in app parameters. */
539 if (ptp_data
->kernel_time_set
== 1)
540 update_kernel_time();
548 /* This function processes PTP packets, implementing slave PTP IEEE1588 L2
552 parse_ptp_frames(uint8_t portid
, struct rte_mbuf
*m
) {
553 struct ptp_header
*ptp_hdr
;
554 struct ether_hdr
*eth_hdr
;
557 eth_hdr
= rte_pktmbuf_mtod(m
, struct ether_hdr
*);
558 eth_type
= rte_be_to_cpu_16(eth_hdr
->ether_type
);
560 if (eth_type
== PTP_PROTOCOL
) {
562 ptp_data
.portid
= portid
;
563 ptp_hdr
= (struct ptp_header
*)(rte_pktmbuf_mtod(m
, char *)
564 + sizeof(struct ether_hdr
));
566 switch (ptp_hdr
->msg_type
) {
568 parse_sync(&ptp_data
, m
->timesync
);
571 parse_fup(&ptp_data
);
574 parse_drsp(&ptp_data
);
575 print_clock_info(&ptp_data
);
584 * The lcore main. This is the main thread that does the work, reading from an
585 * input port and writing to an output port.
587 static __attribute__((noreturn
)) void
595 * Check that the port is on the same NUMA node as the polling thread
596 * for best performance.
598 printf("\nCore %u Waiting for SYNC packets. [Ctrl+C to quit]\n",
601 /* Run until the application is quit or killed. */
604 /* Read packet from RX queues. */
605 for (portid
= 0; portid
< ptp_enabled_port_nb
; portid
++) {
607 portid
= ptp_enabled_ports
[portid
];
608 nb_rx
= rte_eth_rx_burst(portid
, 0, &m
, 1);
610 if (likely(nb_rx
== 0))
613 if (m
->ol_flags
& PKT_RX_IEEE1588_PTP
)
614 parse_ptp_frames(portid
, m
);
622 print_usage(const char *prgname
)
624 printf("%s [EAL options] -- -p PORTMASK -T VALUE\n"
625 " -T VALUE: 0 - Disable, 1 - Enable Linux Clock"
626 " Synchronization (0 default)\n"
627 " -p PORTMASK: hexadecimal bitmask of ports to configure\n",
632 ptp_parse_portmask(const char *portmask
)
637 /* Parse the hexadecimal string. */
638 pm
= strtoul(portmask
, &end
, 16);
640 if ((portmask
[0] == '\0') || (end
== NULL
) || (*end
!= '\0'))
650 parse_ptp_kernel(const char *param
)
655 /* Parse the hexadecimal string. */
656 pm
= strtoul(param
, &end
, 16);
658 if ((param
[0] == '\0') || (end
== NULL
) || (*end
!= '\0'))
666 /* Parse the commandline arguments. */
668 ptp_parse_args(int argc
, char **argv
)
673 char *prgname
= argv
[0];
674 static struct option lgopts
[] = { {NULL
, 0, 0, 0} };
678 while ((opt
= getopt_long(argc
, argvopt
, "p:T:",
679 lgopts
, &option_index
)) != EOF
) {
685 ptp_enabled_port_mask
= ptp_parse_portmask(optarg
);
686 if (ptp_enabled_port_mask
== 0) {
687 printf("invalid portmask\n");
688 print_usage(prgname
);
692 /* Time synchronization. */
694 ret
= parse_ptp_kernel(optarg
);
696 print_usage(prgname
);
700 ptp_data
.kernel_time_set
= ret
;
704 print_usage(prgname
);
709 argv
[optind
-1] = prgname
;
711 optind
= 1; /* Reset getopt lib. */
717 * The main function, which does initialization and calls the per-lcore
721 main(int argc
, char *argv
[])
727 /* Initialize the Environment Abstraction Layer (EAL). */
728 int ret
= rte_eal_init(argc
, argv
);
731 rte_exit(EXIT_FAILURE
, "Error with EAL initialization\n");
733 memset(&ptp_data
, '\0', sizeof(struct ptpv2_data_slave_ordinary
));
738 ret
= ptp_parse_args(argc
, argv
);
740 rte_exit(EXIT_FAILURE
, "Error with PTP initialization\n");
742 /* Check that there is an even number of ports to send/receive on. */
743 nb_ports
= rte_eth_dev_count();
745 /* Creates a new mempool in memory to hold the mbufs. */
746 mbuf_pool
= rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS
* nb_ports
,
747 MBUF_CACHE_SIZE
, 0, RTE_MBUF_DEFAULT_BUF_SIZE
, rte_socket_id());
749 if (mbuf_pool
== NULL
)
750 rte_exit(EXIT_FAILURE
, "Cannot create mbuf pool\n");
752 /* Initialize all ports. */
753 for (portid
= 0; portid
< nb_ports
; portid
++) {
754 if ((ptp_enabled_port_mask
& (1 << portid
)) != 0) {
755 if (port_init(portid
, mbuf_pool
) == 0) {
756 ptp_enabled_ports
[ptp_enabled_port_nb
] = portid
;
757 ptp_enabled_port_nb
++;
759 rte_exit(EXIT_FAILURE
,
760 "Cannot init port %"PRIu8
"\n",
764 printf("Skipping disabled port %u\n", portid
);
767 if (ptp_enabled_port_nb
== 0) {
768 rte_exit(EXIT_FAILURE
,
769 "All available ports are disabled."
770 " Please set portmask.\n");
773 if (rte_lcore_count() > 1)
774 printf("\nWARNING: Too many lcores enabled. Only 1 used.\n");
776 /* Call lcore_main on the master core only. */