From ccd51bd2108fceadccaa4f5c6829a32207d5091f Mon Sep 17 00:00:00 2001
From: Quentin Young <qlyoung@cumulusnetworks.com>
Date: Thu, 26 Apr 2018 18:30:26 -0400
Subject: [PATCH] zebra: misc fixes, perf improvements

* Coalesce multiple write() syscalls into one
* Write larger chunks
* Decrease default read limit to 1000
* Remove unnecessary operations from hot loop (zserv_write)
* Move cross-schedule out of obuf lock
* Use atomic ops to update atomic variable

Signed-off-by: Quentin Young <qlyoung@cumulusnetworks.com>
---
 zebra/zebra_vty.c |  7 +++++--
 zebra/zserv.c     | 50 ++++++++++++++++++++++++++---------------------
 zebra/zserv.h     |  2 +-
 3 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/zebra/zebra_vty.c b/zebra/zebra_vty.c
index a094ca585..a0eb66e84 100644
--- a/zebra/zebra_vty.c
+++ b/zebra/zebra_vty.c
@@ -3360,7 +3360,8 @@ DEFUN_HIDDEN (zebra_packet_process,
 {
 	uint32_t packets = strtoul(argv[2]->arg, NULL, 10);
 
-	zebrad.packets_to_process = packets;
+	atomic_store_explicit(&zebrad.packets_to_process, packets,
+			      memory_order_relaxed);
 
 	return CMD_SUCCESS;
 }
@@ -3373,7 +3374,9 @@ DEFUN_HIDDEN (no_zebra_packet_process,
 	      "Zapi Protocol\n"
 	      "Number of packets to process before relinquishing thread\n")
 {
-	zebrad.packets_to_process = ZEBRA_ZAPI_PACKETS_TO_PROCESS;
+	atomic_store_explicit(&zebrad.packets_to_process,
+			      ZEBRA_ZAPI_PACKETS_TO_PROCESS,
+			      memory_order_relaxed);
 
 	return CMD_SUCCESS;
 }
diff --git a/zebra/zserv.c b/zebra/zserv.c
index 548e0b32c..a264add42 100644
--- a/zebra/zserv.c
+++ b/zebra/zserv.c
@@ -198,18 +198,19 @@ static int zserv_write(struct thread *thread)
 	struct zserv *client = THREAD_ARG(thread);
 	struct stream *msg;
 	uint32_t wcmd;
-	int writerv;
 	struct stream_fifo *cache;
 
 	if (atomic_load_explicit(&client->dead, memory_order_seq_cst))
 		return 0;
 
 	/* If we have any data pending, try to flush it first */
-	switch (buffer_flush_available(client->wb, client->sock)) {
+	switch (buffer_flush_all(client->wb, client->sock)) {
 	case BUFFER_ERROR:
 		goto zwrite_fail;
 	case BUFFER_PENDING:
-		client->last_write_time = monotime(NULL);
+		atomic_store_explicit(&client->last_write_time,
+				      (uint32_t)monotime(NULL),
+				      memory_order_relaxed);
 		zserv_client_event(client, ZSERV_CLIENT_WRITE);
 		return 0;
 	case BUFFER_EMPTY:
@@ -226,32 +227,35 @@ static int zserv_write(struct thread *thread)
 	}
 	pthread_mutex_unlock(&client->obuf_mtx);
 
-	while (stream_fifo_head(cache)) {
-		msg = stream_fifo_pop(cache);
+	if (cache->tail) {
+		msg = cache->tail;
 		stream_set_getp(msg, 0);
-
 		wcmd = stream_getw_from(msg, 6);
-		writerv = buffer_write(client->wb, client->sock,
-				       STREAM_DATA(msg), stream_get_endp(msg));
-
-		switch (writerv) {
-		case BUFFER_ERROR:
-			stream_free(msg);
-			stream_fifo_free(cache);
-			goto zwrite_fail;
-		case BUFFER_PENDING:
-		case BUFFER_EMPTY:
-			break;
-		}
+	}
 
+	while (stream_fifo_head(cache)) {
+		msg = stream_fifo_pop(cache);
+		buffer_put(client->wb, STREAM_DATA(msg), stream_get_endp(msg));
 		stream_free(msg);
 	}
 
-	if (!buffer_empty(client->wb))
-		zserv_client_event(client, ZSERV_CLIENT_WRITE);
-
 	stream_fifo_free(cache);
 
+	/* If we have any data pending, try to flush it first */
+	switch (buffer_flush_all(client->wb, client->sock)) {
+	case BUFFER_ERROR:
+		goto zwrite_fail;
+	case BUFFER_PENDING:
+		atomic_store_explicit(&client->last_write_time,
+				      (uint32_t)monotime(NULL),
+				      memory_order_relaxed);
+		zserv_client_event(client, ZSERV_CLIENT_WRITE);
+		return 0;
+		break;
+	case BUFFER_EMPTY:
+		break;
+	}
+
 	atomic_store_explicit(&client->last_write_cmd, wcmd,
 			      memory_order_relaxed);
 
@@ -538,9 +542,11 @@ int zserv_send_message(struct zserv *client, struct stream *msg)
 	pthread_mutex_lock(&client->obuf_mtx);
 	{
 		stream_fifo_push(client->obuf_fifo, msg);
-		zserv_client_event(client, ZSERV_CLIENT_WRITE);
 	}
 	pthread_mutex_unlock(&client->obuf_mtx);
+
+	zserv_client_event(client, ZSERV_CLIENT_WRITE);
+
 	return 0;
 }
 
diff --git a/zebra/zserv.h b/zebra/zserv.h
index c8b65b6a4..2a681552d 100644
--- a/zebra/zserv.h
+++ b/zebra/zserv.h
@@ -186,7 +186,7 @@ struct zebra_t {
 	/* LSP work queue */
 	struct work_queue *lsp_process_q;
 
-#define ZEBRA_ZAPI_PACKETS_TO_PROCESS 100000
+#define ZEBRA_ZAPI_PACKETS_TO_PROCESS 1000
 	_Atomic uint32_t packets_to_process;
 };
 extern struct zebra_t zebrad;
-- 
2.39.2