linux-gen: sched: per priority burst size configuration

Change burst size configuration to be per priority instead of
only two levels of control (high/low priority). Also maximum
burst size is configurable, so that application may request
a large burst of events without a worry that a large burst of
low priority events is received.

Signed-off-by: Petri Savolainen <petri.savolainen@linaro.org>
Reviewed-by: Bill Fischofer <bill.fischofer@linaro.org>
Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>

2 files changed, 63 insertions, 44 deletions

diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf
index bddc92dd4..4fe02cf29 100644
--- a/config/odp-linux-generic.conf
+++ b/config/odp-linux-generic.conf
@@ -16,7 +16,7 @@
 
 # Mandatory fields
 odp_implementation = "linux-generic"
-config_file_version = "0.0.1"
+config_file_version = "0.1.0"
 
 # Shared memory options
 shm: {
@@ -73,14 +73,17 @@ sched_basic: {
 	# value is the number of threads using the scheduler.
 	prio_spread = 4
 
-	# Default burst sizes for high and low priority queues. The default
-	# and higher priority levels are considered as high. Scheduler
-	# rounds up number of requested events up to these values. In general,
-	# larger requests are not round down. So, larger bursts than these may
-	# received when requested. A large burst size improves throughput,
-	# but decreases application responsiveness to high priority events
-	# due to head of line blocking cause by a burst of low priority
-	# events.
-	burst_size_hi  = 32
-	burst_size_low = 16
+	# Burst size configuration per priority. The first array element
+	# represents the highest queue priority. The scheduler tries to get
+	# burst_size_default[prio] events from a queue and stashes those that
+	# cannot be passed to the application immediately. More events than the
+	# default burst size may be returned from application request, but no
+	# more than burst_size_max[prio].
+	#
+	# Large burst sizes improve throughput, but decrease application
+	# responsiveness to higher priority events due to head of line blocking
+	# caused by a burst of lower priority events.
+	burst_size_default = [ 32,  32,  32,  32,  32, 16,  8, 4]
+	burst_size_max     = [255, 255, 255, 255, 255, 16, 16, 8]
+
 }
diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c
index a285edc3c..99da87eb0 100644
--- a/platform/linux-generic/odp_schedule_basic.c
+++ b/platform/linux-generic/odp_schedule_basic.c
@@ -100,10 +100,9 @@ ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= MAX_SPREAD,
 /* Start of named groups in group mask arrays */
 #define SCHED_GROUP_NAMED (ODP_SCHED_GROUP_CONTROL + 1)
 
-/* Default burst size. Scheduler rounds up number of requested events up to
- * this value. */
-#define BURST_SIZE_MAX  CONFIG_BURST_SIZE
-#define BURST_SIZE_MIN  1
+/* Limits for burst size configuration */
+#define BURST_MAX  255
+#define STASH_SIZE CONFIG_BURST_SIZE
 
 /* Ordered stash size */
 #define MAX_ORDERED_STASH 512
@@ -138,7 +137,7 @@ typedef struct ODP_ALIGNED_CACHE {
 		uint32_t    qi;
 		odp_queue_t queue;
 		ring_t      *ring;
-		odp_event_t ev[BURST_SIZE_MAX];
+		odp_event_t ev[STASH_SIZE];
 	} stash;
 
 	uint32_t grp_epoch;
@@ -193,8 +192,8 @@ typedef struct {
 
 	struct {
 		uint8_t num_spread;
-		uint8_t burst_hi;
-		uint8_t burst_low;
+		uint8_t burst_default[NUM_PRIO];
+		uint8_t burst_max[NUM_PRIO];
 	} config;
 
 	uint32_t       pri_count[NUM_PRIO][MAX_SPREAD];
@@ -249,6 +248,8 @@ static __thread sched_local_t sched_local;
 static int read_config_file(sched_global_t *sched)
 {
 	const char *str;
+	int i;
+	int burst_val[NUM_PRIO];
 	int val = 0;
 
 	ODP_PRINT("Scheduler config:\n");
@@ -267,33 +268,45 @@ static int read_config_file(sched_global_t *sched)
 	sched->config.num_spread = val;
 	ODP_PRINT("  %s: %i\n", str, val);
 
-	str = "sched_basic.burst_size_hi";
-	if (!_odp_libconfig_lookup_int(str, &val)) {
+	str = "sched_basic.burst_size_default";
+	if (_odp_libconfig_lookup_array(str, burst_val, NUM_PRIO) !=
+	    NUM_PRIO) {
 		ODP_ERR("Config option '%s' not found.\n", str);
 		return -1;
 	}
 
-	if (val > BURST_SIZE_MAX || val < BURST_SIZE_MIN) {
-		ODP_ERR("Bad value %s = %u\n", str, val);
-		return -1;
-	}
+	ODP_PRINT("  %s[] =", str);
+	for (i = 0; i < NUM_PRIO; i++) {
+		val = burst_val[i];
+		sched->config.burst_default[i] = val;
+		ODP_PRINT(" %3i", val);
 
-	sched->config.burst_hi = val;
-	ODP_PRINT("  %s: %i\n", str, val);
+		if (val > STASH_SIZE || val < 1) {
+			ODP_ERR("Bad value %i\n", val);
+			return -1;
+		}
+	}
+	ODP_PRINT("\n");
 
-	str = "sched_basic.burst_size_low";
-	if (!_odp_libconfig_lookup_int(str, &val)) {
+	str = "sched_basic.burst_size_max";
+	if (_odp_libconfig_lookup_array(str, burst_val, NUM_PRIO) !=
+	    NUM_PRIO) {
 		ODP_ERR("Config option '%s' not found.\n", str);
 		return -1;
 	}
 
-	if (val > BURST_SIZE_MAX || val < BURST_SIZE_MIN) {
-		ODP_ERR("Bad value %s = %u\n", str, val);
-		return -1;
-	}
+	ODP_PRINT("  %s[] =    ", str);
+	for (i = 0; i < NUM_PRIO; i++) {
+		val = burst_val[i];
+		sched->config.burst_max[i] = val;
+		ODP_PRINT(" %3i", val);
 
-	sched->config.burst_low = val;
-	ODP_PRINT("  %s: %i\n\n", str, val);
+		if (val > BURST_MAX || val < 1) {
+			ODP_ERR("Bad value %i\n", val);
+			return -1;
+		}
+	}
+	ODP_PRINT("\n\n");
 
 	return 0;
 }
@@ -802,7 +815,7 @@ static inline int poll_pktin(uint32_t qi, int direct_recv,
 	odp_buffer_hdr_t **hdr_tbl;
 	int ret;
 	void *q_int;
-	odp_buffer_hdr_t *b_hdr[BURST_SIZE_MAX];
+	odp_buffer_hdr_t *b_hdr[CONFIG_BURST_SIZE];
 
 	hdr_tbl = (odp_buffer_hdr_t **)ev_tbl;
 
@@ -810,8 +823,8 @@ static inline int poll_pktin(uint32_t qi, int direct_recv,
 		hdr_tbl = b_hdr;
 
 		/* Limit burst to max queue enqueue size */
-		if (max_num > BURST_SIZE_MAX)
-			max_num = BURST_SIZE_MAX;
+		if (max_num > CONFIG_BURST_SIZE)
+			max_num = CONFIG_BURST_SIZE;
 	}
 
 	pktio_index = sched->queue[qi].pktio_index;
@@ -866,7 +879,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
 	int ret;
 	int id;
 	uint32_t qi;
-	unsigned int max_burst;
+	uint16_t burst_def;
 	int num_spread = sched->config.num_spread;
 	uint32_t ring_mask = sched->ring_mask;
 
@@ -876,9 +889,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
 		if (sched->pri_mask[prio] == 0)
 			continue;
 
-		max_burst = sched->config.burst_hi;
-		if (prio > ODP_SCHED_PRIO_DEFAULT)
-			max_burst = sched->config.burst_low;
+		burst_def = sched->config.burst_default[prio];
 
 		/* Select the first ring based on weights */
 		id = first;
@@ -889,7 +900,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
 			odp_queue_t handle;
 			ring_t *ring;
 			int pktin;
-			unsigned int max_deq = max_burst;
+			uint16_t max_deq = burst_def;
 			int stashed = 1;
 			odp_event_t *ev_tbl = sched_local.stash.ev;
 
@@ -917,15 +928,20 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
 			sync_ctx = sched_sync_type(qi);
 			ordered  = (sync_ctx == ODP_SCHED_SYNC_ORDERED);
 
-			/* When application's array is larger than max burst
+			/* When application's array is larger than default burst
 			 * size, output all events directly there. Also, ordered
 			 * queues are not stashed locally to improve
 			 * parallelism. Ordered context can only be released
 			 * when the local cache is empty. */
-			if (max_num > max_burst || ordered) {
+			if (max_num > burst_def || ordered) {
+				uint16_t burst_max;
+
+				burst_max = sched->config.burst_max[prio];
 				stashed = 0;
 				ev_tbl  = out_ev;
 				max_deq = max_num;
+				if (max_num > burst_max)
+					max_deq = burst_max;
 			}
 
 			pktin = queue_is_pktin(qi);