linux-gen: sched: add configuration options for per queue type burst sizes

Add new configuration file options for adjusting scheduler burst sizes per
scheduled queue type. Use 32-bit fast path variables for improved
arithmetic operation performance.

Signed-off-by: Matias Elo <matias.elo@nokia.com>
Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com>
Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com>

7 files changed, 108 insertions, 46 deletions

diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf
index 9c5e85242..f8accd07f 100644
--- a/config/odp-linux-generic.conf
+++ b/config/odp-linux-generic.conf
@@ -16,7 +16,7 @@
 
 # Mandatory fields
 odp_implementation = "linux-generic"
-config_file_version = "0.1.21"
+config_file_version = "0.1.22"
 
 # System options
 system: {
@@ -211,6 +211,16 @@ sched_basic: {
 	burst_size_default = [ 32,  32,  32,  32,  32, 16,  8, 4]
 	burst_size_max     = [255, 255, 255, 255, 255, 16, 16, 8]
 
+	# Burst size configuration per priority for each scheduled queue type.
+	# Overrides default values set in 'burst_size_default' and
+	# 'burst_size_max' if != 0.
+	burst_size_parallel     = [0, 0, 0, 0, 0, 0, 0, 0]
+	burst_size_max_parallel = [0, 0, 0, 0, 0, 0, 0, 0]
+	burst_size_atomic       = [0, 0, 0, 0, 0, 0, 0, 0]
+	burst_size_max_atomic   = [0, 0, 0, 0, 0, 0, 0, 0]
+	burst_size_ordered      = [0, 0, 0, 0, 0, 0, 0, 0]
+	burst_size_max_ordered  = [0, 0, 0, 0, 0, 0, 0, 0]
+
 	# Automatically updated schedule groups
 	#
 	# DEPRECATED: use odp_schedule_config() API instead
diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4
index 886cc07e8..03dbc929d 100644
--- a/platform/linux-generic/m4/odp_libconfig.m4
+++ b/platform/linux-generic/m4/odp_libconfig.m4
@@ -3,7 +3,7 @@
 ##########################################################################
 m4_define([_odp_config_version_generation], [0])
 m4_define([_odp_config_version_major], [1])
-m4_define([_odp_config_version_minor], [21])
+m4_define([_odp_config_version_minor], [22])
 
 m4_define([_odp_config_version],
           [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor])
diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c
index 81065a8d5..d7a76d89c 100644
--- a/platform/linux-generic/odp_schedule_basic.c
+++ b/platform/linux-generic/odp_schedule_basic.c
@@ -52,6 +52,14 @@
 /* Spread balancing frequency. Balance every BALANCE_ROUNDS_M1 + 1 scheduling rounds. */
 #define BALANCE_ROUNDS_M1 0xfffff
 
+/* Number of scheduled queue synchronization types */
+#define NUM_SCHED_SYNC 3
+
+/* Queue types used as array indices */
+ODP_STATIC_ASSERT(ODP_SCHED_SYNC_PARALLEL == 0, "ODP_SCHED_SYNC_PARALLEL_value_changed");
+ODP_STATIC_ASSERT(ODP_SCHED_SYNC_ATOMIC == 1, "ODP_SCHED_SYNC_ATOMIC_value_changed");
+ODP_STATIC_ASSERT(ODP_SCHED_SYNC_ORDERED == 2, "ODP_SCHED_SYNC_ORDERED_value_changed");
+
 /* Load of a queue */
 #define QUEUE_LOAD 256
 
@@ -223,8 +231,8 @@ typedef struct ODP_ALIGNED_CACHE {
 
 typedef struct {
 	struct {
-		uint8_t burst_default[NUM_PRIO];
-		uint8_t burst_max[NUM_PRIO];
+		uint8_t burst_default[NUM_SCHED_SYNC][NUM_PRIO];
+		uint8_t burst_max[NUM_SCHED_SYNC][NUM_PRIO];
 		uint8_t num_spread;
 		uint8_t prefer_ratio;
 	} config;
@@ -300,11 +308,46 @@ static sched_global_t *sched;
 /* Thread local scheduler context */
 static __thread sched_local_t sched_local;
 
+static int read_burst_size_conf(uint8_t out_tbl[], const char *conf_str,
+				int min_val, int max_val, int print)
+{
+	int burst_val[NUM_PRIO];
+	const int max_len = 256;
+	const int n = max_len - 1;
+	char line[max_len];
+	int len = 0;
+
+	if (_odp_libconfig_lookup_array(conf_str, burst_val, NUM_PRIO) !=
+	    NUM_PRIO) {
+		ODP_ERR("Config option '%s' not found.\n", conf_str);
+		return -1;
+	}
+
+	char str[strlen(conf_str) + 4];
+
+	snprintf(str, sizeof(str), "%s[]:", conf_str);
+	len += snprintf(&line[len], n - len, "  %-38s", str);
+
+	for (int i = 0; i < NUM_PRIO; i++) {
+		int val = burst_val[i];
+
+		if (val > max_val || val < min_val) {
+			ODP_ERR("Bad value for %s: %i\n", conf_str, val);
+			return -1;
+		}
+		len += snprintf(&line[len], n - len, " %3i", val);
+		if (val > 0)
+			out_tbl[i] = val;
+	}
+	if (print)
+		ODP_PRINT("%s\n", line);
+
+	return 0;
+}
+
 static int read_config_file(sched_global_t *sched)
 {
 	const char *str;
-	int i;
-	int burst_val[NUM_PRIO];
 	int val = 0;
 
 	ODP_PRINT("Scheduler config:\n");
@@ -355,46 +398,48 @@ static int read_config_file(sched_global_t *sched)
 	if (val == 0 || sched->config.num_spread == 1)
 		sched->load_balance = 0;
 
+	/* Initialize default values for all queue types */
 	str = "sched_basic.burst_size_default";
-	if (_odp_libconfig_lookup_array(str, burst_val, NUM_PRIO) !=
-	    NUM_PRIO) {
-		ODP_ERR("Config option '%s' not found.\n", str);
+	if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ATOMIC], str, 1,
+				 STASH_SIZE, 1) ||
+	    read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_PARALLEL], str, 1,
+				 STASH_SIZE, 0) ||
+	    read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ORDERED], str, 1,
+				 STASH_SIZE, 0))
 		return -1;
-	}
 
-	ODP_PRINT("  %s[] =", str);
-	for (i = 0; i < NUM_PRIO; i++) {
-		val = burst_val[i];
-		sched->config.burst_default[i] = val;
-		ODP_PRINT(" %3i", val);
+	str = "sched_basic.burst_size_max";
+	if (read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_ATOMIC], str, 1,
+				 BURST_MAX, 1) ||
+	    read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_PARALLEL], str, 1,
+				 BURST_MAX, 0) ||
+	    read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_ORDERED], str, 1,
+				 BURST_MAX, 0))
+		return -1;
 
-		if (val > STASH_SIZE || val < 1) {
-			ODP_ERR("Bad value %i\n", val);
-			return -1;
-		}
-	}
-	ODP_PRINT("\n");
+	if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ATOMIC],
+				 "sched_basic.burst_size_atomic", 0, STASH_SIZE, 1))
+		return -1;
 
-	str = "sched_basic.burst_size_max";
-	if (_odp_libconfig_lookup_array(str, burst_val, NUM_PRIO) !=
-	    NUM_PRIO) {
-		ODP_ERR("Config option '%s' not found.\n", str);
+	if (read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_ATOMIC],
+				 "sched_basic.burst_size_max_atomic", 0, BURST_MAX, 1))
 		return -1;
-	}
 
-	ODP_PRINT("  %s[] =    ", str);
-	for (i = 0; i < NUM_PRIO; i++) {
-		val = burst_val[i];
-		sched->config.burst_max[i] = val;
-		ODP_PRINT(" %3i", val);
+	if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_PARALLEL],
+				 "sched_basic.burst_size_parallel", 0, STASH_SIZE, 1))
+		return -1;
 
-		if (val > BURST_MAX || val < 1) {
-			ODP_ERR("Bad value %i\n", val);
-			return -1;
-		}
-	}
+	if (read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_PARALLEL],
+				 "sched_basic.burst_size_max_parallel", 0, BURST_MAX, 1))
+		return -1;
 
-	ODP_PRINT("\n");
+	if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ORDERED],
+				 "sched_basic.burst_size_ordered", 0, STASH_SIZE, 1))
+		return -1;
+
+	if (read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_ORDERED],
+				 "sched_basic.burst_size_max_ordered", 0, BURST_MAX, 1))
+		return -1;
 
 	str = "sched_basic.group_enable.all";
 	if (!_odp_libconfig_lookup_int(str, &val)) {
@@ -1245,7 +1290,14 @@ static inline int schedule_grp_prio(odp_queue_t *out_queue, odp_event_t out_ev[]
 	uint32_t qi;
 	int num_spread = sched->config.num_spread;
 	uint32_t ring_mask = sched->ring_mask;
-	uint16_t burst_def = sched->config.burst_default[prio];
+	const uint32_t burst_def_sync[NUM_SCHED_SYNC] = {
+		sched->config.burst_default[ODP_SCHED_SYNC_PARALLEL][prio],
+		sched->config.burst_default[ODP_SCHED_SYNC_ATOMIC][prio],
+		sched->config.burst_default[ODP_SCHED_SYNC_ORDERED][prio]};
+	const uint32_t burst_max_sync[NUM_SCHED_SYNC] = {
+		sched->config.burst_max[ODP_SCHED_SYNC_PARALLEL][prio],
+		sched->config.burst_max[ODP_SCHED_SYNC_ATOMIC][prio],
+		sched->config.burst_max[ODP_SCHED_SYNC_ORDERED][prio]};
 
 	/* Select the first spread based on weights */
 	spr = first_spr;
@@ -1256,7 +1308,7 @@ static inline int schedule_grp_prio(odp_queue_t *out_queue, odp_event_t out_ev[]
 		odp_queue_t handle;
 		ring_u32_t *ring;
 		int pktin;
-		uint16_t max_deq = burst_def;
+		uint32_t max_deq;
 		int stashed = 1;
 		odp_event_t *ev_tbl = sched_local.stash.ev;
 
@@ -1282,16 +1334,16 @@ static inline int schedule_grp_prio(odp_queue_t *out_queue, odp_event_t out_ev[]
 
 		sync_ctx = sched_sync_type(qi);
 		ordered  = (sync_ctx == ODP_SCHED_SYNC_ORDERED);
+		max_deq = burst_def_sync[sync_ctx];
 
 		/* When application's array is larger than default burst
 		 * size, output all events directly there. Also, ordered
 		 * queues are not stashed locally to improve
 		 * parallelism. Ordered context can only be released
 		 * when the local cache is empty. */
-		if (max_num > burst_def || ordered) {
-			uint16_t burst_max;
+		if (max_num > max_deq || ordered) {
+			const uint32_t burst_max = burst_max_sync[sync_ctx];
 
-			burst_max = sched->config.burst_max[prio];
 			stashed = 0;
 			ev_tbl  = out_ev;
 			max_deq = max_num;
diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf
index c7379e38a..261aa0141 100644
--- a/platform/linux-generic/test/inline-timer.conf
+++ b/platform/linux-generic/test/inline-timer.conf
@@ -1,6 +1,6 @@
 # Mandatory fields
 odp_implementation = "linux-generic"
-config_file_version = "0.1.21"
+config_file_version = "0.1.22"
 
 timer: {
 	# Enable inline timer implementation
diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf
index 433899017..8d2d00e63 100644
--- a/platform/linux-generic/test/packet_align.conf
+++ b/platform/linux-generic/test/packet_align.conf
@@ -1,6 +1,6 @@
 # Mandatory fields
 odp_implementation = "linux-generic"
-config_file_version = "0.1.21"
+config_file_version = "0.1.22"
 
 pool: {
 	pkt: {
diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf
index a36c9fc3d..1e0e7cc95 100644
--- a/platform/linux-generic/test/process-mode.conf
+++ b/platform/linux-generic/test/process-mode.conf
@@ -1,6 +1,6 @@
 # Mandatory fields
 odp_implementation = "linux-generic"
-config_file_version = "0.1.21"
+config_file_version = "0.1.22"
 
 # Shared memory options
 shm: {
diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf
index 16654595b..e63ffa2f3 100644
--- a/platform/linux-generic/test/sched-basic.conf
+++ b/platform/linux-generic/test/sched-basic.conf
@@ -1,6 +1,6 @@
 # Mandatory fields
 odp_implementation = "linux-generic"
-config_file_version = "0.1.21"
+config_file_version = "0.1.22"
 
 # Test scheduler with an odd spread value and without dynamic load balance
 sched_basic: {