linux-generic: classification: implements odp_cls_cos_pool_set() api

Adds support for configuring packet pool to a class-of-service.
linux-generic packet parser is enhanced to parse a packet directly from
a memory location rather than from odp_packet_t.

packet receive code is modified to run packet classifier directly from
the stream so that the packet can be allocated from the pool specified
by the CoS.

Signed-off-by: Balasubramanian Manoharan <bala.manoharan@linaro.org>
Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com>
Reviewed-and-tested-by: Bill Fischofer <bill.fischofer@linaro.org>
Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>

5 files changed, 249 insertions, 91 deletions

diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c
index ce19add2b..47745ad83 100644
--- a/platform/linux-generic/pktio/loop.c
+++ b/platform/linux-generic/pktio/loop.c
@@ -12,6 +12,7 @@
 #include <odp.h>
 #include <odp_packet_internal.h>
 #include <odp_packet_io_internal.h>
+#include <odp_classification_internal.h>
 #include <odp_debug_internal.h>
 #include <odp/hints.h>
 
@@ -50,19 +51,35 @@ static int loopback_close(pktio_entry_t *pktio_entry)
 static int loopback_recv(pktio_entry_t *pktio_entry, odp_packet_t pkts[],
 			 unsigned len)
 {
-	int nbr, i;
+	int nbr, i, j;
 	odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX];
 	queue_entry_t *qentry;
 	odp_packet_hdr_t *pkt_hdr;
+	odp_packet_t pkt;
 
+	nbr = 0;
 	qentry = queue_to_qentry(pktio_entry->s.pkt_loop.loopq);
 	nbr = queue_deq_multi(qentry, hdr_tbl, len);
 
-	for (i = 0; i < nbr; ++i) {
-		pkts[i] = _odp_packet_from_buffer(odp_hdr_to_buf(hdr_tbl[i]));
-		pkt_hdr = odp_packet_hdr(pkts[i]);
-		packet_parse_reset(pkts[i]);
-		packet_parse_l2(pkt_hdr);
+	if (pktio_cls_enabled(pktio_entry)) {
+		for (i = 0, j = 0; i < nbr; i++) {
+			pkt = _odp_packet_from_buffer(odp_hdr_to_buf
+						      (hdr_tbl[i]));
+			pkt_hdr = odp_packet_hdr(pkt);
+			packet_parse_reset(pkt_hdr);
+			packet_parse_l2(pkt_hdr);
+			if (0 > _odp_packet_classifier(pktio_entry, pkt))
+				pkts[j++] = pkt;
+		}
+	nbr = j;
+	} else {
+		for (i = 0; i < nbr; ++i) {
+			pkts[i] = _odp_packet_from_buffer(odp_hdr_to_buf
+							  (hdr_tbl[i]));
+			pkt_hdr = odp_packet_hdr(pkts[i]);
+			packet_parse_reset(pkt_hdr);
+			packet_parse_l2(pkt_hdr);
+		}
 	}
 
 	return nbr;
diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c
index d0643230b..781a2c234 100644
--- a/platform/linux-generic/pktio/netmap.c
+++ b/platform/linux-generic/pktio/netmap.c
@@ -20,6 +20,9 @@
 #include <poll.h>
 #include <linux/ethtool.h>
 #include <linux/sockios.h>
+#include <odp_classification_datamodel.h>
+#include <odp_classification_inlines.h>
+#include <odp_classification_internal.h>
 
 #define NETMAP_WITH_LIBS
 #include <net/netmap_user.h>
@@ -197,7 +200,7 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry,
 				    uint16_t len)
 {
 	odp_packet_t pkt;
-	odp_packet_hdr_t *pkt_hdr;
+	int ret;
 
 	if (odp_unlikely(len > pktio_entry->s.pkt_nm.max_frame_len)) {
 		ODP_ERR("RX: frame too big %" PRIu16 " %zu!\n", len,
@@ -210,21 +213,32 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry,
 		return -1;
 	}
 
-	pkt = packet_alloc(pktio_entry->s.pkt_nm.pool, len, 1);
-	if (pkt == ODP_PACKET_INVALID)
+	if (pktio_cls_enabled(pktio_entry)) {
+		ret = _odp_packet_cls_enq(pktio_entry, (uint8_t *)buf,
+					  len, pkt_out);
+		if (ret)
+			return 0;
 		return -1;
+	} else {
+		odp_packet_hdr_t *pkt_hdr;
 
-	pkt_hdr = odp_packet_hdr(pkt);
+		pkt = packet_alloc(pktio_entry->s.pkt_nm.pool, len, 1);
+		if (pkt == ODP_PACKET_INVALID)
+			return -1;
 
-	/* For now copy the data in the mbuf,
-	   worry about zero-copy later */
-	if (odp_packet_copydata_in(pkt, 0, len, buf) != 0) {
-		odp_packet_free(pkt);
-		return -1;
+		pkt_hdr = odp_packet_hdr(pkt);
+
+		/* For now copy the data in the mbuf,
+		   worry about zero-copy later */
+		if (odp_packet_copydata_in(pkt, 0, len, buf) != 0) {
+			odp_packet_free(pkt);
+			return -1;
+		}
+
+		packet_parse_l2(pkt_hdr);
+		*pkt_out = pkt;
 	}
-	packet_parse_l2(pkt_hdr);
 
-	*pkt_out = pkt;
 	return 0;
 }
 
diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c
new file mode 100644
index 000000000..1fb10a052
--- /dev/null
+++ b/platform/linux-generic/pktio/pktio_common.c
@@ -0,0 +1,56 @@
+/* Copyright (c) 2013, Linaro Limited
+ * Copyright (c) 2013, Nokia Solutions and Networks
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <odp_packet_io_internal.h>
+#include <odp_classification_internal.h>
+
+int _odp_packet_cls_enq(pktio_entry_t *pktio_entry,
+			uint8_t *base, uint16_t buf_len,
+			odp_packet_t *pkt_ret)
+{
+	cos_t *cos;
+	odp_packet_t pkt;
+	odp_packet_hdr_t pkt_hdr;
+	int ret;
+	odp_pool_t pool;
+
+	packet_parse_reset(&pkt_hdr);
+
+	_odp_cls_parse(&pkt_hdr, base);
+	cos = pktio_select_cos(pktio_entry, (uint8_t *)base, &pkt_hdr);
+	pool = cos->s.pool->s.pool_hdl;
+
+	/* if No CoS found then drop the packet */
+	if (cos == NULL || cos->s.queue == NULL)
+		return 0;
+
+	pkt = odp_packet_alloc(pool, buf_len);
+	if (odp_unlikely(pkt == ODP_PACKET_INVALID))
+		return 0;
+
+	copy_packet_parser_metadata(&pkt_hdr, odp_packet_hdr(pkt));
+	odp_packet_hdr(pkt)->input = pktio_entry->s.id;
+
+	if (odp_packet_copydata_in(pkt, 0, buf_len, base) != 0) {
+		odp_packet_free(pkt);
+		return 0;
+	}
+
+	/* Parse and set packet header data */
+	odp_packet_pull_tail(pkt, odp_packet_len(pkt) - buf_len);
+	ret = queue_enq(cos->s.queue, odp_buf_to_hdr((odp_buffer_t)pkt), 0);
+	if (ret < 0) {
+		*pkt_ret = pkt;
+		return 1;
+	}
+
+	return 0;
+}
diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c
index ef2e031db..dfdab18d7 100644
--- a/platform/linux-generic/pktio/socket.c
+++ b/platform/linux-generic/pktio/socket.c
@@ -38,6 +38,9 @@
 #include <odp_packet_io_internal.h>
 #include <odp_align_internal.h>
 #include <odp_debug_internal.h>
+#include <odp_classification_datamodel.h>
+#include <odp_classification_inlines.h>
+#include <odp_classification_internal.h>
 #include <odp/hints.h>
 
 #include <odp/helper/eth.h>
@@ -202,6 +205,8 @@ static int sock_close(pktio_entry_t *pktio_entry)
 		return -1;
 	}
 
+	odp_shm_free(pkt_sock->shm);
+
 	return 0;
 }
 
@@ -213,10 +218,13 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev,
 {
 	int sockfd;
 	int err;
+	int i;
 	unsigned int if_idx;
 	struct ifreq ethreq;
 	struct sockaddr_ll sa_ll;
+	char shm_name[ODP_SHM_NAME_LEN];
 	pkt_sock_t *pkt_sock = &pktio_entry->s.pkt_sock;
+	uint8_t *addr;
 
 	/* Init pktio entry */
 	memset(pkt_sock, 0, sizeof(*pkt_sock));
@@ -226,6 +234,20 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev,
 	if (pool == ODP_POOL_INVALID)
 		return -1;
 	pkt_sock->pool = pool;
+	snprintf(shm_name, ODP_SHM_NAME_LEN, "%s-%s", "pktio", netdev);
+	shm_name[ODP_SHM_NAME_LEN - 1] = '\0';
+
+	pkt_sock->shm = odp_shm_reserve(shm_name, PACKET_JUMBO_LEN,
+					  PACKET_JUMBO_LEN *
+					  ODP_PACKET_SOCKET_MAX_BURST_RX, 0);
+	if (pkt_sock->shm == ODP_SHM_INVALID)
+		return -1;
+
+	addr = odp_shm_addr(pkt_sock->shm);
+	for (i = 0; i < ODP_PACKET_SOCKET_MAX_BURST_RX; i++) {
+		pkt_sock->cache_ptr[i] = addr;
+		addr += PACKET_JUMBO_LEN;
+	}
 
 	sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
 	if (sockfd == -1) {
@@ -261,7 +283,6 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev,
 		ODP_ERR("bind(to IF): %s\n", strerror(errno));
 		goto error;
 	}
-
 	return 0;
 
 error:
@@ -319,58 +340,95 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry,
 	const int sockfd = pkt_sock->sockfd;
 	int msgvec_len;
 	struct mmsghdr msgvec[ODP_PACKET_SOCKET_MAX_BURST_RX];
-	struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX][ODP_BUFFER_MAX_SEG];
 	int nb_rx = 0;
 	int recv_msgs;
+	int ret;
+	uint8_t **recv_cache;
 	int i;
 
 	if (odp_unlikely(len > ODP_PACKET_SOCKET_MAX_BURST_RX))
 		return -1;
 
 	memset(msgvec, 0, sizeof(msgvec));
+	recv_cache = pkt_sock->cache_ptr;
 
-	for (i = 0; i < (int)len; i++) {
-		pkt_table[i] = packet_alloc(pkt_sock->pool, 0 /*default*/, 1);
-		if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID))
-			break;
+	if (pktio_cls_enabled(pktio_entry)) {
+		struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX];
 
-		msgvec[i].msg_hdr.msg_iovlen =
-			_rx_pkt_to_iovec(pkt_table[i], iovecs[i]);
-
-		msgvec[i].msg_hdr.msg_iov = iovecs[i];
-	}
-	msgvec_len = i; /* number of successfully allocated pkt buffers */
+		for (i = 0; i < (int)len; i++) {
+			msgvec[i].msg_hdr.msg_iovlen = 1;
+			iovecs[i].iov_base = recv_cache[i];
+			iovecs[i].iov_len = PACKET_JUMBO_LEN;
+			msgvec[i].msg_hdr.msg_iov = &iovecs[i];
+		}
+		/* number of successfully allocated pkt buffers */
+		msgvec_len = i;
+
+		recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len,
+				     MSG_DONTWAIT, NULL);
+		for (i = 0; i < recv_msgs; i++) {
+			void *base = msgvec[i].msg_hdr.msg_iov->iov_base;
+			struct ethhdr *eth_hdr = base;
+			uint16_t pkt_len = msgvec[i].msg_len;
+
+			/* Don't receive packets sent by ourselves */
+			if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac,
+							eth_hdr->h_source)))
+				continue;
+
+			ret = _odp_packet_cls_enq(pktio_entry, base,
+						  pkt_len, &pkt_table[nb_rx]);
+			if (ret)
+				nb_rx++;
+		}
+	} else {
+		struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX]
+				   [ODP_BUFFER_MAX_SEG];
 
-	recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len, MSG_DONTWAIT, NULL);
+		for (i = 0; i < (int)len; i++) {
+			pkt_table[i] = packet_alloc(pkt_sock->pool,
+						    0 /*default*/, 1);
+			if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID))
+				break;
 
-	for (i = 0; i < recv_msgs; i++) {
-		odp_packet_hdr_t *pkt_hdr;
-		void *base = msgvec[i].msg_hdr.msg_iov->iov_base;
-		struct ethhdr *eth_hdr = base;
+			msgvec[i].msg_hdr.msg_iovlen =
+				_rx_pkt_to_iovec(pkt_table[i], iovecs[i]);
 
-		/* Don't receive packets sent by ourselves */
-		if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac,
-						eth_hdr->h_source))) {
-			odp_packet_free(pkt_table[i]);
-			continue;
+			msgvec[i].msg_hdr.msg_iov = iovecs[i];
 		}
 
-		pkt_hdr = odp_packet_hdr(pkt_table[i]);
-
-		odp_packet_pull_tail(pkt_table[i],
-				     odp_packet_len(pkt_table[i]) -
-				     msgvec[i].msg_len);
+		/* number of successfully allocated pkt buffers */
+		msgvec_len = i;
 
-		packet_parse_l2(pkt_hdr);
+		recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len,
+				     MSG_DONTWAIT, NULL);
 
-		pkt_table[nb_rx] = pkt_table[i];
-		nb_rx++;
-	}
+		for (i = 0; i < recv_msgs; i++) {
+			void *base = msgvec[i].msg_hdr.msg_iov->iov_base;
+			struct ethhdr *eth_hdr = base;
+			odp_packet_hdr_t *pkt_hdr;
 
-	/* Free unused pkt buffers */
-	for (; i < msgvec_len; i++)
-		odp_packet_free(pkt_table[i]);
+			/* Don't receive packets sent by ourselves */
+			if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac,
+							eth_hdr->h_source))) {
+				odp_packet_free(pkt_table[i]);
+				continue;
+			}
+			pkt_hdr = odp_packet_hdr(pkt_table[i]);
+			/* Parse and set packet header data */
+			odp_packet_pull_tail(pkt_table[i],
+					     odp_packet_len(pkt_table[i]) -
+					     msgvec[i].msg_len);
+
+			packet_parse_l2(pkt_hdr);
+			pkt_table[nb_rx] = pkt_table[i];
+			nb_rx++;
+		}
 
+		/* Free unused pkt buffers */
+		for (; i < msgvec_len; i++)
+			odp_packet_free(pkt_table[i]);
+	}
 	return nb_rx;
 }
 
@@ -385,7 +443,7 @@ static uint32_t _tx_pkt_to_iovec(odp_packet_t pkt,
 		uint32_t seglen;
 
 		iovecs[iov_count].iov_base = odp_packet_offset(pkt, offset,
-							       &seglen, NULL);
+				&seglen, NULL);
 		iovecs[iov_count].iov_len = seglen;
 		iov_count++;
 		offset += seglen;
@@ -415,7 +473,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry,
 	for (i = 0; i < len; i++) {
 		msgvec[i].msg_hdr.msg_iov = iovecs[i];
 		msgvec[i].msg_hdr.msg_iovlen = _tx_pkt_to_iovec(pkt_table[i],
-								iovecs[i]);
+				iovecs[i]);
 	}
 
 	for (i = 0; i < len; ) {
@@ -423,7 +481,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry,
 		if (odp_unlikely(ret <= -1)) {
 			if (i == 0 && SOCK_ERR_REPORT(errno)) {
 				__odp_errno = errno;
-				ODP_ERR("sendmmsg(): %s\n", strerror(errno));
+			ODP_ERR("sendmmsg(): %s\n", strerror(errno));
 				return -1;
 			}
 			break;
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c
index 2bdb72b35..fcaaaf63f 100644
--- a/platform/linux-generic/pktio/socket_mmap.c
+++ b/platform/linux-generic/pktio/socket_mmap.c
@@ -28,6 +28,9 @@
 #include <odp_packet_internal.h>
 #include <odp_packet_io_internal.h>
 #include <odp_debug_internal.h>
+#include <odp_classification_datamodel.h>
+#include <odp_classification_inlines.h>
+#include <odp_classification_internal.h>
 #include <odp/hints.h>
 
 #include <odp/helper/eth.h>
@@ -108,9 +111,9 @@ static inline void mmap_tx_user_ready(struct tpacket2_hdr *hdr)
 	__sync_synchronize();
 }
 
-static inline unsigned pkt_mmap_v2_rx(int sock, struct ring *ring,
+static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
+				      pkt_sock_mmap_t *pkt_sock,
 				      odp_packet_t pkt_table[], unsigned len,
-				      odp_pool_t pool,
 				      unsigned char if_mac[])
 {
 	union frame_map ppd;
@@ -118,57 +121,68 @@ static inline unsigned pkt_mmap_v2_rx(int sock, struct ring *ring,
 	uint8_t *pkt_buf;
 	int pkt_len;
 	struct ethhdr *eth_hdr;
-	odp_packet_hdr_t *pkt_hdr;
 	unsigned i = 0;
+	uint8_t nb_rx = 0;
+	struct ring *ring;
+	int ret;
 
-	(void)sock;
-
+	ring  = &pkt_sock->rx_ring;
 	frame_num = ring->frame_num;
 
 	while (i < len) {
-		if (mmap_rx_kernel_ready(ring->rd[frame_num].iov_base)) {
-			ppd.raw = ring->rd[frame_num].iov_base;
+		if (!mmap_rx_kernel_ready(ring->rd[frame_num].iov_base))
+			break;
+
+		ppd.raw = ring->rd[frame_num].iov_base;
+		next_frame_num = (frame_num + 1) % ring->rd_num;
+
+		pkt_buf = (uint8_t *)ppd.raw + ppd.v2->tp_h.tp_mac;
+		pkt_len = ppd.v2->tp_h.tp_snaplen;
 
-			next_frame_num = (frame_num + 1) % ring->rd_num;
+		/* Don't receive packets sent by ourselves */
+		eth_hdr = (struct ethhdr *)pkt_buf;
+		if (odp_unlikely(ethaddrs_equal(if_mac,
+						eth_hdr->h_source))) {
+			mmap_rx_user_ready(ppd.raw); /* drop */
+			frame_num = next_frame_num;
+			continue;
+		}
 
-			pkt_buf = (uint8_t *)ppd.raw + ppd.v2->tp_h.tp_mac;
-			pkt_len = ppd.v2->tp_h.tp_snaplen;
+		if (pktio_cls_enabled(pktio_entry)) {
+			ret = _odp_packet_cls_enq(pktio_entry, pkt_buf,
+						  pkt_len, &pkt_table[nb_rx]);
+			if (ret)
+				nb_rx++;
+		} else {
+			odp_packet_hdr_t *hdr;
 
-			/* Don't receive packets sent by ourselves */
-			eth_hdr = (struct ethhdr *)pkt_buf;
-			if (odp_unlikely(ethaddrs_equal(if_mac,
-							eth_hdr->h_source))) {
+			pkt_table[i] = packet_alloc(pkt_sock->pool, pkt_len, 1);
+			if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID)) {
 				mmap_rx_user_ready(ppd.raw); /* drop */
 				frame_num = next_frame_num;
 				continue;
 			}
-
-			pkt_table[i] = packet_alloc(pool, pkt_len, 1);
-			if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID))
-				break;
-
-			pkt_hdr = odp_packet_hdr(pkt_table[i]);
-
-			if (odp_packet_copydata_in(pkt_table[i], 0,
-						   pkt_len, pkt_buf) != 0) {
+			hdr = odp_packet_hdr(pkt_table[i]);
+			ret = odp_packet_copydata_in(pkt_table[i], 0,
+						     pkt_len, pkt_buf);
+			if (ret != 0) {
 				odp_packet_free(pkt_table[i]);
-				break;
+				mmap_rx_user_ready(ppd.raw); /* drop */
+				frame_num = next_frame_num;
+				continue;
 			}
 
-			packet_parse_l2(pkt_hdr);
-
-			mmap_rx_user_ready(ppd.raw);
-
-			frame_num = next_frame_num;
-			i++;
-		} else {
-			break;
+			packet_parse_l2(hdr);
+			nb_rx++;
 		}
+
+		mmap_rx_user_ready(ppd.raw);
+		frame_num = next_frame_num;
+		i++;
 	}
 
 	ring->frame_num = frame_num;
-
-	return i;
+	return nb_rx;
 }
 
 static inline unsigned pkt_mmap_v2_tx(int sock, struct ring *ring,
@@ -502,9 +516,8 @@ static int sock_mmap_recv(pktio_entry_t *pktio_entry,
 {
 	pkt_sock_mmap_t *const pkt_sock = &pktio_entry->s.pkt_sock_mmap;
 
-	return pkt_mmap_v2_rx(pkt_sock->rx_ring.sock, &pkt_sock->rx_ring,
-			      pkt_table, len, pkt_sock->pool,
-			      pkt_sock->if_mac);
+	return pkt_mmap_v2_rx(pktio_entry, pkt_sock,
+			      pkt_table, len, pkt_sock->if_mac);
 }
 
 static int sock_mmap_send(pktio_entry_t *pktio_entry,