diff options
35 files changed, 5329 insertions, 14 deletions
diff --git a/README.DPDK b/README.DPDK new file mode 120000 index 000000000..659f5671d --- /dev/null +++ b/README.DPDK @@ -0,0 +1 @@ +platform/linux-dpdk/README
\ No newline at end of file diff --git a/configure.ac b/configure.ac index 24a3fa131..4cfaca844 100644 --- a/configure.ac +++ b/configure.ac @@ -71,23 +71,15 @@ AC_SUBST(ODP_LIBSO_VERSION) ########################################################################## AC_ARG_WITH([platform], [AS_HELP_STRING([--with-platform=platform], - [select platform to be used, default linux-generic])], + [select platform to be used, default linux-dpdk])], [], - [with_platform=linux-generic + [with_platform=linux-dpdk ]) AC_SUBST([with_platform]) AC_SUBST([platform_with_platform], ["platform/${with_platform}"]) AC_SUBST([platform_with_platform_test], ["platform/${with_platform}/test"]) -if test "${with_platform}" == "linux-generic"; -then - m4_include([./platform/linux-generic/m4/configure.m4]) -else - echo "UNSUPPORTED PLATFORM: ${with_platform}" - exit 1 -fi - AC_ARG_WITH([sdk-install-path], AC_HELP_STRING([--with-sdk-install-path=DIR path to external libs and headers], [(or in the default path if not specified).]), @@ -97,6 +89,17 @@ AC_SUBST(SDK_INSTALL_PATH) AM_CONDITIONAL([SDK_INSTALL_PATH_], [test "x${SDK_INSTALL_PATH_}" = "x1"]) +if test "${with_platform}" == "linux-generic"; +then + m4_include([./platform/linux-generic/m4/configure.m4]) +elif test "${with_platform}" == "linux-dpdk"; +then + m4_include([./platform/linux-dpdk/m4/configure.m4]) +else + echo "UNSUPPORTED PLATFORM: ${with_platform}" + exit 1 +fi + ########################################################################## # Enable/disable Unit tests ########################################################################## @@ -278,7 +281,7 @@ ODP_CFLAGS="$ODP_CFLAGS -W -Wall -Werror -Wstrict-prototypes -Wmissing-prototype ODP_CFLAGS="$ODP_CFLAGS -Wmissing-declarations -Wold-style-definition -Wpointer-arith" ODP_CFLAGS="$ODP_CFLAGS -Wcast-align -Wnested-externs -Wcast-qual -Wformat-nonliteral" ODP_CFLAGS="$ODP_CFLAGS -Wformat-security -Wundef -Wwrite-strings" -ODP_CFLAGS="$ODP_CFLAGS -std=c99" +ODP_CFLAGS="$ODP_CFLAGS -std=gnu99" ########################################################################## # Default include setup @@ -299,7 +302,8 @@ AC_CONFIG_FILES([Makefile pkgconfig/libodp.pc pkgconfig/libodphelper.pc platform/linux-generic/Makefile - platform/linux-generic/test/pktio/Makefile + platform/linux-dpdk/Makefile + platform/linux-dpdk/test/pktio/Makefile scripts/Makefile test/Makefile test/api_test/Makefile diff --git a/example/Makefile.am b/example/Makefile.am index 353f397f0..83413f693 100644 --- a/example/Makefile.am +++ b/example/Makefile.am @@ -1 +1,2 @@ -SUBDIRS = classifier generator ipsec packet timer +#SUBDIRS = classifier generator ipsec packet timer +SUBDIRS = generator ipsec packet timer diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am new file mode 100644 index 000000000..8e32e9031 --- /dev/null +++ b/platform/linux-dpdk/Makefile.am @@ -0,0 +1,177 @@ +include $(top_srcdir)/platform/Makefile.inc + +PLAT_CFLAGS = -msse4.2 +if SDK_INSTALL_PATH_ +PLAT_CFLAGS += -include $(SDK_INSTALL_PATH)/include/rte_config.h +PLAT_CFLAGS += -I$(SDK_INSTALL_PATH)/include +PLAT_CFLAGS += -I$(SDK_INSTALL_PATH)/include/arch +PLAT_CFLAGS += -I$(SDK_INSTALL_PATH)/include/exec-env + +AM_LDFLAGS += -L$(SDK_INSTALL_PATH)/lib +endif + +AM_CFLAGS += $(PLAT_CFLAGS) +AM_CFLAGS += -I$(srcdir)/include +AM_CFLAGS += -I$(top_srcdir)/platform/linux-generic/include +AM_CFLAGS += -I$(top_srcdir)/include +AM_CFLAGS += -I$(top_srcdir)/helper/include + +DPDK_LIBS="-lintel_dpdk -ldl -lm -lpcap" +LIBS += $(DPDK_LIBS) +SUBDIRS = test + +include_HEADERS = \ + $(top_srcdir)/include/odp.h + +odpincludedir= $(includedir)/odp +odpinclude_HEADERS = \ + $(top_srcdir)/platform/linux-generic/include/odp/align.h \ + $(top_srcdir)/platform/linux-generic/include/odp/atomic.h \ + $(top_srcdir)/platform/linux-generic/include/odp/barrier.h \ + $(top_srcdir)/platform/linux-generic/include/odp/buffer.h \ + $(top_srcdir)/platform/linux-generic/include/odp/byteorder.h \ + $(top_srcdir)/platform/linux-generic/include/odp/classification.h \ + $(top_srcdir)/platform/linux-generic/include/odp/compiler.h \ + $(top_srcdir)/platform/linux-generic/include/odp/config.h \ + $(top_srcdir)/platform/linux-generic/include/odp/cpu.h \ + $(top_srcdir)/platform/linux-generic/include/odp/cpumask.h \ + $(top_srcdir)/platform/linux-generic/include/odp/crypto.h \ + $(top_srcdir)/platform/linux-generic/include/odp/debug.h \ + $(top_srcdir)/platform/linux-generic/include/odp/errno.h \ + $(top_srcdir)/platform/linux-generic/include/odp/event.h \ + $(top_srcdir)/platform/linux-generic/include/odp/hints.h \ + $(top_srcdir)/platform/linux-generic/include/odp/init.h \ + $(top_srcdir)/platform/linux-generic/include/odp/packet_flags.h \ + $(srcdir)/include/odp/packet.h \ + $(top_srcdir)/platform/linux-generic/include/odp/packet_io.h \ + $(top_srcdir)/platform/linux-generic/include/odp/pool.h \ + $(top_srcdir)/platform/linux-generic/include/odp/queue.h \ + $(top_srcdir)/platform/linux-generic/include/odp/random.h \ + $(top_srcdir)/platform/linux-generic/include/odp/rwlock.h \ + $(top_srcdir)/platform/linux-generic/include/odp/schedule.h \ + $(top_srcdir)/platform/linux-generic/include/odp/shared_memory.h \ + $(top_srcdir)/platform/linux-generic/include/odp/spinlock.h \ + $(top_srcdir)/platform/linux-generic/include/odp/std_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/sync.h \ + $(top_srcdir)/platform/linux-generic/include/odp/system_info.h \ + $(top_srcdir)/platform/linux-generic/include/odp/thread.h \ + $(top_srcdir)/platform/linux-generic/include/odp/thrmask.h \ + $(top_srcdir)/platform/linux-generic/include/odp/ticketlock.h \ + $(top_srcdir)/platform/linux-generic/include/odp/time.h \ + $(top_srcdir)/platform/linux-generic/include/odp/timer.h \ + $(top_srcdir)/platform/linux-generic/include/odp/version.h + +odpplatincludedir= $(includedir)/odp/plat +odpplatinclude_HEADERS = \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/atomic_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/barrier_types.h \ + $(srcdir)/include/odp/plat/buffer_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/byteorder_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/classification_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/cpumask_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/crypto_types.h \ + $(srcdir)/include/odp/plat/event_types.h \ + $(srcdir)/include/odp/plat/packet_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/packet_io_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/pool_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/queue_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/rwlock_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/schedule_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/shared_memory_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/spinlock_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/strong_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/thrmask_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/ticketlock_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/timer_types.h \ + $(top_srcdir)/platform/linux-generic/include/odp/plat/version_types.h + +odpapiincludedir= $(includedir)/odp/api +odpapiinclude_HEADERS = \ + $(top_srcdir)/include/odp/api/align.h \ + $(top_srcdir)/include/odp/api/atomic.h \ + $(top_srcdir)/include/odp/api/barrier.h \ + $(top_srcdir)/include/odp/api/buffer.h \ + $(top_srcdir)/include/odp/api/byteorder.h \ + $(top_srcdir)/include/odp/api/classification.h \ + $(top_srcdir)/include/odp/api/compiler.h \ + $(top_srcdir)/include/odp/api/config.h \ + $(top_srcdir)/include/odp/api/cpu.h \ + $(top_srcdir)/include/odp/api/cpumask.h \ + $(top_srcdir)/include/odp/api/crypto.h \ + $(top_srcdir)/include/odp/api/debug.h \ + $(top_srcdir)/include/odp/api/errno.h \ + $(top_srcdir)/include/odp/api/event.h \ + $(top_srcdir)/include/odp/api/hints.h \ + $(top_srcdir)/include/odp/api/init.h \ + $(top_srcdir)/include/odp/api/packet.h \ + $(top_srcdir)/include/odp/api/packet_flags.h \ + $(top_srcdir)/include/odp/api/packet_io.h \ + $(top_srcdir)/include/odp/api/pool.h \ + $(top_srcdir)/include/odp/api/queue.h \ + $(top_srcdir)/include/odp/api/random.h \ + $(top_srcdir)/include/odp/api/rwlock.h \ + $(top_srcdir)/include/odp/api/schedule.h \ + $(top_srcdir)/include/odp/api/shared_memory.h \ + $(top_srcdir)/include/odp/api/spinlock.h \ + $(top_srcdir)/include/odp/api/std_types.h \ + $(top_srcdir)/include/odp/api/sync.h \ + $(top_srcdir)/include/odp/api/system_info.h \ + $(top_srcdir)/include/odp/api/thread.h \ + $(top_srcdir)/include/odp/api/thrmask.h \ + $(top_srcdir)/include/odp/api/ticketlock.h \ + $(top_srcdir)/include/odp/api/time.h \ + $(top_srcdir)/include/odp/api/timer.h \ + $(top_srcdir)/include/odp/api/version.h + +noinst_HEADERS = \ + $(srcdir)/include/odp_buffer_internal.h \ + $(top_srcdir)/platform/linux-generic/include/odp_debug_internal.h \ + $(srcdir)/include/odp_packet_dpdk.h \ + $(srcdir)/include/odp_packet_internal.h \ + $(srcdir)/include/odp_packet_io_internal.h \ + $(top_srcdir)/platform/linux-generic/include/odp_packet_io_queue.h \ + $(srcdir)/include/odp_pool_internal.h \ + ${top_srcdir}/platform/linux-generic/Makefile.inc + +subdirheadersdir = $(includedir)/odp/helper +subdirheaders_HEADERS = \ + $(top_srcdir)/helper/include/odp/helper/chksum.h \ + $(top_srcdir)/helper/include/odp/helper/eth.h \ + $(top_srcdir)/helper/include/odp/helper/icmp.h \ + $(top_srcdir)/helper/include/odp/helper/ip.h \ + $(top_srcdir)/helper/include/odp/helper/ipsec.h \ + $(top_srcdir)/helper/include/odp/helper/linux.h \ + $(top_srcdir)/helper/include/odp/helper/ring.h \ + $(top_srcdir)/helper/include/odp/helper/tcp.h \ + $(top_srcdir)/helper/include/odp/helper/udp.h + +__LIB__libodp_la_SOURCES = \ + ../linux-generic/odp_barrier.c \ + odp_buffer.c \ + odp_cpumask.c \ + ../linux-generic/odp_crypto.c \ + ../linux-generic/odp_errno.c \ + ../linux-generic/odp_event.c \ + odp_init.c \ + ../linux-generic/odp_impl.c \ + odp_linux.c \ + odp_packet.c \ + odp_packet_dpdk.c \ + ../linux-generic/odp_packet_flags.c \ + odp_packet_io.c \ + odp_pool.c \ + ../linux-generic/odp_queue.c \ + ../../helper/ring.c \ + ../linux-generic/odp_rwlock.c \ + ../linux-generic/odp_schedule.c \ + ../linux-generic/odp_shared_memory.c \ + ../linux-generic/odp_spinlock.c \ + ../linux-generic/odp_system_info.c \ + odp_thread.c \ + ../linux-generic/odp_thrmask.c \ + ../linux-generic/odp_ticketlock.c \ + ../linux-generic/odp_time.c \ + ../linux-generic/odp_timer.c \ + ../linux-generic/odp_version.c \ + ../linux-generic/odp_weak.c \ + ../linux-generic/arch/@ARCH@/odp_time.c diff --git a/platform/linux-dpdk/Makefile.inc b/platform/linux-dpdk/Makefile.inc new file mode 100644 index 000000000..f681940a4 --- /dev/null +++ b/platform/linux-dpdk/Makefile.inc @@ -0,0 +1 @@ +AM_LDFLAGS += -R$(SDK_INSTALL_PATH)/lib diff --git a/platform/linux-dpdk/README b/platform/linux-dpdk/README new file mode 100644 index 000000000..263a75891 --- /dev/null +++ b/platform/linux-dpdk/README @@ -0,0 +1,291 @@ +Copyright (c) 2014, Linaro Limited +All rights reserved. + +SPDX-License-Identifier: BSD-3-Clause + +1. Rationale +================================================= + +This is an effort to port ODP on top of DPDK to accelerate packet processing on +Intel systems equiped with NIC's that supports DPDK. + +Prerequisites and considerations: +-------------------------------- +- at least an Intel system is needed for compiling and running odp-dpdk +- 8GB of RAM recommended, 4 might be enough too +- it's recommended to obtain an Intel network card (many come in dual port + configuration available to buy, mostly dedicated to server usage) +- it's also possible to use odp-dpdk for evaluation purposes without a DPDK + compatible NIC, using the pcap poll mode driver +- DPDK code must be downloaded, configured and compiled, details below +- odp-dpdk has been compiled and tested on Ubuntu 14.04 LTS with + 3.13.0-29-generic kernel + + +Checking if an Intel NIC is supported by DPDK: +--------------------------------------------- +DPDK only works on a selected range of Intel network cards. To check if an +installed NIC is supported one needs to get the product id and check if DPDK +supports it. + +Getting the product id is possible using lspci: + + lspci -nn | grep Ethernet + +A sample output could be: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +00:19.0 Ethernet controller [0200]: Intel Corporation 82579LM Gigabit Network Connection [8086:1502] (rev 04) +01:00.0 Ethernet controller [0200]: Intel Corporation I350 Gigabit Network Connection [8086:1521] (rev 01) +01:00.1 Ethernet controller [0200]: Intel Corporation I350 Gigabit Network Connection [8086:1521] (rev 01) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The list of known and supported devices can be found in the DPDK code: +http://dpdk.org/browse/dpdk/tree/lib/librte_eal/common/include/rte_pci_dev_ids.h?id=v2.0.0 + +The definition of a known NIC looks like this: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +#define E1000_DEV_ID_PCH2_LV_LM 0x1502 +... +#define E1000_DEV_ID_I350_COPPER 0x1521 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For a card to be supported a further PCI dev declaration must exist: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I350_COPPER) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Notice that 82579LM is known but not actually supported by DPDK. + + +2. Preparing DPDK +================================================= + +Fetching the DPDK code: +---------------------- + git clone http://92.243.14.124/git/dpdk ./<dpdk-dir> + +Right now odp-dpdk only supports DPDK v2.0.0. During upgrade, make sure that PMD +constuctors are correctly and fully referred in refer_constructors(): + git tag -l -- will list all the tags available + git checkout -b 2.0.0 tags/v2.0.0 + +Compile DPDK: +------------ +Please refer to http://dpdk.org/doc for more details on how to build DPDK. +Getting started guide for Linux might be of help. +Best effort is done to provide some help on DPDK cmds below for Ubuntu, where it +has been compiled and tested. + + +This has to be done only once: + cd <dpdk-dir> + make config T=x86_64-native-linuxapp-gcc O=x86_64-native-linuxapp-gcc + +Set CONFIG_RTE_BUILD_COMBINE_LIBS=y and CONFIG_RTE_BUILD_SHARED_LIB=n in +./x86_64-native-linuxapp-gcc/.config file: + cd <dpdk-dir>/x86_64-native-linuxapp-gcc + sed -ri 's,(CONFIG_RTE_BUILD_COMBINE_LIBS=).*,\1y,' .config + +Note: dynamic linking does not work with DPDK v1.7.1, there is a workaround +though but until then it's better to disable shared libs: + sed -ri 's,(CONFIG_RTE_BUILD_SHARED_LIB=).*,\1n,' .config + +Note: to use odp-dpdk without DPDK supported NIC's enable pcap pmd: + sed -ri 's,(CONFIG_RTE_LIBRTE_PMD_PCAP=).*,\1y,' .config + +Note: if non-intel SFP's are used in IXGBE, then: + sed -ri 's,(CONFIG_RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP=).*,\1y,' .config + +Note: a few other configs are also needed for the moment: + sed -ri 's,(CONFIG_RTE_MBUF_SCATTER_GATHER=).*,\1n,' .config + sed -ri 's,(CONFIG_RTE_LIBRTE_PMD_BOND=).*,\1n,' .config + sed -ri 's,(CONFIG_RTE_LIBRTE_IP_FRAG=).*,\1n,' .config + + make install T=x86_64-native-linuxapp-gcc EXTRA_CFLAGS="-fPIC" + +If "conflicting types for skb_set_hash" error happens during DPDK +build, then please knock-off skb_set_hash function from kcompat.h as +shown below. This was seen in Ubuntu 3.13.0-30-generic. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +diff --git a/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h b/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h +index 19df483..78a794a 100644 +--- a/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h ++++ b/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h +@@ -3845,11 +3845,6 @@ static inline struct sk_buff *__kc__vlan_hwaccel_put_tag(struct sk_buff *skb, + #if ( LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0) ) + #ifdef NETIF_F_RXHASH + #define PKT_HASH_TYPE_L3 0 +-static inline void +-skb_set_hash(struct sk_buff *skb, __u32 hash, __always_unused int type) +-{ +- skb->rxhash = hash; +-} + #endif /* NETIF_F_RXHASH */ + #endif /* < 3.14.0 */ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This only ensures building DPDK, but traffic is not tested with this build yet. + + +3. Compile odp-dpdk +================================================= + + cd <odp-dir> + ./bootstrap + ./configure --with-platform=linux-dpdk --with-sdk-install-path=<dpdk-dir>/x86_64-native-linuxapp-gcc + make + + +4. Prepare DPDK for running odp-dpdk examples +================================================= + +Reserve hugepages: +----------------- +To reserve huge pages, which is needed for DPDK, execute following commands +(these are usually needed only once after the system has started): + sudo sh -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages' + +If you are running on a multi-node machine then hugepages have to be reserved on +each node: + ls /sys/devices/system/node + sudo sh -c 'echo 1024 > /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages' + +Mount hugetlbfs: +--------------- + sudo mkdir /mnt/huge + sudo mount -t hugetlbfs nodev /mnt/huge + +Insert DPDK kernel module: +------------------------- +DPDK uses userspace poll mode drivers, so it's necessary to insert a couple of +modules to allow DPDK to map the NIC's registers to userspace: + sudo /sbin/modprobe uio + ulimit -Sn 2048 + + cd <dpdk-dir> + sudo insmod x86_64-native-linuxapp-gcc/kmod/igb_uio.ko + +Bind NIC's to DPDK: +------------------ +The DPDK code contains a tool used to bind drivers to the network cards. + + cd <dpdk-dir> + ./tools/dpdk_nic_bind.py --status + +This command produces output that is similar to the one given below: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Network devices using IGB_UIO driver +==================================== +0000:05:00.0 'Ethernet 10G 2P X520 Adapter' drv=igb_uio unused= +0000:05:00.1 'Ethernet 10G 2P X520 Adapter' drv=igb_uio unused= + +Network devices using kernel driver +=================================== +0000:01:00.0 'NetXtreme II BCM5709 Gigabit Ethernet' if=eth0 drv=bnx2 unused=<none> *Active* +0000:01:00.1 'NetXtreme II BCM5709 Gigabit Ethernet' if=eth1 drv=bnx2 unused=<none> +0000:07:00.0 'T320 10GbE Dual Port Adapter' if=eth2,eth3 drv=cxgb3 unused=<none> + +Other network devices +===================== +<none> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Bind using interface name: +------------------------- +The easiest way is to let the tool automatically switch the regular drivers. For +that the interface must not be active i.e. no IP addresses assigned: + ifconfig eth0 0 + ifconfig eth1 0 + sudo ./tools/dpdk_nic_bind --bind=igb_uio eth0 + sudo ./tools/dpdk_nic_bind --bind=igb_uio eth1 + + +Bind using PCI ids: +------------------ +Another way is to remove the regular drivers and use PCI ids: + sudo rmmod ixgbe + +If the SFP's used are non-intel, then + sudo modprobe ixgbe allow_unsupported_sfp=1 + + sudo ./tools/igb_uio_bind.py --bind=igb_uio 05:00.0 + sudo ./tools/igb_uio_bind.py --bind=igb_uio 05:00.1 + +Unbind network cards from DPDK: +------------------------------ +To restore the NIC's back to kernel use something like this: + sudo ./tools/igb_uio_bind.py --bind=ixgbe 05:00.0 + sudo ./tools/igb_uio_bind.py --bind=ixgbe 05:00.1 + + +5. Running ODP apps +================================================= + +You need to supply the DPDK command line parameters in the ODP_PLATFORM_PARAMS +environment variable: + + export ODP_PLATFORM_PARAMS="-n 4" + +You need to pass "-n [1..4]" at least to specify memory channels. The coremask +(-c) is calculated by ODP-DPDK based on the process affinity at startup. You can +infcluence that with 'taskset'. +Some useful ODP examples and how to run them: + + l2fwding app: + sudo ODP_PLATFORM_PARAMS="-n 4" ./test/performance/odp_l2fwd -i 0,1 -m 0 -c 2 + + loopback app: + sudo ODP_PLATFORM_PARAMS="-n 4" ./example/packet/odp_pktio -i 0,1 -m 0 -c 2 + + -i 0,1 - interface number + -m 0 - burst mode + -c 2 - number of cpus + + +6. Howto debug DPDK apps on the host +================================================= + +For example you need to debug some l2fwd application. Then network configuration +might be: + +<veth1-2> <-----> <veth2-1> (iface0 DPDK L2FWD APP iface1) <veth2-3> <-----> <veth3-2> + +Where: +vethX-Y - virtual devices for host. + +Packet sent to veth1-2 goes to chain and appears on veth3-2 + +Steps: +Recompile with: +CONFIG_RTE_LIBRTE_PMD_PCAP=y + + ip link add veth1-2 type veth peer name veth2-1 + ip link add veth2-3 type veth peer name veth3-2 + ifconfig veth1-2 up -arp + ifconfig veth2-1 up -arp + ifconfig veth2-3 up -arp + ifconfig veth3-2 up -arp + + mount -t hugetlbfs none /mnt/huge + +Finally give l2fwd fake devices: + ./l2fwd -c '0xf' -n 4 --vdev "eth_pcap0,iface=veth2-1" --vdev="eth_pcap1,iface=veth2-3" -- -p 3 + +7. Build with devbuild.sh +================================================= + +scripts/devbuild.sh contains an example script aimed for developers. It uses +the CI scripts from https://git.linaro.org/lng/check-odp.git to build DPDK and +ODP-DPDK. It can also run "make check" or individual unit tests, but you need to +install CUnit as a prerequisite. +If you have build problems, try to run it and see if it works. An example: + export REPOS=${PWD} + git clone https://git.linaro.org/lng/check-odp.git + git clone https://git.linaro.org/lng/odp-dpdk.git + odp-dpdk/scripts/devbuild.sh dpdk + odp-dpdk/scripts/devbuild.sh odp + odp-dpdk/scripts/devbuild.sh odp-check diff --git a/platform/linux-dpdk/include/odp/packet.h b/platform/linux-dpdk/include/odp/packet.h new file mode 100644 index 000000000..7e22c249e --- /dev/null +++ b/platform/linux-dpdk/include/odp/packet.h @@ -0,0 +1,83 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP packet descriptor + */ + +#ifndef ODP_PLAT_PACKET_H_ +#define ODP_PLAT_PACKET_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/std_types.h> +#include <odp/plat/event_types.h> +#include <odp/plat/packet_io_types.h> +#include <odp/plat/packet_types.h> +#include <odp/plat/buffer_types.h> +#include <odp/plat/pool_types.h> + +/** @ingroup odp_packet + * @{ + */ + +extern const unsigned int buf_addr_offset; +extern const unsigned int data_off_offset; +extern const unsigned int pkt_len_offset; +extern const unsigned int seg_len_offset; +extern const unsigned int udata_len_offset; +extern const unsigned int udata_offset; + +/* + * NOTE: These functions are inlined because they are on a performance hot path. + * As we can't force the application to directly include DPDK headers we have to + * export these fields through constants calculated compile time in + * odp_packet.c, where we can see the DPDK definitions. + * + */ +static inline uint32_t odp_packet_len(odp_packet_t pkt) +{ + return *(uint32_t *)((char *)pkt + pkt_len_offset); +} + +static inline uint32_t odp_packet_seg_len(odp_packet_t pkt) +{ + return *(uint16_t *)((char *)pkt + seg_len_offset); +} + +static inline void *odp_packet_user_area(odp_packet_t pkt) +{ + return (void *)((char *)pkt + udata_offset); +} + +static inline uint32_t odp_packet_user_area_size(odp_packet_t pkt) +{ + return *(uint32_t *)((char *)pkt + udata_len_offset); +} + +static inline void *odp_packet_data(odp_packet_t pkt) +{ + char** buf_addr = (char **)((char *)pkt + buf_addr_offset); + uint16_t data_off = *(uint16_t *)((char *)pkt + data_off_offset); + return (void *)(*buf_addr + data_off); +} + + +/** + * @} + */ + +#include <odp/api/packet.h> + +#ifdef __cplusplus +} +#endif + +#endif /* ODP_PLAT_PACKET_H_ */ diff --git a/platform/linux-dpdk/include/odp/plat/buffer_types.h b/platform/linux-dpdk/include/odp/plat/buffer_types.h new file mode 100644 index 000000000..4ac482a6f --- /dev/null +++ b/platform/linux-dpdk/include/odp/plat/buffer_types.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP buffer descriptor + */ + +#ifndef ODP_BUFFER_TYPES_H_ +#define ODP_BUFFER_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/std_types.h> +#include <odp/plat/strong_types.h> + +/** @addtogroup odp_buffer ODP BUFFER + * Operations on a buffer. + * @{ + */ + +/** ODP buffer */ +typedef odp_handle_t odp_buffer_t; + +/** Invalid buffer */ +#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, NULL) + +/** ODP buffer segment */ +typedef odp_handle_t odp_buffer_seg_t; + +/** Invalid segment */ +#define ODP_SEGMENT_INVALID ((odp_buffer_seg_t)ODP_BUFFER_INVALID) + +/** Get printable format of odp_buffer_t */ +static inline uint64_t odp_buffer_to_u64(odp_buffer_t hdl) +{ + return _odp_pri(hdl); +} + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/include/odp/plat/event_types.h b/platform/linux-dpdk/include/odp/plat/event_types.h new file mode 100644 index 000000000..7fe3c6677 --- /dev/null +++ b/platform/linux-dpdk/include/odp/plat/event_types.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + +/** + * @file + * + * ODP event + */ + +#ifndef ODP_EVENT_TYPES_H_ +#define ODP_EVENT_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/std_types.h> +#include <odp/plat/strong_types.h> + +/** @defgroup odp_event ODP EVENT + * Operations on an event. + * @{ + */ + +typedef ODP_HANDLE_T(odp_event_t); + +#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, NULL) + +#define ODP_EVENT_BUFFER 1 +#define ODP_EVENT_PACKET 2 +#define ODP_EVENT_TIMEOUT 3 +#define ODP_EVENT_CRYPTO_COMPL 4 + +/** Get printable format of odp_event_t */ +static inline uint64_t odp_event_to_u64(odp_event_t hdl) +{ + return _odp_pri(hdl); +} + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/include/odp/plat/packet_types.h b/platform/linux-dpdk/include/odp/plat/packet_types.h new file mode 100644 index 000000000..3dfa77848 --- /dev/null +++ b/platform/linux-dpdk/include/odp/plat/packet_types.h @@ -0,0 +1,59 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + +/** + * @file + * + * ODP packet descriptor + */ + +#ifndef ODP_PACKET_TYPES_H_ +#define ODP_PACKET_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/std_types.h> +#include <odp/plat/strong_types.h> + +/** @addtogroup odp_packet ODP PACKET + * Operations on a packet. + * @{ + */ + +typedef ODP_HANDLE_T(odp_packet_t); + +#define ODP_PACKET_INVALID _odp_cast_scalar(odp_packet_t, NULL) + +#define ODP_PACKET_OFFSET_INVALID (0x0fffffff) + +typedef ODP_HANDLE_T(odp_packet_seg_t); + +#define ODP_PACKET_SEG_INVALID _odp_cast_scalar(odp_packet_seg_t, NULL) + +/** Get printable format of odp_packet_t */ +static inline uint64_t odp_packet_to_u64(odp_packet_t hdl) +{ + return _odp_pri(hdl); +} + +/** Get printable format of odp_packet_seg_t */ +static inline uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl) +{ + return _odp_pri(hdl); +} + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/include/odp_buffer_inlines.h b/platform/linux-dpdk/include/odp_buffer_inlines.h new file mode 100644 index 000000000..2867a91d5 --- /dev/null +++ b/platform/linux-dpdk/include/odp_buffer_inlines.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2014, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * Inline functions for ODP buffer mgmt routines - implementation internal + */ + +#ifndef ODP_BUFFER_INLINES_H_ +#define ODP_BUFFER_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_buffer_internal.h> + +static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr) +{ + return (odp_buffer_t)hdr; +} + +static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) +{ + return (odp_buffer_hdr_t *)(void *)buf; +} + +static inline int _odp_buffer_event_type(odp_buffer_t buf) +{ + return odp_buf_to_hdr(buf)->event_type; +} + +static inline void _odp_buffer_event_type_set(odp_buffer_t buf, int ev) +{ + odp_buf_to_hdr(buf)->event_type = ev; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h new file mode 100644 index 000000000..508c3023c --- /dev/null +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -0,0 +1,95 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + +/** + * @file + * + * ODP buffer descriptor - implementation internal + */ + +#ifndef ODP_BUFFER_INTERNAL_H_ +#define ODP_BUFFER_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/std_types.h> +#include <odp/atomic.h> +#include <odp/pool.h> +#include <odp/buffer.h> +#include <odp/debug.h> +#include <odp/align.h> +#include <odp_align_internal.h> +#include <odp/config.h> +#include <odp/byteorder.h> +#include <odp/thread.h> +#include <sys/types.h> +#include <odp/event.h> + +/* DPDK */ +#include <rte_mbuf.h> + +_ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_SEG_LEN_MIN >= 256, + "ODP Segment size must be a minimum of 256 bytes"); + +_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MAX % + ODP_CONFIG_PACKET_SEG_LEN_MIN) == 0, + "Packet max size must be a multiple of segment size"); + +#define ODP_BUFFER_MAX_SEG \ + (ODP_CONFIG_PACKET_BUF_LEN_MAX / ODP_CONFIG_PACKET_SEG_LEN_MIN) + +/* We can optimize storage of small raw buffers within metadata area */ +#define ODP_MAX_INLINE_BUF ((sizeof(void *)) * (ODP_BUFFER_MAX_SEG - 1)) + +typedef union odp_buffer_bits_t { + odp_buffer_t handle; +} odp_buffer_bits_t; + +typedef struct odp_buffer_hdr_t { + struct rte_mbuf mb; /* Underlying DPDK rte_mbuf */ + struct odp_buffer_hdr_t *next; /* next buf in a list */ + odp_buffer_bits_t handle; /* handle */ + int type; /* ODP buffer type; + not DPDK buf type */ + int event_type; /* for reuse as event */ + odp_pool_t pool_hdl; /* buffer pool handle */ + union { + uint64_t buf_u64; /* user u64 */ + void *buf_ctx; /* user context */ + const void *buf_cctx; /* const alias for ctx */ + }; + uint32_t totsize; /* total size of all allocated segs */ + uint32_t index; /* Index in the rte_mempool */ +} odp_buffer_hdr_t; + +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); + +/* + * Buffer type + * + * @param buf Buffer handle + * + * @return Buffer type + */ +int _odp_buffer_type(odp_buffer_t buf); + +/* + * Buffer type set + * + * @param buf Buffer handle + * @param type New type value + * + */ +void _odp_buffer_type_set(odp_buffer_t buf, int type); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/include/odp_packet_dpdk.h b/platform/linux-dpdk/include/odp_packet_dpdk.h new file mode 100644 index 000000000..608e4799b --- /dev/null +++ b/platform/linux-dpdk/include/odp_packet_dpdk.h @@ -0,0 +1,106 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PACKET_DPDK_H +#define ODP_PACKET_DPDK_H + +#include <stdint.h> +#include <net/if.h> + +#include <odp/helper/eth.h> +#include <odp/align.h> +#include <odp/debug.h> +#include <odp/packet.h> +#include <odp_packet_internal.h> +#include <odp/pool.h> +#include <odp_pool_internal.h> +#include <odp_buffer_internal.h> +#include <odp/std_types.h> + +#include <rte_config.h> +#include <rte_memory.h> +#include <rte_memzone.h> +#include <rte_launch.h> +#include <rte_tailq.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_branch_prediction.h> +#include <rte_prefetch.h> +#include <rte_cycles.h> +#include <rte_errno.h> +#include <rte_debug.h> +#include <rte_log.h> +#include <rte_byteorder.h> +#include <rte_pci.h> +#include <rte_random.h> +#include <rte_ether.h> +#include <rte_ethdev.h> +#include <rte_hash.h> +#include <rte_jhash.h> +#include <rte_hash_crc.h> + + +#define ODP_DPDK_MODE_HW 0 +#define ODP_DPDK_MODE_SW 1 + +#define DPDK_BLOCKING_IO + +/* + * RX and TX Prefetch, Host, and Write-back threshold values should be + * carefully set for optimal performance. Consult the network + * controller's datasheet and supporting DPDK documentation for guidance + * on how these parameters should be set. + */ +#define RX_PTHRESH 8 /**< Default values of RX prefetch threshold reg. */ +#define RX_HTHRESH 8 /**< Default values of RX host threshold reg. */ +#define RX_WTHRESH 4 /**< Default values of RX write-back threshold reg. */ + +/* + * These default values are optimized for use with the Intel(R) 82599 10 GbE + * Controller and the DPDK ixgbe PMD. Consider using other values for other + * network controllers and/or network drivers. + */ +#define TX_PTHRESH 36 /**< Default values of TX prefetch threshold reg. */ +#define TX_HTHRESH 0 /**< Default values of TX host threshold reg. */ +#define TX_WTHRESH 0 /**< Default values of TX write-back threshold reg. */ + +#define MAX_PKT_BURST 256 +#define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ +#define RTE_TEST_RX_DESC_DEFAULT 128 +#define RTE_TEST_TX_DESC_DEFAULT 512 + +/** Packet socket using dpdk mmaped rings for both Rx and Tx */ +typedef struct { + odp_pool_t pool; + + /********************************/ + char ifname[32]; + uint8_t portid; + uint16_t queueid; + odp_bool_t vdev_sysc_promisc; /**< promiscuous mode defined with + system call */ +} pkt_dpdk_t; + +/** + * Configure an interface to work in dpdk mode + */ +int setup_pkt_dpdk(pkt_dpdk_t * const pkt_dpdk, const char *netdev, + odp_pool_t pool); + +/** + * Switch interface from dpdk mode to normal mode + */ +int close_pkt_dpdk(pkt_dpdk_t * const pkt_dpdk); + +/** + * Receive packets using dpdk + */ +int recv_pkt_dpdk(pkt_dpdk_t * const pkt_dpdk, odp_packet_t pkt_table[], + unsigned len); + +int odp_init_dpdk(void); +#endif diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h new file mode 100644 index 000000000..4afbb6d0d --- /dev/null +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -0,0 +1,221 @@ +/* Copyright (c) 2014, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + +/** + * @file + * + * ODP packet descriptor - implementation internal + */ + +#ifndef ODP_PACKET_INTERNAL_H_ +#define ODP_PACKET_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/align.h> +#include <odp_debug_internal.h> +#include <odp/debug.h> +#include <odp_buffer_internal.h> +#include <odp_buffer_inlines.h> +#include <odp_pool_internal.h> +#include <odp/packet.h> +#include <odp/packet_io.h> +#include <odp/crypto.h> +#include <odp_crypto_internal.h> + +#include <rte_acl_osdep.h> + +/** + * Packet input & protocol flags + */ +typedef union { + /* All input flags */ + uint32_t all; + + struct { + uint32_t unparsed:1; /**< Set to inticate parse needed */ + + uint32_t l2:1; /**< known L2 protocol present */ + uint32_t l3:1; /**< known L3 protocol present */ + uint32_t l4:1; /**< known L4 protocol present */ + + uint32_t eth:1; /**< Ethernet */ + uint32_t jumbo:1; /**< Jumbo frame */ + uint32_t vlan:1; /**< VLAN hdr found */ + uint32_t vlan_qinq:1; /**< Stacked VLAN found, QinQ */ + + uint32_t snap:1; /**< SNAP */ + uint32_t arp:1; /**< ARP */ + + uint32_t ipv4:1; /**< IPv4 */ + uint32_t ipv6:1; /**< IPv6 */ + uint32_t ipfrag:1; /**< IP fragment */ + uint32_t ipopt:1; /**< IP optional headers */ + uint32_t ipsec:1; /**< IPSec decryption may be needed */ + + uint32_t udp:1; /**< UDP */ + uint32_t tcp:1; /**< TCP */ + uint32_t tcpopt:1; /**< TCP options present */ + uint32_t sctp:1; /**< SCTP */ + uint32_t icmp:1; /**< ICMP */ + }; +} input_flags_t; + +_ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t), + "INPUT_FLAGS_SIZE_ERROR"); + +/** + * Packet error flags + */ +typedef union { + /* All error flags */ + uint32_t all; + + struct { + /* Bitfield flags for each detected error */ + uint32_t app_error:1; /**< Error bit for application use */ + uint32_t frame_len:1; /**< Frame length error */ + uint32_t snap_len:1; /**< Snap length error */ + uint32_t l2_chksum:1; /**< L2 checksum error, checks TBD */ + uint32_t ip_err:1; /**< IP error, checks TBD */ + uint32_t tcp_err:1; /**< TCP error, checks TBD */ + uint32_t udp_err:1; /**< UDP error, checks TBD */ + }; +} error_flags_t; + +_ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t), + "ERROR_FLAGS_SIZE_ERROR"); + +/** + * Packet output flags + */ +typedef union { + /* All output flags */ + uint32_t all; + + struct { + /* Bitfield flags for each output option */ + uint32_t l3_chksum_set:1; /**< L3 chksum bit is valid */ + uint32_t l3_chksum:1; /**< L3 chksum override */ + uint32_t l4_chksum_set:1; /**< L3 chksum bit is valid */ + uint32_t l4_chksum:1; /**< L4 chksum override */ + }; +} output_flags_t; + +_ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), + "OUTPUT_FLAGS_SIZE_ERROR"); + +/** + * Internal Packet header + */ +typedef struct { + /* common buffer header */ + odp_buffer_hdr_t buf_hdr; + + input_flags_t input_flags; + error_flags_t error_flags; + output_flags_t output_flags; + + uint32_t frame_offset; /**< offset to start of frame, even on error */ + uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ + uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ + uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ + uint32_t payload_offset; /**< offset to payload */ + + uint32_t vlan_s_tag; /**< Parsed 1st VLAN header (S-TAG) */ + uint32_t vlan_c_tag; /**< Parsed 2nd VLAN header (C-TAG) */ + uint32_t l3_protocol; /**< Parsed L3 protocol */ + uint32_t l3_len; /**< Layer 3 length */ + uint32_t l4_protocol; /**< Parsed L4 protocol */ + uint32_t l4_len; /**< Layer 4 length */ + odp_pktio_t input; /**< Originating pktio */ + uint32_t uarea_size; /**< User metadata size, it's right after + odp_packet_hdr_t*/ + odp_crypto_generic_op_result_t op_result; /**< Result for crypto */ +} odp_packet_hdr_t __rte_cache_aligned; + +/** + * Return the packet header + */ +static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) +{ + return (odp_packet_hdr_t *)pkt; +} + +/** + * Parse packet and set internal metadata + */ +void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset); + +#define ODP_PACKET_UNPARSED ~0 + +static inline void _odp_packet_reset_parse(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr->input_flags.all = ODP_PACKET_UNPARSED; +} + +static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, + odp_packet_hdr_t *dst_hdr) +{ + dst_hdr->input_flags = src_hdr->input_flags; + dst_hdr->error_flags = src_hdr->error_flags; + dst_hdr->output_flags = src_hdr->output_flags; + + dst_hdr->l2_offset = src_hdr->l2_offset; + dst_hdr->l3_offset = src_hdr->l3_offset; + dst_hdr->l4_offset = src_hdr->l4_offset; + dst_hdr->payload_offset = src_hdr->payload_offset; + + dst_hdr->vlan_s_tag = src_hdr->vlan_s_tag; + dst_hdr->vlan_c_tag = src_hdr->vlan_c_tag; + dst_hdr->l3_protocol = src_hdr->l3_protocol; + dst_hdr->l3_len = src_hdr->l3_len; + dst_hdr->l4_protocol = src_hdr->l4_protocol; + dst_hdr->l4_len = src_hdr->l4_len; +} + +/* Forward declarations */ +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, + odp_packet_t dstpkt, uint32_t dstoffset, + uint32_t len); + +int _odp_packet_parse(odp_packet_hdr_t *pkt_hdr); + +void _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt); + +/* Convert a packet handle to a buffer handle */ +odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt); + +/* Convert a buffer handle to a packet handle */ +odp_packet_t _odp_packet_from_buffer(odp_buffer_t buf); + +/* DPDK will reserve RTE_PKTMBUF_HEADROOM in any case */ +_ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_HEADROOM <= RTE_PKTMBUF_HEADROOM, + "ERROR: Headroom has to be smaller or equal to DPDK"); + +/* We can't enforce tailroom reservation for received packets */ +_ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_TAILROOM == 0, + "ERROR: Tailroom has to be 0, DPDK doesn't support this"); + +/* + * These options are causing a bug in DPDK. It is fixed by Olivier Matz's series + * starting with 1d493 "mbuf: fix data room size calculation in pool init", + * which was upstreamed after 2.0.0. + */ + +#ifdef RTE_LIBRTE_IP_FRAG +_ODP_STATIC_ASSERT(0, "ERROR: IP frags are not supported!"); +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h new file mode 100644 index 000000000..dffc556d0 --- /dev/null +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -0,0 +1,96 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + +/** + * @file + * + * ODP packet IO - implementation internal + */ + +#ifndef ODP_PACKET_IO_INTERNAL_H_ +#define ODP_PACKET_IO_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/spinlock.h> +#include <odp_packet_socket.h> +#include <odp_classification_datamodel.h> +#include <odp_align_internal.h> + +#include <odp/config.h> +#include <odp/hints.h> + +#include <odp_packet_dpdk.h> + +/** + * Packet IO types + */ + +typedef enum { + ODP_PKTIO_TYPE_DPDK = 0x1, + ODP_PKTIO_TYPE_LOOPBACK, +} odp_pktio_type_t; + + +struct pktio_entry { + odp_spinlock_t lock; /**< entry spinlock */ + odp_ticketlock_t rxl; /**< RX ticket lock */ + odp_ticketlock_t txl; /**< TX ticket lock */ + int taken; /**< is entry taken(1) or free(0) */ + int cls_enabled; /**< is classifier enabled */ + odp_pktio_t handle; /**< pktio handle */ + odp_queue_t inq_default; /**< default input queue, if set */ + odp_queue_t outq_default; /**< default out queue */ + odp_queue_t loopq; /**< loopback queue for "loop" device */ + odp_pktio_type_t type; /**< pktio type */ + pkt_dpdk_t pkt_dpdk; /**< using DPDK API for IO */ + classifier_t cls; /**< classifier linked with this pktio*/ + char name[IFNAMSIZ]; /**< name of pktio provided to + pktio_open() */ + odp_bool_t promisc; /**< promiscuous mode state */ +}; + +typedef union { + struct pktio_entry s; + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pktio_entry))]; +} pktio_entry_t; + +typedef struct { + odp_spinlock_t lock; + pktio_entry_t entries[ODP_CONFIG_PKTIO_ENTRIES]; +} pktio_table_t; + +extern void *pktio_entry_ptr[]; + +static inline int pktio_to_id(odp_pktio_t pktio) +{ + return _odp_typeval(pktio) - 1; +} + +static inline pktio_entry_t *get_pktio_entry(odp_pktio_t pktio) +{ + if (odp_unlikely(pktio == ODP_PKTIO_INVALID)) + return NULL; + + if (odp_unlikely(_odp_typeval(pktio) > ODP_CONFIG_PKTIO_ENTRIES)) { + ODP_DBG("pktio limit %d/%d exceed\n", + _odp_typeval(pktio), ODP_CONFIG_PKTIO_ENTRIES); + return NULL; + } + + return pktio_entry_ptr[pktio_to_id(pktio)]; +} + +int pktin_poll(pktio_entry_t *entry); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h new file mode 100644 index 000000000..c2cf813a8 --- /dev/null +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -0,0 +1,98 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + +/** + * @file + * + * ODP buffer pool - internal header + */ + +#ifndef ODP_POOL_INTERNAL_H_ +#define ODP_POOL_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/std_types.h> +#include <odp/pool.h> +#include <odp_buffer_internal.h> +#include <odp/packet_io.h> +#include <odp/align.h> +#include <odp/hints.h> +#include <odp/config.h> +#include <odp/debug.h> +#include <odp_debug_internal.h> +#include <string.h> + +/* for DPDK */ +#include <rte_mempool.h> +#include <rte_memzone.h> + +_ODP_STATIC_ASSERT(ODP_POOL_NAME_LEN == RTE_MEMZONE_NAMESIZE, + "ERROR: Pool name sizes doesn't match"); + +/* Use ticketlock instead of spinlock */ +#define POOL_USE_TICKETLOCK + +/* Extra error checks */ +/* #define POOL_ERROR_CHECK */ + + +#ifdef POOL_USE_TICKETLOCK +#include <odp/ticketlock.h> +#else +#include <odp/spinlock.h> +#endif + + +struct pool_entry_s { +#ifdef POOL_USE_TICKETLOCK + odp_ticketlock_t lock ODP_ALIGNED_CACHE; +#else + odp_spinlock_t lock ODP_ALIGNED_CACHE; +#endif + char name[ODP_POOL_NAME_LEN]; + odp_pool_param_t params; + odp_pool_t pool_hdl; + struct rte_mempool *rte_mempool; +}; + +typedef union pool_entry_u { + struct pool_entry_s s; + + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))]; + +} pool_entry_t; + +extern void *pool_entry_ptr[]; + +static inline uint32_t pool_handle_to_index(odp_pool_t pool_hdl) +{ + return _odp_typeval(pool_hdl); +} + +static inline void *get_pool_entry(uint32_t pool_id) +{ + return pool_entry_ptr[pool_id]; +} + +static inline pool_entry_t *odp_pool_to_entry(odp_pool_t pool) +{ + return (pool_entry_t *)get_pool_entry(pool_handle_to_index(pool)); +} + +static inline odp_pool_t pool_index_to_handle(uint32_t pool_id) +{ + return _odp_cast_scalar(odp_pool_t, pool_id); +} + +#ifdef __cplusplus +} +#endif + +#endif /* ODP_POOL_INTERNAL_H_ */ diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 new file mode 100644 index 000000000..e3a17169b --- /dev/null +++ b/platform/linux-dpdk/m4/configure.m4 @@ -0,0 +1,42 @@ +AC_MSG_CHECKING(for GCC atomic builtins) +AC_LINK_IFELSE( + [AC_LANG_SOURCE( + [[int main() { + int v = 1; + __atomic_fetch_add(&v, 1, __ATOMIC_RELAXED); + __atomic_fetch_sub(&v, 1, __ATOMIC_RELAXED); + __atomic_store_n(&v, 1, __ATOMIC_RELAXED); + __atomic_load_n(&v, __ATOMIC_RELAXED); + return 0; + } + ]])], + AC_MSG_RESULT(yes), + AC_MSG_RESULT(no) + echo "GCC-style __atomic builtins not supported by the compiler." + echo "Use newer version. For gcc > 4.7.0" + exit -1) + +# +# Check that SDK_INSTALL_PATH provided to right dpdk version +# +saved_cflags="$CFLAGS" +CFLAGS="$CFLAGS -I${SDK_INSTALL_PATH}/include" +AC_MSG_CHECKING(for DPDK include files) +AC_LINK_IFELSE( + [AC_LANG_SOURCE( + [[ + #include <rte_config.h> + #include <rte_memory.h> + #include <rte_eal.h> + int main() { + return 0; + } + ]])], + AC_MSG_RESULT(yes), + AC_MSG_RESULT(no) + echo "Unable to find DPDK SDK." + exit -1 + ) +CFLAGS="$saved_cflags" + +AC_CONFIG_FILES([platform/linux-dpdk/test/Makefile]) diff --git a/platform/linux-dpdk/odp_buffer.c b/platform/linux-dpdk/odp_buffer.c new file mode 100644 index 000000000..c82db63cd --- /dev/null +++ b/platform/linux-dpdk/odp_buffer.c @@ -0,0 +1,110 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/buffer.h> +#include <odp_buffer_internal.h> +#include <odp_buffer_inlines.h> +#include <odp_debug_internal.h> + +#include <string.h> +#include <stdio.h> + +odp_buffer_t odp_buffer_from_event(odp_event_t ev) +{ + return (odp_buffer_t)ev; +} + +odp_event_t odp_buffer_to_event(odp_buffer_t buf) +{ + return (odp_event_t)buf; +} + +void *odp_buffer_addr(odp_buffer_t buf) +{ + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); + + return hdr->mb.buf_addr; +} + + +uint32_t odp_buffer_size(odp_buffer_t buf) +{ + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); + struct rte_mbuf *mbuf = (struct rte_mbuf *)hdr; + + return mbuf->buf_len; +} + + +int _odp_buffer_type(odp_buffer_t buf) +{ + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); + + return hdr->type; +} + +void _odp_buffer_type_set(odp_buffer_t buf, int type) +{ + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); + + hdr->type = type; +} + +int odp_buffer_is_valid(odp_buffer_t buf) +{ + /* We could call rte_mbuf_sanity_check, but that panics + * and aborts the program */ + return buf != ODP_BUFFER_INVALID; +} + + +int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) +{ + odp_buffer_hdr_t *hdr; + int len = 0; + + if (!odp_buffer_is_valid(buf)) { + ODP_PRINT("Buffer is not valid.\n"); + return len; + } + + hdr = odp_buf_to_hdr(buf); + + len += snprintf(&str[len], n-len, + "Buffer\n"); + len += snprintf(&str[len], n-len, + " pool %"PRIu64"\n", (int64_t) hdr->mb.pool); + len += snprintf(&str[len], n-len, + " phy_addr %"PRIu64"\n", hdr->mb.buf_physaddr); + len += snprintf(&str[len], n-len, + " addr %p\n", hdr->mb.buf_addr); + len += snprintf(&str[len], n-len, + " size %u\n", hdr->mb.buf_len); +/* + * Comment this out until we upgrade to a DPDK version where the underlying + * bug is fixed. Grep for RTE_MBUF_SCATTER_GATHER + len += snprintf(&str[len], n-len, + " ref_count %i\n", + odp_atomic_load_u32((odp_atomic_u32_t *) +*/ + len += snprintf(&str[len], n-len, + " odp type %i\n", hdr->type); + + return len; +} + + +void odp_buffer_print(odp_buffer_t buf) +{ + int max_len = 512; + char str[max_len]; + int len; + + len = odp_buffer_snprint(str, max_len-1, buf); + str[len] = 0; + + ODP_PRINT("\n%s\n", str); +} diff --git a/platform/linux-dpdk/odp_cpumask.c b/platform/linux-dpdk/odp_cpumask.c new file mode 100644 index 000000000..8aeccad92 --- /dev/null +++ b/platform/linux-dpdk/odp_cpumask.c @@ -0,0 +1,233 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include <sched.h> +#include <pthread.h> + +#include <odp/cpumask.h> +#include <odp_debug_internal.h> + +#include <stdlib.h> +#include <string.h> + +#include <rte_lcore.h> + +void odp_cpumask_from_str(odp_cpumask_t *mask, const char *str_in) +{ + cpu_set_t cpuset; + const char *str = str_in; + const char *p; + int cpu = 0; + int len = strlen(str); + + CPU_ZERO(&cpuset); + odp_cpumask_zero(mask); + + /* Strip leading "0x"/"0X" if present and verify length */ + if ((len >= 2) && ((str[1] == 'x') || (str[1] == 'X'))) { + str += 2; + len -= 2; + } + if (!len) + return; + + /* Walk string from LSB setting cpu bits */ + for (p = str + len - 1; (len > 0) && (cpu < CPU_SETSIZE); p--, len--) { + char c = *p; + int value; + int idx; + + /* Convert hex nibble, abort when invalid value found */ + if ((c >= '0') && (c <= '9')) + value = c - '0'; + else if ((c >= 'A') && (c <= 'F')) + value = c - 'A' + 10; + else if ((c >= 'a') && (c <= 'f')) + value = c - 'a' + 10; + else + return; + + /* Walk converted nibble and set bits in mask */ + for (idx = 0; idx < 4; idx++, cpu++) + if (value & (1 << idx)) + CPU_SET(cpu, &cpuset); + } + + /* Copy the computed mask */ + memcpy(&mask->set, &cpuset, sizeof(cpuset)); +} + +int32_t odp_cpumask_to_str(const odp_cpumask_t *mask, char *str, int32_t len) +{ + char *p = str; + int cpu = odp_cpumask_last(mask); + int nibbles; + int value; + + /* Handle bad string length, need at least 4 chars for "0x0" and + * terminating null char */ + if (len < 4) + return -1; /* Failure */ + + /* Handle no CPU found */ + if (cpu < 0) { + strcpy(str, "0x0"); + return strlen(str) + 1; /* Success */ + } + /* CPU was found and cpu >= 0 */ + + /* Compute number of nibbles in cpumask that have bits set */ + nibbles = (cpu / 4) + 1; + + /* Verify minimum space (account for "0x" and termination) */ + if (len < (3 + nibbles)) + return -1; /* Failure */ + + /* Prefix */ + *p++ = '0'; + *p++ = 'x'; + + /* + * Now we can scan the cpus down to zero and + * build the string one nibble at a time + */ + value = 0; + do { + /* Set bit to go into the current nibble */ + if (CPU_ISSET(cpu, &mask->set)) + value |= 1 << (cpu % 4); + + /* If we are on a nibble boundary flush value to string */ + if (0 == (cpu % 4)) { + if (value < 0xA) + *p++ = '0' + value; + else + *p++ = 'A' + value - 0xA; + value = 0; + } + } while (cpu--); + + /* Terminate the string */ + *p++ = 0; + return p - str; /* Success */ +} + +void odp_cpumask_zero(odp_cpumask_t *mask) +{ + CPU_ZERO(&mask->set); +} + +void odp_cpumask_set(odp_cpumask_t *mask, int cpu) +{ + CPU_SET(cpu, &mask->set); +} + +void odp_cpumask_setall(odp_cpumask_t *mask) +{ + int cpu; + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) + CPU_SET(cpu, &mask->set); +} + +void odp_cpumask_clr(odp_cpumask_t *mask, int cpu) +{ + CPU_CLR(cpu, &mask->set); +} + +int odp_cpumask_isset(const odp_cpumask_t *mask, int cpu) +{ + return CPU_ISSET(cpu, &mask->set); +} + +int odp_cpumask_count(const odp_cpumask_t *mask) +{ + return CPU_COUNT(&mask->set); +} + +void odp_cpumask_and(odp_cpumask_t *dest, const odp_cpumask_t *src1, + const odp_cpumask_t *src2) +{ + CPU_AND(&dest->set, &src1->set, &src2->set); +} + +void odp_cpumask_or(odp_cpumask_t *dest, const odp_cpumask_t *src1, + const odp_cpumask_t *src2) +{ + CPU_OR(&dest->set, &src1->set, &src2->set); +} + +void odp_cpumask_xor(odp_cpumask_t *dest, const odp_cpumask_t *src1, + const odp_cpumask_t *src2) +{ + CPU_XOR(&dest->set, &src1->set, &src2->set); +} + +int odp_cpumask_equal(const odp_cpumask_t *mask1, + const odp_cpumask_t *mask2) +{ + return CPU_EQUAL(&mask1->set, &mask2->set); +} + +void odp_cpumask_copy(odp_cpumask_t *dest, const odp_cpumask_t *src) +{ + memcpy(&dest->set, &src->set, sizeof(src->set)); +} + +int odp_cpumask_first(const odp_cpumask_t *mask) +{ + int cpu; + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) + if (odp_cpumask_isset(mask, cpu)) + return cpu; + return -1; +} + +int odp_cpumask_last(const odp_cpumask_t *mask) +{ + int cpu; + + for (cpu = CPU_SETSIZE - 1; cpu >= 0; cpu--) + if (odp_cpumask_isset(mask, cpu)) + return cpu; + return -1; +} + +int odp_cpumask_next(const odp_cpumask_t *mask, int cpu) +{ + for (cpu += 1; cpu < CPU_SETSIZE; cpu++) + if (odp_cpumask_isset(mask, cpu)) + return cpu; + return -1; +} + +int odp_cpumask_def_worker(odp_cpumask_t *mask, int num) +{ + int i, count = 0; + + odp_cpumask_zero(mask); + + RTE_LCORE_FOREACH_SLAVE(i) { + odp_cpumask_set(mask, i); + if (++count == num) + break; + } + + /* exclude master lcore */ + return count; +} + +int odp_cpumask_def_control(odp_cpumask_t *mask, int num ODP_UNUSED) +{ + odp_cpumask_zero(mask); + /* By default all control threads on CPU 0 */ + odp_cpumask_set(mask, 0); + return 1; +} diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c new file mode 100644 index 000000000..ab8b59c73 --- /dev/null +++ b/platform/linux-dpdk/odp_init.c @@ -0,0 +1,354 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/cpu.h> +#include <odp/init.h> +#include <odp_internal.h> +#include <odp/debug.h> +#include <odp_packet_dpdk.h> +#include <odp_debug_internal.h> +#include <odp/system_info.h> +#include <unistd.h> + +#define PMD_EXT(drv) extern void devinitfn_##drv(void); +PMD_EXT(pmd_af_packet_drv) +PMD_EXT(bond_drv) +PMD_EXT(em_pmd_drv) +PMD_EXT(pmd_igb_drv) +PMD_EXT(pmd_igbvf_drv) +PMD_EXT(rte_enic_driver) +PMD_EXT(rte_fm10k_driver) +PMD_EXT(rte_i40e_driver) +PMD_EXT(rte_i40evf_driver) +PMD_EXT(rte_ixgbe_driver) +PMD_EXT(rte_ixgbevf_driver) +PMD_EXT(rte_mlx4_driver) +PMD_EXT(pmd_null_drv) +PMD_EXT(pmd_pcap_drv) +PMD_EXT(pmd_ring_drv) +PMD_EXT(rte_virtio_driver) +PMD_EXT(rte_vmxnet3_driver) +PMD_EXT(pmd_xenvirt_drv) + +/* + * This function is not called from anywhere, it's only purpose is to make sure + * that if ODP and DPDK are statically linked to an application, the GCC + * constuctors of the PMDs are linked as well. Otherwise the linker would omit + * them. It's not an issue with dynamic linking. */ +void refer_constructors(void); +void refer_constructors(void) { +#ifdef RTE_LIBRTE_PMD_AF_PACKET + devinitfn_pmd_af_packet_drv(); +#endif +#ifdef RTE_LIBRTE_PMD_BOND + devinitfn_bond_drv(); +#endif +#ifdef RTE_LIBRTE_EM_PMD + devinitfn_em_pmd_drv(); +#endif +#ifdef RTE_LIBRTE_IGB_PMD + devinitfn_pmd_igb_drv(); + devinitfn_pmd_igbvf_drv(); +#endif +#ifdef RTE_LIBRTE_ENIC_PMD + devinitfn_rte_enic_driver(); +#endif +#ifdef RTE_LIBRTE_FM10K_PMD + devinitfn_rte_fm10k_driver(); +#endif +#ifdef RTE_LIBRTE_I40E_PMD + devinitfn_rte_i40e_driver(); + devinitfn_rte_i40evf_driver(); +#endif +#ifdef RTE_LIBRTE_IXGBE_PMD + devinitfn_rte_ixgbe_driver(); + devinitfn_rte_ixgbevf_driver(); +#endif +#ifdef RTE_LIBRTE_MLX4_PMD + devinitfn_rte_mlx4_driver(); +#endif +#ifdef RTE_LIBRTE_PMD_NULL + devinitfn_pmd_null_drv(); +#endif +#ifdef RTE_LIBRTE_PMD_PCAP + devinitfn_pmd_pcap_drv(); +#endif +#ifdef RTE_LIBRTE_PMD_RING + devinitfn_pmd_ring_drv(); +#endif +#ifdef RTE_LIBRTE_VIRTIO_PMD + devinitfn_rte_virtio_driver(); +#endif +#ifdef RTE_LIBRTE_VMXNET3_PMD + devinitfn_rte_vmxnet3_driver(); +#endif +#ifdef RTE_LIBRTE_PMD_XENVIRT + devinitfn_pmd_xenvirt_drv(); +#endif +} + +static void parse_dpdk_args(const char *args, int *dst_argc, char ***dst_argv) +{ + char *buf = strdup(args); + int num = 1; + char *delim; + char **argv = calloc(num, sizeof(char *)); + + if (!buf || !argv) + ODP_ABORT("Can't allocate memory!\n"); + + argv[0] = buf; + + while (1) { + delim = strchr(argv[num - 1], ' '); + if (delim == NULL) + break; + argv = realloc(argv, (num + 1) * sizeof(char *)); + if (!argv) + ODP_ABORT("Can't reallocate memory!\n"); + argv[num] = delim + 1; + *delim = 0; + num++; + } + + *dst_argc = num; + *dst_argv = argv; + + return; +} + + +static void print_dpdk_env_help(void) +{ + char **dpdk_argv; + int dpdk_argc, save_optind; + + parse_dpdk_args("--help", &dpdk_argc, &dpdk_argv); + ODP_ERR("Missing: export ODP_PLATFORM_PARAMS=\"EAL options\"\n"); + ODP_ERR("Example: export ODP_PLATFORM_PARAMS=\"-n 4 --no-huge\"\n"); + ODP_ERR("Note: -c argument substitutes automatically from odp coremask\n"); + save_optind = optind; + optind = 1; + rte_eal_init(dpdk_argc, dpdk_argv); + optind = save_optind; + free(dpdk_argv[0]); + free(dpdk_argv); +} + + +int odp_init_dpdk(void) +{ + char **dpdk_argv; + int dpdk_argc; + char *env; + char *new_env; + int core_mask, i, save_optind; + + env = getenv("ODP_PLATFORM_PARAMS"); + if (env == NULL) { + print_dpdk_env_help(); + ODP_ERR("ODP_PLATFORM_PARAMS has to be exported\n"); + return -1; + } + + for (i = 0, core_mask = 0; i < odp_cpu_count(); i++) + core_mask += (0x1 << i); + + new_env = calloc(1, strlen(env) + strlen("odpdpdk -c ") + + sizeof(core_mask) + 1); + + /* first argument is facility log, simply bind it to odpdpdk for now.*/ + sprintf(new_env, "odpdpdk -c 0x%x %s", core_mask, env); + + parse_dpdk_args(new_env, &dpdk_argc, &dpdk_argv); + for (i = 0; i < dpdk_argc; ++i) + ODP_DBG("arg[%d]: %s\n", i, dpdk_argv[i]); + fflush(stdout); + free(new_env); + + /* reset optind, the caller application might have used it */ + save_optind = optind; + optind = 1; + i = rte_eal_init(dpdk_argc, dpdk_argv); + optind = save_optind; + free(dpdk_argv[0]); + free(dpdk_argv); + if (i < 0) { + ODP_ERR("Cannot init the Intel DPDK EAL!\n"); + return -1; + } else if (i != dpdk_argc) { + ODP_DBG("Some DPDK args were not processed!\n"); + ODP_DBG("Passed: %d Consumed %d\n", dpdk_argc, i); + } + ODP_DBG("rte_eal_init OK\n"); + + return 0; +} + +struct odp_global_data_s odp_global_data; + +int odp_init_global(odp_init_t *params ODP_UNUSED, + odp_platform_init_t *platform_params ODP_UNUSED) +{ + odp_global_data.log_fn = odp_override_log; + odp_global_data.abort_fn = odp_override_abort; + + if (params != NULL) { + if (params->log_fn != NULL) + odp_global_data.log_fn = params->log_fn; + if (params->abort_fn != NULL) + odp_global_data.abort_fn = params->abort_fn; + } + + odp_system_info_init(); + + if (odp_init_dpdk()) { + ODP_ERR("ODP dpdk init failed.\n"); + return -1; + } + + if (odp_shm_init_global()) { + ODP_ERR("ODP shm init failed.\n"); + return -1; + } + + if (odp_thread_init_global()) { + ODP_ERR("ODP thread init failed.\n"); + return -1; + } + + if (odp_pool_init_global()) { + ODP_ERR("ODP pool init failed.\n"); + return -1; + } + + if (odp_queue_init_global()) { + ODP_ERR("ODP queue init failed.\n"); + return -1; + } + + if (odp_schedule_init_global()) { + ODP_ERR("ODP schedule init failed.\n"); + return -1; + } + + if (odp_pktio_init_global()) { + ODP_ERR("ODP packet io init failed.\n"); + return -1; + } + +#if 0 /* for now timer is disabled */ + if (odp_timer_init_global()) { + ODP_ERR("ODP timer init failed.\n"); + return -1; + } +#endif + + if (odp_crypto_init_global()) { + ODP_ERR("ODP crypto init failed.\n"); + return -1; + } + +#if 0 /* for now classification is disabled */ + if (odp_classification_init_global()) { + ODP_ERR("ODP classification init failed.\n"); + return -1; + } +#endif + + return 0; +} + +int odp_term_global(void) +{ + int rc = 0; + +#if 0 /* for now classification is disabled */ + if (odp_classification_term_global()) { + ODP_ERR("ODP classificatio term failed.\n"); + rc = -1; + } +#endif + + if (odp_crypto_term_global()) { + ODP_ERR("ODP crypto term failed.\n"); + rc = -1; + } + + if (odp_schedule_term_global()) { + ODP_ERR("ODP schedule term failed.\n"); + rc = -1; + } + + if (odp_pktio_term_global()) { + ODP_ERR("ODP pktio term failed.\n"); + rc = -1; + } + + if (odp_queue_term_global()) { + ODP_ERR("ODP queue term failed.\n"); + rc = -1; + } + + if (odp_thread_term_global()) { + ODP_ERR("ODP thread term failed.\n"); + rc = -1; + } + + if (odp_shm_term_global()) { + ODP_ERR("ODP shm term failed.\n"); + rc = -1; + } + + return rc; +} + +int odp_init_local(odp_thread_type_t thr_type) +{ + if (odp_shm_init_local()) { + ODP_ERR("ODP shm local init failed.\n"); + return -1; + } + + if (odp_thread_init_local(thr_type)) { + ODP_ERR("ODP thread local init failed.\n"); + return -1; + } + + if (odp_pktio_init_local()) { + ODP_ERR("ODP packet io local init failed.\n"); + return -1; + } + + if (odp_schedule_init_local()) { + ODP_ERR("ODP schedule local init failed.\n"); + return -1; + } + + return 0; +} + +int odp_term_local(void) +{ + int rc = 0; + int rc_thd = 0; + + if (odp_schedule_term_local()) { + ODP_ERR("ODP schedule local term failed.\n"); + rc = -1; + } + + rc_thd = odp_thread_term_local(); + if (rc_thd < 0) { + ODP_ERR("ODP thread local term failed.\n"); + rc = -1; + } else { + if (!rc) + rc = rc_thd; + } + + return rc; +} diff --git a/platform/linux-dpdk/odp_linux.c b/platform/linux-dpdk/odp_linux.c new file mode 100644 index 000000000..22cd4292e --- /dev/null +++ b/platform/linux-dpdk/odp_linux.c @@ -0,0 +1,128 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include <sched.h> + +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <assert.h> + +#include <odp/helper/linux.h> +#include <odp_internal.h> +#include <odp/thread.h> +#include <odp/init.h> +#include <odp/system_info.h> +#include <odp_debug_internal.h> + +#include <rte_lcore.h> + +static void *odp_run_start_routine(void *arg) +{ + odp_start_args_t *start_args = arg; + + /* ODP thread local init */ + if (odp_init_local(ODP_THREAD_WORKER)) { + ODP_ERR("Local init failed\n"); + return NULL; + } + + return start_args->start_routine(start_args->arg); +} + + +int odph_linux_pthread_create(odph_linux_pthread_t *thread_tbl, + const odp_cpumask_t *mask_in, + void *(*start_routine) (void *), void *arg) +{ + int i, num; + int cpu; + odp_cpumask_t mask; + int ret; + + odp_cpumask_copy(&mask, mask_in); + num = odp_cpumask_count(&mask); + + memset(thread_tbl, 0, num * sizeof(odph_linux_pthread_t)); + if (num < 1 || num > odp_cpu_count()) { + ODP_ERR("Bad num\n"); + return 0; + } + + cpu = odp_cpumask_first(&mask); + for (i = 0; i < num; i++) { + /* do not lock up current core */ + if ((unsigned)cpu == rte_get_master_lcore()) { + cpu = odp_cpumask_next(&mask, cpu); + continue; + } + + thread_tbl[i].cpu = cpu; + + /* pthread affinity is not set here because, DPDK + * creates, initialises and sets the affinity for pthread + * part of rte_eal_init() + */ + + thread_tbl[i].start_args = malloc(sizeof(odp_start_args_t)); + if (thread_tbl[i].start_args == NULL) + ODP_ABORT("Malloc failed"); + + thread_tbl[i].start_args->start_routine = start_routine; + thread_tbl[i].start_args->arg = arg; + + ret = rte_eal_remote_launch( + (int(*)(void *))odp_run_start_routine, + thread_tbl[i].start_args, cpu); + if (ret != 0) { + ODP_ERR("Failed to start thread on cpu #%d\n", cpu); + free(thread_tbl[i].start_args); + break; + } + + cpu = odp_cpumask_next(&mask, cpu); + } + + if (i != num) + ODP_DBG("Run %d thread instead of %d\n", i, num); + + return i; +} + + +void odph_linux_pthread_join(odph_linux_pthread_t *thread_tbl, int num) +{ + uint32_t lcore_id; + + (void) thread_tbl; + (void) num; + + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + int ret = rte_eal_wait_lcore(lcore_id); + free(thread_tbl[lcore_id].start_args); + if (ret < 0) + return; + } +} + +int odph_linux_process_fork_n(odph_linux_process_t *proc_tbl ODP_UNUSED, + const odp_cpumask_t *mask_in ODP_UNUSED) +{ + ODP_UNIMPLEMENTED(); + ODP_ABORT(""); + return 0; +} + +int odph_linux_process_wait_n(odph_linux_process_t *proc_tbl ODP_UNUSED, + int num ODP_UNUSED) +{ + ODP_UNIMPLEMENTED(); + ODP_ABORT(""); + return 0; +} diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c new file mode 100644 index 000000000..7ff4e7de7 --- /dev/null +++ b/platform/linux-dpdk/odp_packet.c @@ -0,0 +1,966 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/packet.h> +#include <odp_packet_internal.h> +#include <odp/hints.h> +#include <odp/byteorder.h> +#include <odp_debug_internal.h> + +#include <odp/helper/eth.h> +#include <odp/helper/ip.h> +#include <odp/helper/tcp.h> +#include <odp/helper/udp.h> + +#include <string.h> +#include <stdio.h> +#include <stddef.h> + +/* These are the offsets for packet accessors for inlining. */ +const unsigned int buf_addr_offset = offsetof(odp_packet_hdr_t, buf_hdr) + + offsetof(struct odp_buffer_hdr_t, mb) + + offsetof(struct rte_mbuf, buf_addr); +const unsigned int data_off_offset = offsetof(odp_packet_hdr_t, buf_hdr) + + offsetof(struct odp_buffer_hdr_t, mb) + + offsetof(struct rte_mbuf, data_off); + +/* The last bit is an expanded version of offsetof(), to make sure that if + * rte_pktmbuf_[pkt|data]_len() changes, we will either adapt automatically, or + * throw a compile failure + */ +const unsigned int pkt_len_offset = offsetof(odp_packet_hdr_t, buf_hdr) + + offsetof(struct odp_buffer_hdr_t, mb) + + (size_t)&rte_pktmbuf_pkt_len((struct rte_mbuf *)0); +const unsigned int seg_len_offset = offsetof(odp_packet_hdr_t, buf_hdr) + + offsetof(struct odp_buffer_hdr_t, mb) + + (size_t)&rte_pktmbuf_data_len((struct rte_mbuf *)0); + +const unsigned int udata_len_offset = offsetof(odp_packet_hdr_t, uarea_size); +const unsigned int udata_offset = sizeof(odp_packet_hdr_t); + +struct rte_mbuf dummy; +_ODP_STATIC_ASSERT(sizeof(dummy.data_off) == sizeof(uint16_t), + "data_off should be uint16_t"); +_ODP_STATIC_ASSERT(sizeof(dummy.pkt_len) == sizeof(uint32_t), + "pkt_len should be uint32_t"); +_ODP_STATIC_ASSERT(sizeof(dummy.data_len) == sizeof(uint16_t), + "data_len should be uint16_t"); + + +odp_packet_t _odp_packet_from_buffer(odp_buffer_t buf) +{ + return (odp_packet_t)buf; +} + +odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt) +{ + return (odp_buffer_t)pkt; +} + +odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) +{ + odp_packet_t pkt; + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); + uintmax_t totsize = RTE_PKTMBUF_HEADROOM + len; + odp_packet_hdr_t *pkt_hdr; + struct rte_mbuf *mbuf; + + if (pool->s.params.type != ODP_POOL_PACKET) + return ODP_PACKET_INVALID; + + mbuf = rte_pktmbuf_alloc(pool->s.rte_mempool); + if (mbuf == NULL) + return ODP_PACKET_INVALID; + pkt_hdr = (odp_packet_hdr_t *)mbuf; + pkt_hdr->buf_hdr.totsize = mbuf->buf_len; + + if (mbuf->buf_len < totsize) { + intmax_t needed = totsize - mbuf->buf_len; + struct rte_mbuf *curseg = mbuf; + + do { + struct rte_mbuf *nextseg = + rte_pktmbuf_alloc(pool->s.rte_mempool); + + if (nextseg == NULL) { + rte_pktmbuf_free(mbuf); + return ODP_PACKET_INVALID; + } + + curseg->next = nextseg; + curseg = nextseg; + curseg->data_off = 0; + pkt_hdr->buf_hdr.totsize += curseg->buf_len; + needed -= curseg->buf_len; + } while (needed > 0); + } + + pkt = (odp_packet_t)mbuf; + + if (odp_packet_reset(pkt, len) != 0) + return ODP_PACKET_INVALID; + + return pkt; +} + +void odp_packet_free(odp_packet_t pkt) +{ + struct rte_mbuf *mbuf = (struct rte_mbuf *)pkt; + rte_pktmbuf_free(mbuf); +} + +int odp_packet_reset(odp_packet_t pkt, uint32_t len) +{ + odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt); + struct rte_mbuf *ms, *mb = &pkt_hdr->buf_hdr.mb; + uint8_t nb_segs = 0; + int32_t lenleft = len; + char *start; + + if (RTE_PKTMBUF_HEADROOM + len > odp_packet_buf_len(pkt)) { + ODP_DBG("Not enought head room for that packet %d/%d\n", + RTE_PKTMBUF_HEADROOM + len, + odp_packet_buf_len(pkt)); + return -1; + } + + start = (char *)&pkt_hdr->l2_offset; + memset((void *)start, 0, + ODP_OFFSETOF(odp_packet_hdr_t, uarea_size) - + ODP_OFFSETOF(odp_packet_hdr_t, l2_offset)); + + pkt_hdr->l2_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; + pkt_hdr->l3_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; + pkt_hdr->l4_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID; + pkt_hdr->buf_hdr.next = NULL; + + mb->port = 0xff; + mb->pkt_len = len; + mb->data_off = RTE_PKTMBUF_HEADROOM; + mb->vlan_tci = 0; + nb_segs = 1; + + if (RTE_PKTMBUF_HEADROOM + lenleft <= mb->buf_len) { + mb->data_len = lenleft; + } else { + mb->data_len = mb->buf_len - RTE_PKTMBUF_HEADROOM; + lenleft -= mb->data_len; + ms = mb->next; + while (lenleft > 0) { + nb_segs++; + ms->data_len = lenleft <= ms->buf_len ? + lenleft : ms->buf_len; + lenleft -= ms->buf_len; + ms = ms->next; + } + } + + mb->nb_segs = nb_segs; + return 0; +} + +odp_packet_t odp_packet_from_event(odp_event_t ev) +{ + return (odp_packet_t)ev; +} + +odp_event_t odp_packet_to_event(odp_packet_t pkt) +{ + return (odp_event_t)pkt; +} + +void *odp_packet_head(odp_packet_t pkt) +{ + return odp_buffer_addr(_odp_packet_to_buffer(pkt)); +} + +uint32_t odp_packet_headroom(odp_packet_t pkt) +{ + struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); + return rte_pktmbuf_headroom(mb); +} + +uint32_t odp_packet_tailroom(odp_packet_t pkt) +{ + struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); + return rte_pktmbuf_tailroom(rte_pktmbuf_lastseg(mb)); +} + +void *odp_packet_tail(odp_packet_t pkt) +{ + struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); + mb = rte_pktmbuf_lastseg(mb); + return (void *)(rte_pktmbuf_mtod(mb, char *) + mb->data_len); +} + +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) +{ + struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); + return (void *)rte_pktmbuf_prepend(mb, len); +} + +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) +{ + struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); + return (void *)rte_pktmbuf_adj(mb, len); +} + + +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, + odp_packet_seg_t *seg) +{ + struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); + + do { + if (mb->data_len > offset) { + break; + } else { + offset -= mb->data_len; + mb = mb->next; + } + } while (mb); + + if (mb) { + if (len) + *len = mb->data_len - offset; + if (seg) + *seg = (odp_packet_seg_t)mb; + return (void *)(rte_pktmbuf_mtod(mb, char *) + offset); + } else { + return NULL; + } +} + +odp_pool_t odp_packet_pool(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.pool_hdl; +} + +static inline void *packet_offset_to_ptr(odp_packet_t pkt, uint32_t *len, + const size_t offset) +{ + if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID)) + return NULL; + + if (len) + return odp_packet_offset(pkt, offset, len, NULL); + else + return odp_packet_offset(pkt, offset, NULL, NULL); +} + +void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len) +{ + const size_t offset = odp_packet_l2_offset(pkt); + return packet_offset_to_ptr(pkt, len, offset); +} + +uint32_t odp_packet_l2_offset(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + if (pkt_hdr->input_flags.unparsed) + _odp_packet_parse(pkt_hdr); + return pkt_hdr->l2_offset; +} + +int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + if (odp_unlikely(offset >= (odp_packet_len(pkt) - 1))) + return -1; + if (pkt_hdr->input_flags.unparsed) + _odp_packet_parse(pkt_hdr); + pkt_hdr->l2_offset = offset; + return 0; +} + +void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len) +{ + const size_t offset = odp_packet_l3_offset(pkt); + + return packet_offset_to_ptr(pkt, len, offset); +} + +uint32_t odp_packet_l3_offset(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + if (pkt_hdr->input_flags.unparsed) + _odp_packet_parse(pkt_hdr); + return pkt_hdr->l3_offset; +} + +int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + if (odp_unlikely(offset >= (odp_packet_len(pkt) - 1))) + return -1; + if (pkt_hdr->input_flags.unparsed) + _odp_packet_parse(pkt_hdr); + pkt_hdr->l3_offset = offset; + return 0; +} + +void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) +{ + const size_t offset = odp_packet_l4_offset(pkt); + + return packet_offset_to_ptr(pkt, len, offset); +} + +uint32_t odp_packet_l4_offset(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + if (pkt_hdr->input_flags.unparsed) + _odp_packet_parse(pkt_hdr); + return pkt_hdr->l4_offset; +} + +int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + if (odp_unlikely(offset >= (odp_packet_len(pkt) - 1))) + return -1; + if (pkt_hdr->input_flags.unparsed) + _odp_packet_parse(pkt_hdr); + pkt_hdr->l4_offset = offset; + return 0; +} + +int odp_packet_is_segmented(odp_packet_t pkt) +{ + return !rte_pktmbuf_is_contiguous(&odp_packet_hdr(pkt)->buf_hdr.mb); +} + +int odp_packet_num_segs(odp_packet_t pkt) +{ + struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); + return mb->nb_segs; +} + +odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) +{ + return (odp_packet_seg_t)pkt; +} + +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) +{ + struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); + return (odp_packet_seg_t)rte_pktmbuf_lastseg(mb); +} + +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt ODP_UNUSED, + odp_packet_seg_t seg) +{ + struct rte_mbuf *mb = (struct rte_mbuf *)seg; + if (mb->next == NULL) + return ODP_PACKET_SEG_INVALID; + else + return (odp_packet_seg_t)mb->next; +} + +/* + * + * Segment level + * ******************************************************** + * + */ + +void *odp_packet_seg_buf_addr(odp_packet_t pkt ODP_UNUSED, + odp_packet_seg_t seg) +{ + return odp_packet_head((odp_packet_t)seg); +} + +uint32_t odp_packet_seg_buf_len(odp_packet_t pkt ODP_UNUSED, + odp_packet_seg_t seg) +{ + struct rte_mbuf *mb = (struct rte_mbuf *)seg; + return mb->buf_len; +} + +void *odp_packet_seg_data(odp_packet_t pkt ODP_UNUSED, odp_packet_seg_t seg) +{ + return odp_packet_data((odp_packet_t)seg); +} + +uint32_t odp_packet_seg_data_len(odp_packet_t pkt ODP_UNUSED, + odp_packet_seg_t seg) +{ + return odp_packet_seg_len((odp_packet_t)seg); +} + +/* + * + * Manipulation + * ******************************************************** + * + */ + +odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset, + uint32_t len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint32_t pktlen = odp_packet_len(pkt); + odp_packet_t newpkt; + + if (offset > pktlen) + return ODP_PACKET_INVALID; + + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len); + + if (newpkt != ODP_PACKET_INVALID) { + if (_odp_packet_copy_to_packet(pkt, 0, + newpkt, 0, offset) != 0 || + _odp_packet_copy_to_packet(pkt, offset, newpkt, + offset + len, + pktlen - offset) != 0) { + odp_packet_free(newpkt); + newpkt = ODP_PACKET_INVALID; + } else { + _odp_packet_copy_md_to_packet(pkt, newpkt); + odp_packet_free(pkt); + } + } + + return newpkt; +} + +odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset, + uint32_t len) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint32_t pktlen = odp_packet_len(pkt); + odp_packet_t newpkt; + + if (offset > pktlen || offset + len > pktlen) + return ODP_PACKET_INVALID; + + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen - len); + + if (newpkt != ODP_PACKET_INVALID) { + if (_odp_packet_copy_to_packet(pkt, 0, + newpkt, 0, offset) != 0 || + _odp_packet_copy_to_packet(pkt, offset + len, + newpkt, offset, + pktlen - offset - len) != 0) { + odp_packet_free(newpkt); + newpkt = ODP_PACKET_INVALID; + } else { + _odp_packet_copy_md_to_packet(pkt, newpkt); + odp_packet_free(pkt); + } + } + + return newpkt; +} + +/** + * Parser helper function for IPv4 + */ +static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) +{ + odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; + uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); + uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); + uint16_t frag_offset; + + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len); + + if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) || + odp_unlikely(ver != 4) || + (pkt_hdr->l3_len > pkt_hdr->buf_hdr.mb.pkt_len - *offset)) { + pkt_hdr->error_flags.ip_err = 1; + return 0; + } + + *offset += ihl * 4; + *parseptr += ihl * 4; + + if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) + pkt_hdr->input_flags.ipopt = 1; + + /* A packet is a fragment if: + * "more fragments" flag is set (all fragments except the last) + * OR + * "fragment offset" field is nonzero (all fragments except the first) + */ + frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); + if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) + pkt_hdr->input_flags.ipfrag = 1; + + return ipv4->proto; +} + +/** + * Parser helper function for IPv6 + */ +static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) +{ + odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; + odph_ipv6hdr_ext_t *ipv6ext; + + pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); + + /* Basic sanity checks on IPv6 header */ + if ((ipv6->ver_tc_flow >> 28) != 6 || + pkt_hdr->l3_len > pkt_hdr->buf_hdr.mb.pkt_len - *offset) { + pkt_hdr->error_flags.ip_err = 1; + return 0; + } + + /* Skip past IPv6 header */ + *offset += sizeof(odph_ipv6hdr_t); + *parseptr += sizeof(odph_ipv6hdr_t); + + + /* Skip past any IPv6 extension headers */ + if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS || + ipv6->next_hdr == ODPH_IPPROTO_ROUTE) { + pkt_hdr->input_flags.ipopt = 1; + + do { + ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; + uint16_t extlen = 8 + ipv6ext->ext_len * 8; + + *offset += extlen; + *parseptr += extlen; + } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS || + ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) && + *offset < pkt_hdr->buf_hdr.mb.pkt_len); + + if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) { + pkt_hdr->error_flags.ip_err = 1; + return 0; + } + + if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG) + pkt_hdr->input_flags.ipfrag = 1; + + return ipv6ext->next_hdr; + } + + if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) { + pkt_hdr->input_flags.ipopt = 1; + pkt_hdr->input_flags.ipfrag = 1; + } + + return ipv6->next_hdr; +} + +/** + * Parser helper function for TCP + */ +static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) +{ + odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; + + if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) + pkt_hdr->error_flags.tcp_err = 1; + else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t)) + pkt_hdr->input_flags.tcpopt = 1; + + pkt_hdr->l4_len = pkt_hdr->l3_len + + pkt_hdr->l3_offset - pkt_hdr->l4_offset; + + *offset += (uint32_t)tcp->hl * 4; + *parseptr += (uint32_t)tcp->hl * 4; +} + +/** + * Parser helper function for UDP + */ +static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, + uint8_t **parseptr, uint32_t *offset) +{ + odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; + uint32_t udplen = odp_be_to_cpu_16(udp->length); + + if (udplen < sizeof(odph_udphdr_t) || + udplen > (pkt_hdr->l3_len + + pkt_hdr->l3_offset - pkt_hdr->l4_offset)) { + pkt_hdr->error_flags.udp_err = 1; + } + + pkt_hdr->l4_len = udplen; + + *offset += sizeof(odph_udphdr_t); + *parseptr += sizeof(odph_udphdr_t); +} + + +/** + * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP + * + * Internal function: caller is resposible for passing only valid packet handles + * , lengths and offsets (usually done&called in packet input). + * + * @param pkt Packet handle + * @param len Packet length in bytes + * @param frame_offset Byte offset to L2 header + */ +int _odp_packet_parse(odp_packet_hdr_t *pkt_hdr) +{ + odph_ethhdr_t *eth; + odph_vlanhdr_t *vlan; + uint16_t ethtype; + uint8_t *parseptr; + uint32_t offset; + uint8_t ip_proto = 0; + uint32_t len = pkt_hdr->buf_hdr.mb.pkt_len; + odp_packet_t pkt = (odp_packet_t)pkt_hdr; + + /* Reset parser metadata for new parse */ + pkt_hdr->error_flags.all = 0; + pkt_hdr->input_flags.all = 0; + pkt_hdr->output_flags.all = 0; + pkt_hdr->l2_offset = 0; + pkt_hdr->l3_offset = ODP_PACKET_OFFSET_INVALID; + pkt_hdr->l4_offset = ODP_PACKET_OFFSET_INVALID; + pkt_hdr->payload_offset = ODP_PACKET_OFFSET_INVALID; + pkt_hdr->vlan_s_tag = 0; + pkt_hdr->vlan_c_tag = 0; + pkt_hdr->l3_protocol = 0; + pkt_hdr->l4_protocol = 0; + + /* We only support Ethernet for now */ + pkt_hdr->input_flags.eth = 1; + + /* The frame_offset is not relevant for frames from DPDK */ + pkt_hdr->frame_offset = 0; + + /* Detect jumbo frames */ + if (len > ODPH_ETH_LEN_MAX) + pkt_hdr->input_flags.jumbo = 1; + + /* Assume valid L2 header, no CRC/FCS check in SW */ + pkt_hdr->input_flags.l2 = 1; + + eth = (odph_ethhdr_t *)odp_packet_data(pkt); + offset = sizeof(odph_ethhdr_t); + parseptr = (uint8_t *)ð->type; + ethtype = odp_be_to_cpu_16(eth->type); + + /* Parse the VLAN header(s), if present */ + if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { + pkt_hdr->input_flags.vlan_qinq = 1; + pkt_hdr->input_flags.vlan = 1; + vlan = (odph_vlanhdr_t *)(void *)parseptr; + pkt_hdr->vlan_s_tag = ((ethtype << 16) | + odp_be_to_cpu_16(vlan->tci)); + offset += sizeof(odph_vlanhdr_t); + parseptr += sizeof(odph_vlanhdr_t); + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + if (ethtype == ODPH_ETHTYPE_VLAN) { + pkt_hdr->input_flags.vlan = 1; + vlan = (odph_vlanhdr_t *)(void *)parseptr; + pkt_hdr->vlan_c_tag = ((ethtype << 16) | + odp_be_to_cpu_16(vlan->tci)); + offset += sizeof(odph_vlanhdr_t); + parseptr += sizeof(odph_vlanhdr_t); + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + /* Check for SNAP vs. DIX */ + if (ethtype < ODPH_ETH_LEN_MAX) { + pkt_hdr->input_flags.snap = 1; + if (ethtype > len - offset) { + pkt_hdr->error_flags.snap_len = 1; + goto parse_exit; + } + offset += 8; + parseptr += 8; + ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + } + + /* Consume Ethertype for Layer 3 parse */ + parseptr += 2; + + /* Set l3_offset+flag only for known ethtypes */ + pkt_hdr->input_flags.l3 = 1; + pkt_hdr->l3_offset = offset; + pkt_hdr->l3_protocol = ethtype; + + /* Set l3_offset+flag only for known ethtypes */ + switch (ethtype) { + case ODPH_ETHTYPE_IPV4: + pkt_hdr->input_flags.ipv4 = 1; + ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_ETHTYPE_IPV6: + pkt_hdr->input_flags.ipv6 = 1; + pkt_hdr->input_flags.l3 = 1; + ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_ETHTYPE_ARP: + pkt_hdr->input_flags.arp = 1; + ip_proto = 255; /* Reserved invalid by IANA */ + break; + + default: + pkt_hdr->input_flags.l3 = 0; + pkt_hdr->l2_offset = ODP_PACKET_OFFSET_INVALID; + ip_proto = 255; /* Reserved invalid by IANA */ + } + + /* Set l4_offset+flag only for known ip_proto */ + pkt_hdr->input_flags.l4 = 1; + pkt_hdr->l4_offset = offset; + pkt_hdr->l4_protocol = ip_proto; + + /* Parse Layer 4 headers */ + switch (ip_proto) { + case ODPH_IPPROTO_ICMP: + pkt_hdr->input_flags.icmp = 1; + break; + + case ODPH_IPPROTO_TCP: + pkt_hdr->input_flags.tcp = 1; + parse_tcp(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_IPPROTO_UDP: + pkt_hdr->input_flags.udp = 1; + parse_udp(pkt_hdr, &parseptr, &offset); + break; + + case ODPH_IPPROTO_AH: + case ODPH_IPPROTO_ESP: + pkt_hdr->input_flags.ipsec = 1; + break; + + default: + pkt_hdr->input_flags.l4 = 0; + pkt_hdr->l4_offset = ODP_PACKET_OFFSET_INVALID; + break; + } + + /* + * Anything beyond what we parse here is considered payload. + * Note: Payload is really only relevant for TCP and UDP. For + * all other protocols, the payload offset will point to the + * final header (ARP, ICMP, AH, ESP, or IP Fragment). + */ + pkt_hdr->payload_offset = offset; + +parse_exit: + return pkt_hdr->error_flags.all != 0; +} + +void odp_packet_print(odp_packet_t pkt) +{ + int max_len = 512; + char str[max_len]; + uint8_t *p; + int len = 0; + int n = max_len-1; + odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); + + len += snprintf(&str[len], n-len, "Packet "); + len += odp_buffer_snprint(&str[len], n-len, (odp_buffer_t) pkt); + len += snprintf(&str[len], n-len, + " input_flags 0x%x\n", hdr->input_flags.all); + len += snprintf(&str[len], n-len, + " error_flags 0x%x\n", hdr->error_flags.all); + len += snprintf(&str[len], n-len, + " output_flags 0x%x\n", hdr->output_flags.all); + len += snprintf(&str[len], n-len, + " frame_offset %u\n", hdr->frame_offset); + len += snprintf(&str[len], n-len, + " l2_offset %u\n", hdr->l2_offset); + len += snprintf(&str[len], n-len, + " l3_offset %u\n", hdr->l3_offset); + len += snprintf(&str[len], n-len, + " l4_offset %u\n", hdr->l4_offset); + len += snprintf(&str[len], n-len, + " frame_len %u\n", hdr->buf_hdr.mb.pkt_len); + len += snprintf(&str[len], n-len, + " input %" PRIu64 "\n", + odp_pktio_to_u64(hdr->input)); + str[len] = '\0'; + + ODP_ERR("\n%s\n", str); + rte_pktmbuf_dump(stdout, &hdr->buf_hdr.mb, 32); + + p = odp_packet_data(pkt); + ODP_ERR("00000000: %02X %02X %02X %02X %02X %02X %02X %02X\n", + p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7]); + ODP_ERR("00000008: %02X %02X %02X %02X %02X %02X %02X %02X\n", + p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); +} + +void _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) +{ + odp_packet_hdr_t *srchdr = odp_packet_hdr(srcpkt); + odp_packet_hdr_t *dsthdr = odp_packet_hdr(dstpkt); + uint8_t *newstart, *srcstart; + uint32_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr); + + newstart = (uint8_t *)dsthdr + meta_offset; + srcstart = (uint8_t *)srchdr + meta_offset; + + memcpy(newstart, srcstart, + sizeof(odp_packet_hdr_t) - meta_offset); + + dsthdr->buf_hdr.buf_u64 = srchdr->buf_hdr.buf_u64; + + dsthdr->buf_hdr.mb.port = srchdr->buf_hdr.mb.port; + dsthdr->buf_hdr.mb.vlan_tci = srchdr->buf_hdr.mb.vlan_tci; + dsthdr->buf_hdr.mb.hash = srchdr->buf_hdr.mb.hash; + dsthdr->buf_hdr.mb.ol_flags = srchdr->buf_hdr.mb.ol_flags; + + if (odp_packet_user_area_size(dstpkt) != 0) + memcpy(odp_packet_user_area(dstpkt), + odp_packet_user_area(srcpkt), + odp_packet_user_area_size(dstpkt) <= + odp_packet_user_area_size(srcpkt) ? + odp_packet_user_area_size(dstpkt) : + odp_packet_user_area_size(srcpkt)); + + copy_packet_parser_metadata(srchdr, dsthdr); +} + +int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, + odp_packet_t dstpkt, uint32_t dstoffset, + uint32_t len) +{ + void *srcmap; + void *dstmap; + uint32_t cpylen, minseg; + uint32_t srcseglen = 0; /* GCC */ + uint32_t dstseglen = 0; /* GCC */ + + if (srcoffset + len > odp_packet_len(srcpkt) || + dstoffset + len > odp_packet_len(dstpkt)) + return -1; + + while (len > 0) { + srcmap = odp_packet_offset(srcpkt, srcoffset, &srcseglen, NULL); + dstmap = odp_packet_offset(dstpkt, dstoffset, &dstseglen, NULL); + + minseg = dstseglen > srcseglen ? srcseglen : dstseglen; + cpylen = len > minseg ? minseg : len; + memcpy(dstmap, srcmap, cpylen); + + srcoffset += cpylen; + dstoffset += cpylen; + len -= cpylen; + } + + return 0; +} + +odp_packet_t odp_packet_copy(odp_packet_t pkt_src, odp_pool_t pool) +{ + uint32_t pktlen = odp_packet_len(pkt_src); + odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); + + if (newpkt != ODP_PACKET_INVALID) { + /* Must copy metadata first, followed by packet data */ + _odp_packet_copy_md_to_packet(pkt_src, newpkt); + + if (_odp_packet_copy_to_packet(pkt_src, 0, + newpkt, 0, pktlen) != 0) { + odp_packet_free(newpkt); + newpkt = ODP_PACKET_INVALID; + } + } + + return newpkt; +} + +int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset, + uint32_t len, void *dst) +{ + void *mapaddr; + uint32_t seglen = 0; /* GCC */ + uint32_t cpylen; + uint8_t *dstaddr = (uint8_t *)dst; + + if (offset + len > odp_packet_len(pkt)) + return -1; + + while (len > 0) { + mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL); + cpylen = len > seglen ? seglen : len; + memcpy(dstaddr, mapaddr, cpylen); + offset += cpylen; + dstaddr += cpylen; + len -= cpylen; + } + + return 0; +} + +int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset, + uint32_t len, const void *src) +{ + void *mapaddr; + uint32_t seglen = 0; /* GCC */ + uint32_t cpylen; + const uint8_t *srcaddr = (const uint8_t *)src; + + if (offset + len > odp_packet_len(pkt)) + return -1; + + while (len > 0) { + mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL); + cpylen = len > seglen ? seglen : len; + memcpy(mapaddr, srcaddr, cpylen); + offset += cpylen; + srcaddr += cpylen; + len -= cpylen; + } + + return 0; +} + +void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx) +{ + odp_packet_hdr(pkt)->buf_hdr.buf_cctx = ctx; +} + +void *odp_packet_user_ptr(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.buf_ctx; +} + +int odp_packet_is_valid(odp_packet_t pkt) +{ + odp_buffer_t buf = _odp_packet_to_buffer(pkt); + + return odp_buffer_is_valid(buf); +} + +uint32_t odp_packet_buf_len(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->buf_hdr.totsize; +} + +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) +{ + struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); + if (rte_pktmbuf_trim(mb, len)) + return NULL; + else + return odp_packet_tail(pkt); +} + +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) +{ + struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb); + return (void *)rte_pktmbuf_append(mb, len); +} + +odp_pktio_t odp_packet_input(odp_packet_t pkt) +{ + return odp_packet_hdr(pkt)->input; +} diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c new file mode 100644 index 000000000..3c5ba6db3 --- /dev/null +++ b/platform/linux-dpdk/odp_packet_dpdk.c @@ -0,0 +1,215 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#define _GNU_SOURCE +#include <stdio.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/ioctl.h> +#include <sys/mman.h> +#include <poll.h> +#include <unistd.h> +#include <fcntl.h> +#include <string.h> +#include <stdlib.h> + +#include <linux/ethtool.h> +#include <linux/sockios.h> + +#include <odp/api/cpu.h> +#include <odp/hints.h> +#include <odp/thread.h> + +#include <odp/system_info.h> +#include <odp_debug_internal.h> +#include <odp_packet_dpdk.h> +#include <net/if.h> +#include <math.h> + +/* Test if s has only digits or not. Dpdk pktio uses only digits.*/ +static int _dpdk_netdev_is_valid(const char *s) +{ + while (*s) { + if (!isdigit(*s)) + return 0; + s++; + } + + return 1; +} + +static void _dpdk_print_port_mac(uint8_t portid) +{ + struct ether_addr eth_addr; + + memset(ð_addr, 0, sizeof(eth_addr)); + rte_eth_macaddr_get(portid, ð_addr); + ODP_DBG("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n", + (unsigned)portid, + eth_addr.addr_bytes[0], + eth_addr.addr_bytes[1], + eth_addr.addr_bytes[2], + eth_addr.addr_bytes[3], + eth_addr.addr_bytes[4], + eth_addr.addr_bytes[5]); +} + +int setup_pkt_dpdk(pkt_dpdk_t * const pkt_dpdk, const char *netdev, + odp_pool_t pool) +{ + uint8_t portid = 0; + uint16_t nbrxq, nbtxq; + int ret, i; + pool_entry_t *pool_entry = get_pool_entry(_odp_typeval(pool)); + int sid = rte_eth_dev_socket_id(portid); + int socket_id = sid < 0 ? 0 : sid; + uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; + uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; + + struct rte_eth_rxconf rx_conf = { + .rx_thresh = { + .pthresh = RX_PTHRESH, + .hthresh = RX_HTHRESH, + .wthresh = RX_WTHRESH, + }, + }; + + struct rte_eth_txconf tx_conf = { + .tx_thresh = { + .pthresh = TX_PTHRESH, + .hthresh = TX_HTHRESH, + .wthresh = TX_WTHRESH, + }, + .tx_free_thresh = 256, /* Start flushing when the ring + is half full */ + .tx_rs_thresh = 0, /* Use PMD default values */ + /* + * As the example won't handle mult-segments and offload cases, + * set the flag by default. + */ + .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS | + ETH_TXQ_FLAGS_NOOFFLOADS, + }; + + struct rte_eth_conf port_conf = { + .rxmode = { + .mq_mode = ETH_MQ_RX_RSS, + .split_hdr_size = 0, + .header_split = 0, /**< Header Split */ + .hw_ip_checksum = 0, /**< IP checksum offload */ + .hw_vlan_filter = 0, /**< VLAN filtering */ + .jumbo_frame = 1, /**< Jumbo Frame Support */ + .hw_strip_crc = 0, /**< CRC stripp by hardware */ + }, + .rx_adv_conf = { + .rss_conf = { + .rss_key = NULL, + .rss_hf = ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_IPV4 | + ETH_RSS_IPV6 | + ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_NONFRAG_IPV6_UDP, + }, + }, + .txmode = { + .mq_mode = ETH_MQ_TX_NONE, + }, + }; + + /* rx packet len same size as pool segment */ + port_conf.rxmode.max_rx_pkt_len = pool_entry->s.params.pkt.seg_len; + + if (!_dpdk_netdev_is_valid(netdev)) + return -1; + + portid = atoi(netdev); + pkt_dpdk->portid = portid; + pkt_dpdk->pool = pool; + pkt_dpdk->queueid = 0; + + /* On init set it up only to 1 rx and tx queue.*/ + nbtxq = nbrxq = 1; + + ret = rte_eth_dev_configure(portid, nbrxq, nbtxq, &port_conf); + if (ret < 0) { + ODP_ERR("Cannot configure device: err=%d, port=%u\n", + ret, (unsigned)portid); + return -1; + } + + _dpdk_print_port_mac(portid); + + if (nb_rxd + nb_txd > pool_entry->s.params.pkt.num / 4) { + double downrate = (double)(pool_entry->s.params.pkt.num / 4) / + (double)(nb_rxd + nb_txd); + nb_rxd >>= (int)ceil(downrate); + nb_txd >>= (int)ceil(downrate); + ODP_DBG("downrate %f\n", downrate); + ODP_DBG("Descriptors scaled down. RX: %u TX: %u pool: %u\n", + nb_rxd, nb_txd, pool_entry->s.params.pkt.num); + } + /* init one RX queue on each port */ + for (i = 0; i < nbrxq; i++) { + ret = rte_eth_rx_queue_setup(portid, i, nb_rxd, socket_id, + &rx_conf, + pool_entry->s.rte_mempool); + if (ret < 0) { + ODP_ERR("rxq:err=%d, port=%u\n", ret, (unsigned)portid); + return -1; + } + } + + /* init one TX queue on each port */ + for (i = 0; i < nbtxq; i++) { + ret = rte_eth_tx_queue_setup(portid, i, nb_txd, socket_id, + &tx_conf); + if (ret < 0) { + ODP_ERR("txq:err=%d, port=%u\n", ret, (unsigned)portid); + return -1; + } + } + + /* Start device */ + ret = rte_eth_dev_start(portid); + if (ret < 0) { + ODP_ERR("rte_eth_dev_start:err=%d, port=%u\n", + ret, (unsigned)portid); + return -1; + } + + rte_eth_promiscuous_enable(portid); + /* Some DPDK PMD vdev like pcap do not support promisc mode change. Use + * system call for them. */ + if (!rte_eth_promiscuous_get(portid)) + pkt_dpdk->vdev_sysc_promisc = 1; + else + pkt_dpdk->vdev_sysc_promisc = 0; + + rte_eth_allmulticast_enable(portid); + return 0; +} + +int close_pkt_dpdk(pkt_dpdk_t * const pkt_dpdk) +{ + rte_eth_dev_close(pkt_dpdk->portid); + return 0; +} + +int recv_pkt_dpdk(pkt_dpdk_t * const pkt_dpdk, odp_packet_t pkt_table[], + unsigned len) +{ + uint16_t nb_rx, i = 0; + + nb_rx = rte_eth_rx_burst((uint8_t)pkt_dpdk->portid, + (uint16_t)pkt_dpdk->queueid, + (struct rte_mbuf **)pkt_table, (uint16_t)len); + for (i = 0; i < nb_rx; i++) + _odp_packet_reset_parse(pkt_table[i]); + + return nb_rx; +} diff --git a/platform/linux-dpdk/odp_packet_io.c b/platform/linux-dpdk/odp_packet_io.c new file mode 100644 index 000000000..0ade388f6 --- /dev/null +++ b/platform/linux-dpdk/odp_packet_io.c @@ -0,0 +1,937 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/packet_io.h> +#include <odp_packet_io_internal.h> +#include <odp_packet_io_queue.h> +#include <odp/packet.h> +#include <odp_packet_internal.h> +#include <odp_internal.h> +#include <odp/spinlock.h> +#include <odp/shared_memory.h> +#include <odp_packet_socket.h> +#include <odp/config.h> +#include <odp_queue_internal.h> +#include <odp_schedule_internal.h> +#include <odp_debug_internal.h> +#include <odp_buffer_inlines.h> + +#include <string.h> +#include <sys/ioctl.h> +#include <unistd.h> + +/* MTU to be reported for the "loop" interface */ +#define PKTIO_LOOP_MTU 1500 +/* MAC address for the "loop" interface */ +static const char pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01}; + +static pktio_table_t *pktio_tbl; + +/* pktio pointer entries ( for inlines) */ +void *pktio_entry_ptr[ODP_CONFIG_PKTIO_ENTRIES]; + + +int odp_pktio_init_global(void) +{ + char name[ODP_QUEUE_NAME_LEN]; + pktio_entry_t *pktio_entry; + queue_entry_t *queue_entry; + odp_queue_t qid; + int id; + odp_shm_t shm; + + shm = odp_shm_reserve("odp_pktio_entries", + sizeof(pktio_table_t), + sizeof(pktio_entry_t), 0); + pktio_tbl = odp_shm_addr(shm); + + if (pktio_tbl == NULL) + return -1; + + memset(pktio_tbl, 0, sizeof(pktio_table_t)); + + odp_spinlock_init(&pktio_tbl->lock); + + for (id = 1; id <= ODP_CONFIG_PKTIO_ENTRIES; ++id) { + pktio_entry = &pktio_tbl->entries[id - 1]; + + odp_spinlock_init(&pktio_entry->s.lock); + odp_spinlock_init(&pktio_entry->s.cls.lock); + + pktio_entry_ptr[id - 1] = pktio_entry; + /* Create a default output queue for each pktio resource */ + snprintf(name, sizeof(name), "%i-pktio_outq_default", (int)id); + name[ODP_QUEUE_NAME_LEN-1] = '\0'; + + qid = odp_queue_create(name, ODP_QUEUE_TYPE_PKTOUT, NULL); + if (qid == ODP_QUEUE_INVALID) + return -1; + pktio_entry->s.outq_default = qid; + + queue_entry = queue_to_qentry(qid); + queue_entry->s.pktout = _odp_cast_scalar(odp_pktio_t, id); + } + + return 0; +} + +int odp_pktio_term_global(void) +{ + pktio_entry_t *pktio_entry; + int ret = 0; + int id; + + for (id = 1; id <= ODP_CONFIG_PKTIO_ENTRIES; ++id) { + pktio_entry = &pktio_tbl->entries[id - 1]; + odp_queue_destroy(pktio_entry->s.outq_default); + } + + ret = odp_shm_free(odp_shm_lookup("odp_pktio_entries")); + if (ret < 0) + ODP_ERR("shm free failed for odp_pktio_entries"); + + return ret; +} + +int odp_pktio_init_local(void) +{ + return 0; +} + +static int is_free(pktio_entry_t *entry) +{ + return (entry->s.taken == 0); +} + +static void set_free(pktio_entry_t *entry) +{ + entry->s.taken = 0; +} + +static void set_taken(pktio_entry_t *entry) +{ + entry->s.taken = 1; +} + +static void lock_entry(pktio_entry_t *entry) +{ + odp_spinlock_lock(&entry->s.lock); +} + +static void unlock_entry(pktio_entry_t *entry) +{ + odp_spinlock_unlock(&entry->s.lock); +} + +static void init_pktio_entry(pktio_entry_t *entry) +{ + set_taken(entry); + entry->s.inq_default = ODP_QUEUE_INVALID; + memset(&entry->s.pkt_dpdk, 0, sizeof(entry->s.pkt_dpdk)); + /* Save pktio parameters, type is the most useful */ +} + +static odp_pktio_t alloc_lock_pktio_entry(void) +{ + odp_pktio_t id; + pktio_entry_t *entry; + int i; + + for (i = 0; i < ODP_CONFIG_PKTIO_ENTRIES; ++i) { + entry = &pktio_tbl->entries[i]; + if (is_free(entry)) { + /*lock_entry_classifier(entry);*/ + if (is_free(entry)) { + init_pktio_entry(entry); + id = _odp_cast_scalar(odp_pktio_t, i + 1); + return id; /* return with entry locked! */ + } + /*unlock_entry_classifier(entry);*/ + } + } + + return ODP_PKTIO_INVALID; +} + +static int free_pktio_entry(odp_pktio_t id) +{ + pktio_entry_t *entry = get_pktio_entry(id); + + if (entry == NULL) + return -1; + + set_free(entry); + + return 0; +} + +static int init_loop(pktio_entry_t *entry, odp_pktio_t id) +{ + char loopq_name[ODP_QUEUE_NAME_LEN]; + + entry->s.type = ODP_PKTIO_TYPE_LOOPBACK; + snprintf(loopq_name, sizeof(loopq_name), "%" PRIu64 "-pktio_loopq", + odp_pktio_to_u64(id)); + entry->s.loopq = odp_queue_create(loopq_name, + ODP_QUEUE_TYPE_POLL, NULL); + + if (entry->s.loopq == ODP_QUEUE_INVALID) + return -1; + + return 0; +} + +odp_pktio_t odp_pktio_open(const char *dev, odp_pool_t pool) +{ + odp_pktio_t id; + pktio_entry_t *pktio_entry; + int res; + + if (pool == ODP_POOL_INVALID) + return ODP_PKTIO_INVALID; + + id = odp_pktio_lookup(dev); + if (id != ODP_PKTIO_INVALID) { + /* interface is already open */ + __odp_errno = EEXIST; + return ODP_PKTIO_INVALID; + } + + odp_spinlock_lock(&pktio_tbl->lock); + + id = alloc_lock_pktio_entry(); + if (id == ODP_PKTIO_INVALID) { + ODP_ERR("No resources available.\n"); + return ODP_PKTIO_INVALID; + } + /* if successful, alloc_pktio_entry() returns with the entry locked */ + + pktio_entry = get_pktio_entry(id); + if (!pktio_entry) { + odp_spinlock_unlock(&pktio_tbl->lock); + return ODP_PKTIO_INVALID; + } + + if (strcmp(dev, "loop") == 0) { + res = init_loop(pktio_entry, id); + } else { + pktio_entry->s.type = ODP_PKTIO_TYPE_DPDK; + res = setup_pkt_dpdk(&pktio_entry->s.pkt_dpdk, dev, pool); + } + + if (res != 0) { + close_pkt_dpdk(&pktio_entry->s.pkt_dpdk); + /*unlock_entry_classifier(pktio_entry);*/ + free_pktio_entry(id); + odp_spinlock_unlock(&pktio_tbl->lock); + return ODP_PKTIO_INVALID; + } + + snprintf(pktio_entry->s.name, IFNAMSIZ, "%s", dev); + + pktio_entry->s.handle = id; + odp_ticketlock_init(&pktio_entry->s.rxl); + odp_ticketlock_init(&pktio_entry->s.txl); + + unlock_entry(pktio_entry); + /*unlock_entry_classifier(pktio_entry);*/ + odp_spinlock_unlock(&pktio_tbl->lock); + + return id; +} + +int odp_pktio_close(odp_pktio_t id) +{ + pktio_entry_t *entry; + int res = -1; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("No entry\n"); + return -1; + } + + lock_entry(entry); + if (!is_free(entry)) { + odp_ticketlock_lock(&entry->s.rxl); + odp_ticketlock_lock(&entry->s.txl); + + switch (entry->s.type) { + case ODP_PKTIO_TYPE_LOOPBACK: + res = odp_queue_destroy(entry->s.loopq); + break; + case ODP_PKTIO_TYPE_DPDK: + res = close_pkt_dpdk(&entry->s.pkt_dpdk); + break; + default: + break; + } + res |= free_pktio_entry(id); + odp_ticketlock_unlock(&entry->s.txl); + odp_ticketlock_unlock(&entry->s.rxl); + } + + unlock_entry(entry); + + if (res != 0) + return -1; + + return 0; +} + +odp_pktio_t odp_pktio_lookup(const char *dev) +{ + odp_pktio_t id = ODP_PKTIO_INVALID; + pktio_entry_t *entry; + int i; + + odp_spinlock_lock(&pktio_tbl->lock); + + for (i = 1; i <= ODP_CONFIG_PKTIO_ENTRIES; ++i) { + entry = get_pktio_entry(_odp_cast_scalar(odp_pktio_t, i)); + if (!entry || is_free(entry)) + continue; + + lock_entry(entry); + + if (!is_free(entry) && + strncmp(entry->s.name, dev, IFNAMSIZ) == 0) + id = _odp_cast_scalar(odp_pktio_t, i); + + unlock_entry(entry); + + if (id != ODP_PKTIO_INVALID) + break; + } + + odp_spinlock_unlock(&pktio_tbl->lock); + + return id; +} + +static int deq_loopback(pktio_entry_t *pktio_entry, odp_packet_t pkts[], + unsigned len) +{ + int nbr, i; + odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; + queue_entry_t *qentry; + + qentry = queue_to_qentry(pktio_entry->s.loopq); + nbr = queue_deq_multi(qentry, hdr_tbl, len); + + for (i = 0; i < nbr; ++i) { + pkts[i] = _odp_packet_from_buffer(odp_hdr_to_buf(hdr_tbl[i])); + _odp_packet_parse((odp_packet_hdr_t *)pkts[i]); + } + + return nbr; +} + +static unsigned rte_mempool_available(const struct rte_mempool *mp) +{ +#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 + return rte_ring_count(mp->ring) + mp->local_cache[rte_lcore_id()].len; +#else + return rte_ring_count(mp->ring); +#endif +} + +static void _odp_pktio_send_completion(pktio_entry_t *pktio_entry) +{ + int i; + struct rte_mbuf* dummy; + pool_entry_t *pool_entry = + get_pool_entry(_odp_typeval(pktio_entry->s.pkt_dpdk.pool)); + struct rte_mempool *rte_mempool = pool_entry->s.rte_mempool; + pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; + + rte_eth_tx_burst(pkt_dpdk->portid, pkt_dpdk->queueid, &dummy, 0); + + for (i = 0; i < ODP_CONFIG_PKTIO_ENTRIES; ++i) { + pktio_entry_t *entry = &pktio_tbl->entries[i]; + + if (rte_mempool_available(rte_mempool) != 0) + return; + + if (entry == pktio_entry) + continue; + + if (odp_ticketlock_trylock(&entry->s.txl)) { + if (!is_free(entry) && + entry->s.type == ODP_PKTIO_TYPE_DPDK) { + pkt_dpdk = &entry->s.pkt_dpdk; + rte_eth_tx_burst(pkt_dpdk->portid, + pkt_dpdk->queueid, &dummy, 0); + } + odp_ticketlock_unlock(&entry->s.txl); + } + } + + return; +} + +int odp_pktio_recv(odp_pktio_t id, odp_packet_t pkt_table[], int len) +{ + pktio_entry_t *pktio_entry = get_pktio_entry(id); + int pkts, i; + + if (pktio_entry == NULL) + return -1; + + odp_ticketlock_lock(&pktio_entry->s.rxl); + if (odp_likely(pktio_entry->s.type == ODP_PKTIO_TYPE_DPDK)) { + pkts = recv_pkt_dpdk(&pktio_entry->s.pkt_dpdk, pkt_table, len); + if (pkts == 0) { + pool_entry_t *pool_entry = + get_pool_entry(_odp_typeval(pktio_entry->s.pkt_dpdk.pool)); + struct rte_mempool *rte_mempool = + pool_entry->s.rte_mempool; + if (rte_mempool_available(rte_mempool) == 0) + _odp_pktio_send_completion(pktio_entry); + } + } else { + pkts = deq_loopback(pktio_entry, pkt_table, len); + } + odp_ticketlock_unlock(&pktio_entry->s.rxl); + if (pkts >= 0) + for (i = 0; i < pkts; ++i) + odp_packet_hdr(pkt_table[i])->input = id; + return pkts; +} + +static int enq_loopback(pktio_entry_t *pktio_entry, odp_packet_t pkt_tbl[], + unsigned len) +{ + odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; + queue_entry_t *qentry; + unsigned i; + + for (i = 0; i < len; ++i) + hdr_tbl[i] = odp_buf_to_hdr(_odp_packet_to_buffer(pkt_tbl[i])); + + qentry = queue_to_qentry(pktio_entry->s.loopq); + return queue_enq_multi(qentry, hdr_tbl, len); +} + +int odp_pktio_send(odp_pktio_t id, odp_packet_t pkt_table[], int len) +{ + pktio_entry_t *pktio_entry = get_pktio_entry(id); + pkt_dpdk_t *pkt_dpdk; + int pkts; + + if (pktio_entry == NULL) + return -1; + pkt_dpdk = &pktio_entry->s.pkt_dpdk; + + odp_ticketlock_lock(&pktio_entry->s.txl); + if (odp_likely(pktio_entry->s.type == ODP_PKTIO_TYPE_DPDK)) + pkts = rte_eth_tx_burst(pkt_dpdk->portid, pkt_dpdk->queueid, + (struct rte_mbuf **)pkt_table, len); + else + pkts = enq_loopback(pktio_entry, pkt_table, len); + odp_ticketlock_unlock(&pktio_entry->s.txl); + + return pkts; +} + +int odp_pktio_inq_setdef(odp_pktio_t id, odp_queue_t queue) +{ + pktio_entry_t *pktio_entry = get_pktio_entry(id); + queue_entry_t *qentry; + + if (pktio_entry == NULL || queue == ODP_QUEUE_INVALID) + return -1; + + qentry = queue_to_qentry(queue); + + if (qentry->s.type != ODP_QUEUE_TYPE_PKTIN) + return -1; + + lock_entry(pktio_entry); + pktio_entry->s.inq_default = queue; + unlock_entry(pktio_entry); + + switch (qentry->s.type) { + /* Change to ODP_QUEUE_TYPE_POLL when ODP_QUEUE_TYPE_PKTIN is removed */ + case ODP_QUEUE_TYPE_PKTIN: + /* User polls the input queue */ + queue_lock(qentry); + qentry->s.pktin = id; + queue_unlock(qentry); + + /* Uncomment when ODP_QUEUE_TYPE_PKTIN is removed + break; + case ODP_QUEUE_TYPE_SCHED: + */ + /* Packet input through the scheduler */ + if (schedule_pktio_start(id, ODP_SCHED_PRIO_LOWEST)) { + ODP_ERR("Schedule pktio start failed\n"); + return -1; + } + break; + default: + ODP_ABORT("Bad queue type\n"); + } + + return 0; +} + +int odp_pktio_inq_remdef(odp_pktio_t id) +{ + pktio_entry_t *pktio_entry = get_pktio_entry(id); + odp_queue_t queue; + queue_entry_t *qentry; + + if (pktio_entry == NULL) + return -1; + + lock_entry(pktio_entry); + queue = pktio_entry->s.inq_default; + qentry = queue_to_qentry(queue); + + queue_lock(qentry); + qentry->s.pktin = ODP_PKTIO_INVALID; + queue_unlock(qentry); + + pktio_entry->s.inq_default = ODP_QUEUE_INVALID; + unlock_entry(pktio_entry); + + return 0; +} + +odp_queue_t odp_pktio_inq_getdef(odp_pktio_t id) +{ + pktio_entry_t *pktio_entry = get_pktio_entry(id); + + if (pktio_entry == NULL) + return ODP_QUEUE_INVALID; + + return pktio_entry->s.inq_default; +} + +odp_queue_t odp_pktio_outq_getdef(odp_pktio_t id) +{ + pktio_entry_t *pktio_entry = get_pktio_entry(id); + + if (pktio_entry == NULL) + return ODP_QUEUE_INVALID; + + return pktio_entry->s.outq_default; +} + +int pktout_enqueue(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr) +{ + odp_packet_t pkt = _odp_packet_from_buffer((odp_buffer_t) buf_hdr); + int len = 1; + int nbr; + + nbr = odp_pktio_send(qentry->s.pktout, &pkt, len); + return (nbr == len ? 0 : -1); +} + +odp_buffer_hdr_t *pktout_dequeue(queue_entry_t *qentry ODP_UNUSED) +{ + ODP_ABORT("attempted dequeue from a pktout queue"); + return NULL; +} + +int pktout_enq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], + int num) +{ + odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; + int nbr; + int i; + + for (i = 0; i < num; ++i) + pkt_tbl[i] = _odp_packet_from_buffer((odp_buffer_t) buf_hdr[i]); + + nbr = odp_pktio_send(qentry->s.pktout, pkt_tbl, num); + return nbr; +} + +int pktout_deq_multi(queue_entry_t *qentry ODP_UNUSED, + odp_buffer_hdr_t *buf_hdr[] ODP_UNUSED, + int num ODP_UNUSED) +{ + ODP_ABORT("attempted dequeue from a pktout queue"); + return 0; +} + +int pktin_enqueue(queue_entry_t *qentry ODP_UNUSED, + odp_buffer_hdr_t *buf_hdr ODP_UNUSED) +{ + ODP_ABORT("attempted enqueue to a pktin queue"); + return -1; +} + +odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *qentry) +{ + odp_buffer_hdr_t *buf_hdr; + odp_buffer_t buf; + odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; + odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; + int pkts, i, j; + + buf_hdr = queue_deq(qentry); + if (buf_hdr != NULL) + return buf_hdr; + + pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX); + if (pkts <= 0) + return NULL; + + for (i = 0, j = 0; i < pkts; ++i) { + buf = _odp_packet_to_buffer(pkt_tbl[i]); + buf_hdr = odp_buf_to_hdr(buf); +#if 0 /* Classifier not enabled yet */ + if (0 > packet_classifier(qentry->s.pktin, pkt_tbl[i])) +#endif + tmp_hdr_tbl[j++] = buf_hdr; + } + + if (0 == j) + return NULL; + + if (j > 1) + queue_enq_multi(qentry, &tmp_hdr_tbl[1], j-1); + buf_hdr = tmp_hdr_tbl[0]; + return buf_hdr; +} + +int pktin_enq_multi(queue_entry_t *qentry ODP_UNUSED, + odp_buffer_hdr_t *buf_hdr[] ODP_UNUSED, + int num ODP_UNUSED) +{ + ODP_ABORT("attempted enqueue to a pktin queue"); + return 0; +} + +int pktin_deq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int num) +{ + int nbr; + odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; + odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX]; + odp_buffer_hdr_t *tmp_hdr; + odp_buffer_t buf; + int pkts, i, j; + + nbr = queue_deq_multi(qentry, buf_hdr, num); + if (odp_unlikely(nbr > num)) + ODP_ABORT("queue_deq_multi req: %d, returned %d\n", + num, nbr); + + /** queue already has number of requsted buffers, + * do not do receive in that case. + */ + if (nbr == num) + return nbr; + + pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX); + if (pkts <= 0) + return nbr; + + for (i = 0, j = 0; i < pkts; ++i) { + buf = _odp_packet_to_buffer(pkt_tbl[i]); + tmp_hdr = odp_buf_to_hdr(buf); +#if 0 /* Classifier not enabled yet */ + if (0 > packet_classifier(qentry->s.pktin, pkt_tbl[i])) +#endif + tmp_hdr_tbl[j++] = tmp_hdr; + } + + if (j) + queue_enq_multi(qentry, tmp_hdr_tbl, j); + return nbr; +} + +int pktin_poll(pktio_entry_t *entry) +{ + odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; + odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; + int num, num_enq, i; + + if (odp_unlikely(is_free(entry))) + return -1; + + if (odp_unlikely(entry->s.inq_default == ODP_QUEUE_INVALID)) + return -1; + + num = odp_pktio_recv(entry->s.handle, pkt_tbl, QUEUE_MULTI_MAX); + + if (num < 0) { + ODP_ERR("Packet recv error\n"); + return -1; + } + + for (i = 0, num_enq = 0; i < num; ++i) { + odp_buffer_t buf; + odp_buffer_hdr_t *hdr; + + buf = _odp_packet_to_buffer(pkt_tbl[i]); + hdr = odp_buf_to_hdr(buf); + + if (entry->s.cls_enabled) { +#if 0 /* Classifier not enabled yet */ + if (packet_classifier(entry->s.handle, pkt_tbl[i]) < 0) + hdr_tbl[num_enq++] = hdr; +#endif + } else { + hdr_tbl[num_enq++] = hdr; + } + } + + if (num_enq) { + queue_entry_t *qentry; + qentry = queue_to_qentry(entry->s.inq_default); + queue_enq_multi(qentry, hdr_tbl, num_enq); + } + + return 0; +} + +static int _dpdk_vdev_promisc_mode_set(uint8_t port_id, int enable) +{ + struct rte_eth_dev_info dev_info = {0}; + struct ifreq ifr; + int ret; + int sockfd; + + rte_eth_dev_info_get(port_id, &dev_info); + if_indextoname(dev_info.if_index, ifr.ifr_name); + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + + ret = ioctl(sockfd, SIOCGIFFLAGS, &ifr); + if (ret < 0) { + close(sockfd); + ODP_DBG("ioctl SIOCGIFFLAGS error\n"); + return -1; + } + + if (enable) + ifr.ifr_flags |= IFF_PROMISC; + else + ifr.ifr_flags &= ~(IFF_PROMISC); + + ret = ioctl(sockfd, SIOCSIFFLAGS, &ifr); + if (ret < 0) { + close(sockfd); + ODP_DBG("ioctl SIOCSIFFLAGS error\n"); + return -1; + } + + ret = ioctl(sockfd, SIOCGIFMTU, &ifr); + if (ret < 0) { + close(sockfd); + ODP_DBG("ioctl SIOCGIFMTU error\n"); + return -1; + } + + ODP_DBG("vdev promisc set to %d\n", enable); + close(sockfd); + return 0; +} + +int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable) +{ + pktio_entry_t *entry; + uint8_t portid; + int ret; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + entry->s.promisc = enable; + + if (entry->s.type == ODP_PKTIO_TYPE_LOOPBACK) { + unlock_entry(entry); + return 0; + } + + portid = entry->s.pkt_dpdk.portid; + if (enable) + rte_eth_promiscuous_enable(portid); + else + rte_eth_promiscuous_disable(portid); + + if (entry->s.pkt_dpdk.vdev_sysc_promisc) { + ret = _dpdk_vdev_promisc_mode_set(portid, enable); + if (ret < 0) + ODP_DBG("vdev promisc mode fail\n"); + } + + unlock_entry(entry); + return 0; +} + +static int _dpdk_vdev_promisc_mode(uint8_t port_id) +{ + struct rte_eth_dev_info dev_info = {0}; + struct ifreq ifr; + int ret; + int sockfd; + + rte_eth_dev_info_get(port_id, &dev_info); + if_indextoname(dev_info.if_index, ifr.ifr_name); + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + ret = ioctl(sockfd, SIOCGIFFLAGS, &ifr); + close(sockfd); + if (ret < 0) { + ODP_DBG("ioctl SIOCGIFFLAGS error\n"); + return -1; + } + + if (ifr.ifr_flags & IFF_PROMISC) { + ODP_DBG("promisc is 1\n"); + return 1; + } else + return 0; +} + +int odp_pktio_promisc_mode(odp_pktio_t id) +{ + pktio_entry_t *entry; + int promisc; + uint8_t portid; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + portid = entry->s.pkt_dpdk.portid; + + if (entry->s.pkt_dpdk.vdev_sysc_promisc) + promisc = _dpdk_vdev_promisc_mode(portid); + else + promisc = rte_eth_promiscuous_get(portid); + + unlock_entry(entry); + + return promisc; +} + +int odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr, int addr_size) +{ + pktio_entry_t *entry; + + if (addr_size < ETH_ALEN) { + /* Output buffer too small */ + return -1; + } + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + switch (entry->s.type) { + case ODP_PKTIO_TYPE_DPDK: + rte_eth_macaddr_get(entry->s.pkt_dpdk.portid, + (struct ether_addr *)mac_addr); + break; + case ODP_PKTIO_TYPE_LOOPBACK: + memcpy(mac_addr, pktio_loop_mac, ETH_ALEN); + break; + default: + ODP_ABORT("Wrong socket type %d\n", entry->s.type); + } + + unlock_entry(entry); + + return ETH_ALEN; +} + +static int _dpdk_vdev_mtu(uint8_t port_id) +{ + struct rte_eth_dev_info dev_info = {0}; + struct ifreq ifr; + int ret; + int sockfd; + + rte_eth_dev_info_get(port_id, &dev_info); + if_indextoname(dev_info.if_index, ifr.ifr_name); + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + ret = ioctl(sockfd, SIOCGIFMTU, &ifr); + close(sockfd); + if (ret < 0) { + ODP_DBG("ioctl SIOCGIFMTU error\n"); + return -1; + } + + return ifr.ifr_mtu; +} + +int odp_pktio_mtu(odp_pktio_t id) +{ + pktio_entry_t *entry; + uint16_t mtu; + int ret; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + if (entry->s.type == ODP_PKTIO_TYPE_LOOPBACK) { + unlock_entry(entry); + return PKTIO_LOOP_MTU; + } + + ret = rte_eth_dev_get_mtu(entry->s.pkt_dpdk.portid, + &mtu); + if (ret < 0) { + unlock_entry(entry); + return -2; + } + + /* some dpdk PMD vdev does not support getting mtu size, + * try to use system call if dpdk cannot get mtu value. + */ + if (mtu == 0) + mtu = _dpdk_vdev_mtu(entry->s.pkt_dpdk.portid); + + unlock_entry(entry); + + return mtu; +} diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c new file mode 100644 index 000000000..7714c6da5 --- /dev/null +++ b/platform/linux-dpdk/odp_pool.c @@ -0,0 +1,479 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/std_types.h> +#include <odp/pool.h> +#include <odp_pool_internal.h> +#include <odp_buffer_internal.h> +#include <odp_packet_internal.h> +#include <odp_timer_internal.h> +#include <odp_align_internal.h> +#include <odp/shared_memory.h> +#include <odp/align.h> +#include <odp_internal.h> +#include <odp/config.h> +#include <odp/hints.h> +#include <odp/debug.h> +#include <odp_debug_internal.h> + +#include <string.h> +#include <stdlib.h> +#include <math.h> + +/* for DPDK */ +#include <odp_packet_dpdk.h> + +#ifdef POOL_USE_TICKETLOCK +#include <odp/ticketlock.h> +#define LOCK(a) odp_ticketlock_lock(a) +#define UNLOCK(a) odp_ticketlock_unlock(a) +#define LOCK_INIT(a) odp_ticketlock_init(a) +#else +#include <odp/spinlock.h> +#define LOCK(a) odp_spinlock_lock(a) +#define UNLOCK(a) odp_spinlock_unlock(a) +#define LOCK_INIT(a) odp_spinlock_init(a) +#endif + +typedef struct pool_table_t { + pool_entry_t pool[ODP_CONFIG_POOLS]; + +} pool_table_t; + + +/* The pool table ptr - resides in shared memory */ +static pool_table_t *pool_tbl; + +/* Pool entry pointers (for inlining) */ +void *pool_entry_ptr[ODP_CONFIG_POOLS]; + + +int odp_pool_init_global(void) +{ + uint32_t i; + odp_shm_t shm; + + shm = odp_shm_reserve("odp_pools", + sizeof(pool_table_t), + sizeof(pool_entry_t), 0); + + pool_tbl = odp_shm_addr(shm); + + if (pool_tbl == NULL) + return -1; + + memset(pool_tbl, 0, sizeof(pool_table_t)); + + + for (i = 0; i < ODP_CONFIG_POOLS; i++) { + /* init locks */ + pool_entry_t *pool = &pool_tbl->pool[i]; + LOCK_INIT(&pool->s.lock); + pool->s.pool_hdl = pool_index_to_handle(i); + + pool_entry_ptr[i] = pool; + } + + ODP_DBG("\nPool init global\n"); + ODP_DBG(" pool_entry_s size %zu\n", sizeof(struct pool_entry_s)); + ODP_DBG(" pool_entry_t size %zu\n", sizeof(pool_entry_t)); + ODP_DBG(" odp_buffer_hdr_t size %zu\n", sizeof(odp_buffer_hdr_t)); + ODP_DBG("\n"); + + return 0; +} + +struct mbuf_ctor_arg { + uint16_t seg_buf_offset; /* To skip the ODP buf/pkt/tmo header */ + uint16_t seg_buf_size; /* size of user data */ + int type; + int pkt_uarea_size; /* size of user area in bytes */ +}; + +struct mbuf_pool_ctor_arg { + struct rte_pktmbuf_pool_private pkt; + odp_pool_t pool_hdl; +}; + +static void +odp_dpdk_mbuf_pool_ctor(struct rte_mempool *mp, + void *opaque_arg) +{ + struct mbuf_pool_ctor_arg *mbp_priv; + + if (mp->private_data_size < sizeof(struct mbuf_pool_ctor_arg)) { + ODP_ERR("(%s) private_data_size %d < %d", + mp->name, (int) mp->private_data_size, + (int) sizeof(struct mbuf_pool_ctor_arg)); + return; + } + mbp_priv = rte_mempool_get_priv(mp); + *mbp_priv = *((struct mbuf_pool_ctor_arg *)opaque_arg); +} + +/* ODP DPDK mbuf constructor. + * This is a combination of rte_pktmbuf_init in rte_mbuf.c + * and testpmd_mbuf_ctor in testpmd.c + */ +static void +odp_dpdk_mbuf_ctor(struct rte_mempool *mp, + void *opaque_arg, + void *raw_mbuf, + unsigned i) +{ + struct mbuf_ctor_arg *mb_ctor_arg; + struct rte_mbuf *mb = raw_mbuf; + struct odp_buffer_hdr_t *buf_hdr; + struct mbuf_pool_ctor_arg *mbp_ctor_arg = rte_mempool_get_priv(mp); + + /* The rte_mbuf is at the begninning in all cases */ + mb_ctor_arg = (struct mbuf_ctor_arg *)opaque_arg; + mb = (struct rte_mbuf *)raw_mbuf; + + RTE_MBUF_ASSERT(mp->elt_size >= sizeof(struct rte_mbuf)); + + memset(mb, 0, mp->elt_size); + + /* Start of buffer is just after the ODP type specific header + * which contains in the very beginning the rte_mbuf struct */ + mb->buf_addr = (char *)mb + mb_ctor_arg->seg_buf_offset; + mb->buf_physaddr = rte_mempool_virt2phy(mp, mb) + + mb_ctor_arg->seg_buf_offset; + mb->buf_len = mb_ctor_arg->seg_buf_size; + + /* keep some headroom between start of buffer and data */ + if (mb_ctor_arg->type == ODP_POOL_PACKET) { + odp_packet_hdr_t *pkt_hdr; + mb->data_off = RTE_PKTMBUF_HEADROOM; + mb->nb_segs = 1; + mb->port = 0xff; + mb->vlan_tci = 0; + pkt_hdr = (odp_packet_hdr_t *)raw_mbuf; + pkt_hdr->uarea_size = mb_ctor_arg->pkt_uarea_size; + } else { + mb->data_off = 0; + } + + /* init some constant fields */ + mb->pool = mp; + mb->ol_flags = 0; + + /* Save index, might be useful for debugging purposes */ + buf_hdr = (struct odp_buffer_hdr_t *)raw_mbuf; + buf_hdr->index = i; + buf_hdr->handle.handle = (odp_buffer_t)buf_hdr; + buf_hdr->pool_hdl = mbp_ctor_arg->pool_hdl; + buf_hdr->type = mb_ctor_arg->type; + buf_hdr->event_type = mb_ctor_arg->type; +} + +#define CHECK_U16_OVERFLOW(X) do { \ + if (odp_unlikely(X > UINT16_MAX)) { \ + ODP_ERR("Invalid size: %d", X); \ + UNLOCK(&pool->s.lock); \ + return ODP_POOL_INVALID; \ + } \ +} while (0) + +odp_pool_t odp_pool_create(const char *name, odp_pool_param_t *params) +{ + struct mbuf_pool_ctor_arg mbp_ctor_arg; + struct mbuf_ctor_arg mb_ctor_arg; + odp_pool_t pool_hdl = ODP_POOL_INVALID; + unsigned mb_size, i, cache_size; + size_t hdr_size; + pool_entry_t *pool; + uint32_t buf_align, blk_size, headroom, tailroom, seg_len; +#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 + unsigned j; +#endif + + /* Find an unused buffer pool slot and initalize it as requested */ + for (i = 0; i < ODP_CONFIG_POOLS; i++) { + uint32_t num; + pool = get_pool_entry(i); + + LOCK(&pool->s.lock); + if (pool->s.rte_mempool != NULL) { + UNLOCK(&pool->s.lock); + continue; + } + + switch (params->type) { + case ODP_POOL_BUFFER: + buf_align = params->buf.align; + blk_size = params->buf.size; + + /* Validate requested buffer alignment */ + if (buf_align > ODP_CONFIG_BUFFER_ALIGN_MAX || + buf_align != + ODP_ALIGN_ROUNDDOWN_POWER_2(buf_align, buf_align)) { + UNLOCK(&pool->s.lock); + return ODP_POOL_INVALID; + } + + /* Set correct alignment based on input request */ + if (buf_align == 0) + buf_align = ODP_CACHE_LINE_SIZE; + else if (buf_align < ODP_CONFIG_BUFFER_ALIGN_MIN) + buf_align = ODP_CONFIG_BUFFER_ALIGN_MIN; + + /* Optimize small raw buffers */ + if (blk_size > ODP_MAX_INLINE_BUF || + params->buf.align != 0) + blk_size = ODP_ALIGN_ROUNDUP(blk_size, + buf_align); + + hdr_size = sizeof(odp_buffer_hdr_t); + CHECK_U16_OVERFLOW(blk_size); + mbp_ctor_arg.pkt.mbuf_data_room_size = blk_size; + num = params->buf.num; + ODP_DBG("type: buffer name: %s num: " + "%u size: %u align: %u\n", name, num, + params->buf.size, params->buf.align); + break; + case ODP_POOL_PACKET: + headroom = RTE_PKTMBUF_HEADROOM; + tailroom = ODP_CONFIG_PACKET_TAILROOM; + seg_len = params->pkt.seg_len == 0 ? + ODP_CONFIG_PACKET_SEG_LEN_MIN : + (params->pkt.seg_len <= ODP_CONFIG_PACKET_SEG_LEN_MAX ? + params->pkt.seg_len : ODP_CONFIG_PACKET_SEG_LEN_MAX); + + seg_len = ODP_ALIGN_ROUNDUP( + headroom + seg_len + tailroom, + ODP_CONFIG_BUFFER_ALIGN_MIN); + blk_size = params->pkt.len <= seg_len ? seg_len : + ODP_ALIGN_ROUNDUP(params->pkt.len, seg_len); + + /* Reject create if pkt.len needs too many segments */ + if (blk_size / seg_len > ODP_BUFFER_MAX_SEG) { + UNLOCK(&pool->s.lock); + return ODP_POOL_INVALID; + } + + hdr_size = sizeof(odp_packet_hdr_t) + + params->pkt.uarea_size; + mb_ctor_arg.pkt_uarea_size = params->pkt.uarea_size; + CHECK_U16_OVERFLOW(blk_size); + mbp_ctor_arg.pkt.mbuf_data_room_size = blk_size; + num = params->pkt.num; + ODP_DBG("type: packet, name: %s, " + "num: %u, len: %u, seg_len: %u, blk_size %d, " + "uarea_size %d, hdr_size %d\n", + name, num, params->pkt.len, + params->pkt.seg_len, blk_size, + params->pkt.uarea_size, hdr_size); + break; + case ODP_POOL_TIMEOUT: + hdr_size = sizeof(odp_timeout_hdr_t); + mbp_ctor_arg.pkt.mbuf_data_room_size = 0; + num = params->tmo.num; + ODP_DBG("type: tmo name: %s num: %u\n", + name, num); + break; + default: + ODP_ERR("Bad type %i\n", + params->type); + UNLOCK(&pool->s.lock); + return ODP_POOL_INVALID; + break; + } + + mb_ctor_arg.seg_buf_offset = + (uint16_t) ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size); + mb_ctor_arg.seg_buf_size = mbp_ctor_arg.pkt.mbuf_data_room_size; + mb_ctor_arg.type = params->type; + mb_size = mb_ctor_arg.seg_buf_offset + mb_ctor_arg.seg_buf_size; + mbp_ctor_arg.pool_hdl = pool->s.pool_hdl; + + cache_size = 0; +#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 + j = ceil((double)num / RTE_MEMPOOL_CACHE_MAX_SIZE); + j = RTE_MAX(j, 2UL); + for (; j <= (num / 2); ++j) + if ((num % j) == 0) { + cache_size = num / j; + break; + } + if (odp_unlikely(cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE || + (uint32_t) cache_size * 1.5 > num)) { + ODP_ERR("cache_size calc failure: %d\n", cache_size); + cache_size = 0; + } +#endif + ODP_DBG("cache_size %d\n", cache_size); + + pool->s.rte_mempool = + rte_mempool_create(name, + num, + mb_size, + cache_size, + sizeof(struct mbuf_pool_ctor_arg), + odp_dpdk_mbuf_pool_ctor, + &mbp_ctor_arg, + odp_dpdk_mbuf_ctor, + &mb_ctor_arg, + rte_socket_id(), + 0); + if (pool->s.rte_mempool == NULL) { + ODP_ERR("Cannot init DPDK mbuf pool: %s\n", + rte_strerror(rte_errno)); + UNLOCK(&pool->s.lock); + return ODP_POOL_INVALID; + } + /* found free pool */ + if (name == NULL) { + pool->s.name[0] = 0; + } else { + strncpy(pool->s.name, name, + ODP_POOL_NAME_LEN - 1); + pool->s.name[ODP_POOL_NAME_LEN - 1] = 0; + } + + pool->s.params = *params; + UNLOCK(&pool->s.lock); + pool_hdl = pool->s.pool_hdl; + break; + } + + return pool_hdl; +} + + +odp_pool_t odp_pool_lookup(const char *name) +{ + struct rte_mempool *mp = NULL; + odp_pool_t pool_hdl = ODP_POOL_INVALID; + int i; + + mp = rte_mempool_lookup(name); + if (mp == NULL) + return ODP_POOL_INVALID; + + for (i = 0; i < ODP_CONFIG_POOLS; i++) { + pool_entry_t *pool = get_pool_entry(i); + LOCK(&pool->s.lock); + if (pool->s.rte_mempool != mp) { + UNLOCK(&pool->s.lock); + continue; + } + UNLOCK(&pool->s.lock); + pool_hdl = pool->s.pool_hdl; + } + return pool_hdl; +} + + +odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) +{ + uint32_t pool_id = pool_handle_to_index(pool_hdl); + pool_entry_t *pool = get_pool_entry(pool_id); + odp_buffer_t buffer; + if (pool->s.params.type == ODP_POOL_PACKET) { + struct mbuf_pool_ctor_arg *mbp_priv = + rte_mempool_get_priv(pool->s.rte_mempool); + uint32_t mbp_buf_size = mbp_priv->pkt.mbuf_data_room_size; + + buffer = (odp_buffer_t)odp_packet_alloc(pool_hdl, mbp_buf_size); + } + else + buffer = (odp_buffer_t)rte_ctrlmbuf_alloc(pool->s.rte_mempool); + + if ((struct rte_mbuf *)buffer == NULL) { + return ODP_BUFFER_INVALID; + } else { + odp_buf_to_hdr(buffer)->next = NULL; + return buffer; + } +} + + +void odp_buffer_free(odp_buffer_t buf) +{ + odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf); + struct rte_mbuf *mbuf = (struct rte_mbuf *)hdr; + + rte_pktmbuf_free(mbuf); +} + + +void odp_pool_print(odp_pool_t pool_hdl) +{ + uint32_t pool_id = pool_handle_to_index(pool_hdl); + pool_entry_t *pool = get_pool_entry(pool_id); + rte_mempool_dump(stdout, pool->s.rte_mempool); +} + +int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) +{ + uint32_t pool_id = pool_handle_to_index(pool_hdl); + pool_entry_t *pool = get_pool_entry(pool_id); + + if (pool == NULL || info == NULL) + return -1; + + info->name = pool->s.name; + info->params = pool->s.params; + + return 0; +} + +/* + * DPDK doesn't support pool destroy at the moment. Instead we should improve + * odp_pool_create() to try to reuse pools + */ +int odp_pool_destroy(odp_pool_t pool_hdl) +{ + uint32_t pool_id = pool_handle_to_index(pool_hdl); + pool_entry_t *pool = get_pool_entry(pool_id); + struct rte_mem_config *mcfg; + int i; + char name[RTE_MEMZONE_NAMESIZE]; + char ringname[RTE_MEMZONE_NAMESIZE]; + struct rte_mempool *mp; + + ODP_ERR("WARNING! This function doesn't really delete the pool due to " + "lack of DPDK support, it only releases its name. The memzone " + "allocated will be leaked!\n"); + + if ((mp = rte_mempool_lookup(pool->s.name)) == NULL) { + ODP_ERR("Can't find pool with this name!\n"); + return -1; + } + mp->name[0] = 0; + + snprintf(name, sizeof(name), RTE_MEMPOOL_MZ_FORMAT, pool->s.name); + snprintf(ringname, sizeof(ringname), "%s%s", RTE_RING_MZ_PREFIX, name); + + /* This code is based on memzone_lookup_thread_unsafe() from DPDK. We + * are deleting the name of the memzones, so at least we can reuse it + * next time */ + /* get pointer to global configuration */ + mcfg = rte_eal_get_configuration()->mem_config; + rte_rwlock_write_lock(&mcfg->mlock); + /* + * the algorithm is not optimal (linear), but there are few + * zones and this function should be called at init only + */ + for (i = 0; i < RTE_MAX_MEMZONE && mcfg->memzone[i].addr != NULL; i++) { + if (!strncmp(name, mcfg->memzone[i].name, + RTE_MEMZONE_NAMESIZE) || + !strncmp(ringname, mcfg->memzone[i].name, + RTE_MEMZONE_NAMESIZE)) + mcfg->memzone[i].name[0] = 0; + } + rte_rwlock_write_unlock(&mcfg->mlock); + + pool->s.rte_mempool = NULL; + /* The pktio supposed to be closed by now */ + return 0; +} + +odp_pool_t odp_buffer_pool(odp_buffer_t buf) +{ + return odp_buf_to_hdr(buf)->pool_hdl; +} diff --git a/platform/linux-dpdk/odp_thread.c b/platform/linux-dpdk/odp_thread.c new file mode 100644 index 000000000..125c98505 --- /dev/null +++ b/platform/linux-dpdk/odp_thread.c @@ -0,0 +1,218 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include <sched.h> + +#include <odp/thread.h> +#include <odp/thrmask.h> +#include <odp_internal.h> +#include <odp/spinlock.h> +#include <odp/config.h> +#include <odp_debug_internal.h> +#include <odp/shared_memory.h> +#include <odp/align.h> +#include <odp/cpu.h> +#include <rte_lcore.h> + +#include <string.h> +#include <stdio.h> +#include <stdlib.h> + +#define MASK_SIZE_16 ((ODP_CONFIG_MAX_THREADS+15)/16) + +typedef struct { + int thr; + int cpu; + odp_thread_type_t type; +} thread_state_t; + + +typedef struct { + thread_state_t thr[ODP_CONFIG_MAX_THREADS]; + odp_thrmask_t all; + odp_thrmask_t worker; + odp_thrmask_t control; + uint32_t num; + uint32_t num_worker; + uint32_t num_control; + odp_spinlock_t lock; +} thread_globals_t; + + +/* Globals */ +static thread_globals_t *thread_globals; + + +/* Thread local */ +static __thread thread_state_t *this_thread; + + +int odp_thread_init_global(void) +{ + odp_shm_t shm; + + shm = odp_shm_reserve("odp_thread_globals", + sizeof(thread_globals_t), + ODP_CACHE_LINE_SIZE, 0); + + thread_globals = odp_shm_addr(shm); + + if (thread_globals == NULL) + return -1; + + memset(thread_globals, 0, sizeof(thread_globals_t)); + odp_spinlock_init(&thread_globals->lock); + odp_thrmask_zero(&thread_globals->all); + odp_thrmask_zero(&thread_globals->worker); + odp_thrmask_zero(&thread_globals->control); + + return 0; +} + +int odp_thread_term_global(void) +{ + int ret; + + ret = odp_shm_free(odp_shm_lookup("odp_thread_globals")); + if (ret < 0) + ODP_ERR("shm free failed for odp_thread_globals"); + + return ret; +} + +static int alloc_id(odp_thread_type_t type) +{ + int thr; + odp_thrmask_t *all = &thread_globals->all; + + if (thread_globals->num >= ODP_CONFIG_MAX_THREADS) + return -1; + + for (thr = 0; thr < ODP_CONFIG_MAX_THREADS; thr++) { + if (odp_thrmask_isset(all, thr) == 0) { + odp_thrmask_set(all, thr); + + if (type == ODP_THREAD_WORKER) { + odp_thrmask_set(&thread_globals->worker, thr); + thread_globals->num_worker++; + } else { + odp_thrmask_set(&thread_globals->control, thr); + thread_globals->num_control++; + } + + thread_globals->num++; + return thr; + } + } + + return -2; +} + +static int free_id(int thr) +{ + odp_thrmask_t *all = &thread_globals->all; + + if (thr < 0 || thr >= ODP_CONFIG_MAX_THREADS) + return -1; + + if (odp_thrmask_isset(all, thr) == 0) + return -1; + + odp_thrmask_clr(all, thr); + + if (thread_globals->thr[thr].type == ODP_THREAD_WORKER) { + odp_thrmask_clr(&thread_globals->worker, thr); + thread_globals->num_worker--; + } else { + odp_thrmask_clr(&thread_globals->control, thr); + thread_globals->num_control--; + } + + thread_globals->num--; + return thread_globals->num; +} + +int odp_thread_init_local(odp_thread_type_t type) +{ + int id; + int cpu; + + odp_spinlock_lock(&thread_globals->lock); + id = alloc_id(type); + odp_spinlock_unlock(&thread_globals->lock); + + if (id < 0) { + ODP_ERR("Too many threads\n"); + return -1; + } + + cpu = sched_getcpu(); + + if (cpu < 0) { + ODP_ERR("getcpu failed\n"); + return -1; + } + + thread_globals->thr[id].thr = id; + thread_globals->thr[id].cpu = cpu; + thread_globals->thr[id].type = type; + RTE_PER_LCORE(_lcore_id) = cpu; + + this_thread = &thread_globals->thr[id]; + return 0; +} + +int odp_thread_term_local(void) +{ + int num; + int id = this_thread->thr; + + odp_spinlock_lock(&thread_globals->lock); + num = free_id(id); + odp_spinlock_unlock(&thread_globals->lock); + + if (num < 0) { + ODP_ERR("failed to free thread id %i", id); + return -1; + } + + return num; /* return a number of threads left */ +} + +int odp_thread_id(void) +{ + return this_thread->thr; +} + +int odp_thread_count(void) +{ + return thread_globals->num; +} + +odp_thread_type_t odp_thread_type(void) +{ + return this_thread->type; +} + +int odp_cpu_id(void) +{ + return this_thread->cpu; +} + +int odp_thrmask_worker(odp_thrmask_t *mask) +{ + odp_thrmask_copy(mask, &thread_globals->worker); + return thread_globals->num_worker; +} + +int odp_thrmask_control(odp_thrmask_t *mask) +{ + odp_thrmask_copy(mask, &thread_globals->control); + return thread_globals->num_control; +} diff --git a/platform/linux-dpdk/test/Makefile.am b/platform/linux-dpdk/test/Makefile.am new file mode 100644 index 000000000..5f2054bea --- /dev/null +++ b/platform/linux-dpdk/test/Makefile.am @@ -0,0 +1,35 @@ +include $(top_srcdir)/test/Makefile.inc +TESTS_ENVIRONMENT += TEST_DIR=${top_builddir}/test/validation + +ODP_MODULES = pktio + +if test_vald +LOG_COMPILER = $(top_srcdir)/platform/linux-dpdk/test/wrapper-script.sh +TESTS = pktio/pktio_run \ + ${top_builddir}/test/validation/buffer/buffer_main$(EXEEXT) \ +## ${top_builddir}/test/validation/classification/classification_main$(EXEEXT) \ + ${top_builddir}/test/validation/cpumask/cpumask_main$(EXEEXT) \ + ${top_builddir}/test/validation/crypto/crypto_main$(EXEEXT) \ + ${top_builddir}/test/validation/errno/errno_main$(EXEEXT) \ + ${top_builddir}/test/validation/init/init_main_ok$(EXEEXT) \ + ${top_builddir}/test/validation/init/init_main_abort$(EXEEXT) \ + ${top_builddir}/test/validation/init/init_main_log$(EXEEXT) \ + ${top_builddir}/test/validation/packet/packet_main$(EXEEXT) \ + ${top_builddir}/test/validation/pool/pool_main$(EXEEXT) \ + ${top_builddir}/test/validation/queue/queue_main$(EXEEXT) \ + ${top_builddir}/test/validation/random/random_main$(EXEEXT) \ + ${top_builddir}/test/validation/scheduler/scheduler_main$(EXEEXT) \ + ${top_builddir}/test/validation/synchronizers/synchronizers_main$(EXEEXT) \ + ${top_builddir}/test/validation/thread/thread_main$(EXEEXT) \ + ${top_builddir}/test/validation/time/time_main$(EXEEXT) \ + ${top_builddir}/test/validation/timer/timer_main$(EXEEXT) \ + ${top_builddir}/test/validation/shmem/shmem_main$(EXEEXT) \ + ${top_builddir}/test/validation/system/system_main$(EXEEXT) + +SUBDIRS = $(ODP_MODULES) +endif + +#performance tests refer to pktio_env +if test_perf +SUBDIRS = pktio +endif diff --git a/platform/linux-dpdk/test/pktio/.gitignore b/platform/linux-dpdk/test/pktio/.gitignore new file mode 120000 index 000000000..563cb9228 --- /dev/null +++ b/platform/linux-dpdk/test/pktio/.gitignore @@ -0,0 +1 @@ +../../../linux-generic/test/pktio/.gitignore
\ No newline at end of file diff --git a/platform/linux-dpdk/test/pktio/Makefile.am b/platform/linux-dpdk/test/pktio/Makefile.am new file mode 120000 index 000000000..f04861c97 --- /dev/null +++ b/platform/linux-dpdk/test/pktio/Makefile.am @@ -0,0 +1 @@ +../../../linux-generic/test/pktio/Makefile.am
\ No newline at end of file diff --git a/platform/linux-dpdk/test/pktio/pktio_env b/platform/linux-dpdk/test/pktio/pktio_env new file mode 120000 index 000000000..6244f7156 --- /dev/null +++ b/platform/linux-dpdk/test/pktio/pktio_env @@ -0,0 +1 @@ +../../../../platform/linux-generic/test/pktio/pktio_env
\ No newline at end of file diff --git a/platform/linux-dpdk/test/pktio/pktio_run b/platform/linux-dpdk/test/pktio/pktio_run new file mode 100755 index 000000000..28b8d3b8d --- /dev/null +++ b/platform/linux-dpdk/test/pktio/pktio_run @@ -0,0 +1,81 @@ +#!/bin/sh +# +# Copyright (c) 2015, Linaro Limited +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +# directories where pktio_main binary can be found: +# -in the validation dir when running make check (intree or out of tree) +# -in the script directory, when running after 'make install', or +# -in the validation when running standalone (./pktio_run) intree. +# -in the current directory. +# running stand alone out of tree requires setting PATH +PATH=${TEST_DIR}/pktio:$PATH +PATH=$(dirname $0):$PATH +PATH=$(dirname $0)/../../../../test/validation/pktio:$PATH +PATH=.:$PATH + +pktio_main_path=$(which pktio_main${EXEEXT}) +if [ -x "$pktio_main_path" ] ; then + echo "running with pktio_main: $pktio_run_path" +else + echo "cannot find pktio_main: please set you PATH for it." +fi + +# directory where platform test sources are, including scripts +TEST_SRC_DIR=$(dirname $0) + +# exit codes expected by automake for skipped tests +TEST_SKIPPED=77 + +# Use installed pktio env or for make check take it from platform directory +if [ -f "./pktio_env" ]; then + . ./pktio_env +elif [ -f ${TEST_SRC_DIR}/pktio_env ]; then + . ${TEST_SRC_DIR}/pktio_env +else + echo "BUG: unable to find pktio_env!" + echo "pktio_env has to be in current directory or in platform/\$ODP_PLATFORM/test." + echo "ODP_PLATFORM=\"$ODP_PLATFORM\"" + exit 1 +fi + +run_test() +{ + local ret=0 + + pktio_main${EXEEXT} + ret=$? + if [ $ret -ne 0 ]; then + echo "!!! FAILED !!!" + fi + + exit $ret +} + +run() +{ + #need to be root to set the interface: if not, run with default loopback. + if [ "$(id -u)" != "0" ]; then + echo "pktio: using 'loop' device" + pktio_main${EXEEXT} + exit $? + fi + + if [ "$ODP_PKTIO_IF0" = "" ]; then + setup_pktio_env clean + export ODP_PLATFORM_PARAMS="-n 4 --vdev eth_pcap0,iface=$IF0 --vdev eth_pcap1,iface=$IF1" + export ODP_PKTIO_IF0=0 + export ODP_PKTIO_IF1=1 + fi + + run_test +} + +case "$1" in + setup) setup_pktio_env ;; + cleanup) cleanup_pktio_env ;; + *) run ;; +esac diff --git a/platform/linux-dpdk/test/wrapper-script.sh b/platform/linux-dpdk/test/wrapper-script.sh new file mode 100755 index 000000000..6ea4cb2ee --- /dev/null +++ b/platform/linux-dpdk/test/wrapper-script.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +export ODP_PLATFORM_PARAMS=${ODP_PLATFORM_PARAMS:--n 4} +# where to mount huge pages +export HUGEPAGEDIR=${HUGEPAGEDIR:-/mnt/huge} + +if [ ! -d $HUGEPAGEDIR ]; then + sudo mkdir $HUGEPAGEDIR +fi +echo "Mounting hugetlbfs" +sudo mount -t hugetlbfs nodev $HUGEPAGEDIR +sudo sh -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages' +echo "Total number: `cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages`" +echo "Free pages: `cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/free_hugepages`" +echo "running $1!" +$1 +res=$? +echo "Unmounting hugetlbfs" +sleep 0.3 && sudo umount -a -t hugetlbfs +exit $res + diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index d1c2cfc60..80c9233e4 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -247,6 +247,10 @@ void queue_destroy_finalize(queue_entry_t *queue) int odp_queue_destroy(odp_queue_t handle) { queue_entry_t *queue; + + if (handle == ODP_QUEUE_INVALID) + return -1; + queue = queue_to_qentry(handle); LOCK(&queue->s.lock); diff --git a/scripts/devbuild.sh b/scripts/devbuild.sh new file mode 100755 index 000000000..db509978c --- /dev/null +++ b/scripts/devbuild.sh @@ -0,0 +1,104 @@ +#!/bin/sh + +# You can overwrite most of these variables with a wrapper script +# The next 4 variables specify the directories used +export REPOS=${REPOS:-/local/repo/odp} +export CHECK_ODP_DIR=${CHECK_ODP_DIR:-$REPOS/check-odp} +export ROOT_DIR_DPDK=${ROOT_DIR_DPDK:-$REPOS/dpdk} +export ODP_BUILDDIR=${ODP_BUILDDIR:-$REPOS/odp-dpdk} +# These are passed to ODP configure +export EXTRA_FLAGS="${EXTRA_FLAGS:- --enable-debug --enable-debug-print --enable-cunit-support --enable-test-vald --enable-shared=no}" +# where to mount huge pages +export HUGEPAGEDIR=${HUGEPAGEDIR:-/mnt/huge} +# don't do performance tests, they are not working at the moment +export PERF_TEST=0 +# don't build CUnit for us +export VALIDATION=0 +# Number of threads for compiling (make -j NUM_CPUS) +export NUM_CPUS=${NUM_CPUS:-3} +# Don't delete our working directories +export CLEANUP=0 +# Don't run the relocated build test +export RELOCATE_TEST=0 + +if [ -z $1 ]; then + echo "Usage: $0 [dpdk | odp | odp-check | {unit_test} ]" >&2 + echo "Build DPDK, ODP-DPDK or both. You need a successful build of" \ + "the first to build the second." >&2 + echo "odp-check runs all unit tests (make check), but you can run" \ + "them separately as well, e.g. buffer_main." >&2 + echo "The argument after the individual unit test is passed as" \ + "parameter, e.g \"odp_pktio_run setup\"" >&2 + exit 1 +fi + +# Make sure huge pages are released when a unit test crashes "make check" +trap ctrl_c INT + +ctrl_c() { + echo "** Trapped CTRL-C" + if grep -qs "$HUGEPAGEDIR" /proc/mounts; then + echo "** Umounting hugetlbfs" + sleep 1 && sudo umount -a -t hugetlbfs + fi +} + +while [ "$1" != "" ]; +do +case $1 in + dpdk) + cd $CHECK_ODP_DIR + # Build only DPDK + export BUILD_DEPS=2 + ./build-dpdk.sh + if [ $? -ne 0 ]; then + exit 1 + fi + ;; + odp) + cd $CHECK_ODP_DIR + git clean -xfd + # That prevents make check to run + export ARCH=nocheck + # Don't build DPDK + export BUILD_DEPS=0 + ./build-dpdk.sh + if [ $? -ne 0 ]; then + exit 1 + fi + ;; + odp-check) + cd $ODP_BUILDDIR + FOUND=`grep "pktio-p" /proc/net/dev` + if [ -z "$FOUND" ] ; then + sudo ODP_PLATFORM_PARAMS="-n 3" make check + else + sudo ODP_PLATFORM_PARAMS="-n 3 --vdev eth_pcap0,iface=pktio-p1-p0 --vdev eth_pcap1,iface=pktio-p3-p2" ODP_PKTIO_IF0=0 ODP_PKTIO_IF1=1 make check + fi + ;; + *) + export TEST=$1 + shift + cd $CHECK_ODP_DIR/new-build/bin + if [ ! -d $HUGEPAGEDIR ]; then + sudo mkdir $HUGEPAGEDIR + fi + sudo mount -t hugetlbfs nodev $HUGEPAGEDIR + sudo sh -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages' + echo "Total number: `cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages`" + echo "Free pages: `cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/free_hugepages`" + FOUND=`grep "pktio-p" /proc/net/dev` + if [ -z "$FOUND" ] ; then + + sudo ODP_PLATFORM_PARAMS="-n 3" ./$TEST $1 + else + sudo ODP_PLATFORM_PARAMS="-n 3 --vdev eth_pcap0,iface=pktio-p1-p0 --vdev eth_pcap1,iface=pktio-p3-p2" ODP_PKTIO_IF0=0 ODP_PKTIO_IF1=1 ./$TEST $1 + fi + sleep 1 && sudo umount -a -t hugetlbfs + if [ "$1" = "" ]; then + exit + fi + ;; +esac +shift +done diff --git a/test/validation/Makefile.am b/test/validation/Makefile.am index 56ddd64f1..5d8e93b28 100644 --- a/test/validation/Makefile.am +++ b/test/validation/Makefile.am @@ -1,5 +1,5 @@ ODP_MODULES = buffer \ - classification \ +## classification \ cpumask \ crypto \ errno \ |