diff options
Diffstat (limited to 'platform')
59 files changed, 3128 insertions, 646 deletions
diff --git a/platform/Makefile.inc b/platform/Makefile.inc index 9cefe2557..147ba9f4e 100644 --- a/platform/Makefile.inc +++ b/platform/Makefile.inc @@ -41,6 +41,7 @@ odpapiinclude_HEADERS = \ $(top_srcdir)/include/odp/api/packet.h \ $(top_srcdir)/include/odp/api/packet_flags.h \ $(top_srcdir)/include/odp/api/packet_io.h \ + $(top_srcdir)/include/odp/api/packet_io_stats.h \ $(top_srcdir)/include/odp/api/pool.h \ $(top_srcdir)/include/odp/api/queue.h \ $(top_srcdir)/include/odp/api/random.h \ diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 279e5e296..a9090a371 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -2,6 +2,7 @@ #export CUSTOM_STR=https://git.linaro.org/lng/odp.git include $(top_srcdir)/platform/Makefile.inc +include $(top_srcdir)/platform/@with_platform@/Makefile.inc AM_CFLAGS += -I$(srcdir)/include AM_CFLAGS += -I$(top_srcdir)/include @@ -51,7 +52,8 @@ odpinclude_HEADERS = \ $(srcdir)/include/odp/ticketlock.h \ $(srcdir)/include/odp/time.h \ $(srcdir)/include/odp/timer.h \ - $(srcdir)/include/odp/version.h + $(srcdir)/include/odp/version.h \ + $(srcdir)/arch/@ARCH@/odp/cpu_arch.h odpplatincludedir= $(includedir)/odp/plat odpplatinclude_HEADERS = \ @@ -63,6 +65,7 @@ odpplatinclude_HEADERS = \ $(srcdir)/include/odp/plat/cpumask_types.h \ $(srcdir)/include/odp/plat/crypto_types.h \ $(srcdir)/include/odp/plat/event_types.h \ + $(srcdir)/include/odp/plat/init_types.h \ $(srcdir)/include/odp/plat/packet_types.h \ $(srcdir)/include/odp/plat/packet_io_types.h \ $(srcdir)/include/odp/plat/pool_types.h \ @@ -103,11 +106,11 @@ noinst_HEADERS = \ ${srcdir}/include/odp_posix_extensions.h \ ${srcdir}/include/odp_queue_internal.h \ ${srcdir}/include/odp_schedule_internal.h \ - ${srcdir}/include/odp_spin_internal.h \ ${srcdir}/include/odp_timer_internal.h \ ${srcdir}/Makefile.inc __LIB__libodp_la_SOURCES = \ + odp_atomic.c \ odp_barrier.c \ odp_buffer.c \ odp_classification.c \ @@ -123,12 +126,14 @@ __LIB__libodp_la_SOURCES = \ odp_packet.c \ odp_packet_flags.c \ odp_packet_io.c \ + pktio/ethtool.c \ pktio/io_ops.c \ pktio/pktio_common.c \ pktio/loop.c \ pktio/netmap.c \ pktio/socket.c \ pktio/socket_mmap.c \ + pktio/sysfs.c \ pktio/tap.c \ odp_pool.c \ odp_queue.c \ @@ -146,12 +151,18 @@ __LIB__libodp_la_SOURCES = \ odp_timer.c \ odp_version.c \ odp_weak.c \ - arch/@ARCH@/odp_cpu_cycles.c + arch/@ARCH@/odp_cpu_arch.c \ + arch/@ARCH@/odp_sysinfo_parse.c EXTRA_DIST = \ - arch/linux/odp_cpu_cycles.c \ - arch/mips64/odp_cpu_cycles.c \ - arch/x86/odp_cpu_cycles.c + arch/linux/odp_cpu_arch.c \ + arch/linux/odp_sysinfo_parse.c \ + arch/mips64/odp_cpu_arch.c \ + arch/mips64/odp_sysinfo_parse.c \ + arch/powerpc/odp_cpu_arch.c \ + arch/powerpc/odp_sysinfo_parse.c \ + arch/x86/odp_cpu_arch.c \ + arch/x86/odp_sysinfo_parse.c if HAVE_PCAP __LIB__libodp_la_SOURCES += pktio/pcap.c diff --git a/platform/linux-generic/Makefile.inc b/platform/linux-generic/Makefile.inc index e69de29bb..048c8bb78 100644 --- a/platform/linux-generic/Makefile.inc +++ b/platform/linux-generic/Makefile.inc @@ -0,0 +1,2 @@ +AM_CFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/$(ARCH) +AM_CXXFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/$(ARCH) diff --git a/platform/linux-generic/arch/linux/odp/cpu_arch.h b/platform/linux-generic/arch/linux/odp/cpu_arch.h new file mode 100644 index 000000000..1c79f875c --- /dev/null +++ b/platform/linux-generic/arch/linux/odp/cpu_arch.h @@ -0,0 +1,22 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_ARCH_H_ +#define ODP_PLAT_CPU_ARCH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void odp_cpu_pause(void) +{ +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/linux/odp_cpu_cycles.c b/platform/linux-generic/arch/linux/odp_cpu_arch.c index 7509bf28b..3112d0cd0 100644 --- a/platform/linux-generic/arch/linux/odp_cpu_cycles.c +++ b/platform/linux-generic/arch/linux/odp_cpu_arch.c @@ -27,7 +27,7 @@ uint64_t odp_cpu_cycles(void) if (ret != 0) ODP_ABORT("clock_gettime failed\n"); - hz = odp_sys_cpu_hz(); + hz = odp_cpu_hz_max(); sec = (uint64_t)time.tv_sec; ns = (uint64_t)time.tv_nsec; diff --git a/platform/linux-generic/arch/linux/odp_sysinfo_parse.c b/platform/linux-generic/arch/linux/odp_sysinfo_parse.c new file mode 100644 index 000000000..8ff6f48d2 --- /dev/null +++ b/platform/linux-generic/arch/linux/odp_sysinfo_parse.c @@ -0,0 +1,19 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_internal.h> +#include <string.h> + +int odp_cpuinfo_parser(FILE *file ODP_UNUSED, + odp_system_info_t *sysinfo ODP_UNUSED) +{ + return 0; +} + +uint64_t odp_cpu_hz_current(int id ODP_UNUSED) +{ + return 0; +} diff --git a/platform/linux-generic/arch/mips64/odp/cpu_arch.h b/platform/linux-generic/arch/mips64/odp/cpu_arch.h new file mode 100644 index 000000000..3bfa0dcde --- /dev/null +++ b/platform/linux-generic/arch/mips64/odp/cpu_arch.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_ARCH_H_ +#define ODP_PLAT_CPU_ARCH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void odp_cpu_pause(void) +{ + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/mips64/odp_cpu_cycles.c b/platform/linux-generic/arch/mips64/odp_cpu_arch.c index a20a31325..a20a31325 100644 --- a/platform/linux-generic/arch/mips64/odp_cpu_cycles.c +++ b/platform/linux-generic/arch/mips64/odp_cpu_arch.c diff --git a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c b/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c new file mode 100644 index 000000000..53074f7bd --- /dev/null +++ b/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c @@ -0,0 +1,64 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_internal.h> +#include <string.h> + +int odp_cpuinfo_parser(FILE *file, odp_system_info_t *sysinfo) +{ + char str[1024]; + char *pos; + double mhz = 0.0; + uint64_t hz; + int model = 0; + int count = 2; + int id = 0; + + strcpy(sysinfo->cpu_arch_str, "mips64"); + while (fgets(str, sizeof(str), file) != NULL && id < MAX_CPU_NUMBER) { + if (!mhz) { + pos = strstr(str, "BogoMIPS"); + + if (pos) + if (sscanf(pos, "BogoMIPS : %lf", &mhz) == 1) { + /* bogomips seems to be 2x freq */ + hz = (uint64_t)(mhz * 1000000.0 / 2.0); + sysinfo->cpu_hz_max[id] = hz; + count--; + } + } + + if (!model) { + pos = strstr(str, "cpu model"); + + if (pos) { + int len; + + pos = strchr(str, ':'); + strncpy(sysinfo->model_str[id], pos + 2, + sizeof(sysinfo->model_str[id])); + len = strlen(sysinfo->model_str[id]); + sysinfo->model_str[id][len - 1] = 0; + model = 1; + count--; + } + } + + if (count == 0) { + mhz = 0.0; + model = 0; + count = 2; + id++; + } + } + + return 0; +} + +uint64_t odp_cpu_hz_current(int id ODP_UNUSED) +{ + return 0; +} diff --git a/platform/linux-generic/arch/powerpc/odp/cpu_arch.h b/platform/linux-generic/arch/powerpc/odp/cpu_arch.h new file mode 120000 index 000000000..0617d7fa1 --- /dev/null +++ b/platform/linux-generic/arch/powerpc/odp/cpu_arch.h @@ -0,0 +1 @@ +../../linux/odp/cpu_arch.h
\ No newline at end of file diff --git a/platform/linux-generic/arch/powerpc/odp_cpu_arch.c b/platform/linux-generic/arch/powerpc/odp_cpu_arch.c new file mode 120000 index 000000000..c5fe40085 --- /dev/null +++ b/platform/linux-generic/arch/powerpc/odp_cpu_arch.c @@ -0,0 +1 @@ +../linux/odp_cpu_arch.c
\ No newline at end of file diff --git a/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c b/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c new file mode 100644 index 000000000..99457cec4 --- /dev/null +++ b/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c @@ -0,0 +1,63 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_internal.h> +#include <string.h> + +int odp_cpuinfo_parser(FILE *file, odp_system_info_t *sysinfo) +{ + char str[1024]; + char *pos; + double mhz = 0.0; + uint64_t hz; + int model = 0; + int count = 2; + int id = 0; + + strcpy(sysinfo->cpu_arch_str, "powerpc"); + while (fgets(str, sizeof(str), file) != NULL && id < MAX_CPU_NUMBER) { + if (!mhz) { + pos = strstr(str, "clock"); + + if (pos) + if (sscanf(pos, "clock : %lf", &mhz) == 1) { + hz = (uint64_t)(mhz * 1000000.0); + sysinfo->cpu_hz_max[id] = hz; + count--; + } + } + + if (!model) { + pos = strstr(str, "cpu"); + + if (pos) { + int len; + + pos = strchr(str, ':'); + strncpy(sysinfo->model_str[id], pos + 2, + sizeof(sysinfo->model_str[id])); + len = strlen(sysinfo->model_str[id]); + sysinfo->model_str[id][len - 1] = 0; + model = 1; + count--; + } + } + + if (count == 0) { + mhz = 0.0; + model = 0; + count = 2; + id++; + } + } + + return 0; +} + +uint64_t odp_cpu_hz_current(int id ODP_UNUSED) +{ + return 0; +} diff --git a/platform/linux-generic/arch/x86/odp/cpu_arch.h b/platform/linux-generic/arch/x86/odp/cpu_arch.h new file mode 100644 index 000000000..997a95475 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/cpu_arch.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_ARCH_H_ +#define ODP_PLAT_CPU_ARCH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void odp_cpu_pause(void) +{ +#ifdef __SSE2__ + __asm__ __volatile__ ("pause"); +#else + __asm__ __volatile__ ("rep; nop"); +#endif +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp_cpu_cycles.c b/platform/linux-generic/arch/x86/odp_cpu_arch.c index 1c5c0ec7f..1c5c0ec7f 100644 --- a/platform/linux-generic/arch/x86/odp_cpu_cycles.c +++ b/platform/linux-generic/arch/x86/odp_cpu_arch.c diff --git a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c new file mode 100644 index 000000000..816629dfa --- /dev/null +++ b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c @@ -0,0 +1,73 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_internal.h> +#include <string.h> + +int odp_cpuinfo_parser(FILE *file, odp_system_info_t *sysinfo) +{ + char str[1024]; + char *pos; + double ghz = 0.0; + uint64_t hz; + int id = 0; + + strcpy(sysinfo->cpu_arch_str, "x86"); + while (fgets(str, sizeof(str), file) != NULL && id < MAX_CPU_NUMBER) { + pos = strstr(str, "model name"); + if (pos) { + pos = strchr(str, ':'); + strncpy(sysinfo->model_str[id], pos + 2, + sizeof(sysinfo->model_str[id])); + + pos = strchr(sysinfo->model_str[id], '@'); + *(pos - 1) = '\0'; + if (sscanf(pos, "@ %lfGHz", &ghz) == 1) { + hz = (uint64_t)(ghz * 1000000000.0); + sysinfo->cpu_hz_max[id] = hz; + } + id++; + } + } + + return 0; +} + +uint64_t odp_cpu_hz_current(int id) +{ + char str[1024]; + FILE *file; + int cpu; + char *pos; + double mhz = 0.0; + + file = fopen("/proc/cpuinfo", "rt"); + + /* find the correct processor instance */ + while (fgets(str, sizeof(str), file) != NULL) { + pos = strstr(str, "processor"); + if (pos) { + if (sscanf(pos, "processor : %d", &cpu) == 1) + if (cpu == id) + break; + } + } + + /* extract the cpu current speed */ + while (fgets(str, sizeof(str), file) != NULL) { + pos = strstr(str, "cpu MHz"); + if (pos) { + if (sscanf(pos, "cpu MHz : %lf", &mhz) == 1) + break; + } + } + + fclose(file); + if (mhz) + return (uint64_t)(mhz * 1000000.0); + + return 0; +} diff --git a/platform/linux-generic/include/odp/atomic.h b/platform/linux-generic/include/odp/atomic.h index 10cf361b4..e262f4851 100644 --- a/platform/linux-generic/include/odp/atomic.h +++ b/platform/linux-generic/include/odp/atomic.h @@ -84,6 +84,45 @@ static inline void odp_atomic_dec_u32(odp_atomic_u32_t *atom) (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); } +static inline int odp_atomic_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, + uint32_t new_val) +{ + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED); +} + +static inline uint32_t odp_atomic_xchg_u32(odp_atomic_u32_t *atom, + uint32_t new_val) +{ + return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED); +} + +static inline void odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t new_max) +{ + uint32_t old_val; + + old_val = odp_atomic_load_u32(atom); + + while (new_max > old_val) { + if (odp_atomic_cas_u32(atom, &old_val, new_max)) + break; + } +} + +static inline void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) +{ + uint32_t old_val; + + old_val = odp_atomic_load_u32(atom); + + while (new_min < old_val) { + if (odp_atomic_cas_u32(atom, &old_val, new_min)) + break; + } +} + static inline void odp_atomic_init_u64(odp_atomic_u64_t *atom, uint64_t val) { atom->v = val; @@ -185,6 +224,189 @@ static inline void odp_atomic_dec_u64(odp_atomic_u64_t *atom) #endif } +static inline int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, + uint64_t new_val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; +#else + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED); +#endif +} + +static inline uint64_t odp_atomic_xchg_u64(odp_atomic_u64_t *atom, + uint64_t new_val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + return ATOMIC_OP(atom, atom->v = new_val); +#else + return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED); +#endif +} + +static inline void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_max) +{ + uint64_t old_val; + + old_val = odp_atomic_load_u64(atom); + + while (new_max > old_val) { + if (odp_atomic_cas_u64(atom, &old_val, new_max)) + break; + } +} + +static inline void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min) +{ + uint64_t old_val; + + old_val = odp_atomic_load_u64(atom); + + while (new_min < old_val) { + if (odp_atomic_cas_u64(atom, &old_val, new_min)) + break; + } +} + +static inline uint32_t odp_atomic_load_acq_u32(odp_atomic_u32_t *atom) +{ + return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE); +} + +static inline void odp_atomic_store_rel_u32(odp_atomic_u32_t *atom, + uint32_t val) +{ + __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, + uint32_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, + uint32_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, uint32_t new_val) +{ + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); +} + +static inline int odp_atomic_cas_rel_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, uint32_t new_val) +{ + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_RELEASE, + __ATOMIC_RELAXED); +} + +static inline int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, + uint32_t new_val) +{ + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED); +} + +static inline uint64_t odp_atomic_load_acq_u64(odp_atomic_u64_t *atom) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + return ATOMIC_OP(atom, (void)0); +#else + return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE); +#endif +} + +static inline void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, + uint64_t val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + (void)ATOMIC_OP(atom, atom->v = val); +#else + __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE); +#endif +} + +static inline void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + (void)ATOMIC_OP(atom, atom->v += val); +#else + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); +#endif +} + +static inline void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + (void)ATOMIC_OP(atom, atom->v -= val); +#else + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); +#endif +} + +static inline int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, uint64_t new_val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; +#else + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); +#endif +} + +static inline int odp_atomic_cas_rel_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, uint64_t new_val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; +#else + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_RELEASE, + __ATOMIC_RELAXED); +#endif +} + +static inline int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, + uint64_t new_val) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; +#else + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED); +#endif +} + /** * @} */ diff --git a/platform/linux-generic/include/odp/byteorder.h b/platform/linux-generic/include/odp/byteorder.h index 7fc7dc512..6c94556b0 100644 --- a/platform/linux-generic/include/odp/byteorder.h +++ b/platform/linux-generic/include/odp/byteorder.h @@ -25,7 +25,7 @@ extern "C" { * @{ */ -static inline uint16_t odp_be_to_cpu_16(uint16be_t be16) +static inline uint16_t odp_be_to_cpu_16(odp_u16be_t be16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return __odp_builtin_bswap16((__odp_force uint16_t)be16); @@ -34,7 +34,7 @@ static inline uint16_t odp_be_to_cpu_16(uint16be_t be16) #endif } -static inline uint32_t odp_be_to_cpu_32(uint32be_t be32) +static inline uint32_t odp_be_to_cpu_32(odp_u32be_t be32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return __builtin_bswap32((__odp_force uint32_t)be32); @@ -43,7 +43,7 @@ static inline uint32_t odp_be_to_cpu_32(uint32be_t be32) #endif } -static inline uint64_t odp_be_to_cpu_64(uint64be_t be64) +static inline uint64_t odp_be_to_cpu_64(odp_u64be_t be64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return __builtin_bswap64((__odp_force uint64_t)be64); @@ -53,35 +53,35 @@ static inline uint64_t odp_be_to_cpu_64(uint64be_t be64) } -static inline uint16be_t odp_cpu_to_be_16(uint16_t cpu16) +static inline odp_u16be_t odp_cpu_to_be_16(uint16_t cpu16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint16be_t)__odp_builtin_bswap16(cpu16); + return (__odp_force odp_u16be_t)__odp_builtin_bswap16(cpu16); #else - return (__odp_force uint16be_t)cpu16; + return (__odp_force odp_u16be_t)cpu16; #endif } -static inline uint32be_t odp_cpu_to_be_32(uint32_t cpu32) +static inline odp_u32be_t odp_cpu_to_be_32(uint32_t cpu32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint32be_t)__builtin_bswap32(cpu32); + return (__odp_force odp_u32be_t)__builtin_bswap32(cpu32); #else - return (__odp_force uint32be_t)cpu32; + return (__odp_force odp_u32be_t)cpu32; #endif } -static inline uint64be_t odp_cpu_to_be_64(uint64_t cpu64) +static inline odp_u64be_t odp_cpu_to_be_64(uint64_t cpu64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint64be_t)__builtin_bswap64(cpu64); + return (__odp_force odp_u64be_t)__builtin_bswap64(cpu64); #else - return (__odp_force uint64be_t)cpu64; + return (__odp_force odp_u64be_t)cpu64; #endif } -static inline uint16_t odp_le_to_cpu_16(uint16le_t le16) +static inline uint16_t odp_le_to_cpu_16(odp_u16le_t le16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force uint16_t)le16; @@ -90,7 +90,7 @@ static inline uint16_t odp_le_to_cpu_16(uint16le_t le16) #endif } -static inline uint32_t odp_le_to_cpu_32(uint32le_t le32) +static inline uint32_t odp_le_to_cpu_32(odp_u32le_t le32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force uint32_t)le32; @@ -99,7 +99,7 @@ static inline uint32_t odp_le_to_cpu_32(uint32le_t le32) #endif } -static inline uint64_t odp_le_to_cpu_64(uint64le_t le64) +static inline uint64_t odp_le_to_cpu_64(odp_u64le_t le64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force uint64_t)le64; @@ -109,30 +109,30 @@ static inline uint64_t odp_le_to_cpu_64(uint64le_t le64) } -static inline uint16le_t odp_cpu_to_le_16(uint16_t cpu16) +static inline odp_u16le_t odp_cpu_to_le_16(uint16_t cpu16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint16le_t)cpu16; + return (__odp_force odp_u16le_t)cpu16; #else - return (__odp_force uint16le_t)__odp_builtin_bswap16(cpu16); + return (__odp_force odp_u16le_t)__odp_builtin_bswap16(cpu16); #endif } -static inline uint32le_t odp_cpu_to_le_32(uint32_t cpu32) +static inline odp_u32le_t odp_cpu_to_le_32(uint32_t cpu32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint32le_t)cpu32; + return (__odp_force odp_u32le_t)cpu32; #else - return (__odp_force uint32le_t)__builtin_bswap32(cpu32); + return (__odp_force odp_u32le_t)__builtin_bswap32(cpu32); #endif } -static inline uint64le_t odp_cpu_to_le_64(uint64_t cpu64) +static inline odp_u64le_t odp_cpu_to_le_64(uint64_t cpu64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint64le_t)cpu64; + return (__odp_force odp_u64le_t)cpu64; #else - return (__odp_force uint64le_t)__builtin_bswap64(cpu64); + return (__odp_force odp_u64le_t)__builtin_bswap64(cpu64); #endif } diff --git a/platform/linux-generic/include/odp/cpu.h b/platform/linux-generic/include/odp/cpu.h index b5b532005..b98507dd5 100644 --- a/platform/linux-generic/include/odp/cpu.h +++ b/platform/linux-generic/include/odp/cpu.h @@ -17,6 +17,8 @@ extern "C" { #endif +#include <odp/cpu_arch.h> + #include <odp/api/cpu.h> #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/init.h b/platform/linux-generic/include/odp/init.h index 950a4f829..3233e36de 100644 --- a/platform/linux-generic/include/odp/init.h +++ b/platform/linux-generic/include/odp/init.h @@ -17,6 +17,8 @@ extern "C" { #endif +#include <odp/plat/init_types.h> + /** @ingroup odp_initialization * @{ */ diff --git a/platform/linux-generic/include/odp/plat/atomic_types.h b/platform/linux-generic/include/odp/plat/atomic_types.h index 0fe15ed13..bc0bd8bfe 100644 --- a/platform/linux-generic/include/odp/plat/atomic_types.h +++ b/platform/linux-generic/include/odp/plat/atomic_types.h @@ -43,6 +43,21 @@ struct odp_atomic_u32_s { } ODP_ALIGNED(sizeof(uint32_t)); /* Enforce alignement! */; #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + +/** + * @internal + * CAS operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_CAS_OP(ret_ptr, old_val, new_val) \ +({ \ + if (atom->v == (old_val)) { \ + atom->v = (new_val); \ + *(ret_ptr) = 1; \ + } else { \ + *(ret_ptr) = 0; \ + } \ +}) + /** * @internal * Helper macro for lock-based atomic operations on 64-bit integers @@ -52,14 +67,14 @@ struct odp_atomic_u32_s { */ #define ATOMIC_OP(atom, expr) \ ({ \ - uint64_t old_val; \ + uint64_t _old_val; \ /* Loop while lock is already taken, stop when lock becomes clear */ \ while (__atomic_test_and_set(&(atom)->lock, __ATOMIC_ACQUIRE)) \ (void)0; \ - old_val = (atom)->v; \ + _old_val = (atom)->v; \ (expr); /* Perform whatever update is desired */ \ __atomic_clear(&(atom)->lock, __ATOMIC_RELEASE); \ - old_val; /* Return old value */ \ + _old_val; /* Return old value */ \ }) #endif diff --git a/platform/linux-generic/include/odp/plat/byteorder_types.h b/platform/linux-generic/include/odp/plat/byteorder_types.h index cf917b1a2..0a8e4096e 100644 --- a/platform/linux-generic/include/odp/plat/byteorder_types.h +++ b/platform/linux-generic/include/odp/plat/byteorder_types.h @@ -67,17 +67,17 @@ extern "C" { #define ODP_BYTE_ORDER ODP_BIG_ENDIAN #endif -typedef uint16_t __odp_bitwise uint16le_t; -typedef uint16_t __odp_bitwise uint16be_t; +typedef uint16_t __odp_bitwise odp_u16le_t; +typedef uint16_t __odp_bitwise odp_u16be_t; -typedef uint32_t __odp_bitwise uint32le_t; -typedef uint32_t __odp_bitwise uint32be_t; +typedef uint32_t __odp_bitwise odp_u32le_t; +typedef uint32_t __odp_bitwise odp_u32be_t; -typedef uint64_t __odp_bitwise uint64le_t; -typedef uint64_t __odp_bitwise uint64be_t; +typedef uint64_t __odp_bitwise odp_u64le_t; +typedef uint64_t __odp_bitwise odp_u64be_t; -typedef uint16_t __odp_bitwise uint16sum_t; -typedef uint32_t __odp_bitwise uint32sum_t; +typedef uint16_t __odp_bitwise odp_u16sum_t; +typedef uint32_t __odp_bitwise odp_u32sum_t; /** * @} diff --git a/platform/linux-generic/include/odp/plat/init_types.h b/platform/linux-generic/include/odp/plat/init_types.h new file mode 100644 index 000000000..b240c93ca --- /dev/null +++ b/platform/linux-generic/include/odp/plat/init_types.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP initialization. + */ + +#ifndef ODP_INIT_TYPES_H_ +#define ODP_INIT_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @internal platform specific data + */ +typedef struct odp_platform_init_t { +} odp_platform_init_t; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/plat/packet_io_types.h b/platform/linux-generic/include/odp/plat/packet_io_types.h index 3cc64c6d2..934d7de97 100644 --- a/platform/linux-generic/include/odp/plat/packet_io_types.h +++ b/platform/linux-generic/include/odp/plat/packet_io_types.h @@ -21,16 +21,26 @@ extern "C" { #include <odp/std_types.h> #include <odp/plat/strong_types.h> -/** @addtogroup odp_packet_io ODP PACKET IO +/** @addtogroup odp_packet_io * Operations on a packet. * @{ */ typedef ODP_HANDLE_T(odp_pktio_t); -#define ODP_PKTIO_INVALID _odp_cast_scalar(odp_pktio_t, 0) +/** @internal */ +typedef struct odp_pktin_queue_t { + odp_pktio_t pktio; /**< @internal pktio handle */ + int index; /**< @internal pktio queue index */ +} odp_pktin_queue_t; + +/** @internal */ +typedef struct odp_pktout_queue_t { + odp_pktio_t pktio; /**< @internal pktio handle */ + int index; /**< @internal pktio queue index */ +} odp_pktout_queue_t; -#define ODP_PKTIO_ANY _odp_cast_scalar(odp_pktio_t, ~0) +#define ODP_PKTIO_INVALID _odp_cast_scalar(odp_pktio_t, 0) #define ODP_PKTIO_MACADDR_MAXSIZE 16 diff --git a/platform/linux-generic/include/odp/plat/queue_types.h b/platform/linux-generic/include/odp/plat/queue_types.h index a7df15576..40a53e5e6 100644 --- a/platform/linux-generic/include/odp/plat/queue_types.h +++ b/platform/linux-generic/include/odp/plat/queue_types.h @@ -33,14 +33,6 @@ typedef ODP_HANDLE_T(odp_queue_group_t); #define ODP_QUEUE_NAME_LEN 32 - -typedef int odp_queue_type_t; - -#define ODP_QUEUE_TYPE_SCHED 0 -#define ODP_QUEUE_TYPE_POLL 1 -#define ODP_QUEUE_TYPE_PKTIN 2 -#define ODP_QUEUE_TYPE_PKTOUT 3 - /** Get printable format of odp_queue_t */ static inline uint64_t odp_queue_to_u64(odp_queue_t hdl) { diff --git a/platform/linux-generic/include/odp/plat/schedule_types.h b/platform/linux-generic/include/odp/plat/schedule_types.h index 21fcbb84c..a4a352c04 100644 --- a/platform/linux-generic/include/odp/plat/schedule_types.h +++ b/platform/linux-generic/include/odp/plat/schedule_types.h @@ -37,7 +37,7 @@ typedef int odp_schedule_prio_t; typedef int odp_schedule_sync_t; -#define ODP_SCHED_SYNC_NONE 0 +#define ODP_SCHED_SYNC_PARALLEL 0 #define ODP_SCHED_SYNC_ATOMIC 1 #define ODP_SCHED_SYNC_ORDERED 2 diff --git a/platform/linux-generic/include/odp/std_clib.h b/platform/linux-generic/include/odp/std_clib.h index c939c48e9..11c59bec2 100644 --- a/platform/linux-generic/include/odp/std_clib.h +++ b/platform/linux-generic/include/odp/std_clib.h @@ -23,6 +23,11 @@ static inline void *odp_memset(void *ptr, int value, size_t num) return memset(ptr, value, num); } +static inline int odp_memcmp(const void *ptr1, const void *ptr2, size_t num) +{ + return memcmp(ptr1, ptr2, num); +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index ce62368e1..ff3813f73 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -64,12 +64,6 @@ typedef enum { _ODP_MEMMODEL_SC = __ATOMIC_SEQ_CST } _odp_memmodel_t; -/** - * Insert a full memory barrier (fence) in the compiler and instruction - * sequence. - */ -#define _ODP_FULL_BARRIER() __atomic_thread_fence(__ATOMIC_SEQ_CST) - /***************************************************************************** * Operations on 32-bit atomics * _odp_atomic_u32_load_mm - return current value diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h index 5b6520266..27d8a526c 100644 --- a/platform/linux-generic/include/odp_classification_datamodel.h +++ b/platform/linux-generic/include/odp_classification_datamodel.h @@ -54,7 +54,7 @@ Stores the Term and Value mapping for a PMR. The maximum size of value currently supported in 64 bits **/ typedef struct pmr_term_value { - odp_pmr_term_e term; /* PMR Term */ + odp_pmr_term_t term; /* PMR Term */ uint64_t val; /**< Value to be matched */ uint64_t mask; /**< Masked set of bits to be matched */ uint32_t offset; /**< Offset if term == ODP_PMR_CUSTOM_FRAME */ diff --git a/platform/linux-generic/include/odp_classification_inlines.h b/platform/linux-generic/include/odp_classification_inlines.h index 5f0b564f7..96cf77ee2 100644 --- a/platform/linux-generic/include/odp_classification_inlines.h +++ b/platform/linux-generic/include/odp_classification_inlines.h @@ -154,11 +154,28 @@ static inline int verify_pmr_udp_sport(const uint8_t *pkt_addr, return 0; } -static inline int verify_pmr_dmac(const uint8_t *pkt_addr ODP_UNUSED, - odp_packet_hdr_t *pkt_hdr ODP_UNUSED, - pmr_term_value_t *term_value ODP_UNUSED) +static inline int verify_pmr_dmac(const uint8_t *pkt_addr, + odp_packet_hdr_t *pkt_hdr, + pmr_term_value_t *term_value) { - ODP_UNIMPLEMENTED(); + uint64_t dmac = 0; + uint64_t dmac_be = 0; + const odph_ethhdr_t *eth; + + if (!pkt_hdr->input_flags.eth) + return 0; + + eth = (const odph_ethhdr_t *)(pkt_addr + pkt_hdr->l2_offset); + memcpy(&dmac_be, eth->dst.addr, ODPH_ETHADDR_LEN); + dmac = odp_be_to_cpu_64(dmac_be); + /* since we are converting a 48 bit ethernet address from BE to cpu + format using odp_be_to_cpu_64() the last 16 bits needs to be right + shifted */ + if (dmac_be != dmac) + dmac = dmac >> (64 - (ODPH_ETHADDR_LEN * 8)); + + if (term_value->val == (dmac & term_value->mask)) + return 1; return 0; } diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index b22f95698..e75154a51 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -20,16 +20,20 @@ extern "C" { #include <odp/init.h> #include <odp/thread.h> +#include <stdio.h> extern __thread int __odp_errno; +#define MAX_CPU_NUMBER 128 + typedef struct { - uint64_t cpu_hz; + uint64_t cpu_hz_max[MAX_CPU_NUMBER]; uint64_t huge_page_size; uint64_t page_size; int cache_line_size; int cpu_count; - char model_str[128]; + char cpu_arch_str[128]; + char model_str[MAX_CPU_NUMBER][128]; } odp_system_info_t; struct odp_global_data_s { @@ -103,6 +107,9 @@ int odp_time_term_global(void); void _odp_flush_caches(void); +int odp_cpuinfo_parser(FILE *file, odp_system_info_t *sysinfo); +uint64_t odp_cpu_hz_current(int id); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 3ab0bc86f..f871f0a76 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -20,9 +20,6 @@ extern "C" { #include <odp/spinlock.h> #include <odp/ticketlock.h> -#include <odp_packet_socket.h> -#include <odp_packet_netmap.h> -#include <odp_packet_tap.h> #include <odp_classification_datamodel.h> #include <odp_align_internal.h> #include <odp_debug_internal.h> @@ -31,8 +28,16 @@ extern "C" { #include <odp/hints.h> #include <net/if.h> +#define PKTIO_MAX_QUEUES 64 +#include <odp_packet_socket.h> +#include <odp_packet_netmap.h> +#include <odp_packet_tap.h> + #define PKTIO_NAME_LEN 256 +#define PKTIN_INVALID ((odp_pktin_queue_t) {ODP_PKTIO_INVALID, 0}) +#define PKTOUT_INVALID ((odp_pktout_queue_t) {ODP_PKTIO_INVALID, 0}) + /** Determine if a socket read/write error should be reported. Transient errors * that simply require the caller to retry are ignored, the _send/_recv APIs * are non-blocking and it is the caller's responsibility to retry if the @@ -70,7 +75,6 @@ struct pktio_entry { int taken; /**< is entry taken(1) or free(0) */ int cls_enabled; /**< is classifier enabled */ odp_pktio_t handle; /**< pktio handle */ - odp_queue_t inq_default; /**< default input queue, if set */ odp_queue_t outq_default; /**< default out queue */ union { pkt_loop_t pkt_loop; /**< Using loopback for IO */ @@ -88,10 +92,30 @@ struct pktio_entry { STATE_STOP } state; classifier_t cls; /**< classifier linked with this pktio*/ + odp_pktio_stats_t stats; /**< statistic counters for pktio */ + enum { + STATS_SYSFS = 0, + STATS_ETHTOOL, + STATS_UNSUPPORTED + } stats_type; char name[PKTIO_NAME_LEN]; /**< name of pktio provided to pktio_open() */ odp_pktio_t id; odp_pktio_param_t param; + + /* Storage for queue handles + * Multi-queue support is pktio driver specific */ + unsigned num_in_queue; + unsigned num_out_queue; + + struct { + odp_queue_t queue; + odp_pktin_queue_t pktin; + } in_queue[PKTIO_MAX_QUEUES]; + + struct { + odp_pktout_queue_t pktout; + } out_queue[PKTIO_MAX_QUEUES]; }; typedef union { @@ -107,6 +131,7 @@ typedef struct { int is_free(pktio_entry_t *entry); typedef struct pktio_if_ops { + const char *name; int (*init)(void); int (*term)(void); int (*open)(odp_pktio_t pktio, pktio_entry_t *pktio_entry, @@ -114,6 +139,8 @@ typedef struct pktio_if_ops { int (*close)(pktio_entry_t *pktio_entry); int (*start)(pktio_entry_t *pktio_entry); int (*stop)(pktio_entry_t *pktio_entry); + int (*stats)(pktio_entry_t *pktio_entry, odp_pktio_stats_t *stats); + int (*stats_reset)(pktio_entry_t *pktio_entry); int (*recv)(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], unsigned len); int (*send)(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], @@ -122,6 +149,22 @@ typedef struct pktio_if_ops { int (*promisc_mode_set)(pktio_entry_t *pktio_entry, int enable); int (*promisc_mode_get)(pktio_entry_t *pktio_entry); int (*mac_get)(pktio_entry_t *pktio_entry, void *mac_addr); + int (*link_status)(pktio_entry_t *pktio_entry); + int (*capability)(pktio_entry_t *pktio_entry, + odp_pktio_capability_t *capa); + int (*input_queues_config)(pktio_entry_t *pktio_entry, + const odp_pktin_queue_param_t *param); + int (*output_queues_config)(pktio_entry_t *pktio_entry, + const odp_pktout_queue_param_t *p); + int (*in_queues)(pktio_entry_t *entry, odp_queue_t queues[], int num); + int (*pktin_queues)(pktio_entry_t *entry, odp_pktin_queue_t queues[], + int num); + int (*pktout_queues)(pktio_entry_t *entry, odp_pktout_queue_t queues[], + int num); + int (*recv_queue)(pktio_entry_t *entry, int index, + odp_packet_t packets[], int num); + int (*send_queue)(pktio_entry_t *entry, int index, + odp_packet_t packets[], int num); } pktio_if_ops_t; int _odp_packet_cls_enq(pktio_entry_t *pktio_entry, const uint8_t *base, @@ -158,7 +201,25 @@ static inline void pktio_cls_enabled_set(pktio_entry_t *entry, int ena) entry->s.cls_enabled = ena; } -int pktin_poll(pktio_entry_t *entry); +int pktin_poll(pktio_entry_t *entry, int num_queue, int index[]); + +/* + * Dummy single queue implementations of multi-queue API + */ +int single_capability(odp_pktio_capability_t *capa); +int single_input_queues_config(pktio_entry_t *entry, + const odp_pktin_queue_param_t *param); +int single_output_queues_config(pktio_entry_t *entry, + const odp_pktout_queue_param_t *param); +int single_in_queues(pktio_entry_t *entry, odp_queue_t queues[], int num); +int single_pktin_queues(pktio_entry_t *entry, odp_pktin_queue_t queues[], + int num); +int single_pktout_queues(pktio_entry_t *entry, odp_pktout_queue_t queues[], + int num); +int single_recv_queue(pktio_entry_t *entry, int index, odp_packet_t packets[], + int num); +int single_send_queue(pktio_entry_t *entry, int index, odp_packet_t packets[], + int num); extern const pktio_if_ops_t netmap_pktio_ops; extern const pktio_if_ops_t sock_mmsg_pktio_ops; @@ -170,6 +231,13 @@ extern const pktio_if_ops_t pcap_pktio_ops; extern const pktio_if_ops_t tap_pktio_ops; extern const pktio_if_ops_t * const pktio_if_ops[]; +int sysfs_stats(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats); +int sock_stats_fd(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats, + int fd); +int sock_stats_reset_fd(pktio_entry_t *pktio_entry, int fd); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_netmap.h b/platform/linux-generic/include/odp_packet_netmap.h index 0577dfe20..26a8da1eb 100644 --- a/platform/linux-generic/include/odp_packet_netmap.h +++ b/platform/linux-generic/include/odp_packet_netmap.h @@ -7,19 +7,55 @@ #ifndef ODP_PACKET_NETMAP_H #define ODP_PACKET_NETMAP_H +#include <odp/align.h> +#include <odp/debug.h> +#include <odp/packet_io.h> #include <odp/pool.h> +#include <odp/ticketlock.h> +#include <odp_align_internal.h> #include <linux/if_ether.h> +#include <net/if.h> + +#define NM_MAX_DESC 32 + +/** Ring for mapping pktin/pktout queues to netmap descriptors */ +struct netmap_ring_t { + unsigned first; /**< Index of first netmap descriptor */ + unsigned last; /**< Index of last netmap descriptor */ + unsigned num; /**< Number of netmap descriptors */ + /** Netmap metadata for the device */ + struct nm_desc *desc[NM_MAX_DESC]; + unsigned cur; /**< Index of current netmap descriptor */ + odp_ticketlock_t lock; /**< Queue lock */ +}; + +typedef union { + struct netmap_ring_t s; + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct netmap_ring_t))]; +} netmap_ring_t ODP_ALIGNED_CACHE; /** Packet socket using netmap mmaped rings for both Rx and Tx */ typedef struct { odp_pool_t pool; /**< pool to alloc packets from */ size_t max_frame_len; /**< buf_size - sizeof(pkt_hdr) */ - struct nm_desc *rx_desc; /**< netmap meta-data for the device */ - struct nm_desc *tx_desc; /**< netmap meta-data for the device */ uint32_t if_flags; /**< interface flags */ + uint32_t mtu; /**< maximum transmission unit */ int sockfd; /**< control socket */ unsigned char if_mac[ETH_ALEN]; /**< eth mac address */ + char nm_name[IF_NAMESIZE + 7]; /**< netmap:<ifname> */ + odp_pktio_capability_t capa; /**< interface capabilities */ + unsigned cur_rx_queue; /**< current pktin queue */ + uint32_t num_rx_rings; /**< number of nm rx rings */ + uint32_t num_tx_rings; /**< number of nm tx rings */ + unsigned num_rx_desc_rings; /**< number of rx descriptor rings */ + unsigned num_tx_desc_rings; /**< number of tx descriptor rings */ + odp_bool_t lockless_rx; /**< no locking for rx */ + odp_bool_t lockless_tx; /**< no locking for tx */ + /** mapping of pktin queues to netmap rx descriptors */ + netmap_ring_t rx_desc_ring[PKTIO_MAX_QUEUES]; + /** mapping of pktout queues to netmap tx descriptors */ + netmap_ring_t tx_desc_ring[PKTIO_MAX_QUEUES]; } pkt_netmap_t; #endif diff --git a/platform/linux-generic/include/odp_packet_socket.h b/platform/linux-generic/include/odp_packet_socket.h index 1eaafb7e5..a7797d1c0 100644 --- a/platform/linux-generic/include/odp_packet_socket.h +++ b/platform/linux-generic/include/odp_packet_socket.h @@ -18,6 +18,7 @@ #include <odp/debug.h> #include <odp/pool.h> #include <odp/packet.h> +#include <odp/packet_io.h> #include <linux/version.h> @@ -116,4 +117,60 @@ int promisc_mode_set_fd(int fd, const char *name, int enable); */ int promisc_mode_get_fd(int fd, const char *name); +/** + * Return link status of a packet socket (up/down) + */ +int link_status_fd(int fd, const char *name); + +/** + * Get enabled RSS hash protocols of a packet socket + * + * @param fd Socket file descriptor + * @param name Interface name + * @param hash_proto[out] Hash protocols + * + * @returns Number enabled hash protocols + */ +int rss_conf_get_fd(int fd, const char *name, + odp_pktin_hash_proto_t *hash_proto); + +/** + * Get supported RSS hash protocols of a packet socket + * + * Can be both read and modified. + * + * @param fd Socket file descriptor + * @param name Interface name + * @param hash_proto[out] Hash protocols + * + * @returns Number of supported hash protocols + */ +int rss_conf_get_supported_fd(int fd, const char *name, + odp_pktin_hash_proto_t *hash_proto); + +/** + * Set RSS hash protocols of a packet socket + * + * @param fd Socket file descriptor + * @param name Interface name + * @param hash_proto Hash protocols + * + * @retval 0 on success + * @retval <0 on failure + */ +int rss_conf_set_fd(int fd, const char *name, + const odp_pktin_hash_proto_t *proto); + +/** + * Print enabled RSS hash protocols + * + * @param hash_proto Hash protocols + */ +void rss_conf_print(const odp_pktin_hash_proto_t *hash_proto); + +/** + * Get ethtool statistics of a packet socket + */ +int ethtool_stats_get_fd(int fd, const char *name, odp_pktio_stats_t *stats); + #endif diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index ae4836cc8..fdac6067e 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -28,7 +28,6 @@ extern "C" { #include <odp/debug.h> #include <odp/shared_memory.h> #include <odp/atomic.h> -#include <odp_atomic_internal.h> #include <odp/thread.h> #include <string.h> diff --git a/platform/linux-generic/include/odp_queue_internal.h b/platform/linux-generic/include/odp_queue_internal.h index 1cc0ed26e..2d349466f 100644 --- a/platform/linux-generic/include/odp_queue_internal.h +++ b/platform/linux-generic/include/odp_queue_internal.h @@ -76,7 +76,7 @@ struct queue_entry_s { odp_event_t cmd_ev; odp_queue_type_t type; odp_queue_param_t param; - odp_pktio_t pktin; + odp_pktin_queue_t pktin; odp_pktio_t pktout; char name[ODP_QUEUE_NAME_LEN]; uint64_t order_in; diff --git a/platform/linux-generic/include/odp_schedule_internal.h b/platform/linux-generic/include/odp_schedule_internal.h index 6b301cd18..08683941a 100644 --- a/platform/linux-generic/include/odp_schedule_internal.h +++ b/platform/linux-generic/include/odp_schedule_internal.h @@ -23,7 +23,8 @@ extern "C" { int schedule_queue_init(queue_entry_t *qe); void schedule_queue_destroy(queue_entry_t *qe); int schedule_queue(const queue_entry_t *qe); -int schedule_pktio_start(odp_pktio_t pktio, int prio); +void schedule_pktio_start(odp_pktio_t pktio, int num_in_queue, + int in_queue_idx[]); void odp_schedule_release_context(void); #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp_spin_internal.h b/platform/linux-generic/include/odp_spin_internal.h deleted file mode 100644 index 29c524fb1..000000000 --- a/platform/linux-generic/include/odp_spin_internal.h +++ /dev/null @@ -1,58 +0,0 @@ -/* Copyright (c) 2013, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - - - -#ifndef ODP_SPIN_INTERNAL_H_ -#define ODP_SPIN_INTERNAL_H_ - -#ifdef __cplusplus -extern "C" { -#endif - - -/** - * Spin loop for ODP internal use - */ -static inline void odp_spin(void) -{ -#if defined __x86_64__ || defined __i386__ - -#ifdef __SSE2__ - __asm__ __volatile__ ("pause"); -#else - __asm__ __volatile__ ("rep; nop"); -#endif - -#elif defined __arm__ - -#if __ARM_ARCH == 7 - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); -#endif - -#elif defined __OCTEON__ - - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - -#endif -} - - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/odp_atomic.c b/platform/linux-generic/odp_atomic.c new file mode 100644 index 000000000..5b71ecff3 --- /dev/null +++ b/platform/linux-generic/odp_atomic.c @@ -0,0 +1,26 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/atomic.h> + +int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +#else + /* All operations are lock-free */ + if (atomic_op) { + atomic_op->all_bits = ~((uint32_t)0); + atomic_op->op.init = 0; + } + + return 2; +#endif +} diff --git a/platform/linux-generic/odp_barrier.c b/platform/linux-generic/odp_barrier.c index 53d83c0dd..f3525e209 100644 --- a/platform/linux-generic/odp_barrier.c +++ b/platform/linux-generic/odp_barrier.c @@ -6,8 +6,8 @@ #include <odp/barrier.h> #include <odp/sync.h> -#include <odp_spin_internal.h> -#include <odp_atomic_internal.h> +#include <odp/cpu.h> +#include <odp/atomic.h> void odp_barrier_init(odp_barrier_t *barrier, int count) { @@ -27,13 +27,13 @@ void odp_barrier_init(odp_barrier_t *barrier, int count) * the cycle the barrier was in upon entry. Exit is when the * barrier crosses to the other half of the cycle. */ - void odp_barrier_wait(odp_barrier_t *barrier) { uint32_t count; int wasless; - _ODP_FULL_BARRIER(); + odp_mb_full(); + count = odp_atomic_fetch_inc_u32(&barrier->bar); wasless = count < barrier->count; @@ -43,8 +43,8 @@ void odp_barrier_wait(odp_barrier_t *barrier) } else { while ((odp_atomic_load_u32(&barrier->bar) < barrier->count) == wasless) - odp_spin(); + odp_cpu_pause(); } - _ODP_FULL_BARRIER(); + odp_mb_full(); } diff --git a/platform/linux-generic/odp_cpumask_task.c b/platform/linux-generic/odp_cpumask_task.c index 41f2bc949..c5093e051 100644 --- a/platform/linux-generic/odp_cpumask_task.c +++ b/platform/linux-generic/odp_cpumask_task.c @@ -53,3 +53,14 @@ int odp_cpumask_default_control(odp_cpumask_t *mask, int num ODP_UNUSED) odp_cpumask_set(mask, 0); return 1; } + +int odp_cpumask_all_available(odp_cpumask_t *mask) +{ + odp_cpumask_t mask_work, mask_ctrl; + + odp_cpumask_default_worker(&mask_work, 0); + odp_cpumask_default_control(&mask_ctrl, 0); + odp_cpumask_or(mask, &mask_work, &mask_ctrl); + + return odp_cpumask_count(mask); +} diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 540cdb8d2..a3ceec820 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -53,6 +53,8 @@ int odp_pktio_init_global(void) odp_spinlock_init(&pktio_tbl->lock); for (id = 1; id <= ODP_CONFIG_PKTIO_ENTRIES; ++id) { + odp_queue_param_t param; + pktio_entry = &pktio_tbl->entries[id - 1]; odp_ticketlock_init(&pktio_entry->s.rxl); @@ -66,7 +68,10 @@ int odp_pktio_init_global(void) snprintf(name, sizeof(name), "%i-pktio_outq_default", (int)id); name[ODP_QUEUE_NAME_LEN-1] = '\0'; - qid = odp_queue_create(name, ODP_QUEUE_TYPE_PKTOUT, NULL); + odp_queue_param_init(¶m); + param.type = ODP_QUEUE_TYPE_PKTOUT; + + qid = odp_queue_create(name, ¶m); if (qid == ODP_QUEUE_INVALID) return -1; pktio_entry->s.outq_default = qid; @@ -133,9 +138,16 @@ static void unlock_entry_classifier(pktio_entry_t *entry) static void init_pktio_entry(pktio_entry_t *entry) { + int i; + set_taken(entry); pktio_cls_enabled_set(entry, 0); - entry->s.inq_default = ODP_QUEUE_INVALID; + + for (i = 0; i < PKTIO_MAX_QUEUES; i++) { + entry->s.in_queue[i].queue = ODP_QUEUE_INVALID; + entry->s.in_queue[i].pktin = PKTIN_INVALID; + entry->s.out_queue[i].pktout = PKTOUT_INVALID; + } pktio_classifier_init(entry); } @@ -208,6 +220,8 @@ static odp_pktio_t setup_pktio_entry(const char *dev, odp_pool_t pool, if (!ret) { pktio_entry->s.ops = pktio_if_ops[pktio_if]; + ODP_DBG("%s uses %s\n", + dev, pktio_if_ops[pktio_if]->name); break; } } @@ -275,6 +289,18 @@ static int _pktio_close(pktio_entry_t *entry) return 0; } +static void destroy_in_queues(pktio_entry_t *entry, int num) +{ + int i; + + for (i = 0; i < num; i++) { + if (entry->s.in_queue[i].queue != ODP_QUEUE_INVALID) { + odp_queue_destroy(entry->s.in_queue[i].queue); + entry->s.in_queue[i].queue = ODP_QUEUE_INVALID; + } + } +} + int odp_pktio_close(odp_pktio_t id) { pktio_entry_t *entry; @@ -285,6 +311,9 @@ int odp_pktio_close(odp_pktio_t id) return -1; lock_entry(entry); + + destroy_in_queues(entry, entry->s.num_in_queue); + if (!is_free(entry)) { res = _pktio_close(entry); if (res) @@ -298,6 +327,7 @@ int odp_pktio_close(odp_pktio_t id) int odp_pktio_start(odp_pktio_t id) { pktio_entry_t *entry; + odp_pktin_mode_t mode; int res = 0; entry = get_pktio_entry(id); @@ -313,8 +343,26 @@ int odp_pktio_start(odp_pktio_t id) res = entry->s.ops->start(entry); if (!res) entry->s.state = STATE_START; + unlock_entry(entry); + mode = entry->s.param.in_mode; + + if (mode == ODP_PKTIN_MODE_SCHED) { + unsigned i; + + for (i = 0; i < entry->s.num_in_queue; i++) { + int index = i; + + if (entry->s.in_queue[i].queue == ODP_QUEUE_INVALID) { + ODP_ERR("No input queue\n"); + return -1; + } + + schedule_pktio_start(id, 1, &index); + } + } + return res; } @@ -379,8 +427,6 @@ odp_pktio_t odp_pktio_lookup(const char *dev) return id; } - - int odp_pktio_recv(odp_pktio_t id, odp_packet_t pkt_table[], int len) { pktio_entry_t *pktio_entry = get_pktio_entry(id); @@ -448,30 +494,19 @@ int odp_pktio_inq_setdef(odp_pktio_t id, odp_queue_t queue) unlock_entry(pktio_entry); return -1; } - pktio_entry->s.inq_default = queue; + + /* Temporary support for default input queue */ + pktio_entry->s.in_queue[0].queue = queue; + pktio_entry->s.in_queue[0].pktin.pktio = id; + pktio_entry->s.in_queue[0].pktin.index = 0; + pktio_entry->s.num_in_queue = 1; unlock_entry(pktio_entry); - switch (qentry->s.type) { - /* Change to ODP_QUEUE_TYPE_POLL when ODP_QUEUE_TYPE_PKTIN is removed */ - case ODP_QUEUE_TYPE_PKTIN: - /* User polls the input queue */ - queue_lock(qentry); - qentry->s.pktin = id; - queue_unlock(qentry); - - /* Uncomment when ODP_QUEUE_TYPE_PKTIN is removed - break; - case ODP_QUEUE_TYPE_SCHED: - */ - /* Packet input through the scheduler */ - if (schedule_pktio_start(id, ODP_SCHED_PRIO_LOWEST)) { - ODP_ERR("Schedule pktio start failed\n"); - return -1; - } - break; - default: - ODP_ABORT("Bad queue type\n"); - } + /* User polls the input queue */ + queue_lock(qentry); + qentry->s.pktin.pktio = id; + qentry->s.pktin.index = 0; + queue_unlock(qentry); return 0; } @@ -490,14 +525,19 @@ int odp_pktio_inq_remdef(odp_pktio_t id) unlock_entry(pktio_entry); return -1; } - queue = pktio_entry->s.inq_default; + + /* Temporary support for default input queue */ + queue = pktio_entry->s.in_queue[0].queue; qentry = queue_to_qentry(queue); queue_lock(qentry); - qentry->s.pktin = ODP_PKTIO_INVALID; + qentry->s.pktin = PKTIN_INVALID; queue_unlock(qentry); - pktio_entry->s.inq_default = ODP_QUEUE_INVALID; + pktio_entry->s.in_queue[0].queue = ODP_QUEUE_INVALID; + pktio_entry->s.in_queue[0].pktin.pktio = ODP_PKTIO_INVALID; + pktio_entry->s.in_queue[0].pktin.index = 0; + pktio_entry->s.num_in_queue = 0; unlock_entry(pktio_entry); return 0; @@ -510,7 +550,8 @@ odp_queue_t odp_pktio_inq_getdef(odp_pktio_t id) if (pktio_entry == NULL) return ODP_QUEUE_INVALID; - return pktio_entry->s.inq_default; + /* Temporary support for default input queue */ + return pktio_entry->s.in_queue[0].queue; } odp_queue_t odp_pktio_outq_getdef(odp_pktio_t id) @@ -575,15 +616,13 @@ odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *qentry) odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; int pkts, i; - odp_pktio_t pktio; buf_hdr = queue_deq(qentry); if (buf_hdr != NULL) return buf_hdr; - pktio = qentry->s.pktin; + pkts = odp_pktio_recv_queue(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX); - pkts = odp_pktio_recv(pktio, pkt_tbl, QUEUE_MULTI_MAX); if (pkts <= 0) return NULL; @@ -613,7 +652,6 @@ int pktin_deq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int num) odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; odp_buffer_t buf; int pkts, i, j; - odp_pktio_t pktio; nbr = queue_deq_multi(qentry, buf_hdr, num); if (odp_unlikely(nbr > num)) @@ -626,9 +664,7 @@ int pktin_deq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int num) if (nbr == num) return nbr; - pktio = qentry->s.pktin; - - pkts = odp_pktio_recv(pktio, pkt_tbl, QUEUE_MULTI_MAX); + pkts = odp_pktio_recv_queue(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX); if (pkts <= 0) return nbr; @@ -648,43 +684,47 @@ int pktin_deq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int num) return nbr; } -int pktin_poll(pktio_entry_t *entry) +int pktin_poll(pktio_entry_t *entry, int num_queue, int index[]) { odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; - int num, i; + int num, i, idx; odp_buffer_t buf; - odp_pktio_t pktio; - pktio = entry->s.handle; - - if (odp_unlikely(is_free(entry))) + if (odp_unlikely(is_free(entry))) { + ODP_ERR("Bad pktio entry\n"); return -1; + } - if (odp_unlikely(entry->s.inq_default == ODP_QUEUE_INVALID)) + /* Temporarely needed for odp_pktio_inq_remdef() */ + if (odp_unlikely(entry->s.num_in_queue == 0)) return -1; if (entry->s.state == STATE_STOP) return 0; - num = odp_pktio_recv(pktio, pkt_tbl, QUEUE_MULTI_MAX); + for (idx = 0; idx < num_queue; idx++) { + queue_entry_t *qentry; + odp_queue_t queue; + odp_pktin_queue_t pktin = entry->s.in_queue[index[idx]].pktin; - if (num == 0) - return 0; + num = odp_pktio_recv_queue(pktin, pkt_tbl, QUEUE_MULTI_MAX); - if (num < 0) { - ODP_ERR("Packet recv error\n"); - return -1; - } + if (num == 0) + continue; - for (i = 0; i < num; i++) { - buf = _odp_packet_to_buffer(pkt_tbl[i]); - hdr_tbl[i] = odp_buf_to_hdr(buf); - } + if (num < 0) { + ODP_ERR("Packet recv error\n"); + return -1; + } - if (num) { - queue_entry_t *qentry; - qentry = queue_to_qentry(entry->s.inq_default); + for (i = 0; i < num; i++) { + buf = _odp_packet_to_buffer(pkt_tbl[i]); + hdr_tbl[i] = odp_buf_to_hdr(buf); + } + + queue = entry->s.in_queue[index[idx]].queue; + qentry = queue_to_qentry(queue); queue_enq_multi(qentry, hdr_tbl, num, 0); } @@ -800,11 +840,91 @@ int odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr, int addr_size) return ret; } +int odp_pktio_link_status(odp_pktio_t id) +{ + pktio_entry_t *entry; + int ret = -1; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + if (entry->s.ops->link_status) + ret = entry->s.ops->link_status(entry); + unlock_entry(entry); + + return ret; +} + void odp_pktio_param_init(odp_pktio_param_t *params) { memset(params, 0, sizeof(odp_pktio_param_t)); } +void odp_pktin_queue_param_init(odp_pktin_queue_param_t *param) +{ + memset(param, 0, sizeof(odp_pktin_queue_param_t)); + param->op_mode = ODP_PKTIO_OP_MT; +} + +void odp_pktout_queue_param_init(odp_pktout_queue_param_t *param) +{ + memset(param, 0, sizeof(odp_pktout_queue_param_t)); + param->op_mode = ODP_PKTIO_OP_MT; +} + +void odp_pktio_print(odp_pktio_t id) +{ + pktio_entry_t *entry; + uint8_t addr[ETH_ALEN]; + int max_len = 512; + char str[max_len]; + int len = 0; + int n = max_len - 1; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return; + } + + len += snprintf(&str[len], n - len, + "pktio\n"); + len += snprintf(&str[len], n - len, + " handle %" PRIu64 "\n", odp_pktio_to_u64(id)); + len += snprintf(&str[len], n - len, + " name %s\n", entry->s.name); + len += snprintf(&str[len], n - len, + " type %s\n", entry->s.ops->name); + len += snprintf(&str[len], n - len, + " state %s\n", + entry->s.state == STATE_START ? "start" : + (entry->s.state == STATE_STOP ? "stop" : "unknown")); + memset(addr, 0, sizeof(addr)); + odp_pktio_mac_addr(id, addr, ETH_ALEN); + len += snprintf(&str[len], n - len, + " mac %02x:%02x:%02x:%02x:%02x:%02x\n", + addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); + len += snprintf(&str[len], n - len, + " mtu %d\n", odp_pktio_mtu(id)); + len += snprintf(&str[len], n - len, + " promisc %s\n", + odp_pktio_promisc_mode(id) ? "yes" : "no"); + str[len] = '\0'; + + ODP_PRINT("\n%s\n", str); +} + int odp_pktio_term_global(void) { int ret; @@ -852,42 +972,391 @@ int odp_pktio_term_global(void) return ret; } -void odp_pktio_print(odp_pktio_t id) +int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) { pktio_entry_t *entry; - uint8_t addr[ETH_ALEN]; - int max_len = 512; - char str[max_len]; - int len = 0; - int n = max_len - 1; - entry = get_pktio_entry(id); + entry = get_pktio_entry(pktio); if (entry == NULL) { - ODP_DBG("pktio entry %d does not exist\n", id); - return; + ODP_DBG("pktio entry %d does not exist\n", pktio); + return -1; } - len += snprintf(&str[len], n - len, - "pktio\n"); - len += snprintf(&str[len], n - len, - " handle %" PRIu64 "\n", odp_pktio_to_u64(id)); - len += snprintf(&str[len], n - len, - " name %s\n", entry->s.name); - len += snprintf(&str[len], n - len, - " state %s\n", - entry->s.state == STATE_START ? "start" : - (entry->s.state == STATE_STOP ? "stop" : "unknown")); - memset(addr, 0, sizeof(addr)); - odp_pktio_mac_addr(id, addr, ETH_ALEN); - len += snprintf(&str[len], n - len, - " mac %02x:%02x:%02x:%02x:%02x:%02x\n", - addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); - len += snprintf(&str[len], n - len, - " mtu %d\n", odp_pktio_mtu(id)); - len += snprintf(&str[len], n - len, - " promisc %s\n", - odp_pktio_promisc_mode(id) ? "yes" : "no"); - str[len] = '\0'; + if (entry->s.ops->capability) + return entry->s.ops->capability(entry, capa); - ODP_PRINT("\n%s\n", str); + return single_capability(capa); +} + +int odp_pktio_stats(odp_pktio_t pktio, + odp_pktio_stats_t *stats) +{ + pktio_entry_t *entry; + int ret = -1; + + entry = get_pktio_entry(pktio); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", pktio); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + if (entry->s.ops->stats) + ret = entry->s.ops->stats(entry, stats); + unlock_entry(entry); + + return ret; +} + +int odp_pktio_stats_reset(odp_pktio_t pktio) +{ + pktio_entry_t *entry; + int ret = -1; + + entry = get_pktio_entry(pktio); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", pktio); + return -1; + } + + lock_entry(entry); + + if (odp_unlikely(is_free(entry))) { + unlock_entry(entry); + ODP_DBG("already freed pktio\n"); + return -1; + } + + if (entry->s.ops->stats) + ret = entry->s.ops->stats_reset(entry); + unlock_entry(entry); + + return ret; +} + +int odp_pktin_queue_config(odp_pktio_t pktio, + const odp_pktin_queue_param_t *param) +{ + pktio_entry_t *entry; + odp_pktin_mode_t mode; + odp_pktio_capability_t capa; + unsigned num_queues; + unsigned i; + odp_queue_t queue; + + if (param == NULL) { + ODP_DBG("no parameters\n"); + return -1; + } + + entry = get_pktio_entry(pktio); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", pktio); + return -1; + } + + if (entry->s.state != STATE_STOP) { + ODP_DBG("pktio %s: not stopped\n", entry->s.name); + return -1; + } + + mode = entry->s.param.in_mode; + + if (mode == ODP_PKTIN_MODE_DISABLED) { + ODP_DBG("pktio %s: packet input is disabled\n", entry->s.name); + return -1; + } + + num_queues = param->num_queues; + + if (num_queues == 0) { + ODP_DBG("pktio %s: zero input queues\n", entry->s.name); + return -1; + } + + odp_pktio_capability(pktio, &capa); + + if (num_queues > capa.max_input_queues) { + ODP_DBG("pktio %s: too many input queues\n", entry->s.name); + return -1; + } + + /* If re-configuring, destroy old queues */ + if (entry->s.num_in_queue) + destroy_in_queues(entry, entry->s.num_in_queue); + + for (i = 0; i < num_queues; i++) { + if (mode == ODP_PKTIN_MODE_QUEUE || + mode == ODP_PKTIN_MODE_SCHED) { + odp_queue_param_t queue_param; + + memcpy(&queue_param, ¶m->queue_param, + sizeof(odp_queue_param_t)); + + queue_param.type = ODP_QUEUE_TYPE_PLAIN; + + if (mode == ODP_PKTIN_MODE_SCHED) + queue_param.type = ODP_QUEUE_TYPE_SCHED; + + queue = odp_queue_create("pktio_in", + &queue_param); + + if (queue == ODP_QUEUE_INVALID) { + destroy_in_queues(entry, i + 1); + return -1; + } + + if (mode == ODP_PKTIN_MODE_QUEUE) { + queue_entry_t *qentry; + + qentry = queue_to_qentry(queue); + qentry->s.pktin.index = i; + qentry->s.pktin.pktio = pktio; + + qentry->s.enqueue = pktin_enqueue; + qentry->s.dequeue = pktin_dequeue; + qentry->s.enqueue_multi = pktin_enq_multi; + qentry->s.dequeue_multi = pktin_deq_multi; + } + + entry->s.in_queue[i].queue = queue; + } else { + entry->s.in_queue[i].queue = ODP_QUEUE_INVALID; + } + + entry->s.in_queue[i].pktin.index = i; + entry->s.in_queue[i].pktin.pktio = entry->s.handle; + } + + entry->s.num_in_queue = num_queues; + + if (entry->s.ops->input_queues_config) + return entry->s.ops->input_queues_config(entry, param); + + return 0; +} + +int odp_pktout_queue_config(odp_pktio_t pktio, + const odp_pktout_queue_param_t *param) +{ + pktio_entry_t *entry; + odp_pktout_mode_t mode; + odp_pktio_capability_t capa; + unsigned num_queues; + unsigned i; + + if (param == NULL) { + ODP_DBG("no parameters\n"); + return -1; + } + + entry = get_pktio_entry(pktio); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", pktio); + return -1; + } + + if (entry->s.state != STATE_STOP) { + ODP_DBG("pktio %s: not stopped\n", entry->s.name); + return -1; + } + + mode = entry->s.param.out_mode; + + if (mode == ODP_PKTOUT_MODE_DISABLED) { + ODP_DBG("pktio %s: packet output is disabled\n", entry->s.name); + return -1; + } + + if (mode != ODP_PKTOUT_MODE_DIRECT) { + ODP_DBG("pktio %s: bad packet output mode\n", entry->s.name); + return -1; + } + + num_queues = param->num_queues; + + if (num_queues == 0) { + ODP_DBG("pktio %s: zero output queues\n", entry->s.name); + return -1; + } + + odp_pktio_capability(pktio, &capa); + + if (num_queues > capa.max_output_queues) { + ODP_DBG("pktio %s: too many output queues\n", entry->s.name); + return -1; + } + + for (i = 0; i < num_queues; i++) { + entry->s.out_queue[i].pktout.index = i; + entry->s.out_queue[i].pktout.pktio = entry->s.handle; + } + + entry->s.num_out_queue = num_queues; + + if (entry->s.ops->output_queues_config) + return entry->s.ops->output_queues_config(entry, param); + + return 0; +} + +int odp_pktin_event_queue(odp_pktio_t pktio, odp_queue_t queues[], int num) +{ + pktio_entry_t *entry; + odp_pktin_mode_t mode; + + entry = get_pktio_entry(pktio); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", pktio); + return -1; + } + + mode = entry->s.param.in_mode; + + if (mode != ODP_PKTIN_MODE_QUEUE && + mode != ODP_PKTIN_MODE_SCHED) + return -1; + + if (entry->s.ops->in_queues) + return entry->s.ops->in_queues(entry, queues, num); + + return single_in_queues(entry, queues, num); +} + +int odp_pktin_queue(odp_pktio_t pktio, odp_pktin_queue_t queues[], int num) +{ + pktio_entry_t *entry; + odp_pktin_mode_t mode; + + entry = get_pktio_entry(pktio); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", pktio); + return -1; + } + + mode = entry->s.param.in_mode; + + if (mode != ODP_PKTIN_MODE_DIRECT) + return -1; + + if (entry->s.ops->pktin_queues) + return entry->s.ops->pktin_queues(entry, queues, num); + + return single_pktin_queues(entry, queues, num); +} + +int odp_pktout_queue(odp_pktio_t pktio, odp_pktout_queue_t queues[], int num) +{ + pktio_entry_t *entry; + odp_pktout_mode_t mode; + + entry = get_pktio_entry(pktio); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", pktio); + return -1; + } + + mode = entry->s.param.out_mode; + + if (mode != ODP_PKTOUT_MODE_DIRECT) + return -1; + + if (entry->s.ops->pktout_queues) + return entry->s.ops->pktout_queues(entry, queues, num); + + return single_pktout_queues(entry, queues, num); +} + +int odp_pktio_recv_queue(odp_pktin_queue_t queue, odp_packet_t packets[], + int num) +{ + pktio_entry_t *entry; + odp_pktio_t pktio = queue.pktio; + + entry = get_pktio_entry(pktio); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", pktio); + return -1; + } + + if (entry->s.ops->recv_queue) + return entry->s.ops->recv_queue(entry, queue.index, + packets, num); + + return single_recv_queue(entry, queue.index, packets, num); +} + +int odp_pktio_send_queue(odp_pktout_queue_t queue, odp_packet_t packets[], + int num) +{ + pktio_entry_t *entry; + odp_pktio_t pktio = queue.pktio; + + entry = get_pktio_entry(pktio); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", pktio); + return -1; + } + + if (entry->s.ops->send_queue) + return entry->s.ops->send_queue(entry, queue.index, + packets, num); + + return single_send_queue(entry, queue.index, packets, num); +} + +int single_capability(odp_pktio_capability_t *capa) +{ + memset(capa, 0, sizeof(odp_pktio_capability_t)); + capa->max_input_queues = 1; + capa->max_output_queues = 1; + + return 0; +} + +int single_in_queues(pktio_entry_t *entry, odp_queue_t queues[], int num) +{ + if (queues && num > 0) + queues[0] = entry->s.in_queue[0].queue; + + return 1; +} + +int single_pktin_queues(pktio_entry_t *entry, odp_pktin_queue_t queues[], + int num) +{ + if (queues && num > 0) + queues[0] = entry->s.in_queue[0].pktin; + + return 1; +} + +int single_pktout_queues(pktio_entry_t *entry, odp_pktout_queue_t queues[], + int num) +{ + if (queues && num > 0) + queues[0] = entry->s.out_queue[0].pktout; + + return 1; +} + +int single_recv_queue(pktio_entry_t *entry, int index, odp_packet_t packets[], + int num) +{ + (void)index; + return odp_pktio_recv(entry->s.handle, packets, num); +} + +int single_send_queue(pktio_entry_t *entry, int index, odp_packet_t packets[], + int num) +{ + (void)index; + return odp_pktio_send(entry->s.handle, packets, num); } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index a3fcac373..64668cc3d 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -19,7 +19,6 @@ #include <odp/hints.h> #include <odp/thread.h> #include <odp_debug_internal.h> -#include <odp_atomic_internal.h> #include <string.h> #include <stdlib.h> diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index 7f1147825..dbe2d9ce7 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -87,25 +87,29 @@ queue_entry_t *get_qentry(uint32_t queue_id) } static int queue_init(queue_entry_t *queue, const char *name, - odp_queue_type_t type, odp_queue_param_t *param) + const odp_queue_param_t *param) { strncpy(queue->s.name, name, ODP_QUEUE_NAME_LEN - 1); - queue->s.type = type; if (param) { memcpy(&queue->s.param, param, sizeof(odp_queue_param_t)); if (queue->s.param.sched.lock_count > ODP_CONFIG_MAX_ORDERED_LOCKS_PER_QUEUE) return -1; + + if (param->type == ODP_QUEUE_TYPE_SCHED) + queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED; } else { /* Defaults */ - memset(&queue->s.param, 0, sizeof(odp_queue_param_t)); + odp_queue_param_init(&queue->s.param); queue->s.param.sched.prio = ODP_SCHED_PRIO_DEFAULT; queue->s.param.sched.sync = ODP_SCHED_SYNC_ATOMIC; queue->s.param.sched.group = ODP_SCHED_GROUP_ALL; } - switch (type) { + queue->s.type = queue->s.param.type; + + switch (queue->s.type) { case ODP_QUEUE_TYPE_PKTIN: queue->s.enqueue = pktin_enqueue; queue->s.dequeue = pktin_dequeue; @@ -126,6 +130,8 @@ static int queue_init(queue_entry_t *queue, const char *name, break; } + queue->s.pktin = PKTIN_INVALID; + queue->s.head = NULL; queue->s.tail = NULL; @@ -248,12 +254,12 @@ int odp_queue_lock_count(odp_queue_t handle) (int)queue->s.param.sched.lock_count : -1; } -odp_queue_t odp_queue_create(const char *name, odp_queue_type_t type, - odp_queue_param_t *param) +odp_queue_t odp_queue_create(const char *name, const odp_queue_param_t *param) { uint32_t i; queue_entry_t *queue; odp_queue_t handle = ODP_QUEUE_INVALID; + odp_queue_type_t type; for (i = 0; i < ODP_CONFIG_QUEUES; i++) { queue = &queue_tbl->queue[i]; @@ -263,11 +269,13 @@ odp_queue_t odp_queue_create(const char *name, odp_queue_type_t type, LOCK(&queue->s.lock); if (queue->s.status == QUEUE_STATUS_FREE) { - if (queue_init(queue, name, type, param)) { + if (queue_init(queue, name, param)) { UNLOCK(&queue->s.lock); return handle; } + type = queue->s.type; + if (type == ODP_QUEUE_TYPE_SCHED || type == ODP_QUEUE_TYPE_PKTIN) queue->s.status = QUEUE_STATUS_NOTSCHED; @@ -944,6 +952,9 @@ void queue_unlock(queue_entry_t *queue) void odp_queue_param_init(odp_queue_param_t *params) { memset(params, 0, sizeof(odp_queue_param_t)); + params->type = ODP_QUEUE_TYPE_PLAIN; + params->enq_mode = ODP_QUEUE_OP_MT; + params->deq_mode = ODP_QUEUE_OP_MT; } /* These routines exists here rather than in odp_schedule @@ -1099,7 +1110,6 @@ int odp_queue_info(odp_queue_t handle, odp_queue_info_t *info) } info->name = queue->s.name; - info->type = queue->s.type; info->param = queue->s.param; UNLOCK(&queue->s.lock); diff --git a/platform/linux-generic/odp_rwlock.c b/platform/linux-generic/odp_rwlock.c index 47c15ef42..42ad0ccdb 100644 --- a/platform/linux-generic/odp_rwlock.c +++ b/platform/linux-generic/odp_rwlock.c @@ -6,10 +6,8 @@ #include <stdbool.h> #include <odp/atomic.h> -#include <odp_atomic_internal.h> #include <odp/rwlock.h> - -#include <odp_spin_internal.h> +#include <odp/cpu.h> void odp_rwlock_init(odp_rwlock_t *rwlock) { @@ -22,23 +20,20 @@ void odp_rwlock_read_lock(odp_rwlock_t *rwlock) int is_locked = 0; while (is_locked == 0) { - cnt = _odp_atomic_u32_load_mm(&rwlock->cnt, _ODP_MEMMODEL_RLX); + cnt = odp_atomic_load_u32(&rwlock->cnt); /* waiting for read lock */ if ((int32_t)cnt < 0) { - odp_spin(); + odp_cpu_pause(); continue; } - is_locked = _odp_atomic_u32_cmp_xchg_strong_mm(&rwlock->cnt, - &cnt, - cnt + 1, - _ODP_MEMMODEL_ACQ, - _ODP_MEMMODEL_RLX); + is_locked = odp_atomic_cas_acq_u32(&rwlock->cnt, + &cnt, cnt + 1); } } void odp_rwlock_read_unlock(odp_rwlock_t *rwlock) { - _odp_atomic_u32_sub_mm(&rwlock->cnt, 1, _ODP_MEMMODEL_RLS); + odp_atomic_sub_rel_u32(&rwlock->cnt, 1); } void odp_rwlock_write_lock(odp_rwlock_t *rwlock) @@ -48,21 +43,18 @@ void odp_rwlock_write_lock(odp_rwlock_t *rwlock) while (is_locked == 0) { uint32_t zero = 0; - cnt = _odp_atomic_u32_load_mm(&rwlock->cnt, _ODP_MEMMODEL_RLX); + cnt = odp_atomic_load_u32(&rwlock->cnt); /* lock acquired, wait */ if (cnt != 0) { - odp_spin(); + odp_cpu_pause(); continue; } - is_locked = _odp_atomic_u32_cmp_xchg_strong_mm(&rwlock->cnt, - &zero, - (uint32_t)-1, - _ODP_MEMMODEL_ACQ, - _ODP_MEMMODEL_RLX); + is_locked = odp_atomic_cas_acq_u32(&rwlock->cnt, + &zero, (uint32_t)-1); } } void odp_rwlock_write_unlock(odp_rwlock_t *rwlock) { - _odp_atomic_u32_store_mm(&rwlock->cnt, 0, _ODP_MEMMODEL_RLS); + odp_atomic_store_rel_u32(&rwlock->cnt, 0); } diff --git a/platform/linux-generic/odp_schedule.c b/platform/linux-generic/odp_schedule.c index 58f1b1b37..fc54ee5d0 100644 --- a/platform/linux-generic/odp_schedule.c +++ b/platform/linux-generic/odp_schedule.c @@ -19,10 +19,10 @@ #include <odp/time.h> #include <odp/spinlock.h> #include <odp/hints.h> +#include <odp/cpu.h> #include <odp_queue_internal.h> #include <odp_packet_io_internal.h> -#include <odp_spin_internal.h> odp_thrmask_t sched_mask_all; @@ -30,12 +30,17 @@ odp_thrmask_t sched_mask_all; * One per scheduled queue and packet interface */ #define NUM_SCHED_CMD (ODP_CONFIG_QUEUES + ODP_CONFIG_PKTIO_ENTRIES) -/* Scheduler sub queues */ +/* Priority queues per priority */ #define QUEUES_PER_PRIO 4 +/* Packet input poll cmd queues */ +#define POLL_CMD_QUEUES 4 + /* Maximum number of dequeues */ #define MAX_DEQ 4 +/* Maximum number of packet input queues per command */ +#define MAX_PKTIN 8 /* Mask of queues per priority */ typedef uint8_t pri_mask_t; @@ -50,6 +55,13 @@ typedef struct { odp_queue_t pri_queue[ODP_CONFIG_SCHED_PRIOS][QUEUES_PER_PRIO]; pri_mask_t pri_mask[ODP_CONFIG_SCHED_PRIOS]; odp_spinlock_t mask_lock; + + odp_spinlock_t poll_cmd_lock; + struct { + odp_queue_t queue; + uint16_t num; + } poll_cmd[POLL_CMD_QUEUES]; + odp_pool_t pool; odp_shm_t shm; uint32_t pri_count[ODP_CONFIG_SCHED_PRIOS][QUEUES_PER_PRIO]; @@ -69,8 +81,9 @@ typedef struct { struct { odp_pktio_t pktio; + int num; + int index[MAX_PKTIN]; pktio_entry_t *pe; - int prio; }; }; } sched_cmd_t; @@ -80,19 +93,20 @@ typedef struct { typedef struct { + int thr; + int num; + int index; + int pause; + uint32_t pktin_polls; odp_queue_t pri_queue; odp_event_t cmd_ev; - - odp_buffer_hdr_t *buf_hdr[MAX_DEQ]; queue_entry_t *qe; queue_entry_t *origin_qe; + odp_buffer_hdr_t *buf_hdr[MAX_DEQ]; uint64_t order; uint64_t sync[ODP_CONFIG_MAX_ORDERED_LOCKS_PER_QUEUE]; odp_pool_t pool; int enq_called; - int num; - int index; - int pause; int ignore_ordered_context; } sched_local_t; @@ -109,6 +123,7 @@ static void sched_local_init(void) { memset(&sched_local, 0, sizeof(sched_local_t)); + sched_local.thr = odp_thread_id(); sched_local.pri_queue = ODP_QUEUE_INVALID; sched_local.cmd_ev = ODP_EVENT_INVALID; } @@ -163,8 +178,7 @@ int odp_schedule_init_global(void) name[10] = '0' + j / 10; name[11] = '0' + j - 10*(j / 10); - queue = odp_queue_create(name, - ODP_QUEUE_TYPE_POLL, NULL); + queue = odp_queue_create(name, NULL); if (queue == ODP_QUEUE_INVALID) { ODP_ERR("Sched init: Queue create failed.\n"); @@ -176,6 +190,24 @@ int odp_schedule_init_global(void) } } + odp_spinlock_init(&sched->poll_cmd_lock); + for (i = 0; i < POLL_CMD_QUEUES; i++) { + odp_queue_t queue; + char name[] = "odp_poll_cmd_YY"; + + name[13] = '0' + i / 10; + name[14] = '0' + i - 10 * (i / 10); + + queue = odp_queue_create(name, NULL); + + if (queue == ODP_QUEUE_INVALID) { + ODP_ERR("Sched init: Queue create failed.\n"); + return -1; + } + + sched->poll_cmd[i].queue = queue; + } + odp_spinlock_init(&sched->grp_lock); for (i = 0; i < ODP_CONFIG_SCHED_GRPS; i++) { @@ -195,11 +227,11 @@ int odp_schedule_term_global(void) int ret = 0; int rc = 0; int i, j; + odp_event_t ev; for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) { for (j = 0; j < QUEUES_PER_PRIO; j++) { odp_queue_t pri_q; - odp_event_t ev; pri_q = sched->pri_queue[i][j]; @@ -207,25 +239,20 @@ int odp_schedule_term_global(void) ODP_EVENT_INVALID) { odp_buffer_t buf; sched_cmd_t *sched_cmd; + queue_entry_t *qe; + odp_buffer_hdr_t *buf_hdr[1]; + int num; buf = odp_buffer_from_event(ev); sched_cmd = odp_buffer_addr(buf); + qe = sched_cmd->qe; + num = queue_deq_multi(qe, buf_hdr, 1); - if (sched_cmd->cmd == SCHED_CMD_DEQUEUE) { - queue_entry_t *qe; - odp_buffer_hdr_t *buf_hdr[1]; - int num; - - qe = sched_cmd->qe; - num = queue_deq_multi(qe, buf_hdr, 1); + if (num < 0) + queue_destroy_finalize(qe); - if (num < 0) - queue_destroy_finalize(qe); - - if (num > 0) - ODP_ERR("Queue not empty\n"); - } else - odp_buffer_free(buf); + if (num > 0) + ODP_ERR("Queue not empty\n"); } if (odp_queue_destroy(pri_q)) { @@ -235,6 +262,18 @@ int odp_schedule_term_global(void) } } + for (i = 0; i < POLL_CMD_QUEUES; i++) { + odp_queue_t queue = sched->poll_cmd[i].queue; + + while ((ev = odp_queue_deq(queue)) != ODP_EVENT_INVALID) + odp_event_free(ev); + + if (odp_queue_destroy(queue)) { + ODP_ERR("Poll cmd queue destroy failed\n"); + rc = -1; + } + } + if (odp_pool_destroy(sched->pool) != 0) { ODP_ERR("Pool destroy fail.\n"); rc = -1; @@ -273,11 +312,6 @@ static int pri_id_queue(odp_queue_t queue) return (QUEUES_PER_PRIO-1) & (queue_to_id(queue)); } -static int pri_id_pktio(odp_pktio_t pktio) -{ - return (QUEUES_PER_PRIO-1) & (pktio_to_id(pktio)); -} - static odp_queue_t pri_set(int id, int prio) { odp_spinlock_lock(&sched->mask_lock); @@ -308,25 +342,12 @@ static odp_queue_t pri_set_queue(odp_queue_t queue, int prio) return pri_set(id, prio); } -static odp_queue_t pri_set_pktio(odp_pktio_t pktio, int prio) -{ - int id = pri_id_pktio(pktio); - - return pri_set(id, prio); -} - static void pri_clr_queue(odp_queue_t queue, int prio) { int id = pri_id_queue(queue); pri_clr(id, prio); } -static void pri_clr_pktio(odp_pktio_t pktio, int prio) -{ - int id = pri_id_pktio(pktio); - pri_clr(id, prio); -} - int schedule_queue_init(queue_entry_t *qe) { odp_buffer_t buf; @@ -357,30 +378,55 @@ void schedule_queue_destroy(queue_entry_t *qe) qe->s.pri_queue = ODP_QUEUE_INVALID; } -int schedule_pktio_start(odp_pktio_t pktio, int prio) +static int poll_cmd_queue_idx(odp_pktio_t pktio, int in_queue_idx) +{ + return (POLL_CMD_QUEUES - 1) & (pktio_to_id(pktio) ^ in_queue_idx); +} + +void schedule_pktio_start(odp_pktio_t pktio, int num_in_queue, + int in_queue_idx[]) { odp_buffer_t buf; sched_cmd_t *sched_cmd; - odp_queue_t pri_queue; + odp_queue_t queue; + int i, idx; buf = odp_buffer_alloc(sched->pool); if (buf == ODP_BUFFER_INVALID) - return -1; + ODP_ABORT("Sched pool empty\n"); sched_cmd = odp_buffer_addr(buf); sched_cmd->cmd = SCHED_CMD_POLL_PKTIN; sched_cmd->pktio = pktio; + sched_cmd->num = num_in_queue; sched_cmd->pe = get_pktio_entry(pktio); - sched_cmd->prio = prio; - pri_queue = pri_set_pktio(pktio, prio); + if (num_in_queue > MAX_PKTIN) + ODP_ABORT("Too many input queues for scheduler\n"); + + for (i = 0; i < num_in_queue; i++) + sched_cmd->index[i] = in_queue_idx[i]; - if (odp_queue_enq(pri_queue, odp_buffer_to_event(buf))) + idx = poll_cmd_queue_idx(pktio, in_queue_idx[0]); + + odp_spinlock_lock(&sched->poll_cmd_lock); + sched->poll_cmd[idx].num++; + odp_spinlock_unlock(&sched->poll_cmd_lock); + + queue = sched->poll_cmd[idx].queue; + + if (odp_queue_enq(queue, odp_buffer_to_event(buf))) ODP_ABORT("schedule_pktio_start failed\n"); +} +static void schedule_pktio_stop(sched_cmd_t *sched_cmd) +{ + int idx = poll_cmd_queue_idx(sched_cmd->pktio, sched_cmd->index[0]); - return 0; + odp_spinlock_lock(&sched->poll_cmd_lock); + sched->poll_cmd[idx].num--; + odp_spinlock_unlock(&sched->poll_cmd_lock); } void odp_schedule_release_atomic(void) @@ -439,9 +485,12 @@ static int schedule(odp_queue_t *out_queue, odp_event_t out_ev[], unsigned int max_num, unsigned int max_deq) { int i, j; - int thr; int ret; uint32_t k; + int id; + odp_event_t ev; + odp_buffer_t buf; + sched_cmd_t *sched_cmd; if (sched_local.num) { ret = copy_events(out_ev, max_num); @@ -457,21 +506,16 @@ static int schedule(odp_queue_t *out_queue, odp_event_t out_ev[], if (odp_unlikely(sched_local.pause)) return 0; - thr = odp_thread_id(); - + /* Schedule events */ for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) { - int id; if (sched->pri_mask[i] == 0) continue; - id = thr & (QUEUES_PER_PRIO-1); + id = sched_local.thr & (QUEUES_PER_PRIO - 1); for (j = 0; j < QUEUES_PER_PRIO; j++, id++) { odp_queue_t pri_q; - odp_event_t ev; - odp_buffer_t buf; - sched_cmd_t *sched_cmd; queue_entry_t *qe; int num; int qe_grp; @@ -491,28 +535,12 @@ static int schedule(odp_queue_t *out_queue, odp_event_t out_ev[], buf = odp_buffer_from_event(ev); sched_cmd = odp_buffer_addr(buf); - if (sched_cmd->cmd == SCHED_CMD_POLL_PKTIN) { - /* Poll packet input */ - if (pktin_poll(sched_cmd->pe)) { - /* Stop scheduling the pktio */ - pri_clr_pktio(sched_cmd->pktio, - sched_cmd->prio); - odp_buffer_free(buf); - } else { - /* Continue scheduling the pktio */ - if (odp_queue_enq(pri_q, ev)) - ODP_ABORT("schedule failed\n"); - } - - continue; - } - qe = sched_cmd->qe; qe_grp = qe->s.param.sched.group; if (qe_grp > ODP_SCHED_GROUP_ALL && !odp_thrmask_isset(sched->sched_grp[qe_grp].mask, - thr)) { + sched_local.thr)) { /* This thread is not eligible for work from * this queue, so continue scheduling it. */ @@ -580,6 +608,59 @@ static int schedule(odp_queue_t *out_queue, odp_event_t out_ev[], } } + /* + * Poll packet input when there are no events + * * Each thread starts the search for a poll command from its + * preferred command queue. If the queue is empty, it moves to other + * queues. + * * Most of the times, the search stops on the first command found to + * optimize multi-threaded performance. A small portion of polls + * have to do full iteration to avoid packet input starvation when + * there are less threads than command queues. + */ + id = sched_local.thr & (POLL_CMD_QUEUES - 1); + + for (i = 0; i < POLL_CMD_QUEUES; i++, id++) { + odp_queue_t cmd_queue; + + if (id == POLL_CMD_QUEUES) + id = 0; + + if (sched->poll_cmd[id].num == 0) + continue; + + cmd_queue = sched->poll_cmd[id].queue; + ev = odp_queue_deq(cmd_queue); + + if (ev == ODP_EVENT_INVALID) + continue; + + buf = odp_buffer_from_event(ev); + sched_cmd = odp_buffer_addr(buf); + + if (sched_cmd->cmd != SCHED_CMD_POLL_PKTIN) + ODP_ABORT("Bad poll command\n"); + + /* Poll packet input */ + if (pktin_poll(sched_cmd->pe, + sched_cmd->num, + sched_cmd->index)) { + /* Stop scheduling the pktio */ + schedule_pktio_stop(sched_cmd); + odp_buffer_free(buf); + } else { + /* Continue scheduling the pktio */ + if (odp_queue_enq(cmd_queue, ev)) + ODP_ABORT("Poll command enqueue failed\n"); + + /* Do not iterate through all pktin poll command queues + * every time. */ + if (odp_likely(sched_local.pktin_polls & 0xf)) + break; + } + } + + sched_local.pktin_polls++; return 0; } @@ -812,7 +893,7 @@ void odp_schedule_order_lock(unsigned lock_index) * some events in the ordered flow need to lock. */ while (sync != sync_out) { - odp_spin(); + odp_cpu_pause(); sync_out = odp_atomic_load_u64(&origin_qe->s.sync_out[lock_index]); } diff --git a/platform/linux-generic/odp_spinlock.c b/platform/linux-generic/odp_spinlock.c index f16572053..6a16dc4b9 100644 --- a/platform/linux-generic/odp_spinlock.c +++ b/platform/linux-generic/odp_spinlock.c @@ -5,9 +5,8 @@ */ #include <odp/spinlock.h> +#include <odp/cpu.h> #include <odp_atomic_internal.h> -#include <odp_spin_internal.h> - void odp_spinlock_init(odp_spinlock_t *spinlock) { @@ -23,7 +22,7 @@ void odp_spinlock_lock(odp_spinlock_t *spinlock) * the loop will exit when the lock becomes available * and we will retry the TAS operation above */ while (_odp_atomic_flag_load(&spinlock->lock)) - odp_spin(); + odp_cpu_pause(); } diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index a948fce93..42aef8a11 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -25,13 +25,6 @@ #include <dirent.h> - -typedef struct { - const char *cpu_arch_str; - int (*cpuinfo_parser)(FILE *file, odp_system_info_t *sysinfo); - -} odp_compiler_info_t; - #define CACHE_LNSZ_FILE \ "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size" @@ -112,166 +105,6 @@ static int huge_page_size(void) } - -/* - * HW specific /proc/cpuinfo file parsing - */ -#if defined __x86_64__ || defined __i386__ - -static int cpuinfo_x86(FILE *file, odp_system_info_t *sysinfo) -{ - char str[1024]; - char *pos; - double mhz = 0.0; - int model = 0; - int count = 2; - - while (fgets(str, sizeof(str), file) != NULL && count > 0) { - if (!mhz) { - pos = strstr(str, "cpu MHz"); - if (pos) { - sscanf(pos, "cpu MHz : %lf", &mhz); - count--; - } - } - - if (!model) { - pos = strstr(str, "model name"); - if (pos) { - int len; - pos = strchr(str, ':'); - strncpy(sysinfo->model_str, pos+2, - sizeof(sysinfo->model_str)); - len = strlen(sysinfo->model_str); - sysinfo->model_str[len - 1] = 0; - model = 1; - count--; - } - } - } - - sysinfo->cpu_hz = (uint64_t) (mhz * 1000000.0); - - return 0; -} - -#elif defined __arm__ || defined __aarch64__ - -static int cpuinfo_arm(FILE *file ODP_UNUSED, -odp_system_info_t *sysinfo ODP_UNUSED) -{ - return 0; -} - -#elif defined __OCTEON__ - -static int cpuinfo_octeon(FILE *file, odp_system_info_t *sysinfo) -{ - char str[1024]; - char *pos; - double mhz = 0.0; - int model = 0; - int count = 2; - - while (fgets(str, sizeof(str), file) != NULL && count > 0) { - if (!mhz) { - pos = strstr(str, "BogoMIPS"); - - if (pos) { - sscanf(pos, "BogoMIPS : %lf", &mhz); - count--; - } - } - - if (!model) { - pos = strstr(str, "cpu model"); - - if (pos) { - int len; - pos = strchr(str, ':'); - strncpy(sysinfo->model_str, pos+2, - sizeof(sysinfo->model_str)); - len = strlen(sysinfo->model_str); - sysinfo->model_str[len - 1] = 0; - model = 1; - count--; - } - } - } - - /* bogomips seems to be 2x freq */ - sysinfo->cpu_hz = (uint64_t) (mhz * 1000000.0 / 2.0); - - return 0; -} -#elif defined __powerpc__ -static int cpuinfo_powerpc(FILE *file, odp_system_info_t *sysinfo) -{ - char str[1024]; - char *pos; - double mhz = 0.0; - int model = 0; - int count = 2; - - while (fgets(str, sizeof(str), file) != NULL && count > 0) { - if (!mhz) { - pos = strstr(str, "clock"); - - if (pos) { - sscanf(pos, "clock : %lf", &mhz); - count--; - } - } - - if (!model) { - pos = strstr(str, "cpu"); - - if (pos) { - int len; - pos = strchr(str, ':'); - strncpy(sysinfo->model_str, pos+2, - sizeof(sysinfo->model_str)); - len = strlen(sysinfo->model_str); - sysinfo->model_str[len - 1] = 0; - model = 1; - count--; - } - } - - sysinfo->cpu_hz = (uint64_t) (mhz * 1000000.0); - } - - - return 0; -} - -#else - #error GCC target not found -#endif - -static odp_compiler_info_t compiler_info = { - #if defined __x86_64__ || defined __i386__ - .cpu_arch_str = "x86", - .cpuinfo_parser = cpuinfo_x86 - - #elif defined __arm__ || defined __aarch64__ - .cpu_arch_str = "arm", - .cpuinfo_parser = cpuinfo_arm - - #elif defined __OCTEON__ - .cpu_arch_str = "octeon", - .cpuinfo_parser = cpuinfo_octeon - - #elif defined __powerpc__ - .cpu_arch_str = "powerpc", - .cpuinfo_parser = cpuinfo_powerpc - - #else - #error GCC target not found - #endif -}; - - #if defined __x86_64__ || defined __i386__ || defined __OCTEON__ || \ defined __powerpc__ @@ -318,7 +151,7 @@ static int systemcpu(odp_system_info_t *sysinfo) static int systemcpu(odp_system_info_t *sysinfo) { - int ret; + int ret, i; ret = sysconf_cpu_count(); if (ret == 0) { @@ -331,10 +164,14 @@ static int systemcpu(odp_system_info_t *sysinfo) sysinfo->huge_page_size = huge_page_size(); /* Dummy values */ - sysinfo->cpu_hz = 1400000000; sysinfo->cache_line_size = 64; - strncpy(sysinfo->model_str, "UNKNOWN", sizeof(sysinfo->model_str)); + ODP_DBG("Warning: use dummy values for freq and model string\n"); + ODP_DBG("Refer to https://bugs.linaro.org/show_bug.cgi?id=1870\n"); + for (i = 0; i < MAX_CPU_NUMBER; i++) { + sysinfo->cpu_hz_max[i] = 1400000000; + strcpy(sysinfo->model_str[i], "UNKNOWN"); + } return 0; } @@ -358,7 +195,7 @@ int odp_system_info_init(void) return -1; } - compiler_info.cpuinfo_parser(file, &odp_global_data.system_info); + odp_cpuinfo_parser(file, &odp_global_data.system_info); fclose(file); @@ -383,9 +220,29 @@ int odp_system_info_term(void) * Public access functions ************************* */ -uint64_t odp_sys_cpu_hz(void) +uint64_t odp_cpu_hz(void) { - return odp_global_data.system_info.cpu_hz; + int id = sched_getcpu(); + + return odp_cpu_hz_current(id); +} + +uint64_t odp_cpu_hz_id(int id) +{ + return odp_cpu_hz_current(id); +} + +uint64_t odp_cpu_hz_max(void) +{ + return odp_cpu_hz_max_id(0); +} + +uint64_t odp_cpu_hz_max_id(int id) +{ + if (id >= 0 && id < MAX_CPU_NUMBER) + return odp_global_data.system_info.cpu_hz_max[id]; + else + return 0; } uint64_t odp_sys_huge_page_size(void) @@ -398,9 +255,17 @@ uint64_t odp_sys_page_size(void) return odp_global_data.system_info.page_size; } -const char *odp_sys_cpu_model_str(void) +const char *odp_cpu_model_str(void) +{ + return odp_cpu_model_str_id(0); +} + +const char *odp_cpu_model_str_id(int id) { - return odp_global_data.system_info.model_str; + if (id >= 0 && id < MAX_CPU_NUMBER) + return odp_global_data.system_info.model_str[id]; + else + return NULL; } int odp_sys_cache_line_size(void) diff --git a/platform/linux-generic/odp_ticketlock.c b/platform/linux-generic/odp_ticketlock.c index 3e2a4ece1..84b893a5c 100644 --- a/platform/linux-generic/odp_ticketlock.c +++ b/platform/linux-generic/odp_ticketlock.c @@ -6,10 +6,8 @@ #include <odp/ticketlock.h> #include <odp/atomic.h> -#include <odp_atomic_internal.h> #include <odp/sync.h> -#include <odp_spin_internal.h> - +#include <odp/cpu.h> void odp_ticketlock_init(odp_ticketlock_t *ticketlock) { @@ -17,7 +15,6 @@ void odp_ticketlock_init(odp_ticketlock_t *ticketlock) odp_atomic_init_u32(&ticketlock->cur_ticket, 0); } - void odp_ticketlock_lock(odp_ticketlock_t *ticketlock) { uint32_t ticket; @@ -29,9 +26,8 @@ void odp_ticketlock_lock(odp_ticketlock_t *ticketlock) /* Spin waiting for our turn. Use load-acquire so that we acquire * all stores from the previous lock owner */ - while (ticket != _odp_atomic_u32_load_mm(&ticketlock->cur_ticket, - _ODP_MEMMODEL_ACQ)) - odp_spin(); + while (ticket != odp_atomic_load_acq_u32(&ticketlock->cur_ticket)) + odp_cpu_pause(); } int odp_ticketlock_trylock(odp_ticketlock_t *tklock) @@ -55,11 +51,8 @@ int odp_ticketlock_trylock(odp_ticketlock_t *tklock) * If CAS fails, it means some other thread intercepted and * took a ticket which means the lock is not available * anymore */ - if (_odp_atomic_u32_cmp_xchg_strong_mm(&tklock->next_ticket, - &next, - next + 1, - _ODP_MEMMODEL_ACQ, - _ODP_MEMMODEL_RLX)) + if (odp_atomic_cas_acq_u32(&tklock->next_ticket, + &next, next + 1)) return 1; } return 0; @@ -72,17 +65,15 @@ void odp_ticketlock_unlock(odp_ticketlock_t *ticketlock) * 'cur_ticket', we don't need to do this with an (expensive) * atomic RMW operation. Instead load-relaxed the current value * and a store-release of the incremented value */ - uint32_t cur = _odp_atomic_u32_load_mm(&ticketlock->cur_ticket, - _ODP_MEMMODEL_RLX); - _odp_atomic_u32_store_mm(&ticketlock->cur_ticket, cur + 1, - _ODP_MEMMODEL_RLS); + uint32_t cur = odp_atomic_load_u32(&ticketlock->cur_ticket); + + odp_atomic_store_rel_u32(&ticketlock->cur_ticket, cur + 1); #if defined __OCTEON__ odp_sync_stores(); /* SYNCW to flush write buffer */ #endif } - int odp_ticketlock_is_locked(odp_ticketlock_t *ticketlock) { /* Compare 'cur_ticket' with 'next_ticket'. Ideally we should read diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 01339ad86..fe3d40f21 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -27,12 +27,17 @@ #include <stdlib.h> #include <time.h> #include <signal.h> +#include <pthread.h> +#include <unistd.h> +#include <sys/syscall.h> + #include <odp/align.h> #include <odp_align_internal.h> #include <odp/atomic.h> #include <odp_atomic_internal.h> #include <odp/buffer.h> #include <odp_buffer_inlines.h> +#include <odp/cpu.h> #include <odp/pool.h> #include <odp_pool_internal.h> #include <odp/debug.h> @@ -42,7 +47,6 @@ #include <odp_internal.h> #include <odp/queue.h> #include <odp/shared_memory.h> -#include <odp_spin_internal.h> #include <odp/spinlock.h> #include <odp/std_types.h> #include <odp/sync.h> @@ -159,7 +163,6 @@ typedef struct odp_timer_pool_s { tick_buf_t *tick_buf; /* Expiration tick and timeout buffer */ odp_timer *timers; /* User pointer and queue handle (and lock) */ odp_atomic_u32_t high_wm;/* High watermark of allocated timers */ - odp_spinlock_t itimer_running; odp_spinlock_t lock; uint32_t num_alloc;/* Current number of allocated timers */ uint32_t first_free;/* 0..max_timers-1 => free timer */ @@ -169,6 +172,9 @@ typedef struct odp_timer_pool_s { odp_shm_t shm; timer_t timerid; int notify_overrun; + pthread_t timer_thread; /* pthread_t of timer thread */ + pid_t timer_thread_id; /* gettid() for timer thread */ + int timer_thread_exit; /* request to exit for timer thread */ } odp_timer_pool; #define MAX_TIMER_POOLS 255 /* Leave one for ODP_TIMER_INVALID */ @@ -254,26 +260,48 @@ static odp_timer_pool *odp_timer_pool_new( } tp->tp_idx = tp_idx; odp_spinlock_init(&tp->lock); - odp_spinlock_init(&tp->itimer_running); timer_pool[tp_idx] = tp; if (tp->param.clk_src == ODP_CLOCK_CPU) itimer_init(tp); return tp; } +static void block_sigalarm(void) +{ + sigset_t sigset; + + sigemptyset(&sigset); + sigaddset(&sigset, SIGALRM); + sigprocmask(SIG_BLOCK, &sigset, NULL); +} + +static void stop_timer_thread(odp_timer_pool *tp) +{ + int ret; + + ODP_DBG("stop\n"); + tp->timer_thread_exit = 1; + ret = pthread_join(tp->timer_thread, NULL); + if (ret != 0) + ODP_ABORT("unable to join thread, err %d\n", ret); +} + static void odp_timer_pool_del(odp_timer_pool *tp) { odp_spinlock_lock(&tp->lock); timer_pool[tp->tp_idx] = NULL; - /* Wait for itimer thread to stop running */ - odp_spinlock_lock(&tp->itimer_running); + + /* Stop timer triggering */ + if (tp->param.clk_src == ODP_CLOCK_CPU) + itimer_fini(tp); + + stop_timer_thread(tp); + if (tp->num_alloc != 0) { /* It's a programming error to attempt to destroy a */ /* timer pool which is still in use */ ODP_ABORT("%s: timers in use\n", tp->name); } - if (tp->param.clk_src == ODP_CLOCK_CPU) - itimer_fini(tp); int rc = odp_shm_free(tp->shm); if (rc != 0) ODP_ABORT("Failed to free shared memory (%d)\n", rc); @@ -410,7 +438,7 @@ static bool timer_reset(uint32_t idx, while (_odp_atomic_flag_tas(IDX2LOCK(idx))) /* While lock is taken, spin using relaxed loads */ while (_odp_atomic_flag_load(IDX2LOCK(idx))) - odp_spin(); + odp_cpu_pause(); /* Only if there is a timeout buffer can be reset the timer */ if (odp_likely(tb->tmo_buf != ODP_BUFFER_INVALID)) { @@ -457,7 +485,7 @@ static bool timer_reset(uint32_t idx, while (_odp_atomic_flag_tas(IDX2LOCK(idx))) /* While lock is taken, spin using relaxed loads */ while (_odp_atomic_flag_load(IDX2LOCK(idx))) - odp_spin(); + odp_cpu_pause(); /* Swap in new buffer, save any old buffer */ old_buf = tb->tmo_buf; @@ -498,7 +526,7 @@ static odp_buffer_t timer_cancel(odp_timer_pool *tp, while (_odp_atomic_flag_tas(IDX2LOCK(idx))) /* While lock is taken, spin using relaxed loads */ while (_odp_atomic_flag_load(IDX2LOCK(idx))) - odp_spin(); + odp_cpu_pause(); /* Update the timer state (e.g. cancel the current timeout) */ tb->exp_tck.v = new_state; @@ -552,7 +580,7 @@ static unsigned timer_expire(odp_timer_pool *tp, uint32_t idx, uint64_t tick) while (_odp_atomic_flag_tas(IDX2LOCK(idx))) /* While lock is taken, spin using relaxed loads */ while (_odp_atomic_flag_load(IDX2LOCK(idx))) - odp_spin(); + odp_cpu_pause(); /* Proper check for timer expired */ exp_tck = tb->exp_tck.v; if (odp_likely(exp_tck <= tick)) { @@ -632,10 +660,10 @@ static unsigned odp_timer_pool_expire(odp_timer_pool_t tpid, uint64_t tick) * Functions that use Linux/POSIX per-process timers and related facilities *****************************************************************************/ -static void timer_notify(sigval_t sigval) +static void timer_notify(odp_timer_pool *tp) { int overrun; - odp_timer_pool *tp = (odp_timer_pool *)sigval.sival_ptr; + int64_t prev_tick; if (tp->notify_overrun) { overrun = timer_getoverrun(tp->timerid); @@ -653,32 +681,72 @@ static void timer_notify(sigval_t sigval) for (i = 0; i < 32; i += ODP_CACHE_LINE_SIZE / sizeof(array[0])) PREFETCH(&array[i]); #endif - uint64_t prev_tick = odp_atomic_fetch_inc_u64(&tp->cur_tick); - /* Attempt to acquire the lock, check if the old value was clear */ - if (odp_spinlock_trylock(&tp->itimer_running)) { - /* Scan timer array, looking for timers to expire */ - (void)odp_timer_pool_expire(tp, prev_tick); - odp_spinlock_unlock(&tp->itimer_running); - } + prev_tick = odp_atomic_fetch_inc_u64(&tp->cur_tick); + + /* Scan timer array, looking for timers to expire */ + (void)odp_timer_pool_expire(tp, prev_tick); + /* Else skip scan of timers. cur_tick was updated and next itimer * invocation will process older expiration ticks as well */ } +static void *timer_thread(void *arg) +{ + odp_timer_pool *tp = (odp_timer_pool *)arg; + sigset_t sigset; + int ret; + struct timespec tmo; + siginfo_t si; + + tp->timer_thread_id = (pid_t)syscall(SYS_gettid); + + tmo.tv_sec = 0; + tmo.tv_nsec = ODP_TIME_MSEC_IN_NS * 100; + + sigemptyset(&sigset); + /* unblock sigalarm in this thread */ + sigprocmask(SIG_BLOCK, &sigset, NULL); + + sigaddset(&sigset, SIGALRM); + + while (1) { + ret = sigtimedwait(&sigset, &si, &tmo); + if (tp->timer_thread_exit) { + tp->timer_thread_id = 0; + return NULL; + } + if (ret > 0) + timer_notify(tp); + } + + return NULL; +} + static void itimer_init(odp_timer_pool *tp) { struct sigevent sigev; struct itimerspec ispec; uint64_t res, sec, nsec; + int ret; ODP_DBG("Creating POSIX timer for timer pool %s, period %" PRIu64" ns\n", tp->name, tp->param.res_ns); - memset(&sigev, 0, sizeof(sigev)); - memset(&ispec, 0, sizeof(ispec)); + tp->timer_thread_id = 0; + ret = pthread_create(&tp->timer_thread, NULL, timer_thread, tp); + if (ret) + ODP_ABORT("unable to create timer thread\n"); + + /* wait thread set tp->timer_thread_id */ + do { + sched_yield(); + } while (tp->timer_thread_id == 0); - sigev.sigev_notify = SIGEV_THREAD; - sigev.sigev_notify_function = timer_notify; + memset(&sigev, 0, sizeof(sigev)); + sigev.sigev_notify = SIGEV_THREAD_ID; sigev.sigev_value.sival_ptr = tp; + sigev._sigev_un._tid = tp->timer_thread_id; + sigev.sigev_signo = SIGALRM; if (timer_create(CLOCK_MONOTONIC, &sigev, &tp->timerid)) ODP_ABORT("timer_create() returned error %s\n", @@ -688,6 +756,7 @@ static void itimer_init(odp_timer_pool *tp) sec = res / ODP_TIME_SEC_IN_NS; nsec = res - sec * ODP_TIME_SEC_IN_NS; + memset(&ispec, 0, sizeof(ispec)); ispec.it_interval.tv_sec = (time_t)sec; ispec.it_interval.tv_nsec = (long)nsec; ispec.it_value.tv_sec = (time_t)sec; @@ -898,6 +967,9 @@ int odp_timer_init_global(void) ODP_DBG("Using lock-less timer implementation\n"); #endif odp_atomic_init_u32(&num_timer_pools, 0); + + block_sigalarm(); + return 0; } diff --git a/platform/linux-generic/odp_weak.c b/platform/linux-generic/odp_weak.c index 0c59138eb..21fb5ed10 100644 --- a/platform/linux-generic/odp_weak.c +++ b/platform/linux-generic/odp_weak.c @@ -12,7 +12,7 @@ #include <stdarg.h> ODP_WEAK_SYMBOL ODP_PRINTF_FORMAT(2, 3) -int odp_override_log(odp_log_level_e level, const char *fmt, ...) +int odp_override_log(odp_log_level_t level, const char *fmt, ...) { va_list args; int r; diff --git a/platform/linux-generic/pktio/ethtool.c b/platform/linux-generic/pktio/ethtool.c new file mode 100644 index 000000000..1f2943888 --- /dev/null +++ b/platform/linux-generic/pktio/ethtool.c @@ -0,0 +1,164 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <sys/ioctl.h> +#include <netinet/in.h> +#include <linux/sockios.h> +#include <linux/if.h> +#include <linux/ethtool.h> +#include <errno.h> +#include <net/if.h> + +#include <odp.h> +#include <odp_packet_socket.h> +#include <odp_debug_internal.h> + +static struct ethtool_gstrings *get_stringset(int fd, struct ifreq *ifr) +{ + struct { + struct ethtool_sset_info hdr; + uint32_t buf[1]; + } sset_info; + struct ethtool_drvinfo drvinfo; + uint32_t len; + struct ethtool_gstrings *strings; + ptrdiff_t drvinfo_offset = offsetof(struct ethtool_drvinfo, n_stats); + + sset_info.hdr.cmd = ETHTOOL_GSSET_INFO; + sset_info.hdr.reserved = 0; + sset_info.hdr.sset_mask = 1ULL << ETH_SS_STATS; + ifr->ifr_data = &sset_info; + if (ioctl(fd, SIOCETHTOOL, ifr) == 0) { + len = sset_info.hdr.sset_mask ? sset_info.hdr.data[0] : 0; + } else if (errno == EOPNOTSUPP && drvinfo_offset != 0) { + /* Fallback for old kernel versions */ + drvinfo.cmd = ETHTOOL_GDRVINFO; + ifr->ifr_data = &drvinfo; + if (ioctl(fd, SIOCETHTOOL, ifr)) { + __odp_errno = errno; + ODP_ERR("Cannot get stats information\n"); + return NULL; + } + len = *(uint32_t *)(void *)((char *)&drvinfo + drvinfo_offset); + } else { + __odp_errno = errno; + return NULL; + } + + if (!len) { + ODP_ERR("len is zero"); + return NULL; + } + + strings = calloc(1, sizeof(*strings) + len * ETH_GSTRING_LEN); + if (!strings) { + ODP_ERR("alloc failed\n"); + return NULL; + } + + strings->cmd = ETHTOOL_GSTRINGS; + strings->string_set = ETH_SS_STATS; + strings->len = len; + ifr->ifr_data = strings; + if (ioctl(fd, SIOCETHTOOL, ifr)) { + __odp_errno = errno; + ODP_ERR("Cannot get stats information\n"); + free(strings); + return NULL; + } + + return strings; +} + +static int ethtool_stats(int fd, struct ifreq *ifr, odp_pktio_stats_t *stats) +{ + struct ethtool_gstrings *strings; + struct ethtool_stats *estats; + unsigned int n_stats, i; + int err; + int cnts; + + strings = get_stringset(fd, ifr); + if (!strings) + return -1; + + n_stats = strings->len; + if (n_stats < 1) { + ODP_ERR("no stats available\n"); + free(strings); + return -1; + } + + estats = calloc(1, n_stats * sizeof(uint64_t) + + sizeof(struct ethtool_stats)); + if (!estats) { + free(strings); + return -1; + } + + estats->cmd = ETHTOOL_GSTATS; + estats->n_stats = n_stats; + ifr->ifr_data = estats; + err = ioctl(fd, SIOCETHTOOL, ifr); + if (err < 0) { + __odp_errno = errno; + free(strings); + free(estats); + return -1; + } + + cnts = 0; + for (i = 0; i < n_stats; i++) { + char *cnt = (char *)&strings->data[i * ETH_GSTRING_LEN]; + uint64_t val = estats->data[i]; + + if (!strcmp(cnt, "rx_octets")) { + stats->in_octets = val; + cnts++; + } else if (!strcmp(cnt, "rx_ucast_packets")) { + stats->in_ucast_pkts = val; + cnts++; + } else if (!strcmp(cnt, "rx_discards")) { + stats->in_discards = val; + cnts++; + } else if (!strcmp(cnt, "rx_errors")) { + stats->in_errors = val; + cnts++; + } else if (!strcmp(cnt, "tx_octets")) { + stats->out_octets = val; + cnts++; + } else if (!strcmp(cnt, "tx_ucast_packets")) { + stats->out_ucast_pkts = val; + cnts++; + } else if (!strcmp(cnt, "tx_discards")) { + stats->out_discards = val; + cnts++; + } else if (!strcmp(cnt, "tx_errors")) { + stats->out_errors = val; + cnts++; + } + } + + free(strings); + free(estats); + + /* Ethtool strings came from kernel driver. Name of that + * strings is not universal. Current function needs to be updated + * if your driver has different names for counters */ + if (cnts < 8) + return -1; + + return 0; +} + +int ethtool_stats_get_fd(int fd, const char *name, odp_pktio_stats_t *stats) +{ + struct ifreq ifr; + + snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); + + return ethtool_stats(fd, &ifr, stats); +} diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 3e2645c66..e8203ff16 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -20,6 +20,8 @@ /* MAC address for the "loop" interface */ static const char pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01}; +static int loopback_stats_reset(pktio_entry_t *pktio_entry); + static int loopback_open(odp_pktio_t id, pktio_entry_t *pktio_entry, const char *devname, odp_pool_t pool ODP_UNUSED) { @@ -31,11 +33,13 @@ static int loopback_open(odp_pktio_t id, pktio_entry_t *pktio_entry, snprintf(loopq_name, sizeof(loopq_name), "%" PRIu64 "-pktio_loopq", odp_pktio_to_u64(id)); pktio_entry->s.pkt_loop.loopq = - odp_queue_create(loopq_name, ODP_QUEUE_TYPE_POLL, NULL); + odp_queue_create(loopq_name, NULL); if (pktio_entry->s.pkt_loop.loopq == ODP_QUEUE_INVALID) return -1; + loopback_stats_reset(pktio_entry); + return 0; } @@ -63,8 +67,11 @@ static int loopback_recv(pktio_entry_t *pktio_entry, odp_packet_t pkts[], pkt_hdr = odp_packet_hdr(pkt); packet_parse_reset(pkt_hdr); packet_parse_l2(pkt_hdr); - if (0 > _odp_packet_classifier(pktio_entry, pkt)) + if (0 > _odp_packet_classifier(pktio_entry, pkt)) { pkts[j++] = pkt; + pktio_entry->s.stats.in_octets += + odp_packet_len(pkts[i]); + } } nbr = j; } else { @@ -74,9 +81,13 @@ static int loopback_recv(pktio_entry_t *pktio_entry, odp_packet_t pkts[], pkt_hdr = odp_packet_hdr(pkts[i]); packet_parse_reset(pkt_hdr); packet_parse_l2(pkt_hdr); + pktio_entry->s.stats.in_octets += + odp_packet_len(pkts[i]); } } + pktio_entry->s.stats.in_ucast_pkts += nbr; + return nbr; } @@ -86,12 +97,22 @@ static int loopback_send(pktio_entry_t *pktio_entry, odp_packet_t pkt_tbl[], odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; queue_entry_t *qentry; unsigned i; + int ret; + uint32_t bytes = 0; - for (i = 0; i < len; ++i) + for (i = 0; i < len; ++i) { hdr_tbl[i] = odp_buf_to_hdr(_odp_packet_to_buffer(pkt_tbl[i])); + bytes += odp_packet_len(pkt_tbl[i]); + } qentry = queue_to_qentry(pktio_entry->s.pkt_loop.loopq); - return queue_enq_multi(qentry, hdr_tbl, len, 0); + ret = queue_enq_multi(qentry, hdr_tbl, len, 0); + if (ret > 0) { + pktio_entry->s.stats.out_ucast_pkts += ret; + pktio_entry->s.stats.out_octets += bytes; + } + + return ret; } static int loopback_mtu_get(pktio_entry_t *pktio_entry ODP_UNUSED) @@ -119,17 +140,41 @@ static int loopback_promisc_mode_get(pktio_entry_t *pktio_entry) return pktio_entry->s.pkt_loop.promisc ? 1 : 0; } +static int loopback_stats(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats) +{ + memcpy(stats, &pktio_entry->s.stats, sizeof(odp_pktio_stats_t)); + return 0; +} + +static int loopback_stats_reset(pktio_entry_t *pktio_entry ODP_UNUSED) +{ + memset(&pktio_entry->s.stats, 0, sizeof(odp_pktio_stats_t)); + return 0; +} + const pktio_if_ops_t loopback_pktio_ops = { + .name = "loop", .init = NULL, .term = NULL, .open = loopback_open, .close = loopback_close, .start = NULL, .stop = NULL, + .stats = loopback_stats, + .stats_reset = loopback_stats_reset, .recv = loopback_recv, .send = loopback_send, .mtu_get = loopback_mtu_get, .promisc_mode_set = loopback_promisc_mode_set, .promisc_mode_get = loopback_promisc_mode_get, - .mac_get = loopback_mac_addr_get + .mac_get = loopback_mac_addr_get, + .capability = NULL, + .input_queues_config = NULL, + .output_queues_config = NULL, + .in_queues = NULL, + .pktin_queues = NULL, + .pktout_queues = NULL, + .recv_queue = NULL, + .send_queue = NULL }; diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 774e8137d..97fb6c3bc 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -8,9 +8,9 @@ #include <odp_posix_extensions.h> +#include <odp_packet_io_internal.h> #include <odp_packet_netmap.h> #include <odp_packet_socket.h> -#include <odp_packet_io_internal.h> #include <odp_debug_internal.h> #include <odp/helper/eth.h> @@ -22,16 +22,21 @@ #include <odp_classification_inlines.h> #include <odp_classification_internal.h> +/* Disable netmap debug prints */ +#ifndef ND +#define ND(_fmt, ...) do {} while (0) +#define D(_fmt, ...) do {} while (0) +#define RD(lps, format, ...) do {} while (0) +#endif + #define NETMAP_WITH_LIBS #include <net/netmap_user.h> -static struct nm_desc mmap_desc; /** Used to store the mmap address; - filled in first time, used for - subsequent calls to nm_open */ - -#define NM_OPEN_RETRIES 5 +#define NM_WAIT_TIMEOUT 5 /* netmap_wait_for_link() timeout in seconds */ #define NM_INJECT_RETRIES 10 +static int netmap_stats_reset(pktio_entry_t *pktio_entry); + static int netmap_do_ioctl(pktio_entry_t *pktio_entry, unsigned long cmd, int subcmd) { @@ -68,7 +73,7 @@ static int netmap_do_ioctl(pktio_entry_t *pktio_entry, unsigned long cmd, break; case SIOCETHTOOL: if (subcmd == ETHTOOL_GLINK) - return !eval.data; + return eval.data; break; default: break; @@ -80,16 +85,119 @@ done: return err; } -static int netmap_close(pktio_entry_t *pktio_entry) +/** + * Map netmap rings to pktin/pktout queues + * + * @param rings Array of netmap descriptor rings + * @param num_queues Number of pktin/pktout queues + * @param num_rings Number of matching netmap rings + */ +static inline void map_netmap_rings(netmap_ring_t *rings, + unsigned num_queues, unsigned num_rings) +{ + struct netmap_ring_t *desc_ring; + unsigned rings_per_queue; + unsigned remainder; + unsigned mapped_rings; + unsigned i; + unsigned desc_id = 0; + + rings_per_queue = num_rings / num_queues; + remainder = num_rings % num_queues; + + if (remainder) + ODP_DBG("WARNING: Netmap rings mapped unevenly to queues\n"); + + for (i = 0; i < num_queues; i++) { + desc_ring = &rings[i].s; + if (i < remainder) + mapped_rings = rings_per_queue + 1; + else + mapped_rings = rings_per_queue; + + desc_ring->first = desc_id; + desc_ring->cur = desc_id; + desc_ring->last = desc_ring->first + mapped_rings - 1; + desc_ring->num = mapped_rings; + + desc_id = desc_ring->last + 1; + } +} + +static int netmap_input_queues_config(pktio_entry_t *pktio_entry, + const odp_pktin_queue_param_t *p) { pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm; + odp_pktin_mode_t mode = pktio_entry->s.param.in_mode; + unsigned num_queues = p->num_queues; + odp_bool_t lockless; + + /* Scheduler synchronizes input queue polls. Only single thread + * at a time polls a queue */ + if (mode == ODP_PKTIN_MODE_SCHED) + lockless = 1; + else + lockless = (p->op_mode == ODP_PKTIO_OP_MT_UNSAFE); - if (pkt_nm->rx_desc != NULL) { - nm_close(pkt_nm->rx_desc); - mmap_desc.mem = NULL; + if (p->hash_enable && num_queues > 1) { + if (rss_conf_set_fd(pktio_entry->s.pkt_nm.sockfd, + pktio_entry->s.name, &p->hash_proto)) { + ODP_ERR("Failed to configure input hash\n"); + return -1; + } } - if (pkt_nm->tx_desc != NULL) - nm_close(pkt_nm->tx_desc); + + pkt_nm->lockless_rx = lockless; + + return 0; +} + +static int netmap_output_queues_config(pktio_entry_t *pktio_entry, + const odp_pktout_queue_param_t *p) +{ + pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm; + + pkt_nm->lockless_tx = (p->op_mode == ODP_PKTIO_OP_MT_UNSAFE); + + return 0; +} + +/** + * Close netmap descriptors + * + * Can be reopened using netmap_start() function. + * + * @param pktio_entry Packet IO entry + */ +static inline void netmap_close_descriptors(pktio_entry_t *pktio_entry) +{ + int i, j; + pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm; + + for (i = 0; i < PKTIO_MAX_QUEUES; i++) { + for (j = 0; j < NM_MAX_DESC; j++) { + if (pkt_nm->rx_desc_ring[i].s.desc[j] != NULL) { + nm_close(pkt_nm->rx_desc_ring[i].s.desc[j]); + pkt_nm->rx_desc_ring[i].s.desc[j] = NULL; + } + } + for (j = 0; j < NM_MAX_DESC; j++) { + if (pkt_nm->tx_desc_ring[i].s.desc[j] != NULL) { + nm_close(pkt_nm->tx_desc_ring[i].s.desc[j]); + pkt_nm->tx_desc_ring[i].s.desc[j] = NULL; + } + } + } + + pkt_nm->num_rx_desc_rings = 0; + pkt_nm->num_tx_desc_rings = 0; +} + +static int netmap_close(pktio_entry_t *pktio_entry) +{ + pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm; + + netmap_close_descriptors(pktio_entry); if (pkt_nm->sockfd != -1 && close(pkt_nm->sockfd) != 0) { __odp_errno = errno; @@ -99,14 +207,57 @@ static int netmap_close(pktio_entry_t *pktio_entry) return 0; } +static int netmap_link_status(pktio_entry_t *pktio_entry) +{ + return link_status_fd(pktio_entry->s.pkt_nm.sockfd, + pktio_entry->s.name); +} + +/** + * Wait for netmap link to come up + * + * @param pktio_entry Packet IO entry + * + * @retval 1 link is up + * @retval 0 link is down + * @retval <0 on failure + */ +static inline int netmap_wait_for_link(pktio_entry_t *pktio_entry) +{ + int i; + int ret; + + /* Wait for the link to come up */ + for (i = 0; i <= NM_WAIT_TIMEOUT; i++) { + ret = netmap_link_status(pktio_entry); + if (ret == -1) + return -1; + /* nm_open() causes the physical link to reset. When using a + * direct attached loopback cable there may be a small delay + * until the opposing end's interface comes back up again. In + * this case without the additional sleep pktio validation + * tests fail. */ + sleep(1); + if (ret == 1) + return 1; + } + ODP_DBG("%s link is down\n", pktio_entry->s.name); + return 0; +} + static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, const char *netdev, odp_pool_t pool) { - char ifname[IFNAMSIZ + 7]; /* netmap:<ifname> */ + int i; int err; int sockfd; - int i; + int mtu; + uint32_t buf_size; pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm; + struct nm_desc *desc; + struct netmap_ring *ring; + odp_pktin_hash_proto_t hash_proto; + odp_pktio_stats_t cur_stats; if (getenv("ODP_PKTIO_DISABLE_NETMAP")) return -1; @@ -126,25 +277,39 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, snprintf(pktio_entry->s.name, sizeof(pktio_entry->s.name), "%s", netdev); - snprintf(ifname, sizeof(ifname), "netmap:%s", netdev); - - if (mmap_desc.mem == NULL) - pkt_nm->rx_desc = nm_open(ifname, NULL, NETMAP_NO_TX_POLL, - NULL); - else - pkt_nm->rx_desc = nm_open(ifname, NULL, NETMAP_NO_TX_POLL | - NM_OPEN_NO_MMAP, &mmap_desc); - pkt_nm->tx_desc = nm_open(ifname, NULL, NM_OPEN_NO_MMAP, &mmap_desc); + snprintf(pkt_nm->nm_name, sizeof(pkt_nm->nm_name), "netmap:%s", + netdev); - if (pkt_nm->rx_desc == NULL || pkt_nm->tx_desc == NULL) { - ODP_ERR("nm_open(%s) failed\n", ifname); + /* Dummy open here to check if netmap module is available and to read + * capability info. */ + desc = nm_open(pkt_nm->nm_name, NULL, 0, NULL); + if (desc == NULL) { + ODP_ERR("nm_open(%s) failed\n", pkt_nm->nm_name); goto error; } - - if (mmap_desc.mem == NULL) { - mmap_desc.mem = pkt_nm->rx_desc->mem; - mmap_desc.memsize = pkt_nm->rx_desc->memsize; + if (desc->nifp->ni_rx_rings > NM_MAX_DESC) { + ODP_ERR("Unable to store all rx rings\n"); + nm_close(desc); + goto error; } + pkt_nm->num_rx_rings = desc->nifp->ni_rx_rings; + pkt_nm->capa.max_input_queues = PKTIO_MAX_QUEUES; + if (desc->nifp->ni_rx_rings < PKTIO_MAX_QUEUES) + pkt_nm->capa.max_input_queues = desc->nifp->ni_rx_rings; + + if (desc->nifp->ni_tx_rings > NM_MAX_DESC) { + ODP_ERR("Unable to store all tx rings\n"); + nm_close(desc); + goto error; + } + pkt_nm->num_tx_rings = desc->nifp->ni_tx_rings; + pkt_nm->capa.max_output_queues = PKTIO_MAX_QUEUES; + if (desc->nifp->ni_tx_rings < PKTIO_MAX_QUEUES) + pkt_nm->capa.max_output_queues = desc->nifp->ni_tx_rings; + + ring = NETMAP_RXRING(desc->nifp, desc->cur_rx_ring); + buf_size = ring->nr_buf_size; + nm_close(desc); sockfd = socket(AF_INET, SOCK_DGRAM, 0); if (sockfd == -1) { @@ -153,6 +318,22 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, } pkt_nm->sockfd = sockfd; + /* Use either interface MTU (+ ethernet header length) or netmap buffer + * size as MTU, whichever is smaller. */ + mtu = mtu_get_fd(pktio_entry->s.pkt_nm.sockfd, pktio_entry->s.name) + + ODPH_ETHHDR_LEN; + if (mtu < 0) { + ODP_ERR("Unable to read interface MTU\n"); + goto error; + } + pkt_nm->mtu = ((uint32_t)mtu < buf_size) ? (uint32_t)mtu : buf_size; + + /* Check if RSS is supported. If not, set 'max_input_queues' to 1. */ + if (rss_conf_get_supported_fd(sockfd, netdev, &hash_proto) == 0) { + ODP_DBG("RSS not supported\n"); + pkt_nm->capa.max_input_queues = 1; + } + err = netmap_do_ioctl(pktio_entry, SIOCGIFFLAGS, 0); if (err) goto error; @@ -163,29 +344,148 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, if (err) goto error; - /* Wait for the link to come up */ - for (i = 0; i < NM_OPEN_RETRIES; i++) { - err = netmap_do_ioctl(pktio_entry, SIOCETHTOOL, ETHTOOL_GLINK); - /* nm_open() causes the physical link to reset. When using a - * direct attached loopback cable there may be a small delay - * until the opposing end's interface comes back up again. In - * this case without the additional sleep pktio validation - * tests fail. */ - sleep(1); - if (err == 0) - return 0; + for (i = 0; i < PKTIO_MAX_QUEUES; i++) { + odp_ticketlock_init(&pkt_nm->rx_desc_ring[i].s.lock); + odp_ticketlock_init(&pkt_nm->tx_desc_ring[i].s.lock); } - ODP_ERR("%s didn't come up\n", pktio_entry->s.name); + + /* netmap uses only ethtool to get statistics counters */ + err = ethtool_stats_get_fd(pktio_entry->s.pkt_nm.sockfd, + pktio_entry->s.name, + &cur_stats); + if (err) { + ODP_ERR("netmap pktio %s does not support statistics counters\n", + pktio_entry->s.name); + pktio_entry->s.stats_type = STATS_UNSUPPORTED; + } else { + pktio_entry->s.stats_type = STATS_ETHTOOL; + } + + (void)netmap_stats_reset(pktio_entry); + + return 0; error: netmap_close(pktio_entry); return -1; } +static int netmap_start(pktio_entry_t *pktio_entry) +{ + pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm; + netmap_ring_t *desc_ring; + struct nm_desc base_desc; + unsigned i; + unsigned j; + uint64_t flags; + odp_pktin_mode_t in_mode = pktio_entry->s.param.in_mode; + odp_pktout_mode_t out_mode = pktio_entry->s.param.out_mode; + + /* If no pktin/pktout queues have been configured. Configure one + * for each direction. */ + if (!pktio_entry->s.num_in_queue && + in_mode != ODP_PKTIN_MODE_DISABLED) { + odp_pktin_queue_param_t param; + + odp_pktin_queue_param_init(¶m); + param.num_queues = 1; + if (odp_pktin_queue_config(pktio_entry->s.handle, ¶m)) + return -1; + } + if (!pktio_entry->s.num_out_queue && + out_mode == ODP_PKTOUT_MODE_DIRECT) { + odp_pktout_queue_param_t param; + + odp_pktout_queue_param_init(¶m); + param.num_queues = 1; + if (odp_pktout_queue_config(pktio_entry->s.handle, ¶m)) + return -1; + } + + if (pkt_nm->num_rx_desc_rings == pktio_entry->s.num_in_queue && + pkt_nm->num_tx_desc_rings == pktio_entry->s.num_out_queue) + return (netmap_wait_for_link(pktio_entry) == 1) ? 0 : -1; + + netmap_close_descriptors(pktio_entry); + + /* Map pktin/pktout queues to netmap rings */ + if (pktio_entry->s.num_in_queue) + map_netmap_rings(pkt_nm->rx_desc_ring, + pktio_entry->s.num_in_queue, + pkt_nm->num_rx_rings); + if (pktio_entry->s.num_out_queue) + /* Enough to map only one netmap tx ring per pktout queue */ + map_netmap_rings(pkt_nm->tx_desc_ring, + pktio_entry->s.num_out_queue, + pktio_entry->s.num_out_queue); + + base_desc.self = &base_desc; + base_desc.mem = NULL; + memcpy(base_desc.req.nr_name, pktio_entry->s.name, + sizeof(pktio_entry->s.name)); + base_desc.req.nr_flags &= ~NR_REG_MASK; + base_desc.req.nr_flags |= NR_REG_ONE_NIC; + base_desc.req.nr_ringid = 0; + + /* Only the first rx descriptor does mmap */ + desc_ring = pkt_nm->rx_desc_ring; + flags = NM_OPEN_IFNAME | NETMAP_NO_TX_POLL; + desc_ring[0].s.desc[0] = nm_open(pkt_nm->nm_name, NULL, flags, + &base_desc); + if (desc_ring[0].s.desc[0] == NULL) { + ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name); + goto error; + } + /* Open rest of the rx descriptors (one per netmap ring) */ + flags = NM_OPEN_IFNAME | NETMAP_NO_TX_POLL | NM_OPEN_NO_MMAP; + for (i = 0; i < pktio_entry->s.num_in_queue; i++) { + for (j = desc_ring[i].s.first; j <= desc_ring[i].s.last; j++) { + if (i == 0 && j == 0) /* First already opened */ + continue; + base_desc.req.nr_ringid = j; + desc_ring[i].s.desc[j] = nm_open(pkt_nm->nm_name, NULL, + flags, &base_desc); + if (desc_ring[i].s.desc[j] == NULL) { + ODP_ERR("nm_start(%s) failed\n", + pkt_nm->nm_name); + goto error; + } + } + } + /* Open tx descriptors */ + desc_ring = pkt_nm->tx_desc_ring; + flags = NM_OPEN_IFNAME | NM_OPEN_NO_MMAP; + for (i = 0; i < pktio_entry->s.num_out_queue; i++) { + for (j = desc_ring[i].s.first; j <= desc_ring[i].s.last; j++) { + base_desc.req.nr_ringid = j; + desc_ring[i].s.desc[j] = nm_open(pkt_nm->nm_name, NULL, + flags, &base_desc); + if (desc_ring[i].s.desc[j] == NULL) { + ODP_ERR("nm_start(%s) failed\n", + pkt_nm->nm_name); + goto error; + } + } + } + pkt_nm->num_rx_desc_rings = pktio_entry->s.num_in_queue; + pkt_nm->num_tx_desc_rings = pktio_entry->s.num_out_queue; + /* Wait for the link to come up */ + return (netmap_wait_for_link(pktio_entry) == 1) ? 0 : -1; + +error: + netmap_close_descriptors(pktio_entry); + return -1; +} + +static int netmap_stop(pktio_entry_t *pktio_entry ODP_UNUSED) +{ + return 0; +} + /** * Create ODP packet from netmap packet * - * @param pktio_entry Packet IO handle + * @param pktio_entry Packet IO entry * @param pkt_out Storage for new ODP packet handle * @param buf Netmap buffer address * @param len Netmap buffer length @@ -234,37 +534,56 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, } packet_parse_l2(pkt_hdr); + + pkt_hdr->input = pktio_entry->s.handle; + *pkt_out = pkt; } return 0; } -static int netmap_recv(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], - unsigned num) +static int netmap_recv_queue(pktio_entry_t *pktio_entry, int index, + odp_packet_t pkt_table[], int num) { - struct netmap_ring *ring; - struct nm_desc *desc = pktio_entry->s.pkt_nm.rx_desc; - struct pollfd polld; char *buf; + struct netmap_ring *ring; + struct nm_desc *desc; + pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm; + unsigned first_desc_id = pkt_nm->rx_desc_ring[index].s.first; + unsigned last_desc_id = pkt_nm->rx_desc_ring[index].s.last; + unsigned desc_id; + int num_desc = pkt_nm->rx_desc_ring[index].s.num; int i; - int num_rings = desc->last_rx_ring - desc->first_rx_ring + 1; - int ring_id = desc->cur_rx_ring; - unsigned num_rx = 0; + int num_rx = 0; + int max_fd = 0; uint32_t slot_id; + fd_set empty_rings; - polld.fd = desc->fd; - polld.events = POLLIN; + if (odp_unlikely(pktio_entry->s.state == STATE_STOP)) + return 0; - for (i = 0; i < num_rings && num_rx != num; i++) { - ring_id = desc->cur_rx_ring + i; + FD_ZERO(&empty_rings); - if (ring_id > desc->last_rx_ring) - ring_id = desc->first_rx_ring; + if (!pkt_nm->lockless_rx) + odp_ticketlock_lock(&pkt_nm->rx_desc_ring[index].s.lock); - ring = NETMAP_RXRING(desc->nifp, ring_id); + desc_id = pkt_nm->rx_desc_ring[index].s.cur; - while (!nm_ring_empty(ring) && num_rx != num) { + for (i = 0; i < num_desc && num_rx != num; i++) { + if (desc_id > last_desc_id) + desc_id = first_desc_id; + + desc = pkt_nm->rx_desc_ring[index].s.desc[desc_id]; + ring = NETMAP_RXRING(desc->nifp, desc->cur_rx_ring); + + while (num_rx != num) { + if (nm_ring_empty(ring)) { + FD_SET(desc->fd, &empty_rings); + if (desc->fd > max_fd) + max_fd = desc->fd; + break; + } slot_id = ring->cur; buf = NETMAP_BUF(ring, ring->slot[slot_id].buf_idx); @@ -277,51 +596,127 @@ static int netmap_recv(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], ring->cur = nm_ring_next(ring, slot_id); ring->head = ring->cur; } + desc_id++; } - desc->cur_rx_ring = ring_id; + pkt_nm->rx_desc_ring[index].s.cur = desc_id; + + if (num_rx != num) { + struct timeval tout = {.tv_sec = 0, .tv_usec = 0}; - if (num_rx == 0) { - if (odp_unlikely(poll(&polld, 1, 0) < 0)) - ODP_ERR("RX: poll error\n"); + if (select(max_fd + 1, &empty_rings, NULL, NULL, &tout) == -1) + ODP_ERR("RX: select error\n"); } + if (!pkt_nm->lockless_rx) + odp_ticketlock_unlock(&pkt_nm->rx_desc_ring[index].s.lock); + return num_rx; } -static int netmap_send(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], +static int netmap_recv(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], unsigned num) { + unsigned i; + unsigned num_rx = 0; + unsigned queue_id = pktio_entry->s.pkt_nm.cur_rx_queue; + unsigned num_queues = pktio_entry->s.num_in_queue; + unsigned pkts_left = num; + odp_packet_t *pkt_table_cur = pkt_table; + + for (i = 0; i < num_queues && num_rx != num; i++) { + if (queue_id >= num_queues) + queue_id = 0; + + pkt_table_cur = &pkt_table[num_rx]; + pkts_left = num - num_rx; + + num_rx += netmap_recv_queue(pktio_entry, queue_id, + pkt_table_cur, pkts_left); + queue_id++; + } + pktio_entry->s.pkt_nm.cur_rx_queue = queue_id; + + return num_rx; +} + +static int netmap_send_queue(pktio_entry_t *pktio_entry, int index, + odp_packet_t pkt_table[], int num) +{ + pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm; struct pollfd polld; - struct nm_desc *nm_desc = pktio_entry->s.pkt_nm.tx_desc; - unsigned i, nb_tx; - uint8_t *frame; - uint32_t frame_len; + struct nm_desc *desc; + struct netmap_ring *ring; + int i; + int nb_tx; + int desc_id; + odp_packet_t pkt; + uint32_t pkt_len; + unsigned slot_id; + char *buf; + + if (odp_unlikely(pktio_entry->s.state == STATE_STOP)) + return 0; - polld.fd = nm_desc->fd; + /* Only one netmap tx ring per pktout queue */ + desc_id = pkt_nm->tx_desc_ring[index].s.cur; + desc = pkt_nm->tx_desc_ring[index].s.desc[desc_id]; + ring = NETMAP_TXRING(desc->nifp, desc->cur_tx_ring); + + if (!pkt_nm->lockless_tx) + odp_ticketlock_lock(&pkt_nm->tx_desc_ring[index].s.lock); + + polld.fd = desc->fd; polld.events = POLLOUT; for (nb_tx = 0; nb_tx < num; nb_tx++) { - frame_len = 0; - frame = odp_packet_l2_ptr(pkt_table[nb_tx], &frame_len); + pkt = pkt_table[nb_tx]; + pkt_len = odp_packet_len(pkt); + + if (pkt_len > pkt_nm->mtu) { + if (nb_tx == 0) + __odp_errno = EMSGSIZE; + break; + } for (i = 0; i < NM_INJECT_RETRIES; i++) { - if (nm_inject(nm_desc, frame, frame_len) == 0) + if (nm_ring_empty(ring)) { poll(&polld, 1, 0); - else + continue; + } + slot_id = ring->cur; + ring->slot[slot_id].flags = 0; + ring->slot[slot_id].len = pkt_len; + + buf = NETMAP_BUF(ring, ring->slot[slot_id].buf_idx); + + if (odp_packet_copydata_out(pkt, 0, pkt_len, buf)) { + i = NM_INJECT_RETRIES; break; - } - if (odp_unlikely(i == NM_INJECT_RETRIES)) { - ioctl(nm_desc->fd, NIOCTXSYNC, NULL); + } + ring->cur = nm_ring_next(ring, slot_id); + ring->head = ring->cur; break; } + if (i == NM_INJECT_RETRIES) + break; + odp_packet_free(pkt); } /* Send pending packets */ poll(&polld, 1, 0); - for (i = 0; i < nb_tx; i++) - odp_packet_free(pkt_table[i]); + if (!pkt_nm->lockless_tx) + odp_ticketlock_unlock(&pkt_nm->tx_desc_ring[index].s.lock); + + if (odp_unlikely(nb_tx == 0 && __odp_errno != 0)) + return -1; return nb_tx; } +static int netmap_send(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], + unsigned num) +{ + return netmap_send_queue(pktio_entry, 0, pkt_table, num); +} + static int netmap_mac_addr_get(pktio_entry_t *pktio_entry, void *mac_addr) { memcpy(mac_addr, pktio_entry->s.pkt_nm.if_mac, ETH_ALEN); @@ -330,7 +725,7 @@ static int netmap_mac_addr_get(pktio_entry_t *pktio_entry, void *mac_addr) static int netmap_mtu_get(pktio_entry_t *pktio_entry) { - return mtu_get_fd(pktio_entry->s.pkt_nm.sockfd, pktio_entry->s.name); + return pktio_entry->s.pkt_nm.mtu; } static int netmap_promisc_mode_set(pktio_entry_t *pktio_entry, @@ -346,19 +741,105 @@ static int netmap_promisc_mode_get(pktio_entry_t *pktio_entry) pktio_entry->s.name); } +static int netmap_capability(pktio_entry_t *pktio_entry, + odp_pktio_capability_t *capa) +{ + *capa = pktio_entry->s.pkt_nm.capa; + return 0; +} + +static int netmap_in_queues(pktio_entry_t *pktio_entry, odp_queue_t queues[], + int num) +{ + int i; + int num_queues = pktio_entry->s.num_in_queue; + + if (queues && num > 0) { + for (i = 0; i < num && i < num_queues; i++) + queues[i] = pktio_entry->s.in_queue[i].queue; + } + + return num_queues; +} + +static int netmap_pktin_queues(pktio_entry_t *pktio_entry, + odp_pktin_queue_t queues[], int num) +{ + int i; + int num_queues = pktio_entry->s.num_in_queue; + + if (queues && num > 0) { + for (i = 0; i < num && i < num_queues; i++) + queues[i] = pktio_entry->s.in_queue[i].pktin; + } + + return num_queues; +} + +static int netmap_pktout_queues(pktio_entry_t *pktio_entry, + odp_pktout_queue_t queues[], int num) +{ + int i; + int num_queues = pktio_entry->s.num_out_queue; + + if (queues && num > 0) { + for (i = 0; i < num && i < num_queues; i++) + queues[i] = pktio_entry->s.out_queue[i].pktout; + } + + return num_queues; +} + +static int netmap_stats(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats) +{ + if (pktio_entry->s.stats_type == STATS_UNSUPPORTED) { + memset(stats, 0, sizeof(*stats)); + return 0; + } + + return sock_stats_fd(pktio_entry, + stats, + pktio_entry->s.pkt_nm.sockfd); +} + +static int netmap_stats_reset(pktio_entry_t *pktio_entry) +{ + if (pktio_entry->s.stats_type == STATS_UNSUPPORTED) { + memset(&pktio_entry->s.stats, 0, + sizeof(odp_pktio_stats_t)); + return 0; + } + + return sock_stats_reset_fd(pktio_entry, + pktio_entry->s.pkt_nm.sockfd); +} + const pktio_if_ops_t netmap_pktio_ops = { + .name = "netmap", .init = NULL, .term = NULL, .open = netmap_open, .close = netmap_close, - .start = NULL, - .stop = NULL, + .start = netmap_start, + .stop = netmap_stop, + .link_status = netmap_link_status, + .stats = netmap_stats, + .stats_reset = netmap_stats_reset, .recv = netmap_recv, .send = netmap_send, .mtu_get = netmap_mtu_get, .promisc_mode_set = netmap_promisc_mode_set, .promisc_mode_get = netmap_promisc_mode_get, - .mac_get = netmap_mac_addr_get + .mac_get = netmap_mac_addr_get, + .capability = netmap_capability, + .input_queues_config = netmap_input_queues_config, + .output_queues_config = netmap_output_queues_config, + .in_queues = netmap_in_queues, + .pktin_queues = netmap_pktin_queues, + .pktout_queues = netmap_pktout_queues, + .recv_queue = netmap_recv_queue, + .send_queue = netmap_send_queue }; #endif /* ODP_NETMAP */ diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index 6511132b6..c22cce09a 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -49,6 +49,8 @@ #define PKTIO_PCAP_MTU (64 * 1024) static const char pcap_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x04}; +static int pcapif_stats_reset(pktio_entry_t *pktio_entry); + static int _pcapif_parse_devname(pkt_pcap_t *pcap, const char *devname) { char *tok; @@ -154,6 +156,8 @@ static int pcapif_init(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, if (ret == 0 && (!pcap->rx && !pcap->tx_dump)) ret = -1; + (void)pcapif_stats_reset(pktio_entry); + return ret; } @@ -249,6 +253,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, odp_packet_t pkts[], } packet_parse_l2(pkt_hdr); + pktio_entry->s.stats.in_octets += pkt_hdr->frame_len; pkts[i] = pkt; pkt = ODP_PACKET_INVALID; @@ -259,6 +264,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, odp_packet_t pkts[], if (pkt != ODP_PACKET_INVALID) odp_packet_free(pkt); + pktio_entry->s.stats.in_ucast_pkts += i; return i; } @@ -291,7 +297,9 @@ static int pcapif_send_pkt(pktio_entry_t *pktio_entry, odp_packet_t pkts[], ODP_ASSERT(pktio_entry->s.state == STATE_START); for (i = 0; i < len; ++i) { - if (odp_packet_len(pkts[i]) > PKTIO_PCAP_MTU) { + int pkt_len = odp_packet_len(pkts[i]); + + if (pkt_len > PKTIO_PCAP_MTU) { if (i == 0) return -1; break; @@ -300,9 +308,12 @@ static int pcapif_send_pkt(pktio_entry_t *pktio_entry, odp_packet_t pkts[], if (_pcapif_dump_pkt(pcap, pkts[i]) != 0) break; + pktio_entry->s.stats.out_octets += pkt_len; odp_packet_free(pkts[i]); } + pktio_entry->s.stats.out_ucast_pkts += i; + return i; } @@ -367,13 +378,37 @@ static int pcapif_promisc_mode_get(pktio_entry_t *pktio_entry) return pktio_entry->s.pkt_pcap.promisc; } +static int pcapif_stats_reset(pktio_entry_t *pktio_entry) +{ + memset(&pktio_entry->s.stats, 0, sizeof(odp_pktio_stats_t)); + return 0; +} + +static int pcapif_stats(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats) +{ + memcpy(stats, &pktio_entry->s.stats, sizeof(odp_pktio_stats_t)); + return 0; +} + const pktio_if_ops_t pcap_pktio_ops = { + .name = "pcap", .open = pcapif_init, .close = pcapif_close, + .stats = pcapif_stats, + .stats_reset = pcapif_stats_reset, .recv = pcapif_recv_pkt, .send = pcapif_send_pkt, .mtu_get = pcapif_mtu_get, .promisc_mode_set = pcapif_promisc_mode_set, .promisc_mode_get = pcapif_promisc_mode_get, - .mac_get = pcapif_mac_addr_get + .mac_get = pcapif_mac_addr_get, + .capability = NULL, + .input_queues_config = NULL, + .output_queues_config = NULL, + .in_queues = NULL, + .pktin_queues = NULL, + .pktout_queues = NULL, + .recv_queue = NULL, + .send_queue = NULL }; diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c index be9db330f..1adc5f2ea 100644 --- a/platform/linux-generic/pktio/pktio_common.c +++ b/platform/linux-generic/pktio/pktio_common.c @@ -51,3 +51,75 @@ int _odp_packet_cls_enq(pktio_entry_t *pktio_entry, return 0; } + +int sock_stats_reset_fd(pktio_entry_t *pktio_entry, int fd) +{ + int err = 0; + odp_pktio_stats_t cur_stats; + + if (pktio_entry->s.stats_type == STATS_UNSUPPORTED) { + memset(&pktio_entry->s.stats, 0, + sizeof(odp_pktio_stats_t)); + return 0; + } + + memset(&cur_stats, 0, sizeof(odp_pktio_stats_t)); + + if (pktio_entry->s.stats_type == STATS_ETHTOOL) { + (void)ethtool_stats_get_fd(fd, + pktio_entry->s.name, + &cur_stats); + } else if (pktio_entry->s.stats_type == STATS_SYSFS) { + err = sysfs_stats(pktio_entry, &cur_stats); + if (err != 0) + ODP_ERR("stats error\n"); + } + + if (err == 0) + memcpy(&pktio_entry->s.stats, &cur_stats, + sizeof(odp_pktio_stats_t)); + + return err; +} + +int sock_stats_fd(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats, + int fd) +{ + odp_pktio_stats_t cur_stats; + int ret = 0; + + if (pktio_entry->s.stats_type == STATS_UNSUPPORTED) + return 0; + + memset(&cur_stats, 0, sizeof(odp_pktio_stats_t)); + if (pktio_entry->s.stats_type == STATS_ETHTOOL) { + (void)ethtool_stats_get_fd(fd, + pktio_entry->s.name, + &cur_stats); + } else if (pktio_entry->s.stats_type == STATS_SYSFS) { + sysfs_stats(pktio_entry, &cur_stats); + } + + stats->in_octets = cur_stats.in_octets - + pktio_entry->s.stats.in_octets; + stats->in_ucast_pkts = cur_stats.in_ucast_pkts - + pktio_entry->s.stats.in_ucast_pkts; + stats->in_discards = cur_stats.in_discards - + pktio_entry->s.stats.in_discards; + stats->in_errors = cur_stats.in_errors - + pktio_entry->s.stats.in_errors; + stats->in_unknown_protos = cur_stats.in_unknown_protos - + pktio_entry->s.stats.in_unknown_protos; + + stats->out_octets = cur_stats.out_octets - + pktio_entry->s.stats.out_octets; + stats->out_ucast_pkts = cur_stats.out_ucast_pkts - + pktio_entry->s.stats.out_ucast_pkts; + stats->out_discards = cur_stats.out_discards - + pktio_entry->s.stats.out_discards; + stats->out_errors = cur_stats.out_errors - + pktio_entry->s.stats.out_errors; + + return ret; +} diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 1417fb421..ed338c2c6 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -29,6 +29,8 @@ #include <sys/ioctl.h> #include <errno.h> #include <sys/syscall.h> +#include <linux/ethtool.h> +#include <linux/sockios.h> #include <odp.h> #include <odp_packet_socket.h> @@ -44,6 +46,8 @@ #include <odp/helper/eth.h> #include <odp/helper/ip.h> +static int sock_stats_reset(pktio_entry_t *pktio_entry); + /** Provide a sendmmsg wrapper for systems with no libc or kernel support. * As it is implemented as a weak symbol, it has zero effect on systems * with both. @@ -191,6 +195,255 @@ int promisc_mode_get_fd(int fd, const char *name) return !!(ifr.ifr_flags & IFF_PROMISC); } +int link_status_fd(int fd, const char *name) +{ + struct ifreq ifr; + int ret; + + snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); + ret = ioctl(fd, SIOCGIFFLAGS, &ifr); + if (ret < 0) { + __odp_errno = errno; + ODP_DBG("ioctl(SIOCGIFFLAGS): %s: \"%s\".\n", strerror(errno), + ifr.ifr_name); + return -1; + } + + return !!(ifr.ifr_flags & IFF_RUNNING); +} + +/** + * Get enabled hash options of a packet socket + * + * @param fd Socket file descriptor + * @param name Interface name + * @param flow_type Packet flow type + * @param options[out] Enabled hash options + * + * @retval 0 on success + * @retval <0 on failure + */ +static inline int get_rss_hash_options(int fd, const char *name, + uint32_t flow_type, uint64_t *options) +{ + struct ifreq ifr; + struct ethtool_rxnfc rsscmd; + + memset(&rsscmd, 0, sizeof(rsscmd)); + *options = 0; + + snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); + + rsscmd.cmd = ETHTOOL_GRXFH; + rsscmd.flow_type = flow_type; + + ifr.ifr_data = (caddr_t)&rsscmd; + + if (ioctl(fd, SIOCETHTOOL, &ifr) < 0) + return -1; + + *options = rsscmd.data; + return 0; +} + +int rss_conf_get_fd(int fd, const char *name, + odp_pktin_hash_proto_t *hash_proto) +{ + uint64_t options; + int rss_enabled = 0; + + memset(hash_proto, 0, sizeof(odp_pktin_hash_proto_t)); + + get_rss_hash_options(fd, name, IPV4_FLOW, &options); + if ((options & RXH_IP_SRC) && (options & RXH_IP_DST)) { + hash_proto->proto.ipv4 = 1; + rss_enabled++; + } + get_rss_hash_options(fd, name, TCP_V4_FLOW, &options); + if ((options & RXH_IP_SRC) && (options & RXH_IP_DST) && + (options & RXH_L4_B_0_1) && (options & RXH_L4_B_2_3)) { + hash_proto->proto.ipv4_tcp = 1; + rss_enabled++; + } + get_rss_hash_options(fd, name, UDP_V4_FLOW, &options); + if ((options & RXH_IP_SRC) && (options & RXH_IP_DST) && + (options & RXH_L4_B_0_1) && (options & RXH_L4_B_2_3)) { + hash_proto->proto.ipv4_udp = 1; + rss_enabled++; + } + get_rss_hash_options(fd, name, IPV6_FLOW, &options); + if ((options & RXH_IP_SRC) && (options & RXH_IP_DST)) { + hash_proto->proto.ipv6 = 1; + rss_enabled++; + } + get_rss_hash_options(fd, name, TCP_V6_FLOW, &options); + if ((options & RXH_IP_SRC) && (options & RXH_IP_DST) && + (options & RXH_L4_B_0_1) && (options & RXH_L4_B_2_3)) { + hash_proto->proto.ipv6_tcp = 1; + rss_enabled++; + } + get_rss_hash_options(fd, name, UDP_V6_FLOW, &options); + if ((options & RXH_IP_SRC) && (options & RXH_IP_DST) && + (options & RXH_L4_B_0_1) && (options & RXH_L4_B_2_3)) { + hash_proto->proto.ipv6_udp = 1; + rss_enabled++; + } + return rss_enabled; +} + +/** + * Set hash options of a packet socket + * + * @param fd Socket file descriptor + * @param name Interface name + * @param flow_type Packet flow type + * @param options Hash options + * + * @retval 0 on success + * @retval <0 on failure + */ +static inline int set_rss_hash(int fd, const char *name, + uint32_t flow_type, uint64_t options) +{ + struct ifreq ifr; + struct ethtool_rxnfc rsscmd; + + memset(&rsscmd, 0, sizeof(rsscmd)); + + snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); + + rsscmd.cmd = ETHTOOL_SRXFH; + rsscmd.flow_type = flow_type; + rsscmd.data = options; + + ifr.ifr_data = (caddr_t)&rsscmd; + + if (ioctl(fd, SIOCETHTOOL, &ifr) < 0) + return -1; + + return 0; +} + +int rss_conf_set_fd(int fd, const char *name, + const odp_pktin_hash_proto_t *hash_proto) +{ + uint64_t options; + odp_pktin_hash_proto_t cur_hash; + + /* Compare to currently set hash protocols */ + rss_conf_get_fd(fd, name, &cur_hash); + + if (hash_proto->proto.ipv4_udp && !cur_hash.proto.ipv4_udp) { + options = RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3; + if (set_rss_hash(fd, name, UDP_V4_FLOW, options)) + return -1; + } + if (hash_proto->proto.ipv4_tcp && !cur_hash.proto.ipv4_tcp) { + options = RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3; + if (set_rss_hash(fd, name, TCP_V4_FLOW, options)) + return -1; + } + if (hash_proto->proto.ipv6_udp && !cur_hash.proto.ipv6_udp) { + options = RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3; + if (set_rss_hash(fd, name, UDP_V6_FLOW, options)) + return -1; + } + if (hash_proto->proto.ipv6_tcp && !cur_hash.proto.ipv6_tcp) { + options = RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3; + if (set_rss_hash(fd, name, TCP_V6_FLOW, options)) + return -1; + } + if (hash_proto->proto.ipv4 && !cur_hash.proto.ipv4) { + options = RXH_IP_SRC | RXH_IP_DST; + if (set_rss_hash(fd, name, IPV4_FLOW, options)) + return -1; + } + if (hash_proto->proto.ipv6 && !cur_hash.proto.ipv6) { + options = RXH_IP_SRC | RXH_IP_DST; + if (set_rss_hash(fd, name, IPV6_FLOW, options)) + return -1; + } + return 0; +} + +int rss_conf_get_supported_fd(int fd, const char *name, + odp_pktin_hash_proto_t *hash_proto) +{ + uint64_t options; + int rss_supported = 0; + + memset(hash_proto, 0, sizeof(odp_pktin_hash_proto_t)); + + if (!get_rss_hash_options(fd, name, IPV4_FLOW, &options)) { + if (!set_rss_hash(fd, name, IPV4_FLOW, options)) { + hash_proto->proto.ipv4 = 1; + rss_supported++; + } + } + if (!get_rss_hash_options(fd, name, TCP_V4_FLOW, &options)) { + if (!set_rss_hash(fd, name, TCP_V4_FLOW, options)) { + hash_proto->proto.ipv4_tcp = 1; + rss_supported++; + } + } + if (!get_rss_hash_options(fd, name, UDP_V4_FLOW, &options)) { + if (!set_rss_hash(fd, name, UDP_V4_FLOW, options)) { + hash_proto->proto.ipv4_udp = 1; + rss_supported++; + } + } + if (!get_rss_hash_options(fd, name, IPV6_FLOW, &options)) { + if (!set_rss_hash(fd, name, IPV6_FLOW, options)) { + hash_proto->proto.ipv6 = 1; + rss_supported++; + } + } + if (!get_rss_hash_options(fd, name, TCP_V6_FLOW, &options)) { + if (!set_rss_hash(fd, name, TCP_V6_FLOW, options)) { + hash_proto->proto.ipv6_tcp = 1; + rss_supported++; + } + } + if (!get_rss_hash_options(fd, name, UDP_V6_FLOW, &options)) { + if (!set_rss_hash(fd, name, UDP_V6_FLOW, options)) { + hash_proto->proto.ipv6_udp = 1; + rss_supported++; + } + } + return rss_supported; +} + +void rss_conf_print(const odp_pktin_hash_proto_t *hash_proto) +{ int max_len = 512; + char str[max_len]; + int len = 0; + int n = max_len - 1; + + len += snprintf(&str[len], n - len, "RSS conf\n"); + + if (hash_proto->proto.ipv4) + len += snprintf(&str[len], n - len, + " IPV4\n"); + if (hash_proto->proto.ipv4_tcp) + len += snprintf(&str[len], n - len, + " IPV4 TCP\n"); + if (hash_proto->proto.ipv4_udp) + len += snprintf(&str[len], n - len, + " IPV4 UDP\n"); + if (hash_proto->proto.ipv6) + len += snprintf(&str[len], n - len, + " IPV6\n"); + if (hash_proto->proto.ipv6_tcp) + len += snprintf(&str[len], n - len, + " IPV6 TCP\n"); + if (hash_proto->proto.ipv6_udp) + len += snprintf(&str[len], n - len, + " IPV6 UDP\n"); + str[len] = '\0'; + + ODP_PRINT("\n%s\n", str); +} + /* * ODP_PACKET_SOCKET_MMSG: */ @@ -223,6 +476,7 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, char shm_name[ODP_SHM_NAME_LEN]; pkt_sock_t *pkt_sock = &pktio_entry->s.pkt_sock; uint8_t *addr; + odp_pktio_stats_t cur_stats; /* Init pktio entry */ memset(pkt_sock, 0, sizeof(*pkt_sock)); @@ -281,6 +535,27 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, ODP_ERR("bind(to IF): %s\n", strerror(errno)); goto error; } + + err = ethtool_stats_get_fd(pktio_entry->s.pkt_sock.sockfd, + pktio_entry->s.name, + &cur_stats); + if (err != 0) { + err = sysfs_stats(pktio_entry, &cur_stats); + if (err != 0) { + pktio_entry->s.stats_type = STATS_UNSUPPORTED; + ODP_DBG("pktio: %s unsupported stats\n", + pktio_entry->s.name); + } else { + pktio_entry->s.stats_type = STATS_SYSFS; + } + } else { + pktio_entry->s.stats_type = STATS_ETHTOOL; + } + + err = sock_stats_reset(pktio_entry); + if (err != 0) + goto error; + return 0; error: @@ -531,17 +806,60 @@ static int sock_promisc_mode_get(pktio_entry_t *pktio_entry) pktio_entry->s.name); } +static int sock_link_status(pktio_entry_t *pktio_entry) +{ + return link_status_fd(pktio_entry->s.pkt_sock.sockfd, + pktio_entry->s.name); +} + +static int sock_stats(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats) +{ + if (pktio_entry->s.stats_type == STATS_UNSUPPORTED) { + memset(stats, 0, sizeof(*stats)); + return 0; + } + + return sock_stats_fd(pktio_entry, + stats, + pktio_entry->s.pkt_sock.sockfd); +} + +static int sock_stats_reset(pktio_entry_t *pktio_entry) +{ + if (pktio_entry->s.stats_type == STATS_UNSUPPORTED) { + memset(&pktio_entry->s.stats, 0, + sizeof(odp_pktio_stats_t)); + return 0; + } + + return sock_stats_reset_fd(pktio_entry, + pktio_entry->s.pkt_sock.sockfd); +} + const pktio_if_ops_t sock_mmsg_pktio_ops = { + .name = "socket", .init = NULL, .term = NULL, .open = sock_mmsg_open, .close = sock_close, .start = NULL, .stop = NULL, + .stats = sock_stats, + .stats_reset = sock_stats_reset, .recv = sock_mmsg_recv, .send = sock_mmsg_send, .mtu_get = sock_mtu_get, .promisc_mode_set = sock_promisc_mode_set, .promisc_mode_get = sock_promisc_mode_get, - .mac_get = sock_mac_addr_get + .mac_get = sock_mac_addr_get, + .link_status = sock_link_status, + .capability = NULL, + .input_queues_config = NULL, + .output_queues_config = NULL, + .in_queues = NULL, + .pktin_queues = NULL, + .pktout_queues = NULL, + .recv_queue = NULL, + .send_queue = NULL }; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 1d2eb3c9c..07fbee63e 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -444,6 +444,7 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, { int if_idx; int ret = 0; + odp_pktio_stats_t cur_stats; if (getenv("ODP_PKTIO_DISABLE_SOCKET_MMAP")) return -1; @@ -503,6 +504,27 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, goto error; } + ret = ethtool_stats_get_fd(pktio_entry->s.pkt_sock_mmap.sockfd, + pktio_entry->s.name, + &cur_stats); + if (ret != 0) { + ret = sysfs_stats(pktio_entry, &cur_stats); + if (ret != 0) { + pktio_entry->s.stats_type = STATS_UNSUPPORTED; + ODP_DBG("pktio: %s unsupported stats\n", + pktio_entry->s.name); + } else { + pktio_entry->s.stats_type = STATS_SYSFS; + } + } else { + pktio_entry->s.stats_type = STATS_ETHTOOL; + } + + ret = sock_stats_reset_fd(pktio_entry, + pktio_entry->s.pkt_sock_mmap.sockfd); + if (ret != 0) + goto error; + return 0; error: @@ -553,17 +575,60 @@ static int sock_mmap_promisc_mode_get(pktio_entry_t *pktio_entry) pktio_entry->s.name); } +static int sock_mmap_link_status(pktio_entry_t *pktio_entry) +{ + return link_status_fd(pktio_entry->s.pkt_sock_mmap.sockfd, + pktio_entry->s.name); +} + +static int sock_mmap_stats(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats) +{ + if (pktio_entry->s.stats_type == STATS_UNSUPPORTED) { + memset(stats, 0, sizeof(*stats)); + return 0; + } + + return sock_stats_fd(pktio_entry, + stats, + pktio_entry->s.pkt_sock_mmap.sockfd); +} + +static int sock_mmap_stats_reset(pktio_entry_t *pktio_entry) +{ + if (pktio_entry->s.stats_type == STATS_UNSUPPORTED) { + memset(&pktio_entry->s.stats, 0, + sizeof(odp_pktio_stats_t)); + return 0; + } + + return sock_stats_reset_fd(pktio_entry, + pktio_entry->s.pkt_sock_mmap.sockfd); +} + const pktio_if_ops_t sock_mmap_pktio_ops = { + .name = "socket_mmap", .init = NULL, .term = NULL, .open = sock_mmap_open, .close = sock_mmap_close, .start = NULL, .stop = NULL, + .stats = sock_mmap_stats, + .stats_reset = sock_mmap_stats_reset, .recv = sock_mmap_recv, .send = sock_mmap_send, .mtu_get = sock_mmap_mtu_get, .promisc_mode_set = sock_mmap_promisc_mode_set, .promisc_mode_get = sock_mmap_promisc_mode_get, - .mac_get = sock_mmap_mac_addr_get + .mac_get = sock_mmap_mac_addr_get, + .link_status = sock_mmap_link_status, + .capability = NULL, + .input_queues_config = NULL, + .output_queues_config = NULL, + .in_queues = NULL, + .pktin_queues = NULL, + .pktout_queues = NULL, + .recv_queue = NULL, + .send_queue = NULL }; diff --git a/platform/linux-generic/pktio/sysfs.c b/platform/linux-generic/pktio/sysfs.c new file mode 100644 index 000000000..4e5c02837 --- /dev/null +++ b/platform/linux-generic/pktio/sysfs.c @@ -0,0 +1,76 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp.h> +#include <odp_packet_io_internal.h> +#include <errno.h> +#include <string.h> + +static int sysfs_get_val(const char *fname, uint64_t *val) +{ + FILE *file; + char str[128]; + int ret = -1; + + file = fopen(fname, "rt"); + if (file == NULL) { + __odp_errno = errno; + /* do not print debug err if sysfs is not supported by + * kernel driver. + */ + if (errno != ENOENT) + ODP_ERR("fopen %s: %s\n", fname, strerror(errno)); + return 0; + } + + if (fgets(str, sizeof(str), file) != NULL) + ret = sscanf(str, "%" SCNx64, val); + + (void)fclose(file); + + if (ret != 1) { + ODP_ERR("read %s\n", fname); + return -1; + } + + return 0; +} + +int sysfs_stats(pktio_entry_t *pktio_entry, + odp_pktio_stats_t *stats) +{ + char fname[256]; + const char *dev = pktio_entry->s.name; + int ret = 0; + + sprintf(fname, "/sys/class/net/%s/statistics/rx_bytes", dev); + ret -= sysfs_get_val(fname, &stats->in_octets); + + sprintf(fname, "/sys/class/net/%s/statistics/rx_packets", dev); + ret -= sysfs_get_val(fname, &stats->in_ucast_pkts); + + sprintf(fname, "/sys/class/net/%s/statistics/rx_droppped", dev); + ret -= sysfs_get_val(fname, &stats->in_discards); + + sprintf(fname, "/sys/class/net/%s/statistics/rx_errors", dev); + ret -= sysfs_get_val(fname, &stats->in_errors); + + /* stats->in_unknown_protos is not supported in sysfs */ + + sprintf(fname, "/sys/class/net/%s/statistics/tx_bytes", dev); + ret -= sysfs_get_val(fname, &stats->out_octets); + + sprintf(fname, "/sys/class/net/%s/statistics/tx_packets", dev); + ret -= sysfs_get_val(fname, &stats->out_ucast_pkts); + + sprintf(fname, "/sys/class/net/%s/statistics/tx_dropped", dev); + ret -= sysfs_get_val(fname, &stats->out_discards); + + sprintf(fname, "/sys/class/net/%s/statistics/tx_errors", dev); + ret -= sysfs_get_val(fname, &stats->out_errors); + + return ret; +} diff --git a/platform/linux-generic/test/Makefile.am b/platform/linux-generic/test/Makefile.am index e62987297..db923b8c7 100644 --- a/platform/linux-generic/test/Makefile.am +++ b/platform/linux-generic/test/Makefile.am @@ -6,6 +6,8 @@ ODP_MODULES = pktio if test_vald TESTS = pktio/pktio_run \ pktio/pktio_run_tap \ + ${top_builddir}/test/validation/atomic/atomic_main$(EXEEXT) \ + ${top_builddir}/test/validation/barrier/barrier_main$(EXEEXT) \ ${top_builddir}/test/validation/buffer/buffer_main$(EXEEXT) \ ${top_builddir}/test/validation/classification/classification_main$(EXEEXT) \ ${top_builddir}/test/validation/config/config_main$(EXEEXT) \ @@ -16,13 +18,13 @@ TESTS = pktio/pktio_run \ ${top_builddir}/test/validation/init/init_main_ok$(EXEEXT) \ ${top_builddir}/test/validation/init/init_main_abort$(EXEEXT) \ ${top_builddir}/test/validation/init/init_main_log$(EXEEXT) \ + ${top_builddir}/test/validation/lock/lock_main$(EXEEXT) \ ${top_builddir}/test/validation/packet/packet_main$(EXEEXT) \ ${top_builddir}/test/validation/pool/pool_main$(EXEEXT) \ ${top_builddir}/test/validation/queue/queue_main$(EXEEXT) \ ${top_builddir}/test/validation/random/random_main$(EXEEXT) \ ${top_builddir}/test/validation/scheduler/scheduler_main$(EXEEXT) \ ${top_builddir}/test/validation/std_clib/std_clib_main$(EXEEXT) \ - ${top_builddir}/test/validation/synchronizers/synchronizers_main$(EXEEXT) \ ${top_builddir}/test/validation/thread/thread_main$(EXEEXT) \ ${top_builddir}/test/validation/time/time_main$(EXEEXT) \ ${top_builddir}/test/validation/timer/timer_main$(EXEEXT) \ |