diff options
Diffstat (limited to 'platform')
55 files changed, 1799 insertions, 451 deletions
diff --git a/platform/Makefile.inc b/platform/Makefile.inc index 5d589b1b0..ef9724baa 100644 --- a/platform/Makefile.inc +++ b/platform/Makefile.inc @@ -32,6 +32,7 @@ odpapiinclude_HEADERS = \ $(top_srcdir)/include/odp/api/debug.h \ $(top_srcdir)/include/odp/api/errno.h \ $(top_srcdir)/include/odp/api/event.h \ + $(top_srcdir)/include/odp/api/hash.h \ $(top_srcdir)/include/odp/api/hints.h \ $(top_srcdir)/include/odp/api/init.h \ $(top_srcdir)/include/odp/api/packet.h \ @@ -41,10 +42,13 @@ odpapiinclude_HEADERS = \ $(top_srcdir)/include/odp/api/queue.h \ $(top_srcdir)/include/odp/api/random.h \ $(top_srcdir)/include/odp/api/rwlock.h \ + $(top_srcdir)/include/odp/api/rwlock_recursive.h \ $(top_srcdir)/include/odp/api/schedule.h \ $(top_srcdir)/include/odp/api/schedule_types.h \ $(top_srcdir)/include/odp/api/shared_memory.h \ $(top_srcdir)/include/odp/api/spinlock.h \ + $(top_srcdir)/include/odp/api/spinlock_recursive.h \ + $(top_srcdir)/include/odp/api/std_clib.h \ $(top_srcdir)/include/odp/api/std_types.h \ $(top_srcdir)/include/odp/api/sync.h \ $(top_srcdir)/include/odp/api/system_info.h \ diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 4639ebcc0..806419359 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -26,6 +26,7 @@ odpinclude_HEADERS = \ $(srcdir)/include/odp/debug.h \ $(srcdir)/include/odp/errno.h \ $(srcdir)/include/odp/event.h \ + $(srcdir)/include/odp/hash.h \ $(srcdir)/include/odp/hints.h \ $(srcdir)/include/odp/init.h \ $(srcdir)/include/odp/packet_flags.h \ @@ -35,10 +36,13 @@ odpinclude_HEADERS = \ $(srcdir)/include/odp/queue.h \ $(srcdir)/include/odp/random.h \ $(srcdir)/include/odp/rwlock.h \ + $(srcdir)/include/odp/rwlock_recursive.h \ $(srcdir)/include/odp/schedule.h \ $(srcdir)/include/odp/schedule_types.h \ $(srcdir)/include/odp/shared_memory.h \ $(srcdir)/include/odp/spinlock.h \ + $(srcdir)/include/odp/spinlock_recursive.h \ + $(srcdir)/include/odp/std_clib.h \ $(srcdir)/include/odp/std_types.h \ $(srcdir)/include/odp/sync.h \ $(srcdir)/include/odp/system_info.h \ @@ -64,10 +68,13 @@ odpplatinclude_HEADERS = \ $(srcdir)/include/odp/plat/pool_types.h \ $(srcdir)/include/odp/plat/queue_types.h \ $(srcdir)/include/odp/plat/rwlock_types.h \ + $(srcdir)/include/odp/plat/rwlock_recursive_types.h \ $(srcdir)/include/odp/plat/schedule_types.h \ $(srcdir)/include/odp/plat/shared_memory_types.h \ $(srcdir)/include/odp/plat/spinlock_types.h \ + $(srcdir)/include/odp/plat/spinlock_recursive_types.h \ $(srcdir)/include/odp/plat/strong_types.h \ + $(srcdir)/include/odp/plat/thread_types.h \ $(srcdir)/include/odp/plat/thrmask_types.h \ $(srcdir)/include/odp/plat/ticketlock_types.h \ $(srcdir)/include/odp/plat/time_types.h \ @@ -82,7 +89,6 @@ noinst_HEADERS = \ ${srcdir}/include/odp_classification_datamodel.h \ ${srcdir}/include/odp_classification_inlines.h \ ${srcdir}/include/odp_classification_internal.h \ - ${srcdir}/include/odp_config_internal.h \ ${srcdir}/include/odp_crypto_internal.h \ ${srcdir}/include/odp_debug_internal.h \ ${srcdir}/include/odp_forward_typedefs_internal.h \ @@ -98,7 +104,6 @@ noinst_HEADERS = \ ${srcdir}/include/odp_schedule_internal.h \ ${srcdir}/include/odp_spin_internal.h \ ${srcdir}/include/odp_timer_internal.h \ - ${srcdir}/include/odp_cpu_internal.h \ ${srcdir}/Makefile.inc __LIB__libodp_la_SOURCES = \ @@ -111,12 +116,14 @@ __LIB__libodp_la_SOURCES = \ odp_crypto.c \ odp_errno.c \ odp_event.c \ + odp_hash.c \ odp_init.c \ odp_impl.c \ odp_packet.c \ odp_packet_flags.c \ odp_packet_io.c \ pktio/io_ops.c \ + pktio/pktio_common.c \ pktio/loop.c \ pktio/netmap.c \ pktio/socket.c \ @@ -125,9 +132,11 @@ __LIB__libodp_la_SOURCES = \ odp_pool.c \ odp_queue.c \ odp_rwlock.c \ + odp_rwlock_recursive.c \ odp_schedule.c \ odp_shared_memory.c \ odp_spinlock.c \ + odp_spinlock_recursive.c \ odp_system_info.c \ odp_thread.c \ odp_thrmask.c \ diff --git a/platform/linux-generic/include/odp/atomic.h b/platform/linux-generic/include/odp/atomic.h index e47a280bf..deb4039b3 100644 --- a/platform/linux-generic/include/odp/atomic.h +++ b/platform/linux-generic/include/odp/atomic.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/align.h> #include <odp/plat/atomic_types.h> -/** @ingroup odp_synchronizers +/** @ingroup odp_atomic * @{ */ diff --git a/platform/linux-generic/include/odp/barrier.h b/platform/linux-generic/include/odp/barrier.h index 7ea5a6b99..42df8595a 100644 --- a/platform/linux-generic/include/odp/barrier.h +++ b/platform/linux-generic/include/odp/barrier.h @@ -21,14 +21,6 @@ extern "C" { #include <odp/plat/shared_memory_types.h> #include <odp/plat/barrier_types.h> -/** @ingroup odp_synchronizers - * @{ - */ - -/** - * @} - */ - #include <odp/api/barrier.h> #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp/hash.h index 4f20ff8e0..df8add002 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp/hash.h @@ -7,20 +7,25 @@ /** * @file * - * Linux-generic platform internal configuration + * ODP Hash function */ -#ifndef ODP_CONFIG_INTERNAL_H_ -#define ODP_CONFIG_INTERNAL_H_ +#ifndef ODP_PLAT_HASH_H_ +#define ODP_PLAT_HASH_H_ #ifdef __cplusplus extern "C" { #endif +/** @ingroup odp_hash + * @{ + */ + /** - * Maximum number of threads + * @} */ -#define _ODP_INTERNAL_MAX_THREADS 128 + +#include <odp/api/hash.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp/plat/atomic_types.h b/platform/linux-generic/include/odp/plat/atomic_types.h index 3cdcab8bd..0f6c353a1 100644 --- a/platform/linux-generic/include/odp/plat/atomic_types.h +++ b/platform/linux-generic/include/odp/plat/atomic_types.h @@ -62,18 +62,10 @@ struct odp_atomic_u32_s { }) #endif -/** @addtogroup odp_synchronizers - * @{ - */ - typedef struct odp_atomic_u64_s odp_atomic_u64_t; typedef struct odp_atomic_u32_s odp_atomic_u32_t; -/** - * @} - */ - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp/plat/barrier_types.h b/platform/linux-generic/include/odp/plat/barrier_types.h index c8c978dca..b8e1d97bb 100644 --- a/platform/linux-generic/include/odp/plat/barrier_types.h +++ b/platform/linux-generic/include/odp/plat/barrier_types.h @@ -30,16 +30,8 @@ struct odp_barrier_s { odp_atomic_u32_t bar; /**< Barrier counter */ }; -/** @addtogroup odp_synchronizers - * @{ - */ - typedef struct odp_barrier_s odp_barrier_t; -/** - * @} - */ - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp/plat/rwlock_recursive_types.h b/platform/linux-generic/include/odp/plat/rwlock_recursive_types.h new file mode 100644 index 000000000..474751cf0 --- /dev/null +++ b/platform/linux-generic/include/odp/plat/rwlock_recursive_types.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP recursive read/write lock + */ + +#ifndef ODP_RWLOCK_RECURSIVE_TYPES_H_ +#define ODP_RWLOCK_RECURSIVE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/rwlock.h> +#include <odp/std_types.h> +#include <odp/thread.h> + +/** @internal */ +struct odp_rwlock_recursive_s { + odp_rwlock_t lock; /**< the lock */ + int wr_owner; /**< write owner thread */ + uint32_t wr_cnt; /**< write recursion count */ + uint8_t rd_cnt[ODP_THREAD_COUNT_MAX]; /**< read recursion count */ +}; + +typedef struct odp_rwlock_recursive_s odp_rwlock_recursive_t; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/plat/rwlock_types.h b/platform/linux-generic/include/odp/plat/rwlock_types.h index bd46e5722..35d65decb 100644 --- a/platform/linux-generic/include/odp/plat/rwlock_types.h +++ b/platform/linux-generic/include/odp/plat/rwlock_types.h @@ -20,10 +20,7 @@ extern "C" { #include <odp/atomic.h> -/** - * @internal - * ODP rwlock - */ +/** @internal */ struct odp_rwlock_s { odp_atomic_u32_t cnt; /**< lock count 0 lock not taken @@ -31,16 +28,8 @@ struct odp_rwlock_s { >0 read lock(s) taken */ }; -/** @addtogroup odp_synchronizers - * @{ - */ - typedef struct odp_rwlock_s odp_rwlock_t; -/** - * @} - */ - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp/plat/spinlock_recursive_types.h b/platform/linux-generic/include/odp/plat/spinlock_recursive_types.h new file mode 100644 index 000000000..2809277bd --- /dev/null +++ b/platform/linux-generic/include/odp/plat/spinlock_recursive_types.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP recursive spinlock + */ + +#ifndef ODP_SPINLOCK_RECURSIVE_TYPES_H_ +#define ODP_SPINLOCK_RECURSIVE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/spinlock.h> +#include <odp/std_types.h> + +/** @internal */ +struct odp_spinlock_recursive_s { + odp_spinlock_t lock; /**< the lock */ + int owner; /**< thread owning the lock */ + uint32_t cnt; /**< recursion count */ +}; + +typedef struct odp_spinlock_recursive_s odp_spinlock_recursive_t; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/plat/spinlock_types.h b/platform/linux-generic/include/odp/plat/spinlock_types.h index 83d306b5b..3e0231d1d 100644 --- a/platform/linux-generic/include/odp/plat/spinlock_types.h +++ b/platform/linux-generic/include/odp/plat/spinlock_types.h @@ -20,25 +20,13 @@ extern "C" { #include <odp/std_types.h> -/** - * @internal - * ODP spinlock - */ +/** @internal */ struct odp_spinlock_s { char lock; /**< lock flag, should match odp_atomic_flag_t */ }; - -/** @addtogroup odp_synchronizers - * @{ - */ - typedef struct odp_spinlock_s odp_spinlock_t; -/** - * @} - */ - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp/plat/thread_types.h b/platform/linux-generic/include/odp/plat/thread_types.h new file mode 100644 index 000000000..33af45983 --- /dev/null +++ b/platform/linux-generic/include/odp/plat/thread_types.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP thread + */ + +#ifndef ODP_THREAD_TYPES_H_ +#define ODP_THREAD_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup odp_thread + * @{ + */ + +#define ODP_THREAD_COUNT_MAX 128 + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/plat/ticketlock_types.h b/platform/linux-generic/include/odp/plat/ticketlock_types.h index be9308556..73f970542 100644 --- a/platform/linux-generic/include/odp/plat/ticketlock_types.h +++ b/platform/linux-generic/include/odp/plat/ticketlock_types.h @@ -20,25 +20,14 @@ extern "C" { #include <odp/atomic.h> -/** - * @internal - * ODP ticketlock - */ +/** @internal */ struct odp_ticketlock_s { odp_atomic_u32_t next_ticket; /**< Next ticket */ odp_atomic_u32_t cur_ticket; /**< Current ticket */ }; -/** @addtogroup odp_synchronizers - * @{ - */ - typedef struct odp_ticketlock_s odp_ticketlock_t; -/** - * @} - */ - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp/plat/time_types.h b/platform/linux-generic/include/odp/plat/time_types.h index 9ba1508c7..14c35f0bf 100644 --- a/platform/linux-generic/include/odp/plat/time_types.h +++ b/platform/linux-generic/include/odp/plat/time_types.h @@ -21,9 +21,16 @@ extern "C" { * @{ **/ -typedef uint64_t odp_time_t; +/** + * @internal Time structure used to isolate linux-generic implementation from + * the linux timespec structure, which is dependent on _POSIX_C_SOURCE level. + */ +typedef struct odp_time_t { + int64_t tv_sec; /**< @internal Seconds */ + int64_t tv_nsec; /**< @internal Nanoseconds */ +} odp_time_t; -#define ODP_TIME_NULL ((odp_time_t)0) +#define ODP_TIME_NULL ((odp_time_t){0, 0}) /** * @} diff --git a/platform/linux-generic/include/odp/rwlock.h b/platform/linux-generic/include/odp/rwlock.h index ca88ff7db..f9d8f5fd4 100644 --- a/platform/linux-generic/include/odp/rwlock.h +++ b/platform/linux-generic/include/odp/rwlock.h @@ -19,14 +19,6 @@ extern "C" { #include <odp/plat/rwlock_types.h> -/** @ingroup odp_synchronizers - * @{ - */ - -/** - * @} - */ - #include <odp/api/rwlock.h> #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/rwlock_recursive.h b/platform/linux-generic/include/odp/rwlock_recursive.h new file mode 100644 index 000000000..e9dadc878 --- /dev/null +++ b/platform/linux-generic/include/odp/rwlock_recursive.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP resursive read/write lock + */ + +#ifndef ODP_PLAT_RWLOCK_RECURSIVE_H_ +#define ODP_PLAT_RWLOCK_RECURSIVE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/plat/rwlock_recursive_types.h> + +#include <odp/api/rwlock_recursive.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/spinlock.h b/platform/linux-generic/include/odp/spinlock.h index 7dbb1c4bf..046fcfada 100644 --- a/platform/linux-generic/include/odp/spinlock.h +++ b/platform/linux-generic/include/odp/spinlock.h @@ -19,14 +19,6 @@ extern "C" { #include <odp/plat/spinlock_types.h> -/** @ingroup odp_synchronizers - * @{ - */ - -/** - * @} - */ - #include <odp/api/spinlock.h> #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/spinlock_recursive.h b/platform/linux-generic/include/odp/spinlock_recursive.h new file mode 100644 index 000000000..e8a996837 --- /dev/null +++ b/platform/linux-generic/include/odp/spinlock_recursive.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP resursive spinlock + */ + +#ifndef ODP_PLAT_SPINLOCK_RECURSIVE_H_ +#define ODP_PLAT_SPINLOCK_RECURSIVE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/plat/spinlock_recursive_types.h> + +#include <odp/api/spinlock_recursive.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/std_clib.h b/platform/linux-generic/include/odp/std_clib.h new file mode 100644 index 000000000..c939c48e9 --- /dev/null +++ b/platform/linux-generic/include/odp/std_clib.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_STD_CLIB_H_ +#define ODP_PLAT_STD_CLIB_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/std_types.h> + +static inline void *odp_memcpy(void *dst, const void *src, size_t num) +{ + return memcpy(dst, src, num); +} + +static inline void *odp_memset(void *ptr, int value, size_t num) +{ + return memset(ptr, value, num); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/sync.h b/platform/linux-generic/include/odp/sync.h index 8cdab21ef..b2995e19c 100644 --- a/platform/linux-generic/include/odp/sync.h +++ b/platform/linux-generic/include/odp/sync.h @@ -17,10 +17,25 @@ extern "C" { #endif -/** @ingroup odp_synchronizers +/** @ingroup odp_barrier * @{ */ +static inline void odp_mb_release(void) +{ + __atomic_thread_fence(__ATOMIC_RELEASE); +} + +static inline void odp_mb_acquire(void) +{ + __atomic_thread_fence(__ATOMIC_ACQUIRE); +} + +static inline void odp_mb_full(void) +{ + __atomic_thread_fence(__ATOMIC_SEQ_CST); +} + /** * @} */ diff --git a/platform/linux-generic/include/odp/thread.h b/platform/linux-generic/include/odp/thread.h index e5215852a..522ca2599 100644 --- a/platform/linux-generic/include/odp/thread.h +++ b/platform/linux-generic/include/odp/thread.h @@ -17,13 +17,7 @@ extern "C" { #endif -/** @ingroup odp_thread ODP THREAD - * @{ - */ - -/** - * @} - */ +#include <odp/plat/thread_types.h> #include <odp/api/thread.h> diff --git a/platform/linux-generic/include/odp/ticketlock.h b/platform/linux-generic/include/odp/ticketlock.h index 658e27fe4..d3491519a 100644 --- a/platform/linux-generic/include/odp/ticketlock.h +++ b/platform/linux-generic/include/odp/ticketlock.h @@ -19,15 +19,6 @@ extern "C" { #include <odp/plat/ticketlock_types.h> -/** @ingroup odp_synchronizers - * Operations on ticket locks. - * @{ - */ - -/** - * @} - */ - #include <odp/api/ticketlock.h> #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index 06fc16b1b..ce62368e1 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -25,11 +25,6 @@ extern "C" { #endif -/** @addtogroup odp_synchronizers - * Atomic operations. - * @{ - */ - /** * Pointer atomic type */ @@ -598,10 +593,6 @@ static inline void _odp_atomic_flag_clear(_odp_atomic_flag_t *flag) __atomic_clear(flag, __ATOMIC_RELEASE); } -/** - * @} - */ - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 74a0b5c4f..abdd320b4 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -30,7 +30,6 @@ extern "C" { #include <odp/thread.h> #include <odp/event.h> #include <odp_forward_typedefs_internal.h> -#include <odp_config_internal.h> #define ODP_BITSIZE(x) \ ((x) <= 2 ? 1 : \ @@ -144,7 +143,7 @@ struct odp_buffer_hdr_t { /** @internal Compile time assert that the * allocator field can handle any allocator id*/ -_ODP_STATIC_ASSERT(INT16_MAX >= _ODP_INTERNAL_MAX_THREADS, +_ODP_STATIC_ASSERT(INT16_MAX >= ODP_THREAD_COUNT_MAX, "ODP_BUFFER_HDR_T__ALLOCATOR__SIZE_ERROR"); typedef struct odp_buffer_hdr_stride { diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h index 4358fcaed..5b6520266 100644 --- a/platform/linux-generic/include/odp_classification_datamodel.h +++ b/platform/linux-generic/include/odp_classification_datamodel.h @@ -70,9 +70,8 @@ struct cos_s { union pmr_u *pmr; /* Chained PMR */ union cos_u *linked_cos; /* CoS linked with the PMR */ uint32_t valid; /* validity Flag */ - odp_drop_e drop_policy; /* Associated Drop Policy */ + odp_cls_drop_t drop_policy; /* Associated Drop Policy */ odp_queue_group_t queue_group; /* Associated Queue Group */ - odp_queue_t queue_id; /* Associated Queue handle */ odp_cos_flow_set_t flow_set; /* Assigned Flow Set */ char name[ODP_COS_NAME_LEN]; /* name */ size_t headroom; /* Headroom for this CoS */ diff --git a/platform/linux-generic/include/odp_classification_inlines.h b/platform/linux-generic/include/odp_classification_inlines.h index 7f13530c0..e9739aafb 100644 --- a/platform/linux-generic/include/odp_classification_inlines.h +++ b/platform/linux-generic/include/odp_classification_inlines.h @@ -39,15 +39,16 @@ static inline int verify_pmr_packet_len(odp_packet_hdr_t *pkt_hdr, return 0; } -static inline int verify_pmr_ip_proto(uint8_t *pkt_addr, + +static inline int verify_pmr_ip_proto(const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr, pmr_term_value_t *term_value) { - odph_ipv4hdr_t *ip; + const odph_ipv4hdr_t *ip; uint8_t proto; if (!pkt_hdr->input_flags.ipv4) return 0; - ip = (odph_ipv4hdr_t *)(pkt_addr + pkt_hdr->l3_offset); + ip = (const odph_ipv4hdr_t *)(pkt_addr + pkt_hdr->l3_offset); proto = ip->proto; if (term_value->val == (proto & term_value->mask)) return 1; @@ -55,15 +56,15 @@ static inline int verify_pmr_ip_proto(uint8_t *pkt_addr, return 0; } -static inline int verify_pmr_ipv4_saddr(uint8_t *pkt_addr, +static inline int verify_pmr_ipv4_saddr(const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr, pmr_term_value_t *term_value) { - odph_ipv4hdr_t *ip; + const odph_ipv4hdr_t *ip; uint32_t ipaddr; if (!pkt_hdr->input_flags.ipv4) return 0; - ip = (odph_ipv4hdr_t *)(pkt_addr + pkt_hdr->l3_offset); + ip = (const odph_ipv4hdr_t *)(pkt_addr + pkt_hdr->l3_offset); ipaddr = odp_be_to_cpu_32(ip->src_addr); if (term_value->val == (ipaddr & term_value->mask)) return 1; @@ -71,15 +72,15 @@ static inline int verify_pmr_ipv4_saddr(uint8_t *pkt_addr, return 0; } -static inline int verify_pmr_ipv4_daddr(uint8_t *pkt_addr, +static inline int verify_pmr_ipv4_daddr(const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr, pmr_term_value_t *term_value) { - odph_ipv4hdr_t *ip; + const odph_ipv4hdr_t *ip; uint32_t ipaddr; if (!pkt_hdr->input_flags.ipv4) return 0; - ip = (odph_ipv4hdr_t *)(pkt_addr + pkt_hdr->l3_offset); + ip = (const odph_ipv4hdr_t *)(pkt_addr + pkt_hdr->l3_offset); ipaddr = odp_be_to_cpu_32(ip->dst_addr); if (term_value->val == (ipaddr & term_value->mask)) return 1; @@ -87,15 +88,15 @@ static inline int verify_pmr_ipv4_daddr(uint8_t *pkt_addr, return 0; } -static inline int verify_pmr_tcp_sport(uint8_t *pkt_addr, +static inline int verify_pmr_tcp_sport(const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr, pmr_term_value_t *term_value) { uint16_t sport; - odph_tcphdr_t *tcp; + const odph_tcphdr_t *tcp; if (!pkt_hdr->input_flags.tcp) return 0; - tcp = (odph_tcphdr_t *)(pkt_addr + pkt_hdr->l4_offset); + tcp = (const odph_tcphdr_t *)(pkt_addr + pkt_hdr->l4_offset); sport = odp_be_to_cpu_16(tcp->src_port); if (term_value->val == (sport & term_value->mask)) return 1; @@ -103,15 +104,15 @@ static inline int verify_pmr_tcp_sport(uint8_t *pkt_addr, return 0; } -static inline int verify_pmr_tcp_dport(uint8_t *pkt_addr, +static inline int verify_pmr_tcp_dport(const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr, pmr_term_value_t *term_value) { uint16_t dport; - odph_tcphdr_t *tcp; + const odph_tcphdr_t *tcp; if (!pkt_hdr->input_flags.tcp) return 0; - tcp = (odph_tcphdr_t *)(pkt_addr + pkt_hdr->l4_offset); + tcp = (const odph_tcphdr_t *)(pkt_addr + pkt_hdr->l4_offset); dport = odp_be_to_cpu_16(tcp->dst_port); if (term_value->val == (dport & term_value->mask)) return 1; @@ -119,30 +120,32 @@ static inline int verify_pmr_tcp_dport(uint8_t *pkt_addr, return 0; } -static inline int verify_pmr_udp_dport(uint8_t *pkt_addr, +static inline int verify_pmr_udp_dport(const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr, pmr_term_value_t *term_value) { uint16_t dport; - odph_udphdr_t *udp; + const odph_udphdr_t *udp; if (!pkt_hdr->input_flags.udp) return 0; - udp = (odph_udphdr_t *)(pkt_addr + pkt_hdr->l4_offset); + udp = (const odph_udphdr_t *)(pkt_addr + pkt_hdr->l4_offset); dport = odp_be_to_cpu_16(udp->dst_port); if (term_value->val == (dport & term_value->mask)) return 1; return 0; } -static inline int verify_pmr_udp_sport(uint8_t *pkt_addr, + +static inline int verify_pmr_udp_sport(const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr, pmr_term_value_t *term_value) { uint16_t sport; - odph_udphdr_t *udp; + const odph_udphdr_t *udp; + if (!pkt_hdr->input_flags.udp) return 0; - udp = (odph_udphdr_t *)(pkt_addr + pkt_hdr->l4_offset); + udp = (const odph_udphdr_t *)(pkt_addr + pkt_hdr->l4_offset); sport = odp_be_to_cpu_16(udp->src_port); if (term_value->val == (sport & term_value->mask)) return 1; @@ -150,7 +153,7 @@ static inline int verify_pmr_udp_sport(uint8_t *pkt_addr, return 0; } -static inline int verify_pmr_dmac(uint8_t *pkt_addr ODP_UNUSED, +static inline int verify_pmr_dmac(const uint8_t *pkt_addr ODP_UNUSED, odp_packet_hdr_t *pkt_hdr ODP_UNUSED, pmr_term_value_t *term_value ODP_UNUSED) { @@ -158,28 +161,31 @@ static inline int verify_pmr_dmac(uint8_t *pkt_addr ODP_UNUSED, return 0; } -static inline int verify_pmr_ipv6_saddr(uint8_t *pkt_addr ODP_UNUSED, +static inline int verify_pmr_ipv6_saddr(const uint8_t *pkt_addr ODP_UNUSED, odp_packet_hdr_t *pkt_hdr ODP_UNUSED, pmr_term_value_t *term_value ODP_UNUSED) { ODP_UNIMPLEMENTED(); return 0; } -static inline int verify_pmr_ipv6_daddr(uint8_t *pkt_addr ODP_UNUSED, + +static inline int verify_pmr_ipv6_daddr(const uint8_t *pkt_addr ODP_UNUSED, odp_packet_hdr_t *pkt_hdr ODP_UNUSED, pmr_term_value_t *term_value ODP_UNUSED) { ODP_UNIMPLEMENTED(); return 0; } -static inline int verify_pmr_vlan_id_0(uint8_t *pkt_addr ODP_UNUSED, + +static inline int verify_pmr_vlan_id_0(const uint8_t *pkt_addr ODP_UNUSED, odp_packet_hdr_t *pkt_hdr ODP_UNUSED, pmr_term_value_t *term_value ODP_UNUSED) { ODP_UNIMPLEMENTED(); return 0; } -static inline int verify_pmr_vlan_id_x(uint8_t *pkt_addr ODP_UNUSED, + +static inline int verify_pmr_vlan_id_x(const uint8_t *pkt_addr ODP_UNUSED, odp_packet_hdr_t *pkt_hdr ODP_UNUSED, pmr_term_value_t *term_value ODP_UNUSED) { @@ -187,7 +193,7 @@ static inline int verify_pmr_vlan_id_x(uint8_t *pkt_addr ODP_UNUSED, return 0; } -static inline int verify_pmr_ipsec_spi(uint8_t *pkt_addr, +static inline int verify_pmr_ipsec_spi(const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr, pmr_term_value_t *term_value) { @@ -199,11 +205,11 @@ static inline int verify_pmr_ipsec_spi(uint8_t *pkt_addr, pkt_addr += pkt_hdr->l4_offset; if (pkt_hdr->l4_protocol == ODPH_IPPROTO_AH) { - odph_ahhdr_t *ahhdr = (odph_ahhdr_t *)pkt_addr; + const odph_ahhdr_t *ahhdr = (const odph_ahhdr_t *)pkt_addr; spi = odp_be_to_cpu_32(ahhdr->spi); } else if (pkt_hdr->l4_protocol == ODPH_IPPROTO_ESP) { - odph_esphdr_t *esphdr = (odph_esphdr_t *)pkt_addr; + const odph_esphdr_t *esphdr = (const odph_esphdr_t *)pkt_addr; spi = odp_be_to_cpu_32(esphdr->spi); } else { @@ -216,7 +222,7 @@ static inline int verify_pmr_ipsec_spi(uint8_t *pkt_addr, return 0; } -static inline int verify_pmr_ld_vni(uint8_t *pkt_addr ODP_UNUSED, +static inline int verify_pmr_ld_vni(const uint8_t *pkt_addr ODP_UNUSED, odp_packet_hdr_t *pkt_hdr ODP_UNUSED, pmr_term_value_t *term_value ODP_UNUSED) { @@ -224,7 +230,7 @@ static inline int verify_pmr_ld_vni(uint8_t *pkt_addr ODP_UNUSED, return 0; } -static inline int verify_pmr_custom_frame(uint8_t *pkt_addr, +static inline int verify_pmr_custom_frame(const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr, pmr_term_value_t *term_value) { @@ -244,14 +250,15 @@ static inline int verify_pmr_custom_frame(uint8_t *pkt_addr, return 0; } -static inline int verify_pmr_eth_type_0(uint8_t *pkt_addr ODP_UNUSED, +static inline int verify_pmr_eth_type_0(const uint8_t *pkt_addr ODP_UNUSED, odp_packet_hdr_t *pkt_hdr ODP_UNUSED, pmr_term_value_t *term_value ODP_UNUSED) { ODP_UNIMPLEMENTED(); return 0; } -static inline int verify_pmr_eth_type_x(uint8_t *pkt_addr ODP_UNUSED, + +static inline int verify_pmr_eth_type_x(const uint8_t *pkt_addr ODP_UNUSED, odp_packet_hdr_t *pkt_hdr ODP_UNUSED, pmr_term_value_t *term_value ODP_UNUSED) { diff --git a/platform/linux-generic/include/odp_classification_internal.h b/platform/linux-generic/include/odp_classification_internal.h index 55932802c..1e8f291a6 100644 --- a/platform/linux-generic/include/odp_classification_internal.h +++ b/platform/linux-generic/include/odp_classification_internal.h @@ -40,8 +40,8 @@ with the PKTIO interface. Returns the default cos if the packet does not match any PMR Returns the error_cos if the packet has an error **/ -cos_t *pktio_select_cos(pktio_entry_t *pktio, uint8_t *pkt_addr, - odp_packet_hdr_t *pkt_hdr); +cos_t *pktio_select_cos(pktio_entry_t *pktio, const uint8_t *pkt_addr, + odp_packet_hdr_t *pkt_hdr); /** @internal @@ -51,7 +51,7 @@ Select a CoS for the given Packet based on QoS values This function returns the COS object matching the L2 and L3 QoS based on the l3_preference value of the pktio **/ -cos_t *match_qos_cos(pktio_entry_t *entry, uint8_t *pkt_addr, +cos_t *match_qos_cos(pktio_entry_t *entry, const uint8_t *pkt_addr, odp_packet_hdr_t *hdr); /** Packet Classifier @@ -59,8 +59,17 @@ Packet Classifier Start function for Packet Classifier This function calls Classifier module internal functions for a given packet and enqueues the packet to specific Queue based on PMR and CoS selected. +The packet is allocated from the pool associated with the CoS **/ int packet_classifier(odp_pktio_t pktio, odp_packet_t pkt); + +/** +@internal + +Same as packet classifier uses linux-generic internal pktio struct +**/ +int _odp_packet_classifier(pktio_entry_t *entry, odp_packet_t pkt); + /** Packet IO classifier init @@ -78,7 +87,7 @@ This function gets called recursively to check the chained PMR Term value with the packet. **/ -cos_t *match_pmr_cos(cos_t *cos, uint8_t *pkt_addr, pmr_t *pmr, +cos_t *match_pmr_cos(cos_t *cos, const uint8_t *pkt_addr, pmr_t *pmr, odp_packet_hdr_t *hdr); /** @internal @@ -86,7 +95,7 @@ CoS associated with L3 QoS value This function returns the CoS associated with L3 QoS value **/ -cos_t *match_qos_l3_cos(pmr_l3_cos_t *l3_cos, uint8_t *pkt_addr, +cos_t *match_qos_l3_cos(pmr_l3_cos_t *l3_cos, const uint8_t *pkt_addr, odp_packet_hdr_t *hdr); /** @@ -95,7 +104,7 @@ CoS associated with L2 QoS value This function returns the CoS associated with L2 QoS value **/ -cos_t *match_qos_l2_cos(pmr_l2_cos_t *l2_cos, uint8_t *pkt_addr, +cos_t *match_qos_l2_cos(pmr_l2_cos_t *l2_cos, const uint8_t *pkt_addr, odp_packet_hdr_t *hdr); /** @internal @@ -165,7 +174,7 @@ This function goes through each PMR_TERM value in pmr_t structure and calls verification function for each term.Returns 1 if PMR matches or 0 Otherwise. **/ -int verify_pmr(pmr_t *pmr, uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr); +int verify_pmr(pmr_t *pmr, const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr); #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_cpu_internal.h b/platform/linux-generic/include/odp_cpu_internal.h deleted file mode 100644 index 664e2df5d..000000000 --- a/platform/linux-generic/include/odp_cpu_internal.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (c) 2015, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_CPU_INTERNAL_H_ -#define ODP_CPU_INTERNAL_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <odp/cpu.h> - -static inline -uint64_t _odp_cpu_cycles_diff(uint64_t c2, uint64_t c1) -{ - if (odp_likely(c2 >= c1)) - return c2 - c1; - - return c2 + (odp_cpu_cycles_max() - c1) + 1; -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/include/odp_crypto_internal.h b/platform/linux-generic/include/odp_crypto_internal.h index b9128a426..7b104afa8 100644 --- a/platform/linux-generic/include/odp_crypto_internal.h +++ b/platform/linux-generic/include/odp_crypto_internal.h @@ -50,6 +50,9 @@ struct odp_crypto_generic_session { struct { AES_KEY key; } aes; + struct { + EVP_CIPHER_CTX *ctx; + } aes_gcm; } data; crypto_func_t func; } cipher; diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index 010b82f1f..113b7004d 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -77,6 +77,8 @@ int odp_schedule_term_local(void); int odp_timer_init_global(void); int odp_timer_disarm_all(void); +int odp_time_global_init(void); + void _odp_flush_caches(void); #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 6f1521ca3..12e9cca10 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -28,6 +28,8 @@ extern "C" { #include <odp/crypto.h> #include <odp_crypto_internal.h> +#define PACKET_JUMBO_LEN (9 * 1024) + /** * Packet input & protocol flags */ @@ -243,7 +245,7 @@ void packet_parse_l2(odp_packet_hdr_t *pkt_hdr); int packet_parse_full(odp_packet_hdr_t *pkt_hdr); /* Reset parser metadata for a new parse */ -void packet_parse_reset(odp_packet_t pkt); +void packet_parse_reset(odp_packet_hdr_t *pkt_hdr); /* Convert a packet handle to a buffer handle */ odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt); @@ -251,6 +253,10 @@ odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt); /* Convert a buffer handle to a packet handle */ odp_packet_t _odp_packet_from_buffer(odp_buffer_t buf); +int _odp_parse_common(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr); + +int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index de295578f..4d739528d 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -88,6 +88,7 @@ struct pktio_entry { classifier_t cls; /**< classifier linked with this pktio*/ char name[PKTIO_NAME_LEN]; /**< name of pktio provided to pktio_open() */ + odp_pktio_t id; odp_pktio_param_t param; }; @@ -119,6 +120,9 @@ typedef struct pktio_if_ops { int (*mac_get)(pktio_entry_t *pktio_entry, void *mac_addr); } pktio_if_ops_t; +int _odp_packet_cls_enq(pktio_entry_t *pktio_entry, const uint8_t *base, + uint16_t buf_len, odp_packet_t *pkt_ret); + extern void *pktio_entry_ptr[]; static inline int pktio_to_id(odp_pktio_t pktio) diff --git a/platform/linux-generic/include/odp_packet_socket.h b/platform/linux-generic/include/odp_packet_socket.h index a5e0eb381..1eaafb7e5 100644 --- a/platform/linux-generic/include/odp_packet_socket.h +++ b/platform/linux-generic/include/odp_packet_socket.h @@ -44,6 +44,8 @@ typedef struct { int sockfd; /**< socket descriptor */ odp_pool_t pool; /**< pool to alloc packets from */ unsigned char if_mac[ETH_ALEN]; /**< IF eth mac addr */ + uint8_t *cache_ptr[ODP_PACKET_SOCKET_MAX_BURST_RX]; + odp_shm_t shm; } pkt_sock_t; /** packet mmap ring */ diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 94bf1faee..b12bca8f6 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -29,6 +29,7 @@ extern "C" { #include <odp/shared_memory.h> #include <odp/atomic.h> #include <odp_atomic_internal.h> +#include <odp/thread.h> #include <string.h> /** @@ -139,7 +140,7 @@ struct pool_entry_s { uint32_t headroom; uint32_t tailroom; - local_cache_t local_cache[_ODP_INTERNAL_MAX_THREADS] ODP_ALIGNED_CACHE; + local_cache_t local_cache[ODP_THREAD_COUNT_MAX] ODP_ALIGNED_CACHE; }; typedef union pool_entry_u { diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 1b6d5931a..da195ad8b 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -154,9 +154,32 @@ int odp_classification_term_global(void) return rc; } -odp_cos_t odp_cos_create(const char *name) +void odp_cls_cos_param_init(odp_cls_cos_param_t *param) +{ + param->queue = ODP_QUEUE_INVALID; + param->pool = ODP_POOL_INVALID; + param->drop_policy = ODP_COS_DROP_NEVER; +} + +odp_cos_t odp_cls_cos_create(const char *name, odp_cls_cos_param_t *param) { int i; + queue_entry_t *queue; + pool_entry_t *pool; + odp_cls_drop_t drop_policy; + + /* Packets are dropped if Queue or Pool is invalid*/ + if (param->queue == ODP_QUEUE_INVALID) + queue = NULL; + else + queue = queue_to_qentry(param->queue); + + if (param->pool == ODP_POOL_INVALID) + pool = NULL; + else + pool = odp_pool_to_entry(param->pool); + + drop_policy = param->drop_policy; for (i = 0; i < ODP_COS_MAX_ENTRY; i++) { LOCK(&cos_tbl->cos_entry[i].s.lock); @@ -166,16 +189,18 @@ odp_cos_t odp_cos_create(const char *name) cos_tbl->cos_entry[i].s.name[ODP_COS_NAME_LEN - 1] = 0; cos_tbl->cos_entry[i].s.pmr = NULL; cos_tbl->cos_entry[i].s.linked_cos = NULL; - cos_tbl->cos_entry[i].s.queue = NULL; - cos_tbl->cos_entry[i].s.pool = NULL; + cos_tbl->cos_entry[i].s.queue = queue; + cos_tbl->cos_entry[i].s.pool = pool; cos_tbl->cos_entry[i].s.flow_set = 0; cos_tbl->cos_entry[i].s.headroom = 0; cos_tbl->cos_entry[i].s.valid = 1; + cos_tbl->cos_entry[i].s.drop_policy = drop_policy; UNLOCK(&cos_tbl->cos_entry[i].s.lock); return _odp_cast_scalar(odp_cos_t, i); } UNLOCK(&cos_tbl->cos_entry[i].s.lock); } + ODP_ERR("ODP_COS_MAX_ENTRY reached"); return ODP_COS_INVALID; } @@ -274,8 +299,10 @@ int odp_cos_queue_set(odp_cos_t cos_id, odp_queue_t queue_id) } /* Locking is not required as intermittent stale data during CoS modification is acceptable*/ - cos->s.queue = queue_to_qentry(queue_id); - cos->s.queue_id = queue_id; + if (queue_id == ODP_QUEUE_INVALID) + cos->s.queue = NULL; + else + cos->s.queue = queue_to_qentry(queue_id); return 0; } @@ -288,10 +315,13 @@ odp_queue_t odp_cos_queue(odp_cos_t cos_id) return ODP_QUEUE_INVALID; } - return cos->s.queue_id; + if (!cos->s.queue) + return ODP_QUEUE_INVALID; + + return cos->s.queue->s.handle; } -int odp_cos_drop_set(odp_cos_t cos_id, odp_drop_e drop_policy) +int odp_cos_drop_set(odp_cos_t cos_id, odp_cls_drop_t drop_policy) { cos_t *cos = get_cos_entry(cos_id); @@ -305,7 +335,7 @@ int odp_cos_drop_set(odp_cos_t cos_id, odp_drop_e drop_policy) return 0; } -odp_drop_e odp_cos_drop(odp_cos_t cos_id) +odp_cls_drop_t odp_cos_drop(odp_cos_t cos_id) { cos_t *cos = get_cos_entry(cos_id); @@ -661,7 +691,41 @@ int odp_pktio_pmr_match_set_cos(odp_pmr_set_t pmr_set_id, odp_pktio_t src_pktio, return 0; } -int verify_pmr(pmr_t *pmr, uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr) +int odp_cls_cos_pool_set(odp_cos_t cos_id, odp_pool_t pool_id) +{ + cos_t *cos; + + cos = get_cos_entry(cos_id); + if (cos == NULL) { + ODP_ERR("Invalid odp_cos_t handle"); + return -1; + } + + if (pool_id == ODP_POOL_INVALID) + cos->s.pool = NULL; + else + cos->s.pool = odp_pool_to_entry(pool_id); + + return 0; +} + +odp_pool_t odp_cls_cos_pool(odp_cos_t cos_id) +{ + cos_t *cos; + + cos = get_cos_entry(cos_id); + if (cos == NULL) { + ODP_ERR("Invalid odp_cos_t handle"); + return ODP_POOL_INVALID; + } + + if (!cos->s.pool) + return ODP_POOL_INVALID; + + return cos->s.pool->s.pool_hdl; +} + +int verify_pmr(pmr_t *pmr, const uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr) { int pmr_failure = 0; int num_pmr; @@ -779,7 +843,7 @@ int verify_pmr(pmr_t *pmr, uint8_t *pkt_addr, odp_packet_hdr_t *pkt_hdr) return true; } -cos_t *match_pmr_cos(cos_t *cos, uint8_t *pkt_addr, pmr_t *pmr, +cos_t *match_pmr_cos(cos_t *cos, const uint8_t *pkt_addr, pmr_t *pmr, odp_packet_hdr_t *hdr) { cos_t *retcos = NULL; @@ -824,15 +888,14 @@ int pktio_classifier_init(pktio_entry_t *entry) return 0; } -int packet_classifier(odp_pktio_t pktio, odp_packet_t pkt) +int _odp_packet_classifier(pktio_entry_t *entry, odp_packet_t pkt) { - pktio_entry_t *entry; queue_entry_t *queue; cos_t *cos; odp_packet_hdr_t *pkt_hdr; + odp_packet_t new_pkt; uint8_t *pkt_addr; - entry = get_pktio_entry(pktio); if (entry == NULL) return -1; @@ -844,13 +907,42 @@ int packet_classifier(odp_pktio_t pktio, odp_packet_t pkt) if (cos == NULL) return -1; + if (cos->s.pool == NULL) { + odp_packet_free(pkt); + return -1; + } + + if (cos->s.queue == NULL) { + odp_packet_free(pkt); + return -1; + } + + if (odp_packet_pool(pkt) != cos->s.pool->s.pool_hdl) { + new_pkt = odp_packet_copy(pkt, cos->s.pool->s.pool_hdl); + odp_packet_free(pkt); + if (new_pkt == ODP_PACKET_INVALID) + return -1; + } else { + new_pkt = pkt; + } + /* Enqueuing the Packet based on the CoS */ queue = cos->s.queue; - return queue_enq(queue, odp_buf_to_hdr((odp_buffer_t)pkt), 0); + return queue_enq(queue, odp_buf_to_hdr((odp_buffer_t)new_pkt), 0); +} + +int packet_classifier(odp_pktio_t pktio, odp_packet_t pkt) +{ + pktio_entry_t *entry; + + entry = get_pktio_entry(pktio); + if (entry == NULL) + return -1; + return _odp_packet_classifier(entry, pkt); } -cos_t *pktio_select_cos(pktio_entry_t *entry, uint8_t *pkt_addr, - odp_packet_hdr_t *pkt_hdr) +cos_t *pktio_select_cos(pktio_entry_t *entry, const uint8_t *pkt_addr, + odp_packet_hdr_t *pkt_hdr) { pmr_t *pmr; cos_t *cos; @@ -882,20 +974,20 @@ cos_t *pktio_select_cos(pktio_entry_t *entry, uint8_t *pkt_addr, return cls->default_cos; } -cos_t *match_qos_l3_cos(pmr_l3_cos_t *l3_cos, uint8_t *pkt_addr, +cos_t *match_qos_l3_cos(pmr_l3_cos_t *l3_cos, const uint8_t *pkt_addr, odp_packet_hdr_t *hdr) { uint8_t dscp; cos_t *cos = NULL; - odph_ipv4hdr_t *ipv4; - odph_ipv6hdr_t *ipv6; + const odph_ipv4hdr_t *ipv4; + const odph_ipv6hdr_t *ipv6; if (hdr->input_flags.l3 && hdr->input_flags.ipv4) { - ipv4 = (odph_ipv4hdr_t *)(pkt_addr + hdr->l3_offset); + ipv4 = (const odph_ipv4hdr_t *)(pkt_addr + hdr->l3_offset); dscp = ODPH_IPV4HDR_DSCP(ipv4->tos); cos = l3_cos->cos[dscp]; } else if (hdr->input_flags.l3 && hdr->input_flags.ipv6) { - ipv6 = (odph_ipv6hdr_t *)(pkt_addr + hdr->l3_offset); + ipv6 = (const odph_ipv6hdr_t *)(pkt_addr + hdr->l3_offset); dscp = ODPH_IPV6HDR_DSCP(ipv6->ver_tc_flow); cos = l3_cos->cos[dscp]; } @@ -903,18 +995,18 @@ cos_t *match_qos_l3_cos(pmr_l3_cos_t *l3_cos, uint8_t *pkt_addr, return cos; } -cos_t *match_qos_l2_cos(pmr_l2_cos_t *l2_cos, uint8_t *pkt_addr, +cos_t *match_qos_l2_cos(pmr_l2_cos_t *l2_cos, const uint8_t *pkt_addr, odp_packet_hdr_t *hdr) { cos_t *cos = NULL; - odph_ethhdr_t *eth; - odph_vlanhdr_t *vlan; + const odph_ethhdr_t *eth; + const odph_vlanhdr_t *vlan; uint16_t qos; if (hdr->input_flags.l2 && hdr->input_flags.vlan && hdr->input_flags.eth) { - eth = (odph_ethhdr_t *)(pkt_addr + hdr->l2_offset); - vlan = (odph_vlanhdr_t *)(ð->type); + eth = (const odph_ethhdr_t *)(pkt_addr + hdr->l2_offset); + vlan = (const odph_vlanhdr_t *)(ð->type); qos = odp_be_to_cpu_16(vlan->tci); qos = ((qos >> 13) & 0x07); cos = l2_cos->cos[qos]; @@ -922,7 +1014,7 @@ cos_t *match_qos_l2_cos(pmr_l2_cos_t *l2_cos, uint8_t *pkt_addr, return cos; } -cos_t *match_qos_cos(pktio_entry_t *entry, uint8_t *pkt_addr, +cos_t *match_qos_cos(pktio_entry_t *entry, const uint8_t *pkt_addr, odp_packet_hdr_t *hdr) { classifier_t *cls = &entry->s.cls; diff --git a/platform/linux-generic/odp_cpu.c b/platform/linux-generic/odp_cpu.c index 636f811f8..4bd6a9a07 100644 --- a/platform/linux-generic/odp_cpu.c +++ b/platform/linux-generic/odp_cpu.c @@ -6,9 +6,11 @@ #include <odp/cpu.h> #include <odp/hints.h> -#include <odp_cpu_internal.h> uint64_t odp_cpu_cycles_diff(uint64_t c2, uint64_t c1) { - return _odp_cpu_cycles_diff(c2, c1); + if (odp_likely(c2 >= c1)) + return c2 - c1; + + return c2 + (odp_cpu_cycles_max() - c1) + 1; } diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index 17fced930..65e850343 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -292,6 +292,170 @@ int process_aes_params(odp_crypto_generic_session_t *session, } static +odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_params_t *params, + odp_crypto_generic_session_t *session) +{ + uint8_t *data = odp_packet_data(params->out_pkt); + uint32_t plain_len = params->cipher_range.length; + uint8_t *aad_head = data + params->auth_range.offset; + uint8_t *aad_tail = data + params->cipher_range.offset + + params->cipher_range.length; + uint32_t auth_len = params->auth_range.length; + unsigned char iv_enc[AES_BLOCK_SIZE]; + void *iv_ptr; + uint8_t *tag = data + params->hash_result_offset; + + if (params->override_iv_ptr) + iv_ptr = params->override_iv_ptr; + else if (session->cipher.iv.data) + iv_ptr = session->cipher.iv.data; + else + return ODP_CRYPTO_ALG_ERR_IV_INVALID; + + /* All cipher data must be part of the authentication */ + if (params->auth_range.offset > params->cipher_range.offset || + params->auth_range.offset + auth_len < + params->cipher_range.offset + plain_len) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* + * Create a copy of the IV. The DES library modifies IV + * and if we are processing packets on parallel threads + * we could get corruption. + */ + memcpy(iv_enc, iv_ptr, AES_BLOCK_SIZE); + + /* Adjust pointer for beginning of area to cipher/auth */ + uint8_t *plaindata = data + params->cipher_range.offset; + + /* Encrypt it */ + EVP_CIPHER_CTX *ctx = session->cipher.data.aes_gcm.ctx; + int cipher_len = 0; + + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_enc); + + /* Authenticate header data (if any) without encrypting them */ + if (aad_head < plaindata) { + EVP_EncryptUpdate(ctx, NULL, &cipher_len, + aad_head, plaindata - aad_head); + } + + EVP_EncryptUpdate(ctx, plaindata, &cipher_len, + plaindata, plain_len); + cipher_len = plain_len; + + /* Authenticate footer data (if any) without encrypting them */ + if (aad_head + auth_len > plaindata + plain_len) { + EVP_EncryptUpdate(ctx, NULL, NULL, aad_tail, + auth_len - (aad_tail - aad_head)); + } + + EVP_EncryptFinal_ex(ctx, plaindata + cipher_len, &cipher_len); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag); + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static +odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_params_t *params, + odp_crypto_generic_session_t *session) +{ + uint8_t *data = odp_packet_data(params->out_pkt); + uint32_t cipher_len = params->cipher_range.length; + uint8_t *aad_head = data + params->auth_range.offset; + uint8_t *aad_tail = data + params->cipher_range.offset + + params->cipher_range.length; + uint32_t auth_len = params->auth_range.length; + unsigned char iv_enc[AES_BLOCK_SIZE]; + void *iv_ptr; + uint8_t *tag = data + params->hash_result_offset; + + if (params->override_iv_ptr) + iv_ptr = params->override_iv_ptr; + else if (session->cipher.iv.data) + iv_ptr = session->cipher.iv.data; + else + return ODP_CRYPTO_ALG_ERR_IV_INVALID; + + /* All cipher data must be part of the authentication */ + if (params->auth_range.offset > params->cipher_range.offset || + params->auth_range.offset + auth_len < + params->cipher_range.offset + cipher_len) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* + * Create a copy of the IV. The DES library modifies IV + * and if we are processing packets on parallel threads + * we could get corruption. + */ + memcpy(iv_enc, iv_ptr, AES_BLOCK_SIZE); + + /* Adjust pointer for beginning of area to cipher/auth */ + uint8_t *cipherdata = data + params->cipher_range.offset; + /* Encrypt it */ + EVP_CIPHER_CTX *ctx = session->cipher.data.aes_gcm.ctx; + int plain_len = 0; + + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_enc); + + /* Authenticate header data (if any) without encrypting them */ + if (aad_head < cipherdata) { + EVP_DecryptUpdate(ctx, NULL, &plain_len, + aad_head, cipherdata - aad_head); + } + + EVP_DecryptUpdate(ctx, cipherdata, &plain_len, + cipherdata, cipher_len); + plain_len = cipher_len; + + /* Authenticate footer data (if any) without encrypting them */ + if (aad_head + auth_len > cipherdata + cipher_len) { + EVP_DecryptUpdate(ctx, NULL, NULL, aad_tail, + auth_len - (aad_tail - aad_head)); + } + + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag); + + if (EVP_DecryptFinal_ex(ctx, cipherdata + cipher_len, &plain_len) < 0) + return ODP_CRYPTO_ALG_ERR_ICV_CHECK; + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static +int process_aes_gcm_params(odp_crypto_generic_session_t *session, + odp_crypto_session_params_t *params) +{ + /* Verify Key len is 16 */ + if (params->cipher_key.length != 16) + return -1; + + /* Set function */ + EVP_CIPHER_CTX *ctx = + session->cipher.data.aes_gcm.ctx = EVP_CIPHER_CTX_new(); + + if (ODP_CRYPTO_OP_ENCODE == params->op) { + session->cipher.func = aes_gcm_encrypt; + EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL); + } else { + session->cipher.func = aes_gcm_decrypt; + EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL); + } + + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + params->iv.length, NULL); + if (ODP_CRYPTO_OP_ENCODE == params->op) { + EVP_EncryptInit_ex(ctx, NULL, NULL, + params->cipher_key.data, NULL); + } else { + EVP_DecryptInit_ex(ctx, NULL, NULL, + params->cipher_key.data, NULL); + } + + return 0; +} + +static odp_crypto_alg_err_t des_encrypt(odp_crypto_op_params_t *params, odp_crypto_generic_session_t *session) { @@ -479,6 +643,15 @@ odp_crypto_session_create(odp_crypto_session_params_t *params, case ODP_CIPHER_ALG_AES128_CBC: rc = process_aes_params(session, params); break; + case ODP_CIPHER_ALG_AES128_GCM: + /* AES-GCM requires to do both auth and + * cipher at the same time */ + if (params->auth_alg != ODP_AUTH_ALG_AES128_GCM) { + rc = -1; + break; + } + rc = process_aes_gcm_params(session, params); + break; default: rc = -1; } @@ -501,6 +674,16 @@ odp_crypto_session_create(odp_crypto_session_params_t *params, case ODP_AUTH_ALG_SHA256_128: rc = process_sha256_params(session, params, 128); break; + case ODP_AUTH_ALG_AES128_GCM: + /* AES-GCM requires to do both auth and + * cipher at the same time */ + if (params->cipher_alg != ODP_CIPHER_ALG_AES128_GCM) { + rc = -1; + break; + } + session->auth.func = null_crypto_routine; + rc = 0; + break; default: rc = -1; } diff --git a/platform/linux-generic/odp_hash.c b/platform/linux-generic/odp_hash.c new file mode 100644 index 000000000..6bee4ef6c --- /dev/null +++ b/platform/linux-generic/odp_hash.c @@ -0,0 +1,487 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2013 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <odp/hash.h> +#include <odp/std_types.h> + +static const uint32_t crc32c_tables[8][256] = {{ + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, + 0x26A1E7E8, 0xD4CA64EB, 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, + 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, 0x105EC76F, 0xE235446C, + 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, + 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, 0x5D1D08BF, 0xAF768BBC, + 0xBC267848, 0x4E4DFB4B, 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, 0xAA64D611, 0x580F5512, + 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, + 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, + 0x1642AE59, 0xE4292D5A, 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, + 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, 0x417B1DBC, 0xB3109EBF, + 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F, + 0xED03A29B, 0x1F682198, 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, + 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, 0xDBFC821C, 0x2997011F, + 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, + 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, 0xA65C047D, 0x5437877E, + 0x4767748A, 0xB50CF789, 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, 0x7198540D, 0x83F3D70E, + 0x90A324FA, 0x62C8A7F9, 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, + 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, + 0xDDE0EB2A, 0x2F8B6829, 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, + 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, 0x082F63B7, 0xFA44E0B4, + 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B, + 0xB4091BFF, 0x466298FC, 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, + 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, 0xA24BB5A6, 0x502036A5, + 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, + 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, 0xEF087A76, 0x1D63F975, + 0x0E330A81, 0xFC588982, 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, 0x38CC2A06, 0xCAA7A905, + 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, + 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8, + 0xE52CC12C, 0x1747422F, 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, + 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, 0xD3D3E1AB, 0x21B862A8, + 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, + 0x7FAB5E8C, 0x8DC0DD8F, 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, + 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, 0x69E9F0D5, 0x9B8273D6, + 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, + 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 0x34F4F86A, 0xC69F7B69, + 0xD5CF889D, 0x27A40B9E, 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, +}, +{ + 0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, 0x4E8A61DC, 0x5D28F9AB, + 0x69CF5132, 0x7A6DC945, 0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, + 0xD39EA264, 0xC03C3A13, 0xF4DB928A, 0xE7790AFD, 0x3FC5F181, 0x2C6769F6, + 0x1880C16F, 0x0B225918, 0x714F905D, 0x62ED082A, 0x560AA0B3, 0x45A838C4, + 0xA2D13239, 0xB173AA4E, 0x859402D7, 0x96369AA0, 0xEC5B53E5, 0xFFF9CB92, + 0xCB1E630B, 0xD8BCFB7C, 0x7F8BE302, 0x6C297B75, 0x58CED3EC, 0x4B6C4B9B, + 0x310182DE, 0x22A31AA9, 0x1644B230, 0x05E62A47, 0xE29F20BA, 0xF13DB8CD, + 0xC5DA1054, 0xD6788823, 0xAC154166, 0xBFB7D911, 0x8B507188, 0x98F2E9FF, + 0x404E1283, 0x53EC8AF4, 0x670B226D, 0x74A9BA1A, 0x0EC4735F, 0x1D66EB28, + 0x298143B1, 0x3A23DBC6, 0xDD5AD13B, 0xCEF8494C, 0xFA1FE1D5, 0xE9BD79A2, + 0x93D0B0E7, 0x80722890, 0xB4958009, 0xA737187E, 0xFF17C604, 0xECB55E73, + 0xD852F6EA, 0xCBF06E9D, 0xB19DA7D8, 0xA23F3FAF, 0x96D89736, 0x857A0F41, + 0x620305BC, 0x71A19DCB, 0x45463552, 0x56E4AD25, 0x2C896460, 0x3F2BFC17, + 0x0BCC548E, 0x186ECCF9, 0xC0D23785, 0xD370AFF2, 0xE797076B, 0xF4359F1C, + 0x8E585659, 0x9DFACE2E, 0xA91D66B7, 0xBABFFEC0, 0x5DC6F43D, 0x4E646C4A, + 0x7A83C4D3, 0x69215CA4, 0x134C95E1, 0x00EE0D96, 0x3409A50F, 0x27AB3D78, + 0x809C2506, 0x933EBD71, 0xA7D915E8, 0xB47B8D9F, 0xCE1644DA, 0xDDB4DCAD, + 0xE9537434, 0xFAF1EC43, 0x1D88E6BE, 0x0E2A7EC9, 0x3ACDD650, 0x296F4E27, + 0x53028762, 0x40A01F15, 0x7447B78C, 0x67E52FFB, 0xBF59D487, 0xACFB4CF0, + 0x981CE469, 0x8BBE7C1E, 0xF1D3B55B, 0xE2712D2C, 0xD69685B5, 0xC5341DC2, + 0x224D173F, 0x31EF8F48, 0x050827D1, 0x16AABFA6, 0x6CC776E3, 0x7F65EE94, + 0x4B82460D, 0x5820DE7A, 0xFBC3FAF9, 0xE861628E, 0xDC86CA17, 0xCF245260, + 0xB5499B25, 0xA6EB0352, 0x920CABCB, 0x81AE33BC, 0x66D73941, 0x7575A136, + 0x419209AF, 0x523091D8, 0x285D589D, 0x3BFFC0EA, 0x0F186873, 0x1CBAF004, + 0xC4060B78, 0xD7A4930F, 0xE3433B96, 0xF0E1A3E1, 0x8A8C6AA4, 0x992EF2D3, + 0xADC95A4A, 0xBE6BC23D, 0x5912C8C0, 0x4AB050B7, 0x7E57F82E, 0x6DF56059, + 0x1798A91C, 0x043A316B, 0x30DD99F2, 0x237F0185, 0x844819FB, 0x97EA818C, + 0xA30D2915, 0xB0AFB162, 0xCAC27827, 0xD960E050, 0xED8748C9, 0xFE25D0BE, + 0x195CDA43, 0x0AFE4234, 0x3E19EAAD, 0x2DBB72DA, 0x57D6BB9F, 0x447423E8, + 0x70938B71, 0x63311306, 0xBB8DE87A, 0xA82F700D, 0x9CC8D894, 0x8F6A40E3, + 0xF50789A6, 0xE6A511D1, 0xD242B948, 0xC1E0213F, 0x26992BC2, 0x353BB3B5, + 0x01DC1B2C, 0x127E835B, 0x68134A1E, 0x7BB1D269, 0x4F567AF0, 0x5CF4E287, + 0x04D43CFD, 0x1776A48A, 0x23910C13, 0x30339464, 0x4A5E5D21, 0x59FCC556, + 0x6D1B6DCF, 0x7EB9F5B8, 0x99C0FF45, 0x8A626732, 0xBE85CFAB, 0xAD2757DC, + 0xD74A9E99, 0xC4E806EE, 0xF00FAE77, 0xE3AD3600, 0x3B11CD7C, 0x28B3550B, + 0x1C54FD92, 0x0FF665E5, 0x759BACA0, 0x663934D7, 0x52DE9C4E, 0x417C0439, + 0xA6050EC4, 0xB5A796B3, 0x81403E2A, 0x92E2A65D, 0xE88F6F18, 0xFB2DF76F, + 0xCFCA5FF6, 0xDC68C781, 0x7B5FDFFF, 0x68FD4788, 0x5C1AEF11, 0x4FB87766, + 0x35D5BE23, 0x26772654, 0x12908ECD, 0x013216BA, 0xE64B1C47, 0xF5E98430, + 0xC10E2CA9, 0xD2ACB4DE, 0xA8C17D9B, 0xBB63E5EC, 0x8F844D75, 0x9C26D502, + 0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, 0x0A104FA2, 0x19B2D7D5, + 0x2D557F4C, 0x3EF7E73B, 0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, + 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483, +}, +{ + 0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, 0x9EDEA41A, 0x3B9F3664, + 0xD1B1F617, 0x74F06469, 0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, + 0xA68F9ADF, 0x03CE08A1, 0xE9E0C8D2, 0x4CA15AAC, 0x70A27D8A, 0xD5E3EFF4, + 0x3FCD2F87, 0x9A8CBDF9, 0xEE7CD990, 0x4B3D4BEE, 0xA1138B9D, 0x045219E3, + 0x48F3434F, 0xEDB2D131, 0x079C1142, 0xA2DD833C, 0xD62DE755, 0x736C752B, + 0x9942B558, 0x3C032726, 0xE144FB14, 0x4405696A, 0xAE2BA919, 0x0B6A3B67, + 0x7F9A5F0E, 0xDADBCD70, 0x30F50D03, 0x95B49F7D, 0xD915C5D1, 0x7C5457AF, + 0x967A97DC, 0x333B05A2, 0x47CB61CB, 0xE28AF3B5, 0x08A433C6, 0xADE5A1B8, + 0x91E6869E, 0x34A714E0, 0xDE89D493, 0x7BC846ED, 0x0F382284, 0xAA79B0FA, + 0x40577089, 0xE516E2F7, 0xA9B7B85B, 0x0CF62A25, 0xE6D8EA56, 0x43997828, + 0x37691C41, 0x92288E3F, 0x78064E4C, 0xDD47DC32, 0xC76580D9, 0x622412A7, + 0x880AD2D4, 0x2D4B40AA, 0x59BB24C3, 0xFCFAB6BD, 0x16D476CE, 0xB395E4B0, + 0xFF34BE1C, 0x5A752C62, 0xB05BEC11, 0x151A7E6F, 0x61EA1A06, 0xC4AB8878, + 0x2E85480B, 0x8BC4DA75, 0xB7C7FD53, 0x12866F2D, 0xF8A8AF5E, 0x5DE93D20, + 0x29195949, 0x8C58CB37, 0x66760B44, 0xC337993A, 0x8F96C396, 0x2AD751E8, + 0xC0F9919B, 0x65B803E5, 0x1148678C, 0xB409F5F2, 0x5E273581, 0xFB66A7FF, + 0x26217BCD, 0x8360E9B3, 0x694E29C0, 0xCC0FBBBE, 0xB8FFDFD7, 0x1DBE4DA9, + 0xF7908DDA, 0x52D11FA4, 0x1E704508, 0xBB31D776, 0x511F1705, 0xF45E857B, + 0x80AEE112, 0x25EF736C, 0xCFC1B31F, 0x6A802161, 0x56830647, 0xF3C29439, + 0x19EC544A, 0xBCADC634, 0xC85DA25D, 0x6D1C3023, 0x8732F050, 0x2273622E, + 0x6ED23882, 0xCB93AAFC, 0x21BD6A8F, 0x84FCF8F1, 0xF00C9C98, 0x554D0EE6, + 0xBF63CE95, 0x1A225CEB, 0x8B277743, 0x2E66E53D, 0xC448254E, 0x6109B730, + 0x15F9D359, 0xB0B84127, 0x5A968154, 0xFFD7132A, 0xB3764986, 0x1637DBF8, + 0xFC191B8B, 0x595889F5, 0x2DA8ED9C, 0x88E97FE2, 0x62C7BF91, 0xC7862DEF, + 0xFB850AC9, 0x5EC498B7, 0xB4EA58C4, 0x11ABCABA, 0x655BAED3, 0xC01A3CAD, + 0x2A34FCDE, 0x8F756EA0, 0xC3D4340C, 0x6695A672, 0x8CBB6601, 0x29FAF47F, + 0x5D0A9016, 0xF84B0268, 0x1265C21B, 0xB7245065, 0x6A638C57, 0xCF221E29, + 0x250CDE5A, 0x804D4C24, 0xF4BD284D, 0x51FCBA33, 0xBBD27A40, 0x1E93E83E, + 0x5232B292, 0xF77320EC, 0x1D5DE09F, 0xB81C72E1, 0xCCEC1688, 0x69AD84F6, + 0x83834485, 0x26C2D6FB, 0x1AC1F1DD, 0xBF8063A3, 0x55AEA3D0, 0xF0EF31AE, + 0x841F55C7, 0x215EC7B9, 0xCB7007CA, 0x6E3195B4, 0x2290CF18, 0x87D15D66, + 0x6DFF9D15, 0xC8BE0F6B, 0xBC4E6B02, 0x190FF97C, 0xF321390F, 0x5660AB71, + 0x4C42F79A, 0xE90365E4, 0x032DA597, 0xA66C37E9, 0xD29C5380, 0x77DDC1FE, + 0x9DF3018D, 0x38B293F3, 0x7413C95F, 0xD1525B21, 0x3B7C9B52, 0x9E3D092C, + 0xEACD6D45, 0x4F8CFF3B, 0xA5A23F48, 0x00E3AD36, 0x3CE08A10, 0x99A1186E, + 0x738FD81D, 0xD6CE4A63, 0xA23E2E0A, 0x077FBC74, 0xED517C07, 0x4810EE79, + 0x04B1B4D5, 0xA1F026AB, 0x4BDEE6D8, 0xEE9F74A6, 0x9A6F10CF, 0x3F2E82B1, + 0xD50042C2, 0x7041D0BC, 0xAD060C8E, 0x08479EF0, 0xE2695E83, 0x4728CCFD, + 0x33D8A894, 0x96993AEA, 0x7CB7FA99, 0xD9F668E7, 0x9557324B, 0x3016A035, + 0xDA386046, 0x7F79F238, 0x0B899651, 0xAEC8042F, 0x44E6C45C, 0xE1A75622, + 0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, 0x437AD51E, 0xE63B4760, + 0x0C158713, 0xA954156D, 0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, + 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8, +}, +{ + 0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, 0x7B2231F3, 0xA6679B4B, + 0xC4451272, 0x1900B8CA, 0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, + 0x8D665215, 0x5023F8AD, 0x32017194, 0xEF44DB2C, 0xE964B13D, 0x34211B85, + 0x560392BC, 0x8B463804, 0x924680CE, 0x4F032A76, 0x2D21A34F, 0xF06409F7, + 0x1F20D2DB, 0xC2657863, 0xA047F15A, 0x7D025BE2, 0x6402E328, 0xB9474990, + 0xDB65C0A9, 0x06206A11, 0xD725148B, 0x0A60BE33, 0x6842370A, 0xB5079DB2, + 0xAC072578, 0x71428FC0, 0x136006F9, 0xCE25AC41, 0x2161776D, 0xFC24DDD5, + 0x9E0654EC, 0x4343FE54, 0x5A43469E, 0x8706EC26, 0xE524651F, 0x3861CFA7, + 0x3E41A5B6, 0xE3040F0E, 0x81268637, 0x5C632C8F, 0x45639445, 0x98263EFD, + 0xFA04B7C4, 0x27411D7C, 0xC805C650, 0x15406CE8, 0x7762E5D1, 0xAA274F69, + 0xB327F7A3, 0x6E625D1B, 0x0C40D422, 0xD1057E9A, 0xABA65FE7, 0x76E3F55F, + 0x14C17C66, 0xC984D6DE, 0xD0846E14, 0x0DC1C4AC, 0x6FE34D95, 0xB2A6E72D, + 0x5DE23C01, 0x80A796B9, 0xE2851F80, 0x3FC0B538, 0x26C00DF2, 0xFB85A74A, + 0x99A72E73, 0x44E284CB, 0x42C2EEDA, 0x9F874462, 0xFDA5CD5B, 0x20E067E3, + 0x39E0DF29, 0xE4A57591, 0x8687FCA8, 0x5BC25610, 0xB4868D3C, 0x69C32784, + 0x0BE1AEBD, 0xD6A40405, 0xCFA4BCCF, 0x12E11677, 0x70C39F4E, 0xAD8635F6, + 0x7C834B6C, 0xA1C6E1D4, 0xC3E468ED, 0x1EA1C255, 0x07A17A9F, 0xDAE4D027, + 0xB8C6591E, 0x6583F3A6, 0x8AC7288A, 0x57828232, 0x35A00B0B, 0xE8E5A1B3, + 0xF1E51979, 0x2CA0B3C1, 0x4E823AF8, 0x93C79040, 0x95E7FA51, 0x48A250E9, + 0x2A80D9D0, 0xF7C57368, 0xEEC5CBA2, 0x3380611A, 0x51A2E823, 0x8CE7429B, + 0x63A399B7, 0xBEE6330F, 0xDCC4BA36, 0x0181108E, 0x1881A844, 0xC5C402FC, + 0xA7E68BC5, 0x7AA3217D, 0x52A0C93F, 0x8FE56387, 0xEDC7EABE, 0x30824006, + 0x2982F8CC, 0xF4C75274, 0x96E5DB4D, 0x4BA071F5, 0xA4E4AAD9, 0x79A10061, + 0x1B838958, 0xC6C623E0, 0xDFC69B2A, 0x02833192, 0x60A1B8AB, 0xBDE41213, + 0xBBC47802, 0x6681D2BA, 0x04A35B83, 0xD9E6F13B, 0xC0E649F1, 0x1DA3E349, + 0x7F816A70, 0xA2C4C0C8, 0x4D801BE4, 0x90C5B15C, 0xF2E73865, 0x2FA292DD, + 0x36A22A17, 0xEBE780AF, 0x89C50996, 0x5480A32E, 0x8585DDB4, 0x58C0770C, + 0x3AE2FE35, 0xE7A7548D, 0xFEA7EC47, 0x23E246FF, 0x41C0CFC6, 0x9C85657E, + 0x73C1BE52, 0xAE8414EA, 0xCCA69DD3, 0x11E3376B, 0x08E38FA1, 0xD5A62519, + 0xB784AC20, 0x6AC10698, 0x6CE16C89, 0xB1A4C631, 0xD3864F08, 0x0EC3E5B0, + 0x17C35D7A, 0xCA86F7C2, 0xA8A47EFB, 0x75E1D443, 0x9AA50F6F, 0x47E0A5D7, + 0x25C22CEE, 0xF8878656, 0xE1873E9C, 0x3CC29424, 0x5EE01D1D, 0x83A5B7A5, + 0xF90696D8, 0x24433C60, 0x4661B559, 0x9B241FE1, 0x8224A72B, 0x5F610D93, + 0x3D4384AA, 0xE0062E12, 0x0F42F53E, 0xD2075F86, 0xB025D6BF, 0x6D607C07, + 0x7460C4CD, 0xA9256E75, 0xCB07E74C, 0x16424DF4, 0x106227E5, 0xCD278D5D, + 0xAF050464, 0x7240AEDC, 0x6B401616, 0xB605BCAE, 0xD4273597, 0x09629F2F, + 0xE6264403, 0x3B63EEBB, 0x59416782, 0x8404CD3A, 0x9D0475F0, 0x4041DF48, + 0x22635671, 0xFF26FCC9, 0x2E238253, 0xF36628EB, 0x9144A1D2, 0x4C010B6A, + 0x5501B3A0, 0x88441918, 0xEA669021, 0x37233A99, 0xD867E1B5, 0x05224B0D, + 0x6700C234, 0xBA45688C, 0xA345D046, 0x7E007AFE, 0x1C22F3C7, 0xC167597F, + 0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, 0xBC65029D, 0x6120A825, + 0x0302211C, 0xDE478BA4, 0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, + 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842, +}, +{ + 0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, 0xE045BEB0, 0xD854D11C, + 0x906761E8, 0xA8760E44, 0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, + 0x2522B521, 0x1D33DA8D, 0x55006A79, 0x6D1105D5, 0x8F2261D3, 0xB7330E7F, + 0xFF00BE8B, 0xC711D127, 0x6F67DF63, 0x5776B0CF, 0x1F45003B, 0x27546F97, + 0x4A456A42, 0x725405EE, 0x3A67B51A, 0x0276DAB6, 0xAA00D4F2, 0x9211BB5E, + 0xDA220BAA, 0xE2336406, 0x1BA8B557, 0x23B9DAFB, 0x6B8A6A0F, 0x539B05A3, + 0xFBED0BE7, 0xC3FC644B, 0x8BCFD4BF, 0xB3DEBB13, 0xDECFBEC6, 0xE6DED16A, + 0xAEED619E, 0x96FC0E32, 0x3E8A0076, 0x069B6FDA, 0x4EA8DF2E, 0x76B9B082, + 0x948AD484, 0xAC9BBB28, 0xE4A80BDC, 0xDCB96470, 0x74CF6A34, 0x4CDE0598, + 0x04EDB56C, 0x3CFCDAC0, 0x51EDDF15, 0x69FCB0B9, 0x21CF004D, 0x19DE6FE1, + 0xB1A861A5, 0x89B90E09, 0xC18ABEFD, 0xF99BD151, 0x37516AAE, 0x0F400502, + 0x4773B5F6, 0x7F62DA5A, 0xD714D41E, 0xEF05BBB2, 0xA7360B46, 0x9F2764EA, + 0xF236613F, 0xCA270E93, 0x8214BE67, 0xBA05D1CB, 0x1273DF8F, 0x2A62B023, + 0x625100D7, 0x5A406F7B, 0xB8730B7D, 0x806264D1, 0xC851D425, 0xF040BB89, + 0x5836B5CD, 0x6027DA61, 0x28146A95, 0x10050539, 0x7D1400EC, 0x45056F40, + 0x0D36DFB4, 0x3527B018, 0x9D51BE5C, 0xA540D1F0, 0xED736104, 0xD5620EA8, + 0x2CF9DFF9, 0x14E8B055, 0x5CDB00A1, 0x64CA6F0D, 0xCCBC6149, 0xF4AD0EE5, + 0xBC9EBE11, 0x848FD1BD, 0xE99ED468, 0xD18FBBC4, 0x99BC0B30, 0xA1AD649C, + 0x09DB6AD8, 0x31CA0574, 0x79F9B580, 0x41E8DA2C, 0xA3DBBE2A, 0x9BCAD186, + 0xD3F96172, 0xEBE80EDE, 0x439E009A, 0x7B8F6F36, 0x33BCDFC2, 0x0BADB06E, + 0x66BCB5BB, 0x5EADDA17, 0x169E6AE3, 0x2E8F054F, 0x86F90B0B, 0xBEE864A7, + 0xF6DBD453, 0xCECABBFF, 0x6EA2D55C, 0x56B3BAF0, 0x1E800A04, 0x269165A8, + 0x8EE76BEC, 0xB6F60440, 0xFEC5B4B4, 0xC6D4DB18, 0xABC5DECD, 0x93D4B161, + 0xDBE70195, 0xE3F66E39, 0x4B80607D, 0x73910FD1, 0x3BA2BF25, 0x03B3D089, + 0xE180B48F, 0xD991DB23, 0x91A26BD7, 0xA9B3047B, 0x01C50A3F, 0x39D46593, + 0x71E7D567, 0x49F6BACB, 0x24E7BF1E, 0x1CF6D0B2, 0x54C56046, 0x6CD40FEA, + 0xC4A201AE, 0xFCB36E02, 0xB480DEF6, 0x8C91B15A, 0x750A600B, 0x4D1B0FA7, + 0x0528BF53, 0x3D39D0FF, 0x954FDEBB, 0xAD5EB117, 0xE56D01E3, 0xDD7C6E4F, + 0xB06D6B9A, 0x887C0436, 0xC04FB4C2, 0xF85EDB6E, 0x5028D52A, 0x6839BA86, + 0x200A0A72, 0x181B65DE, 0xFA2801D8, 0xC2396E74, 0x8A0ADE80, 0xB21BB12C, + 0x1A6DBF68, 0x227CD0C4, 0x6A4F6030, 0x525E0F9C, 0x3F4F0A49, 0x075E65E5, + 0x4F6DD511, 0x777CBABD, 0xDF0AB4F9, 0xE71BDB55, 0xAF286BA1, 0x9739040D, + 0x59F3BFF2, 0x61E2D05E, 0x29D160AA, 0x11C00F06, 0xB9B60142, 0x81A76EEE, + 0xC994DE1A, 0xF185B1B6, 0x9C94B463, 0xA485DBCF, 0xECB66B3B, 0xD4A70497, + 0x7CD10AD3, 0x44C0657F, 0x0CF3D58B, 0x34E2BA27, 0xD6D1DE21, 0xEEC0B18D, + 0xA6F30179, 0x9EE26ED5, 0x36946091, 0x0E850F3D, 0x46B6BFC9, 0x7EA7D065, + 0x13B6D5B0, 0x2BA7BA1C, 0x63940AE8, 0x5B856544, 0xF3F36B00, 0xCBE204AC, + 0x83D1B458, 0xBBC0DBF4, 0x425B0AA5, 0x7A4A6509, 0x3279D5FD, 0x0A68BA51, + 0xA21EB415, 0x9A0FDBB9, 0xD23C6B4D, 0xEA2D04E1, 0x873C0134, 0xBF2D6E98, + 0xF71EDE6C, 0xCF0FB1C0, 0x6779BF84, 0x5F68D028, 0x175B60DC, 0x2F4A0F70, + 0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, 0x2D3CD5C6, 0x152DBA6A, + 0x5D1E0A9E, 0x650F6532, 0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, + 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3, +}, +{ + 0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, 0xB2F53777, 0x5DC55C6E, + 0x697997B4, 0x8649FCAD, 0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, + 0xD2F32F68, 0x3DC34471, 0x097F8FAB, 0xE64FE4B2, 0xC00C303E, 0x2F3C5B27, + 0x1B8090FD, 0xF4B0FBE4, 0x72F90749, 0x9DC96C50, 0xA975A78A, 0x4645CC93, + 0xA00A2821, 0x4F3A4338, 0x7B8688E2, 0x94B6E3FB, 0x12FF1F56, 0xFDCF744F, + 0xC973BF95, 0x2643D48C, 0x85F4168D, 0x6AC47D94, 0x5E78B64E, 0xB148DD57, + 0x370121FA, 0xD8314AE3, 0xEC8D8139, 0x03BDEA20, 0xE5F20E92, 0x0AC2658B, + 0x3E7EAE51, 0xD14EC548, 0x570739E5, 0xB83752FC, 0x8C8B9926, 0x63BBF23F, + 0x45F826B3, 0xAAC84DAA, 0x9E748670, 0x7144ED69, 0xF70D11C4, 0x183D7ADD, + 0x2C81B107, 0xC3B1DA1E, 0x25FE3EAC, 0xCACE55B5, 0xFE729E6F, 0x1142F576, + 0x970B09DB, 0x783B62C2, 0x4C87A918, 0xA3B7C201, 0x0E045BEB, 0xE13430F2, + 0xD588FB28, 0x3AB89031, 0xBCF16C9C, 0x53C10785, 0x677DCC5F, 0x884DA746, + 0x6E0243F4, 0x813228ED, 0xB58EE337, 0x5ABE882E, 0xDCF77483, 0x33C71F9A, + 0x077BD440, 0xE84BBF59, 0xCE086BD5, 0x213800CC, 0x1584CB16, 0xFAB4A00F, + 0x7CFD5CA2, 0x93CD37BB, 0xA771FC61, 0x48419778, 0xAE0E73CA, 0x413E18D3, + 0x7582D309, 0x9AB2B810, 0x1CFB44BD, 0xF3CB2FA4, 0xC777E47E, 0x28478F67, + 0x8BF04D66, 0x64C0267F, 0x507CEDA5, 0xBF4C86BC, 0x39057A11, 0xD6351108, + 0xE289DAD2, 0x0DB9B1CB, 0xEBF65579, 0x04C63E60, 0x307AF5BA, 0xDF4A9EA3, + 0x5903620E, 0xB6330917, 0x828FC2CD, 0x6DBFA9D4, 0x4BFC7D58, 0xA4CC1641, + 0x9070DD9B, 0x7F40B682, 0xF9094A2F, 0x16392136, 0x2285EAEC, 0xCDB581F5, + 0x2BFA6547, 0xC4CA0E5E, 0xF076C584, 0x1F46AE9D, 0x990F5230, 0x763F3929, + 0x4283F2F3, 0xADB399EA, 0x1C08B7D6, 0xF338DCCF, 0xC7841715, 0x28B47C0C, + 0xAEFD80A1, 0x41CDEBB8, 0x75712062, 0x9A414B7B, 0x7C0EAFC9, 0x933EC4D0, + 0xA7820F0A, 0x48B26413, 0xCEFB98BE, 0x21CBF3A7, 0x1577387D, 0xFA475364, + 0xDC0487E8, 0x3334ECF1, 0x0788272B, 0xE8B84C32, 0x6EF1B09F, 0x81C1DB86, + 0xB57D105C, 0x5A4D7B45, 0xBC029FF7, 0x5332F4EE, 0x678E3F34, 0x88BE542D, + 0x0EF7A880, 0xE1C7C399, 0xD57B0843, 0x3A4B635A, 0x99FCA15B, 0x76CCCA42, + 0x42700198, 0xAD406A81, 0x2B09962C, 0xC439FD35, 0xF08536EF, 0x1FB55DF6, + 0xF9FAB944, 0x16CAD25D, 0x22761987, 0xCD46729E, 0x4B0F8E33, 0xA43FE52A, + 0x90832EF0, 0x7FB345E9, 0x59F09165, 0xB6C0FA7C, 0x827C31A6, 0x6D4C5ABF, + 0xEB05A612, 0x0435CD0B, 0x308906D1, 0xDFB96DC8, 0x39F6897A, 0xD6C6E263, + 0xE27A29B9, 0x0D4A42A0, 0x8B03BE0D, 0x6433D514, 0x508F1ECE, 0xBFBF75D7, + 0x120CEC3D, 0xFD3C8724, 0xC9804CFE, 0x26B027E7, 0xA0F9DB4A, 0x4FC9B053, + 0x7B757B89, 0x94451090, 0x720AF422, 0x9D3A9F3B, 0xA98654E1, 0x46B63FF8, + 0xC0FFC355, 0x2FCFA84C, 0x1B736396, 0xF443088F, 0xD200DC03, 0x3D30B71A, + 0x098C7CC0, 0xE6BC17D9, 0x60F5EB74, 0x8FC5806D, 0xBB794BB7, 0x544920AE, + 0xB206C41C, 0x5D36AF05, 0x698A64DF, 0x86BA0FC6, 0x00F3F36B, 0xEFC39872, + 0xDB7F53A8, 0x344F38B1, 0x97F8FAB0, 0x78C891A9, 0x4C745A73, 0xA344316A, + 0x250DCDC7, 0xCA3DA6DE, 0xFE816D04, 0x11B1061D, 0xF7FEE2AF, 0x18CE89B6, + 0x2C72426C, 0xC3422975, 0x450BD5D8, 0xAA3BBEC1, 0x9E87751B, 0x71B71E02, + 0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, 0xE501FDF9, 0x0A3196E0, + 0x3E8D5D3A, 0xD1BD3623, 0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, + 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C, +}, +{ + 0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, 0xA5E0C5D1, 0xCDE3E919, + 0x75E69C41, 0x1DE5B089, 0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, + 0xEBCD3882, 0x83CE144A, 0x3BCB6112, 0x53C84DDA, 0x9C5BFAA6, 0xF458D66E, + 0x4C5DA336, 0x245E8FFE, 0x39BB3F77, 0x51B813BF, 0xE9BD66E7, 0x81BE4A2F, + 0xD27607F5, 0xBA752B3D, 0x02705E65, 0x6A7372AD, 0x7796C224, 0x1F95EEEC, + 0xA7909BB4, 0xCF93B77C, 0x3D5B83BD, 0x5558AF75, 0xED5DDA2D, 0x855EF6E5, + 0x98BB466C, 0xF0B86AA4, 0x48BD1FFC, 0x20BE3334, 0x73767EEE, 0x1B755226, + 0xA370277E, 0xCB730BB6, 0xD696BB3F, 0xBE9597F7, 0x0690E2AF, 0x6E93CE67, + 0xA100791B, 0xC90355D3, 0x7106208B, 0x19050C43, 0x04E0BCCA, 0x6CE39002, + 0xD4E6E55A, 0xBCE5C992, 0xEF2D8448, 0x872EA880, 0x3F2BDDD8, 0x5728F110, + 0x4ACD4199, 0x22CE6D51, 0x9ACB1809, 0xF2C834C1, 0x7AB7077A, 0x12B42BB2, + 0xAAB15EEA, 0xC2B27222, 0xDF57C2AB, 0xB754EE63, 0x0F519B3B, 0x6752B7F3, + 0x349AFA29, 0x5C99D6E1, 0xE49CA3B9, 0x8C9F8F71, 0x917A3FF8, 0xF9791330, + 0x417C6668, 0x297F4AA0, 0xE6ECFDDC, 0x8EEFD114, 0x36EAA44C, 0x5EE98884, + 0x430C380D, 0x2B0F14C5, 0x930A619D, 0xFB094D55, 0xA8C1008F, 0xC0C22C47, + 0x78C7591F, 0x10C475D7, 0x0D21C55E, 0x6522E996, 0xDD279CCE, 0xB524B006, + 0x47EC84C7, 0x2FEFA80F, 0x97EADD57, 0xFFE9F19F, 0xE20C4116, 0x8A0F6DDE, + 0x320A1886, 0x5A09344E, 0x09C17994, 0x61C2555C, 0xD9C72004, 0xB1C40CCC, + 0xAC21BC45, 0xC422908D, 0x7C27E5D5, 0x1424C91D, 0xDBB77E61, 0xB3B452A9, + 0x0BB127F1, 0x63B20B39, 0x7E57BBB0, 0x16549778, 0xAE51E220, 0xC652CEE8, + 0x959A8332, 0xFD99AFFA, 0x459CDAA2, 0x2D9FF66A, 0x307A46E3, 0x58796A2B, + 0xE07C1F73, 0x887F33BB, 0xF56E0EF4, 0x9D6D223C, 0x25685764, 0x4D6B7BAC, + 0x508ECB25, 0x388DE7ED, 0x808892B5, 0xE88BBE7D, 0xBB43F3A7, 0xD340DF6F, + 0x6B45AA37, 0x034686FF, 0x1EA33676, 0x76A01ABE, 0xCEA56FE6, 0xA6A6432E, + 0x6935F452, 0x0136D89A, 0xB933ADC2, 0xD130810A, 0xCCD53183, 0xA4D61D4B, + 0x1CD36813, 0x74D044DB, 0x27180901, 0x4F1B25C9, 0xF71E5091, 0x9F1D7C59, + 0x82F8CCD0, 0xEAFBE018, 0x52FE9540, 0x3AFDB988, 0xC8358D49, 0xA036A181, + 0x1833D4D9, 0x7030F811, 0x6DD54898, 0x05D66450, 0xBDD31108, 0xD5D03DC0, + 0x8618701A, 0xEE1B5CD2, 0x561E298A, 0x3E1D0542, 0x23F8B5CB, 0x4BFB9903, + 0xF3FEEC5B, 0x9BFDC093, 0x546E77EF, 0x3C6D5B27, 0x84682E7F, 0xEC6B02B7, + 0xF18EB23E, 0x998D9EF6, 0x2188EBAE, 0x498BC766, 0x1A438ABC, 0x7240A674, + 0xCA45D32C, 0xA246FFE4, 0xBFA34F6D, 0xD7A063A5, 0x6FA516FD, 0x07A63A35, + 0x8FD9098E, 0xE7DA2546, 0x5FDF501E, 0x37DC7CD6, 0x2A39CC5F, 0x423AE097, + 0xFA3F95CF, 0x923CB907, 0xC1F4F4DD, 0xA9F7D815, 0x11F2AD4D, 0x79F18185, + 0x6414310C, 0x0C171DC4, 0xB412689C, 0xDC114454, 0x1382F328, 0x7B81DFE0, + 0xC384AAB8, 0xAB878670, 0xB66236F9, 0xDE611A31, 0x66646F69, 0x0E6743A1, + 0x5DAF0E7B, 0x35AC22B3, 0x8DA957EB, 0xE5AA7B23, 0xF84FCBAA, 0x904CE762, + 0x2849923A, 0x404ABEF2, 0xB2828A33, 0xDA81A6FB, 0x6284D3A3, 0x0A87FF6B, + 0x17624FE2, 0x7F61632A, 0xC7641672, 0xAF673ABA, 0xFCAF7760, 0x94AC5BA8, + 0x2CA92EF0, 0x44AA0238, 0x594FB2B1, 0x314C9E79, 0x8949EB21, 0xE14AC7E9, + 0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, 0x8B39B544, 0xE33A998C, + 0x5B3FECD4, 0x333CC01C, 0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, + 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F, +}, +{ + 0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, 0x211D826D, 0x6821FF4A, + 0xB3657823, 0xFA590504, 0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, + 0x632686B7, 0x2A1AFB90, 0xF15E7CF9, 0xB86201DE, 0x847609B4, 0xCD4A7493, + 0x160EF3FA, 0x5F328EDD, 0xA56B8BD9, 0xEC57F6FE, 0x37137197, 0x7E2F0CB0, + 0xC64D0D6E, 0x8F717049, 0x5435F720, 0x1D098A07, 0xE7508F03, 0xAE6CF224, + 0x7528754D, 0x3C14086A, 0x0D006599, 0x443C18BE, 0x9F789FD7, 0xD644E2F0, + 0x2C1DE7F4, 0x65219AD3, 0xBE651DBA, 0xF759609D, 0x4F3B6143, 0x06071C64, + 0xDD439B0D, 0x947FE62A, 0x6E26E32E, 0x271A9E09, 0xFC5E1960, 0xB5626447, + 0x89766C2D, 0xC04A110A, 0x1B0E9663, 0x5232EB44, 0xA86BEE40, 0xE1579367, + 0x3A13140E, 0x732F6929, 0xCB4D68F7, 0x827115D0, 0x593592B9, 0x1009EF9E, + 0xEA50EA9A, 0xA36C97BD, 0x782810D4, 0x31146DF3, 0x1A00CB32, 0x533CB615, + 0x8878317C, 0xC1444C5B, 0x3B1D495F, 0x72213478, 0xA965B311, 0xE059CE36, + 0x583BCFE8, 0x1107B2CF, 0xCA4335A6, 0x837F4881, 0x79264D85, 0x301A30A2, + 0xEB5EB7CB, 0xA262CAEC, 0x9E76C286, 0xD74ABFA1, 0x0C0E38C8, 0x453245EF, + 0xBF6B40EB, 0xF6573DCC, 0x2D13BAA5, 0x642FC782, 0xDC4DC65C, 0x9571BB7B, + 0x4E353C12, 0x07094135, 0xFD504431, 0xB46C3916, 0x6F28BE7F, 0x2614C358, + 0x1700AEAB, 0x5E3CD38C, 0x857854E5, 0xCC4429C2, 0x361D2CC6, 0x7F2151E1, + 0xA465D688, 0xED59ABAF, 0x553BAA71, 0x1C07D756, 0xC743503F, 0x8E7F2D18, + 0x7426281C, 0x3D1A553B, 0xE65ED252, 0xAF62AF75, 0x9376A71F, 0xDA4ADA38, + 0x010E5D51, 0x48322076, 0xB26B2572, 0xFB575855, 0x2013DF3C, 0x692FA21B, + 0xD14DA3C5, 0x9871DEE2, 0x4335598B, 0x0A0924AC, 0xF05021A8, 0xB96C5C8F, + 0x6228DBE6, 0x2B14A6C1, 0x34019664, 0x7D3DEB43, 0xA6796C2A, 0xEF45110D, + 0x151C1409, 0x5C20692E, 0x8764EE47, 0xCE589360, 0x763A92BE, 0x3F06EF99, + 0xE44268F0, 0xAD7E15D7, 0x572710D3, 0x1E1B6DF4, 0xC55FEA9D, 0x8C6397BA, + 0xB0779FD0, 0xF94BE2F7, 0x220F659E, 0x6B3318B9, 0x916A1DBD, 0xD856609A, + 0x0312E7F3, 0x4A2E9AD4, 0xF24C9B0A, 0xBB70E62D, 0x60346144, 0x29081C63, + 0xD3511967, 0x9A6D6440, 0x4129E329, 0x08159E0E, 0x3901F3FD, 0x703D8EDA, + 0xAB7909B3, 0xE2457494, 0x181C7190, 0x51200CB7, 0x8A648BDE, 0xC358F6F9, + 0x7B3AF727, 0x32068A00, 0xE9420D69, 0xA07E704E, 0x5A27754A, 0x131B086D, + 0xC85F8F04, 0x8163F223, 0xBD77FA49, 0xF44B876E, 0x2F0F0007, 0x66337D20, + 0x9C6A7824, 0xD5560503, 0x0E12826A, 0x472EFF4D, 0xFF4CFE93, 0xB67083B4, + 0x6D3404DD, 0x240879FA, 0xDE517CFE, 0x976D01D9, 0x4C2986B0, 0x0515FB97, + 0x2E015D56, 0x673D2071, 0xBC79A718, 0xF545DA3F, 0x0F1CDF3B, 0x4620A21C, + 0x9D642575, 0xD4585852, 0x6C3A598C, 0x250624AB, 0xFE42A3C2, 0xB77EDEE5, + 0x4D27DBE1, 0x041BA6C6, 0xDF5F21AF, 0x96635C88, 0xAA7754E2, 0xE34B29C5, + 0x380FAEAC, 0x7133D38B, 0x8B6AD68F, 0xC256ABA8, 0x19122CC1, 0x502E51E6, + 0xE84C5038, 0xA1702D1F, 0x7A34AA76, 0x3308D751, 0xC951D255, 0x806DAF72, + 0x5B29281B, 0x1215553C, 0x230138CF, 0x6A3D45E8, 0xB179C281, 0xF845BFA6, + 0x021CBAA2, 0x4B20C785, 0x906440EC, 0xD9583DCB, 0x613A3C15, 0x28064132, + 0xF342C65B, 0xBA7EBB7C, 0x4027BE78, 0x091BC35F, 0xD25F4436, 0x9B633911, + 0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, 0x866AB316, 0xCF56CE31, + 0x14124958, 0x5D2E347F, 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, + 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5, +} }; + +#define CRC32_UPD(crc, n) \ + (crc32c_tables[(n)][(crc) & 0xff] ^ \ + crc32c_tables[(n) - 1][((crc) >> 8) & 0xff]) + +static inline uint32_t crc32c_u32(uint32_t data, uint32_t init_val) +{ + uint32_t crc, term1, term2; + + crc = init_val; + crc ^= data; + + term1 = CRC32_UPD(crc, 3); + term2 = crc >> 16; + crc = term1 ^ CRC32_UPD(term2, 1); + + return crc; +} + +static inline uint32_t crc32c_u64(uint64_t data, uint32_t init_val) +{ + union { + uint64_t u64; + uint32_t u32[2]; + } d; + d.u64 = data; + + uint32_t crc, term1, term2; + + crc = init_val; + crc ^= d.u32[0]; + + term1 = CRC32_UPD(crc, 7); + term2 = crc >> 16; + crc = term1 ^ CRC32_UPD(term2, 5); + term1 = CRC32_UPD(d.u32[1], 3); + term2 = d.u32[1] >> 16; + crc ^= term1 ^ CRC32_UPD(term2, 1); + + return crc; +} + +uint32_t odp_hash_crc32c(const void *data, uint32_t data_len, + uint32_t init_val) +{ + size_t i; + uint64_t temp = 0; + uintptr_t pd = (uintptr_t)data; + + for (i = 0; i < data_len / 8; i++) { + init_val = crc32c_u64(*(const uint64_t *)pd, init_val); + pd += 8; + } + + switch (7 - (data_len & 0x07)) { + case 0: + temp |= (uint64_t)*((const uint8_t *)pd + 6) << 48; + /* Fallthrough */ + case 1: + temp |= (uint64_t)*((const uint8_t *)pd + 5) << 40; + /* Fallthrough */ + case 2: + temp |= (uint64_t)*((const uint8_t *)pd + 4) << 32; + temp |= *(const uint32_t *)pd; + init_val = crc32c_u64(temp, init_val); + break; + case 3: + init_val = crc32c_u32(*(const uint32_t *)pd, init_val); + break; + case 4: + temp |= *((const uint8_t *)pd + 2) << 16; + /* Fallthrough */ + case 5: + temp |= *((const uint8_t *)pd + 1) << 8; + /* Fallthrough */ + case 6: + temp |= *(const uint8_t *)pd; + init_val = crc32c_u32(temp, init_val); + /* Fallthrough */ + default: + break; + } + + return init_val; +} diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 4616e0342..ea9974287 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -24,6 +24,10 @@ int odp_init_global(const odp_init_t *params, odp_global_data.abort_fn = params->abort_fn; } + if (odp_time_global_init()) { + ODP_ERR("ODP time init failed.\n"); + return -1; + } if (odp_system_info_init()) { ODP_ERR("ODP system_info init failed.\n"); diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 07c740cfa..db85b5e5e 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -32,10 +32,8 @@ static inline void packet_parse_disable(odp_packet_hdr_t *pkt_hdr) pkt_hdr->input_flags.parsed_all = 1; } -void packet_parse_reset(odp_packet_t pkt) +void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - /* Reset parser metadata before new parse */ pkt_hdr->error_flags.all = 0; pkt_hdr->input_flags.all = 0; @@ -780,9 +778,9 @@ int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset, * Parser helper function for IPv4 */ static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, - uint8_t **parseptr, uint32_t *offset) + const uint8_t **parseptr, uint32_t *offset) { - odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr; + const odph_ipv4hdr_t *ipv4 = (const odph_ipv4hdr_t *)*parseptr; uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl); uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl); uint16_t frag_offset; @@ -818,10 +816,10 @@ static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr, * Parser helper function for IPv6 */ static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, - uint8_t **parseptr, uint32_t *offset) + const uint8_t **parseptr, uint32_t *offset) { - odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr; - odph_ipv6hdr_ext_t *ipv6ext; + const odph_ipv6hdr_t *ipv6 = (const odph_ipv6hdr_t *)*parseptr; + const odph_ipv6hdr_ext_t *ipv6ext; pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len); @@ -843,7 +841,7 @@ static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, pkt_hdr->input_flags.ipopt = 1; do { - ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr; + ipv6ext = (const odph_ipv6hdr_ext_t *)*parseptr; uint16_t extlen = 8 + ipv6ext->ext_len * 8; *offset += extlen; @@ -875,9 +873,9 @@ static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr, * Parser helper function for TCP */ static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, - uint8_t **parseptr, uint32_t *offset) + const uint8_t **parseptr, uint32_t *offset) { - odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr; + const odph_tcphdr_t *tcp = (const odph_tcphdr_t *)*parseptr; if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t)) pkt_hdr->error_flags.tcp_err = 1; @@ -895,9 +893,9 @@ static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr, * Parser helper function for UDP */ static inline void parse_udp(odp_packet_hdr_t *pkt_hdr, - uint8_t **parseptr, uint32_t *offset) + const uint8_t **parseptr, uint32_t *offset) { - odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr; + const odph_udphdr_t *udp = (const odph_udphdr_t *)*parseptr; uint32_t udplen = odp_be_to_cpu_16(udp->length); if (udplen < sizeof(odph_udphdr_t) || @@ -932,46 +930,55 @@ void packet_parse_l2(odp_packet_hdr_t *pkt_hdr) pkt_hdr->input_flags.parsed_l2 = 1; } -/** - * Simple packet parser - */ -int packet_parse_full(odp_packet_hdr_t *pkt_hdr) +int _odp_parse_common(odp_packet_hdr_t *pkt_hdr, const uint8_t *ptr) { - odph_ethhdr_t *eth; - odph_vlanhdr_t *vlan; + const odph_ethhdr_t *eth; + const odph_vlanhdr_t *vlan; uint16_t ethtype; - uint8_t *parseptr; uint32_t offset, seglen; uint8_t ip_proto = 0; + const uint8_t *parseptr; + offset = sizeof(odph_ethhdr_t); if (packet_parse_l2_not_done(pkt_hdr)) packet_parse_l2(pkt_hdr); - eth = (odph_ethhdr_t *)packet_map(pkt_hdr, 0, &seglen); - offset = sizeof(odph_ethhdr_t); - parseptr = (uint8_t *)ð->type; - ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + if (ptr == NULL) { + eth = (odph_ethhdr_t *)packet_map(pkt_hdr, 0, &seglen); + parseptr = (const uint8_t *)ð->type; + ethtype = odp_be_to_cpu_16(*((const uint16_t *) + (const void *)parseptr)); + } else { + eth = (const odph_ethhdr_t *)ptr; + parseptr = (const uint8_t *)ð->type; + ethtype = odp_be_to_cpu_16(*((const uint16_t *) + (const void *)parseptr)); + } + /* Parse the VLAN header(s), if present */ if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) { pkt_hdr->input_flags.vlan_qinq = 1; pkt_hdr->input_flags.vlan = 1; - vlan = (odph_vlanhdr_t *)(void *)parseptr; + + vlan = (const odph_vlanhdr_t *)(const void *)parseptr; pkt_hdr->vlan_s_tag = ((ethtype << 16) | odp_be_to_cpu_16(vlan->tci)); offset += sizeof(odph_vlanhdr_t); parseptr += sizeof(odph_vlanhdr_t); - ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + ethtype = odp_be_to_cpu_16(*((const uint16_t *) + (const void *)parseptr)); } if (ethtype == ODPH_ETHTYPE_VLAN) { pkt_hdr->input_flags.vlan = 1; - vlan = (odph_vlanhdr_t *)(void *)parseptr; + vlan = (const odph_vlanhdr_t *)(const void *)parseptr; pkt_hdr->vlan_c_tag = ((ethtype << 16) | odp_be_to_cpu_16(vlan->tci)); offset += sizeof(odph_vlanhdr_t); parseptr += sizeof(odph_vlanhdr_t); - ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + ethtype = odp_be_to_cpu_16(*((const uint16_t *) + (const void *)parseptr)); } /* Check for SNAP vs. DIX */ @@ -983,7 +990,8 @@ int packet_parse_full(odp_packet_hdr_t *pkt_hdr) } offset += 8; parseptr += 8; - ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr)); + ethtype = odp_be_to_cpu_16(*((const uint16_t *) + (const void *)parseptr)); } /* Consume Ethertype for Layer 3 parse */ @@ -1061,3 +1069,16 @@ parse_exit: pkt_hdr->input_flags.parsed_all = 1; return pkt_hdr->error_flags.all != 0; } + +int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr) +{ + return _odp_parse_common(pkt_hdr, parseptr); +} + +/** + * Simple packet parser + */ +int packet_parse_full(odp_packet_hdr_t *pkt_hdr) +{ + return _odp_parse_common(pkt_hdr, NULL); +} diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 1e9d08c02..267aa0137 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -196,6 +196,7 @@ static odp_pktio_t setup_pktio_entry(const char *dev, odp_pool_t pool, return ODP_PKTIO_INVALID; memcpy(&pktio_entry->s.param, param, sizeof(odp_pktio_param_t)); + pktio_entry->s.id = id; for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { ret = pktio_if_ops[pktio_if]->open(id, pktio_entry, dev, pool); @@ -568,7 +569,7 @@ odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *qentry) odp_buffer_t buf; odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; - int pkts, i, j; + int pkts, i; odp_pktio_t pktio; buf_hdr = queue_deq(qentry); @@ -581,27 +582,13 @@ odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *qentry) if (pkts <= 0) return NULL; - if (pktio_cls_enabled(get_pktio_entry(pktio))) { - for (i = 0, j = 0; i < pkts; i++) { - if (0 > packet_classifier(pktio, pkt_tbl[i])) { - buf = _odp_packet_to_buffer(pkt_tbl[i]); - hdr_tbl[j++] = odp_buf_to_hdr(buf); - } - } - } else { - for (i = 0; i < pkts; i++) { - buf = _odp_packet_to_buffer(pkt_tbl[i]); - hdr_tbl[i] = odp_buf_to_hdr(buf); - } - - j = pkts; + for (i = 0; i < pkts; i++) { + buf = _odp_packet_to_buffer(pkt_tbl[i]); + hdr_tbl[i] = odp_buf_to_hdr(buf); } - if (0 == j) - return NULL; - - if (j > 1) - queue_enq_multi(qentry, &hdr_tbl[1], j - 1, 0); + if (pkts > 1) + queue_enq_multi(qentry, &hdr_tbl[1], pkts - 1, 0); buf_hdr = hdr_tbl[0]; return buf_hdr; } @@ -640,27 +627,15 @@ int pktin_deq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int num) if (pkts <= 0) return nbr; - if (pktio_cls_enabled(get_pktio_entry(pktio))) { - for (i = 0, j = 0; i < pkts; i++) { - if (0 > packet_classifier(pktio, pkt_tbl[i])) { - buf = _odp_packet_to_buffer(pkt_tbl[i]); - if (nbr < num) - buf_hdr[nbr++] = odp_buf_to_hdr(buf); - else - hdr_tbl[j++] = odp_buf_to_hdr(buf); - } - } - } else { - /* Fill in buf_hdr first */ - for (i = 0; i < pkts && nbr < num; i++, nbr++) { - buf = _odp_packet_to_buffer(pkt_tbl[i]); - buf_hdr[nbr] = odp_buf_to_hdr(buf); - } - /* Queue the rest for later */ - for (j = 0; i < pkts; i++, j++) { - buf = _odp_packet_to_buffer(pkt_tbl[i]); - hdr_tbl[j++] = odp_buf_to_hdr(buf); - } + /* Fill in buf_hdr first */ + for (i = 0; i < pkts && nbr < num; i++, nbr++) { + buf = _odp_packet_to_buffer(pkt_tbl[i]); + buf_hdr[nbr] = odp_buf_to_hdr(buf); + } + /* Queue the rest for later */ + for (j = 0; i < pkts; i++, j++) { + buf = _odp_packet_to_buffer(pkt_tbl[i]); + hdr_tbl[j++] = odp_buf_to_hdr(buf); } if (j) @@ -672,7 +647,7 @@ int pktin_poll(pktio_entry_t *entry) { odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; - int num, num_enq, i; + int num, i; odp_buffer_t buf; odp_pktio_t pktio; @@ -697,26 +672,15 @@ int pktin_poll(pktio_entry_t *entry) return -1; } - if (pktio_cls_enabled(entry)) { - for (i = 0, num_enq = 0; i < num; i++) { - if (packet_classifier(pktio, pkt_tbl[i]) < 0) { - buf = _odp_packet_to_buffer(pkt_tbl[i]); - hdr_tbl[num_enq++] = odp_buf_to_hdr(buf); - } - } - } else { - for (i = 0; i < num; i++) { - buf = _odp_packet_to_buffer(pkt_tbl[i]); - hdr_tbl[i] = odp_buf_to_hdr(buf); - } - - num_enq = num; + for (i = 0; i < num; i++) { + buf = _odp_packet_to_buffer(pkt_tbl[i]); + hdr_tbl[i] = odp_buf_to_hdr(buf); } - if (num_enq) { + if (num) { queue_entry_t *qentry; qentry = queue_to_qentry(entry->s.inq_default); - queue_enq_multi(qentry, hdr_tbl, num_enq, 0); + queue_enq_multi(qentry, hdr_tbl, num, 0); } return 0; @@ -882,3 +846,43 @@ int odp_pktio_term_global(void) return ret; } + +void odp_pktio_print(odp_pktio_t id) +{ + pktio_entry_t *entry; + uint8_t addr[ETH_ALEN]; + int max_len = 512; + char str[max_len]; + int len = 0; + int n = max_len - 1; + + entry = get_pktio_entry(id); + if (entry == NULL) { + ODP_DBG("pktio entry %d does not exist\n", id); + return; + } + + len += snprintf(&str[len], n - len, + "pktio\n"); + len += snprintf(&str[len], n - len, + " handle %" PRIu64 "\n", odp_pktio_to_u64(id)); + len += snprintf(&str[len], n - len, + " name %s\n", entry->s.name); + len += snprintf(&str[len], n - len, + " state %s\n", + entry->s.state == STATE_START ? "start" : + (entry->s.state == STATE_STOP ? "stop" : "unknown")); + memset(addr, 0, sizeof(addr)); + odp_pktio_mac_addr(id, addr, ETH_ALEN); + len += snprintf(&str[len], n - len, + " mac %02x:%02x:%02x:%02x:%02x:%02x\n", + addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); + len += snprintf(&str[len], n - len, + " mtu %d\n", odp_pktio_mtu(id)); + len += snprintf(&str[len], n - len, + " promisc %s\n", + odp_pktio_promisc_mode(id) ? "yes" : "no"); + str[len] = '\0'; + + ODP_PRINT("\n%s\n", str); +} diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 9859ff6d9..84d35bfba 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -17,6 +17,7 @@ #include <odp_internal.h> #include <odp/config.h> #include <odp/hints.h> +#include <odp/thread.h> #include <odp_debug_internal.h> #include <odp_atomic_internal.h> @@ -463,7 +464,7 @@ int odp_pool_destroy(odp_pool_t pool_hdl) } /* Make sure local caches are empty */ - for (i = 0; i < _ODP_INTERNAL_MAX_THREADS; i++) + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) flush_cache(&pool->s.local_cache[i], &pool->s); /* Call fails if pool has allocated buffers */ diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index db3399ce3..e39176c65 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -355,9 +355,9 @@ int odp_queue_context_set(odp_queue_t handle, void *context) { queue_entry_t *queue; queue = queue_to_qentry(handle); - odp_sync_stores(); + odp_mb_full(); queue->s.param.context = context; - odp_sync_stores(); + odp_mb_full(); return 0; } diff --git a/platform/linux-generic/odp_rwlock_recursive.c b/platform/linux-generic/odp_rwlock_recursive.c new file mode 100644 index 000000000..e3a383caf --- /dev/null +++ b/platform/linux-generic/odp_rwlock_recursive.c @@ -0,0 +1,70 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/rwlock_recursive.h> +#include <odp/thread.h> +#include <string.h> + +#define NO_OWNER (-1) + +void odp_rwlock_recursive_init(odp_rwlock_recursive_t *rlock) +{ + memset(rlock, 0, sizeof(odp_rwlock_recursive_t)); + odp_rwlock_init(&rlock->lock); + rlock->wr_owner = NO_OWNER; +} + +/* Multiple readers can recurse the lock concurrently */ +void odp_rwlock_recursive_read_lock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->rd_cnt[thr]) { + rlock->rd_cnt[thr]++; + return; + } + + odp_rwlock_read_lock(&rlock->lock); + rlock->rd_cnt[thr] = 1; +} + +void odp_rwlock_recursive_read_unlock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + rlock->rd_cnt[thr]--; + + if (rlock->rd_cnt[thr] > 0) + return; + + odp_rwlock_read_unlock(&rlock->lock); +} + +/* Only one writer can recurse the lock */ +void odp_rwlock_recursive_write_lock(odp_rwlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->wr_owner == thr) { + rlock->wr_cnt++; + return; + } + + odp_rwlock_write_lock(&rlock->lock); + rlock->wr_owner = thr; + rlock->wr_cnt = 1; +} + +void odp_rwlock_recursive_write_unlock(odp_rwlock_recursive_t *rlock) +{ + rlock->wr_cnt--; + + if (rlock->wr_cnt > 0) + return; + + rlock->wr_owner = NO_OWNER; + odp_rwlock_write_unlock(&rlock->lock); +} diff --git a/platform/linux-generic/odp_schedule.c b/platform/linux-generic/odp_schedule.c index 9b7cd57ff..58f1b1b37 100644 --- a/platform/linux-generic/odp_schedule.c +++ b/platform/linux-generic/odp_schedule.c @@ -588,11 +588,10 @@ static int schedule_loop(odp_queue_t *out_queue, uint64_t wait, odp_event_t out_ev[], unsigned int max_num, unsigned int max_deq) { - odp_time_t start_time, time, diff; + odp_time_t next, wtime; + int first = 1; int ret; - start_time = ODP_TIME_NULL; - while (1) { ret = schedule(out_queue, out_ev, max_num, max_deq); @@ -605,15 +604,14 @@ static int schedule_loop(odp_queue_t *out_queue, uint64_t wait, if (wait == ODP_SCHED_NO_WAIT) break; - if (!odp_time_cmp(ODP_TIME_NULL, start_time)) { - start_time = odp_time_local(); + if (first) { + wtime = odp_time_local_from_ns(wait); + next = odp_time_sum(odp_time_local(), wtime); + first = 0; continue; } - time = odp_time_local(); - diff = odp_time_diff(time, start_time); - - if (odp_time_cmp(wait, diff) < 0) + if (odp_time_cmp(next, odp_time_local()) < 0) break; } @@ -654,7 +652,7 @@ void odp_schedule_resume(void) uint64_t odp_schedule_wait_time(uint64_t ns) { - return odp_time_to_u64(odp_time_local_from_ns(ns)); + return ns; } diff --git a/platform/linux-generic/odp_spinlock_recursive.c b/platform/linux-generic/odp_spinlock_recursive.c new file mode 100644 index 000000000..8ffe6b3dd --- /dev/null +++ b/platform/linux-generic/odp_spinlock_recursive.c @@ -0,0 +1,70 @@ +/* Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/spinlock_recursive.h> +#include <odp/thread.h> + +#define NO_OWNER (-1) + +void odp_spinlock_recursive_init(odp_spinlock_recursive_t *rlock) +{ + odp_spinlock_init(&rlock->lock); + rlock->owner = NO_OWNER; + rlock->cnt = 0; +} + +void odp_spinlock_recursive_lock(odp_spinlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->owner == thr) { + rlock->cnt++; + return; + } + + odp_spinlock_lock(&rlock->lock); + rlock->owner = thr; + rlock->cnt = 1; +} + +int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->owner == thr) { + rlock->cnt++; + return 1; + } + + if (odp_spinlock_trylock(&rlock->lock)) { + rlock->owner = thr; + rlock->cnt = 1; + return 1; + } else { + return 0; + } +} + +void odp_spinlock_recursive_unlock(odp_spinlock_recursive_t *rlock) +{ + rlock->cnt--; + + if (rlock->cnt > 0) + return; + + rlock->owner = NO_OWNER; + odp_spinlock_unlock(&rlock->lock); +} + +int odp_spinlock_recursive_is_locked(odp_spinlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->owner == thr) + return 1; + + return odp_spinlock_is_locked(&rlock->lock); +} diff --git a/platform/linux-generic/odp_thread.c b/platform/linux-generic/odp_thread.c index 2f3934374..06a5dcd5d 100644 --- a/platform/linux-generic/odp_thread.c +++ b/platform/linux-generic/odp_thread.c @@ -18,7 +18,6 @@ #include <odp/shared_memory.h> #include <odp/align.h> #include <odp/cpu.h> -#include <odp_config_internal.h> #include <string.h> #include <stdio.h> @@ -32,7 +31,7 @@ typedef struct { typedef struct { - thread_state_t thr[_ODP_INTERNAL_MAX_THREADS]; + thread_state_t thr[ODP_THREAD_COUNT_MAX]; union { /* struct order must be kept in sync with schedule_types.h */ struct { @@ -102,10 +101,10 @@ static int alloc_id(odp_thread_type_t type) int thr; odp_thrmask_t *all = &thread_globals->all; - if (thread_globals->num >= _ODP_INTERNAL_MAX_THREADS) + if (thread_globals->num >= ODP_THREAD_COUNT_MAX) return -1; - for (thr = 0; thr < _ODP_INTERNAL_MAX_THREADS; thr++) { + for (thr = 0; thr < ODP_THREAD_COUNT_MAX; thr++) { if (odp_thrmask_isset(all, thr) == 0) { odp_thrmask_set(all, thr); @@ -129,7 +128,7 @@ static int free_id(int thr) { odp_thrmask_t *all = &thread_globals->all; - if (thr < 0 || thr >= _ODP_INTERNAL_MAX_THREADS) + if (thr < 0 || thr >= ODP_THREAD_COUNT_MAX) return -1; if (odp_thrmask_isset(all, thr) == 0) @@ -207,7 +206,7 @@ int odp_thread_count(void) int odp_thread_count_max(void) { - return _ODP_INTERNAL_MAX_THREADS; + return ODP_THREAD_COUNT_MAX; } odp_thread_type_t odp_thread_type(void) diff --git a/platform/linux-generic/odp_thrmask.c b/platform/linux-generic/odp_thrmask.c index b62d03c7a..154ae81b6 100644 --- a/platform/linux-generic/odp_thrmask.c +++ b/platform/linux-generic/odp_thrmask.c @@ -8,7 +8,6 @@ #define _GNU_SOURCE #endif -#include <odp_config_internal.h> #include <odp/thrmask.h> #include <odp/cpumask.h> diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c index 181d7323e..2cb84f2ab 100644 --- a/platform/linux-generic/odp_time.c +++ b/platform/linux-generic/odp_time.c @@ -6,81 +6,199 @@ #define _POSIX_C_SOURCE 200809L +#include <time.h> #include <odp/time.h> #include <odp/hints.h> -#include <odp/system_info.h> -#include <odp/cpu.h> -#include <odp_cpu_internal.h> +#include <odp_debug_internal.h> -#define GIGA 1000000000 +typedef union { + odp_time_t ex; + struct timespec in; +} _odp_time_t; + +static odp_time_t start_time; static inline -uint64_t time_to_tick(odp_time_t time) +uint64_t time_to_ns(odp_time_t time) { - return (uint64_t)time; + uint64_t ns; + + ns = time.tv_sec * ODP_TIME_SEC_IN_NS; + ns += time.tv_nsec; + + return ns; } -static inline -odp_time_t tick_to_time(uint64_t tick) +static inline odp_time_t time_diff(odp_time_t t2, odp_time_t t1) +{ + odp_time_t time; + + time.tv_sec = t2.tv_sec - t1.tv_sec; + time.tv_nsec = t2.tv_nsec - t1.tv_nsec; + + if (time.tv_nsec < 0) { + time.tv_nsec += ODP_TIME_SEC_IN_NS; + --time.tv_sec; + } + + return time; +} + +static inline odp_time_t time_local(void) +{ + int ret; + _odp_time_t time; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time.in); + if (odp_unlikely(ret != 0)) + ODP_ABORT("clock_gettime failed\n"); + + return time_diff(time.ex, start_time); +} + +static inline int time_cmp(odp_time_t t2, odp_time_t t1) +{ + if (t2.tv_sec < t1.tv_sec) + return -1; + + if (t2.tv_sec > t1.tv_sec) + return 1; + + return t2.tv_nsec - t1.tv_nsec; +} + +static inline odp_time_t time_sum(odp_time_t t1, odp_time_t t2) { - return (odp_time_t)tick; + odp_time_t time; + + time.tv_sec = t2.tv_sec + t1.tv_sec; + time.tv_nsec = t2.tv_nsec + t1.tv_nsec; + + if (time.tv_nsec >= (long)ODP_TIME_SEC_IN_NS) { + time.tv_nsec -= ODP_TIME_SEC_IN_NS; + ++time.tv_sec; + } + + return time; +} + +static inline odp_time_t time_local_from_ns(uint64_t ns) +{ + odp_time_t time; + + time.tv_sec = ns / ODP_TIME_SEC_IN_NS; + time.tv_nsec = ns - time.tv_sec * ODP_TIME_SEC_IN_NS; + + return time; +} + +static inline void time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = time_local(); + } while (time_cmp(time, cur) > 0); +} + +static inline uint64_t time_local_res(void) +{ + int ret; + struct timespec tres; + + ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); + if (odp_unlikely(ret != 0)) + ODP_ABORT("clock_getres failed\n"); + + return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; } odp_time_t odp_time_local(void) { - return tick_to_time(odp_cpu_cycles()); + return time_local(); +} + +odp_time_t odp_time_global(void) +{ + return time_local(); } odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1) { - return tick_to_time(_odp_cpu_cycles_diff(t2, t1)); + return time_diff(t2, t1); } uint64_t odp_time_to_ns(odp_time_t time) { - uint64_t hz = odp_sys_cpu_hz(); - uint64_t tick = time_to_tick(time); - - if (tick > (UINT64_MAX / GIGA)) - return (tick / hz) * GIGA; + return time_to_ns(time); +} - return (tick * GIGA) / hz; +odp_time_t odp_time_local_from_ns(uint64_t ns) +{ + return time_local_from_ns(ns); } +odp_time_t odp_time_global_from_ns(uint64_t ns) +{ + return time_local_from_ns(ns); +} -odp_time_t odp_time_local_from_ns(uint64_t ns) +int odp_time_cmp(odp_time_t t2, odp_time_t t1) { - uint64_t hz = odp_sys_cpu_hz(); + return time_cmp(t2, t1); +} - if (ns > (UINT64_MAX / hz)) - return tick_to_time((ns / GIGA) * hz); +odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) +{ + return time_sum(t1, t2); +} - return tick_to_time((ns * hz) / GIGA); +uint64_t odp_time_local_res(void) +{ + return time_local_res(); } -int odp_time_cmp(odp_time_t t2, odp_time_t t1) +uint64_t odp_time_global_res(void) { - uint64_t tick1 = time_to_tick(t1); - uint64_t tick2 = time_to_tick(t2); + return time_local_res(); +} - if (tick1 < tick2) - return 1; +void odp_time_wait_ns(uint64_t ns) +{ + odp_time_t cur = time_local(); + odp_time_t wait = time_local_from_ns(ns); + odp_time_t end_time = time_sum(cur, wait); - if (tick1 > tick2) - return -1; + time_wait_until(end_time); +} - return 0; +void odp_time_wait_until(odp_time_t time) +{ + return time_wait_until(time); } -odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) +uint64_t odp_time_to_u64(odp_time_t time) { - uint64_t tick1 = time_to_tick(t1); - uint64_t tick2 = time_to_tick(t2); + int ret; + struct timespec tres; + uint64_t resolution; - return tick_to_time(tick1 + tick2); + ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); + if (odp_unlikely(ret != 0)) + ODP_ABORT("clock_getres failed\n"); + + resolution = (uint64_t)tres.tv_nsec; + + return time_to_ns(time) / resolution; } -uint64_t odp_time_to_u64(odp_time_t hdl) +int odp_time_global_init(void) { - return time_to_tick(hdl); + int ret; + _odp_time_t time; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time.in); + start_time = ret ? ODP_TIME_NULL : time.ex; + + return ret; } diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index ce19add2b..47745ad83 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -12,6 +12,7 @@ #include <odp.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> +#include <odp_classification_internal.h> #include <odp_debug_internal.h> #include <odp/hints.h> @@ -50,19 +51,35 @@ static int loopback_close(pktio_entry_t *pktio_entry) static int loopback_recv(pktio_entry_t *pktio_entry, odp_packet_t pkts[], unsigned len) { - int nbr, i; + int nbr, i, j; odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; queue_entry_t *qentry; odp_packet_hdr_t *pkt_hdr; + odp_packet_t pkt; + nbr = 0; qentry = queue_to_qentry(pktio_entry->s.pkt_loop.loopq); nbr = queue_deq_multi(qentry, hdr_tbl, len); - for (i = 0; i < nbr; ++i) { - pkts[i] = _odp_packet_from_buffer(odp_hdr_to_buf(hdr_tbl[i])); - pkt_hdr = odp_packet_hdr(pkts[i]); - packet_parse_reset(pkts[i]); - packet_parse_l2(pkt_hdr); + if (pktio_cls_enabled(pktio_entry)) { + for (i = 0, j = 0; i < nbr; i++) { + pkt = _odp_packet_from_buffer(odp_hdr_to_buf + (hdr_tbl[i])); + pkt_hdr = odp_packet_hdr(pkt); + packet_parse_reset(pkt_hdr); + packet_parse_l2(pkt_hdr); + if (0 > _odp_packet_classifier(pktio_entry, pkt)) + pkts[j++] = pkt; + } + nbr = j; + } else { + for (i = 0; i < nbr; ++i) { + pkts[i] = _odp_packet_from_buffer(odp_hdr_to_buf + (hdr_tbl[i])); + pkt_hdr = odp_packet_hdr(pkts[i]); + packet_parse_reset(pkt_hdr); + packet_parse_l2(pkt_hdr); + } } return nbr; diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index d0643230b..11ebd5581 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -20,6 +20,9 @@ #include <poll.h> #include <linux/ethtool.h> #include <linux/sockios.h> +#include <odp_classification_datamodel.h> +#include <odp_classification_inlines.h> +#include <odp_classification_internal.h> #define NETMAP_WITH_LIBS #include <net/netmap_user.h> @@ -197,7 +200,7 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, uint16_t len) { odp_packet_t pkt; - odp_packet_hdr_t *pkt_hdr; + int ret; if (odp_unlikely(len > pktio_entry->s.pkt_nm.max_frame_len)) { ODP_ERR("RX: frame too big %" PRIu16 " %zu!\n", len, @@ -210,21 +213,32 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, return -1; } - pkt = packet_alloc(pktio_entry->s.pkt_nm.pool, len, 1); - if (pkt == ODP_PACKET_INVALID) + if (pktio_cls_enabled(pktio_entry)) { + ret = _odp_packet_cls_enq(pktio_entry, (const uint8_t *)buf, + len, pkt_out); + if (ret) + return 0; return -1; + } else { + odp_packet_hdr_t *pkt_hdr; - pkt_hdr = odp_packet_hdr(pkt); + pkt = packet_alloc(pktio_entry->s.pkt_nm.pool, len, 1); + if (pkt == ODP_PACKET_INVALID) + return -1; - /* For now copy the data in the mbuf, - worry about zero-copy later */ - if (odp_packet_copydata_in(pkt, 0, len, buf) != 0) { - odp_packet_free(pkt); - return -1; + pkt_hdr = odp_packet_hdr(pkt); + + /* For now copy the data in the mbuf, + worry about zero-copy later */ + if (odp_packet_copydata_in(pkt, 0, len, buf) != 0) { + odp_packet_free(pkt); + return -1; + } + + packet_parse_l2(pkt_hdr); + *pkt_out = pkt; } - packet_parse_l2(pkt_hdr); - *pkt_out = pkt; return 0; } diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c new file mode 100644 index 000000000..5729db6eb --- /dev/null +++ b/platform/linux-generic/pktio/pktio_common.c @@ -0,0 +1,57 @@ +/* Copyright (c) 2013, Linaro Limited + * Copyright (c) 2013, Nokia Solutions and Networks + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include <odp_packet_io_internal.h> +#include <odp_classification_internal.h> + +int _odp_packet_cls_enq(pktio_entry_t *pktio_entry, + const uint8_t *base, uint16_t buf_len, + odp_packet_t *pkt_ret) +{ + cos_t *cos; + odp_packet_t pkt; + odp_packet_hdr_t pkt_hdr; + int ret; + odp_pool_t pool; + + packet_parse_reset(&pkt_hdr); + + _odp_cls_parse(&pkt_hdr, base); + cos = pktio_select_cos(pktio_entry, base, &pkt_hdr); + + /* if No CoS found then drop the packet */ + if (cos == NULL || cos->s.queue == NULL || cos->s.pool == NULL) + return 0; + + pool = cos->s.pool->s.pool_hdl; + + pkt = odp_packet_alloc(pool, buf_len); + if (odp_unlikely(pkt == ODP_PACKET_INVALID)) + return 0; + + copy_packet_parser_metadata(&pkt_hdr, odp_packet_hdr(pkt)); + odp_packet_hdr(pkt)->input = pktio_entry->s.id; + + if (odp_packet_copydata_in(pkt, 0, buf_len, base) != 0) { + odp_packet_free(pkt); + return 0; + } + + /* Parse and set packet header data */ + odp_packet_pull_tail(pkt, odp_packet_len(pkt) - buf_len); + ret = queue_enq(cos->s.queue, odp_buf_to_hdr((odp_buffer_t)pkt), 0); + if (ret < 0) { + *pkt_ret = pkt; + return 1; + } + + return 0; +} diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index ef2e031db..dfdab18d7 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -38,6 +38,9 @@ #include <odp_packet_io_internal.h> #include <odp_align_internal.h> #include <odp_debug_internal.h> +#include <odp_classification_datamodel.h> +#include <odp_classification_inlines.h> +#include <odp_classification_internal.h> #include <odp/hints.h> #include <odp/helper/eth.h> @@ -202,6 +205,8 @@ static int sock_close(pktio_entry_t *pktio_entry) return -1; } + odp_shm_free(pkt_sock->shm); + return 0; } @@ -213,10 +218,13 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, { int sockfd; int err; + int i; unsigned int if_idx; struct ifreq ethreq; struct sockaddr_ll sa_ll; + char shm_name[ODP_SHM_NAME_LEN]; pkt_sock_t *pkt_sock = &pktio_entry->s.pkt_sock; + uint8_t *addr; /* Init pktio entry */ memset(pkt_sock, 0, sizeof(*pkt_sock)); @@ -226,6 +234,20 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, if (pool == ODP_POOL_INVALID) return -1; pkt_sock->pool = pool; + snprintf(shm_name, ODP_SHM_NAME_LEN, "%s-%s", "pktio", netdev); + shm_name[ODP_SHM_NAME_LEN - 1] = '\0'; + + pkt_sock->shm = odp_shm_reserve(shm_name, PACKET_JUMBO_LEN, + PACKET_JUMBO_LEN * + ODP_PACKET_SOCKET_MAX_BURST_RX, 0); + if (pkt_sock->shm == ODP_SHM_INVALID) + return -1; + + addr = odp_shm_addr(pkt_sock->shm); + for (i = 0; i < ODP_PACKET_SOCKET_MAX_BURST_RX; i++) { + pkt_sock->cache_ptr[i] = addr; + addr += PACKET_JUMBO_LEN; + } sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); if (sockfd == -1) { @@ -261,7 +283,6 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, ODP_ERR("bind(to IF): %s\n", strerror(errno)); goto error; } - return 0; error: @@ -319,58 +340,95 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, const int sockfd = pkt_sock->sockfd; int msgvec_len; struct mmsghdr msgvec[ODP_PACKET_SOCKET_MAX_BURST_RX]; - struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX][ODP_BUFFER_MAX_SEG]; int nb_rx = 0; int recv_msgs; + int ret; + uint8_t **recv_cache; int i; if (odp_unlikely(len > ODP_PACKET_SOCKET_MAX_BURST_RX)) return -1; memset(msgvec, 0, sizeof(msgvec)); + recv_cache = pkt_sock->cache_ptr; - for (i = 0; i < (int)len; i++) { - pkt_table[i] = packet_alloc(pkt_sock->pool, 0 /*default*/, 1); - if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID)) - break; + if (pktio_cls_enabled(pktio_entry)) { + struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX]; - msgvec[i].msg_hdr.msg_iovlen = - _rx_pkt_to_iovec(pkt_table[i], iovecs[i]); - - msgvec[i].msg_hdr.msg_iov = iovecs[i]; - } - msgvec_len = i; /* number of successfully allocated pkt buffers */ + for (i = 0; i < (int)len; i++) { + msgvec[i].msg_hdr.msg_iovlen = 1; + iovecs[i].iov_base = recv_cache[i]; + iovecs[i].iov_len = PACKET_JUMBO_LEN; + msgvec[i].msg_hdr.msg_iov = &iovecs[i]; + } + /* number of successfully allocated pkt buffers */ + msgvec_len = i; + + recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len, + MSG_DONTWAIT, NULL); + for (i = 0; i < recv_msgs; i++) { + void *base = msgvec[i].msg_hdr.msg_iov->iov_base; + struct ethhdr *eth_hdr = base; + uint16_t pkt_len = msgvec[i].msg_len; + + /* Don't receive packets sent by ourselves */ + if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, + eth_hdr->h_source))) + continue; + + ret = _odp_packet_cls_enq(pktio_entry, base, + pkt_len, &pkt_table[nb_rx]); + if (ret) + nb_rx++; + } + } else { + struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX] + [ODP_BUFFER_MAX_SEG]; - recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len, MSG_DONTWAIT, NULL); + for (i = 0; i < (int)len; i++) { + pkt_table[i] = packet_alloc(pkt_sock->pool, + 0 /*default*/, 1); + if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID)) + break; - for (i = 0; i < recv_msgs; i++) { - odp_packet_hdr_t *pkt_hdr; - void *base = msgvec[i].msg_hdr.msg_iov->iov_base; - struct ethhdr *eth_hdr = base; + msgvec[i].msg_hdr.msg_iovlen = + _rx_pkt_to_iovec(pkt_table[i], iovecs[i]); - /* Don't receive packets sent by ourselves */ - if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, - eth_hdr->h_source))) { - odp_packet_free(pkt_table[i]); - continue; + msgvec[i].msg_hdr.msg_iov = iovecs[i]; } - pkt_hdr = odp_packet_hdr(pkt_table[i]); - - odp_packet_pull_tail(pkt_table[i], - odp_packet_len(pkt_table[i]) - - msgvec[i].msg_len); + /* number of successfully allocated pkt buffers */ + msgvec_len = i; - packet_parse_l2(pkt_hdr); + recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len, + MSG_DONTWAIT, NULL); - pkt_table[nb_rx] = pkt_table[i]; - nb_rx++; - } + for (i = 0; i < recv_msgs; i++) { + void *base = msgvec[i].msg_hdr.msg_iov->iov_base; + struct ethhdr *eth_hdr = base; + odp_packet_hdr_t *pkt_hdr; - /* Free unused pkt buffers */ - for (; i < msgvec_len; i++) - odp_packet_free(pkt_table[i]); + /* Don't receive packets sent by ourselves */ + if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, + eth_hdr->h_source))) { + odp_packet_free(pkt_table[i]); + continue; + } + pkt_hdr = odp_packet_hdr(pkt_table[i]); + /* Parse and set packet header data */ + odp_packet_pull_tail(pkt_table[i], + odp_packet_len(pkt_table[i]) - + msgvec[i].msg_len); + + packet_parse_l2(pkt_hdr); + pkt_table[nb_rx] = pkt_table[i]; + nb_rx++; + } + /* Free unused pkt buffers */ + for (; i < msgvec_len; i++) + odp_packet_free(pkt_table[i]); + } return nb_rx; } @@ -385,7 +443,7 @@ static uint32_t _tx_pkt_to_iovec(odp_packet_t pkt, uint32_t seglen; iovecs[iov_count].iov_base = odp_packet_offset(pkt, offset, - &seglen, NULL); + &seglen, NULL); iovecs[iov_count].iov_len = seglen; iov_count++; offset += seglen; @@ -415,7 +473,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry, for (i = 0; i < len; i++) { msgvec[i].msg_hdr.msg_iov = iovecs[i]; msgvec[i].msg_hdr.msg_iovlen = _tx_pkt_to_iovec(pkt_table[i], - iovecs[i]); + iovecs[i]); } for (i = 0; i < len; ) { @@ -423,7 +481,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry, if (odp_unlikely(ret <= -1)) { if (i == 0 && SOCK_ERR_REPORT(errno)) { __odp_errno = errno; - ODP_ERR("sendmmsg(): %s\n", strerror(errno)); + ODP_ERR("sendmmsg(): %s\n", strerror(errno)); return -1; } break; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 2bdb72b35..fcaaaf63f 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -28,6 +28,9 @@ #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> #include <odp_debug_internal.h> +#include <odp_classification_datamodel.h> +#include <odp_classification_inlines.h> +#include <odp_classification_internal.h> #include <odp/hints.h> #include <odp/helper/eth.h> @@ -108,9 +111,9 @@ static inline void mmap_tx_user_ready(struct tpacket2_hdr *hdr) __sync_synchronize(); } -static inline unsigned pkt_mmap_v2_rx(int sock, struct ring *ring, +static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, + pkt_sock_mmap_t *pkt_sock, odp_packet_t pkt_table[], unsigned len, - odp_pool_t pool, unsigned char if_mac[]) { union frame_map ppd; @@ -118,57 +121,68 @@ static inline unsigned pkt_mmap_v2_rx(int sock, struct ring *ring, uint8_t *pkt_buf; int pkt_len; struct ethhdr *eth_hdr; - odp_packet_hdr_t *pkt_hdr; unsigned i = 0; + uint8_t nb_rx = 0; + struct ring *ring; + int ret; - (void)sock; - + ring = &pkt_sock->rx_ring; frame_num = ring->frame_num; while (i < len) { - if (mmap_rx_kernel_ready(ring->rd[frame_num].iov_base)) { - ppd.raw = ring->rd[frame_num].iov_base; + if (!mmap_rx_kernel_ready(ring->rd[frame_num].iov_base)) + break; + + ppd.raw = ring->rd[frame_num].iov_base; + next_frame_num = (frame_num + 1) % ring->rd_num; + + pkt_buf = (uint8_t *)ppd.raw + ppd.v2->tp_h.tp_mac; + pkt_len = ppd.v2->tp_h.tp_snaplen; - next_frame_num = (frame_num + 1) % ring->rd_num; + /* Don't receive packets sent by ourselves */ + eth_hdr = (struct ethhdr *)pkt_buf; + if (odp_unlikely(ethaddrs_equal(if_mac, + eth_hdr->h_source))) { + mmap_rx_user_ready(ppd.raw); /* drop */ + frame_num = next_frame_num; + continue; + } - pkt_buf = (uint8_t *)ppd.raw + ppd.v2->tp_h.tp_mac; - pkt_len = ppd.v2->tp_h.tp_snaplen; + if (pktio_cls_enabled(pktio_entry)) { + ret = _odp_packet_cls_enq(pktio_entry, pkt_buf, + pkt_len, &pkt_table[nb_rx]); + if (ret) + nb_rx++; + } else { + odp_packet_hdr_t *hdr; - /* Don't receive packets sent by ourselves */ - eth_hdr = (struct ethhdr *)pkt_buf; - if (odp_unlikely(ethaddrs_equal(if_mac, - eth_hdr->h_source))) { + pkt_table[i] = packet_alloc(pkt_sock->pool, pkt_len, 1); + if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID)) { mmap_rx_user_ready(ppd.raw); /* drop */ frame_num = next_frame_num; continue; } - - pkt_table[i] = packet_alloc(pool, pkt_len, 1); - if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID)) - break; - - pkt_hdr = odp_packet_hdr(pkt_table[i]); - - if (odp_packet_copydata_in(pkt_table[i], 0, - pkt_len, pkt_buf) != 0) { + hdr = odp_packet_hdr(pkt_table[i]); + ret = odp_packet_copydata_in(pkt_table[i], 0, + pkt_len, pkt_buf); + if (ret != 0) { odp_packet_free(pkt_table[i]); - break; + mmap_rx_user_ready(ppd.raw); /* drop */ + frame_num = next_frame_num; + continue; } - packet_parse_l2(pkt_hdr); - - mmap_rx_user_ready(ppd.raw); - - frame_num = next_frame_num; - i++; - } else { - break; + packet_parse_l2(hdr); + nb_rx++; } + + mmap_rx_user_ready(ppd.raw); + frame_num = next_frame_num; + i++; } ring->frame_num = frame_num; - - return i; + return nb_rx; } static inline unsigned pkt_mmap_v2_tx(int sock, struct ring *ring, @@ -502,9 +516,8 @@ static int sock_mmap_recv(pktio_entry_t *pktio_entry, { pkt_sock_mmap_t *const pkt_sock = &pktio_entry->s.pkt_sock_mmap; - return pkt_mmap_v2_rx(pkt_sock->rx_ring.sock, &pkt_sock->rx_ring, - pkt_table, len, pkt_sock->pool, - pkt_sock->if_mac); + return pkt_mmap_v2_rx(pktio_entry, pkt_sock, + pkt_table, len, pkt_sock->if_mac); } static int sock_mmap_send(pktio_entry_t *pktio_entry, diff --git a/platform/linux-generic/test/Makefile.am b/platform/linux-generic/test/Makefile.am index 594aa11eb..e62987297 100644 --- a/platform/linux-generic/test/Makefile.am +++ b/platform/linux-generic/test/Makefile.am @@ -12,6 +12,7 @@ TESTS = pktio/pktio_run \ ${top_builddir}/test/validation/cpumask/cpumask_main$(EXEEXT) \ ${top_builddir}/test/validation/crypto/crypto_main$(EXEEXT) \ ${top_builddir}/test/validation/errno/errno_main$(EXEEXT) \ + ${top_builddir}/test/validation/hash/hash_main$(EXEEXT) \ ${top_builddir}/test/validation/init/init_main_ok$(EXEEXT) \ ${top_builddir}/test/validation/init/init_main_abort$(EXEEXT) \ ${top_builddir}/test/validation/init/init_main_log$(EXEEXT) \ @@ -20,6 +21,7 @@ TESTS = pktio/pktio_run \ ${top_builddir}/test/validation/queue/queue_main$(EXEEXT) \ ${top_builddir}/test/validation/random/random_main$(EXEEXT) \ ${top_builddir}/test/validation/scheduler/scheduler_main$(EXEEXT) \ + ${top_builddir}/test/validation/std_clib/std_clib_main$(EXEEXT) \ ${top_builddir}/test/validation/synchronizers/synchronizers_main$(EXEEXT) \ ${top_builddir}/test/validation/thread/thread_main$(EXEEXT) \ ${top_builddir}/test/validation/time/time_main$(EXEEXT) \ |