1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
/* Copyright (c) 2016-2018, Linaro Limited
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include "config.h"
/*
* This file handles the lower end of the ishm memory allocator:
* It performs the physical mappings.
*/
#include <odp_posix_extensions.h>
#include <odp_config_internal.h>
#include <odp_internal.h>
#include <odp/api/align.h>
#include <odp/api/system_info.h>
#include <odp/api/debug.h>
#include <odp_debug_internal.h>
#include <odp_align_internal.h>
#include <odp_ishm_internal.h>
#include <odp_ishmphy_internal.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <odp_ishmphy_internal.h>
static void *common_va_address;
static uint64_t common_va_len;
#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif
/* Book some virtual address space
* This function is called at odp_init_global() time to pre-book some
* virtual address space inherited by all odpthreads (i.e. descendant
* processes and threads) and later used to guarantee the unicity the
* the mapping VA address when memory is reserver with the _ODP_ISHM_SINGLE_VA
* flag.
* returns the address of the mapping or NULL on error.
*/
void *_odp_ishmphy_book_va(uintptr_t len, intptr_t align)
{
void *addr;
addr = mmap(NULL, len + align, PROT_NONE,
MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
if (addr == MAP_FAILED) {
ODP_ERR("_ishmphy_book_va failure\n");
return NULL;
}
if (mprotect(addr, len, PROT_NONE))
ODP_ERR("failure for protect\n");
ODP_DBG("VA Reserved: %p, len=%p\n", addr, len + align);
common_va_address = addr;
common_va_len = len;
/* return the nearest aligned address: */
return (void *)(((uintptr_t)addr + align - 1) & (-align));
}
/* Un-book some virtual address space
* This function is called at odp_term_global() time to unbook
* the virtual address space booked by _ishmphy_book_va()
*/
int _odp_ishmphy_unbook_va(void)
{
int ret;
ret = munmap(common_va_address, common_va_len);
if (ret)
ODP_ERR("_unishmphy_book_va failure\n");
return ret;
}
/*
* do a mapping:
* Performs a mapping of the provided file descriptor to the process VA
* space. If the _ODP_ISHM_SINGLE_VA flag is set, 'start' is assumed to be
* the VA address where the mapping is to be done.
* If the flag is not set, a new VA address is taken.
* returns the address of the mapping or NULL on error.
*/
void *_odp_ishmphy_map(int fd, void *start, uint64_t size,
int flags)
{
void *mapped_addr_tmp, *mapped_addr;
int mmap_flags = 0;
if (flags & _ODP_ISHM_SINGLE_VA) {
if (!start) {
ODP_ERR("failure: missing address\n");
return NULL;
}
/* maps over fragment of reserved VA: */
/* first, try a normal map. If that works, remap it where it
* should (on the prereverved space), and remove the initial
* normal mapping:
* This is because it turned out that if a mapping fails
* on a the prereserved virtual address space, then
* the prereserved address space which was tried to be mapped
* on becomes available to the kernel again! This was not
* according to expectations: the assumption was that if a
* mapping fails, the system should remain unchanged, but this
* is obvioulsy not true (at least for huge pages when
* exhausted).
* So the strategy is to first map at a non reserved place
* (which can then be freed and returned to the kernel on
* failure) and peform a new map to the prereserved space on
* success (which is then guaranteed to work).
* The initial free maping can then be removed.
*/
mapped_addr = MAP_FAILED;
mapped_addr_tmp = mmap(NULL, size, PROT_READ | PROT_WRITE,
MAP_SHARED | mmap_flags, fd, 0);
if (mapped_addr_tmp != MAP_FAILED) {
/* If OK, do new map at right fixed location... */
mapped_addr = mmap(start,
size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_FIXED | mmap_flags,
fd, 0);
if (mapped_addr != start)
ODP_ERR("new map failed:%s\n", strerror(errno));
/* ... and remove initial mapping: */
if (munmap(mapped_addr_tmp, size))
ODP_ERR("munmap failed:%s\n", strerror(errno));
}
} else {
/* just do a new mapping in the VA space: */
mapped_addr = mmap(NULL, size, PROT_READ | PROT_WRITE,
MAP_SHARED | mmap_flags, fd, 0);
if ((mapped_addr >= common_va_address) &&
((char *)mapped_addr <
(char *)common_va_address + common_va_len)) {
ODP_ERR("VA SPACE OVERLAP!\n");
}
}
if (mapped_addr == MAP_FAILED) {
ODP_ERR("mmap failed:%s\n", strerror(errno));
return NULL;
}
/* if locking is requested, lock it...*/
if (flags & _ODP_ISHM_LOCK) {
if (mlock(mapped_addr, size)) {
if (munmap(mapped_addr, size))
ODP_ERR("munmap failed:%s\n", strerror(errno));
ODP_ERR("mlock failed:%s\n", strerror(errno));
return NULL;
}
}
return mapped_addr;
}
/* free a mapping:
* If the _ODP_ISHM_SINGLE_VA flag was given at creation time the virtual
* address range must be returned to the preoallocated "pool". this is
* done by mapping non accessibly memory there (hence blocking the VA but
* releasing the physical memory).
* If the _ODP_ISHM_SINGLE_VA flag was not given, both physical memory and
* virtual address space are realeased by calling the normal munmap.
* return 0 on success or -1 on error.
*/
int _odp_ishmphy_unmap(void *start, uint64_t len, int flags)
{
void *addr;
int ret;
int mmap_flgs;
mmap_flgs = MAP_SHARED | MAP_FIXED | MAP_ANONYMOUS | MAP_NORESERVE;
/* if locking was requested, unlock...*/
if (flags & _ODP_ISHM_LOCK)
munlock(start, len);
if (flags & _ODP_ISHM_SINGLE_VA) {
/* map unnaccessible memory overwrites previous mapping
* and free the physical memory, but guarantees to block
* the VA range from other mappings
*/
addr = mmap(start, len, PROT_NONE, mmap_flgs, -1, 0);
if (addr == MAP_FAILED) {
ODP_ERR("_ishmphy_free failure for ISHM_SINGLE_VA\n");
return -1;
}
if (mprotect(start, len, PROT_NONE))
ODP_ERR("_ishmphy_free failure for protect\n");
return 0;
}
/* just release the mapping */
ret = munmap(start, len);
if (ret)
ODP_ERR("_ishmphy_free failure: %s\n", strerror(errno));
return ret;
}
|
