ARGOBOTS  c6511494322293e01714f56f341b8d2b22c1e3c1
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Macros Groups
abti_mem.h
Go to the documentation of this file.
1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 /*
3  * See COPYRIGHT in top-level directory.
4  */
5 
6 #ifndef ABTI_MEM_H_INCLUDED
7 #define ABTI_MEM_H_INCLUDED
8 
9 /* Memory allocation */
10 
11 /* Round desc_size up to the cacheline size. The last four bytes will be
12  * used to determine whether the descriptor is allocated externally (i.e.,
13  * malloc()) or taken from a memory pool. */
14 #define ABTI_MEM_POOL_DESC_ELEM_SIZE \
15  ((sizeof(ABTI_thread) + ABT_CONFIG_STATIC_CACHELINE_SIZE - 1) & \
16  (~(ABT_CONFIG_STATIC_CACHELINE_SIZE - 1)))
17 
18 enum {
19  ABTI_MEM_LP_MALLOC = 0,
20  ABTI_MEM_LP_MMAP_RP,
21  ABTI_MEM_LP_MMAP_HP_RP,
22  ABTI_MEM_LP_MMAP_HP_THP,
23  ABTI_MEM_LP_THP
24 };
25 
26 void ABTI_mem_init(ABTI_global *p_global);
27 void ABTI_mem_init_local(ABTI_xstream *p_local_xstream);
28 void ABTI_mem_finalize(ABTI_global *p_global);
29 void ABTI_mem_finalize_local(ABTI_xstream *p_local_xstream);
30 int ABTI_mem_check_lp_alloc(int lp_alloc);
31 
32 /* Inline functions */
33 ABTU_ret_err static inline int
34 ABTI_mem_alloc_nythread_malloc(ABTI_thread **pp_thread)
35 {
36  ABTI_thread *p_thread;
37  int abt_errno =
38  ABTU_malloc(ABTI_MEM_POOL_DESC_ELEM_SIZE, (void **)&p_thread);
39  ABTI_CHECK_ERROR(abt_errno);
40  p_thread->type = ABTI_THREAD_TYPE_MEM_MALLOC_DESC;
41  *pp_thread = p_thread;
42  return ABT_SUCCESS;
43 }
44 
45 #ifdef ABT_CONFIG_USE_MEM_POOL
46 ABTU_ret_err static inline int
47 ABTI_mem_alloc_nythread_mempool(ABTI_local *p_local, ABTI_thread **pp_thread)
48 {
49  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
50  if (ABTI_IS_EXT_THREAD_ENABLED && p_local_xstream == NULL) {
51  /* For external threads */
52  return ABTI_mem_alloc_nythread_malloc(pp_thread);
53  }
54  /* Find the page that has an empty block */
55  ABTI_thread *p_thread;
56  int abt_errno = ABTI_mem_pool_alloc(&p_local_xstream->mem_pool_desc,
57  (void **)&p_thread);
58  ABTI_CHECK_ERROR(abt_errno);
59  p_thread->type = ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC;
60  *pp_thread = p_thread;
61  return ABT_SUCCESS;
62 }
63 #endif
64 
65 ABTU_ret_err static inline int ABTI_mem_alloc_nythread(ABTI_local *p_local,
66  ABTI_thread **pp_thread)
67 {
68 #ifdef ABT_CONFIG_USE_MEM_POOL
69  return ABTI_mem_alloc_nythread_mempool(p_local, pp_thread);
70 #else
71  return ABTI_mem_alloc_nythread_malloc(pp_thread);
72 #endif
73 }
74 
75 static inline void ABTI_mem_free_nythread(ABTI_local *p_local,
76  ABTI_thread *p_thread)
77 {
78  /* Return stack. */
79 #ifdef ABT_CONFIG_USE_MEM_POOL
80  if (p_thread->type & ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC) {
81  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
82  /* Came from a memory pool. */
83 #ifndef ABT_CONFIG_DISABLE_EXT_THREAD
84  if (p_local_xstream == NULL) {
85  /* Return a stack to the global pool. */
86  ABTI_spinlock_acquire(&gp_ABTI_global->mem_pool_desc_lock);
87  ABTI_mem_pool_free(&gp_ABTI_global->mem_pool_desc_ext, p_thread);
88  ABTI_spinlock_release(&gp_ABTI_global->mem_pool_desc_lock);
89  return;
90  }
91 #endif
92  ABTI_mem_pool_free(&p_local_xstream->mem_pool_desc, p_thread);
93  return;
94  }
95 #endif
96  /* p_thread was allocated by malloc() */
97  ABTU_free(p_thread);
98 }
99 
100 #ifdef ABT_CONFIG_USE_MEM_POOL
101 ABTU_ret_err static inline int ABTI_mem_alloc_ythread_mempool_desc_stack_impl(
102  ABTI_mem_pool_local_pool *p_mem_pool_stack, size_t stacksize,
103  ABTI_ythread **pp_ythread, void **pp_stack)
104 {
105  /* stacksize must be a multiple of ABT_CONFIG_STATIC_CACHELINE_SIZE. */
106  ABTI_ASSERT((stacksize & (ABT_CONFIG_STATIC_CACHELINE_SIZE - 1)) == 0);
107  void *p_ythread;
108  int abt_errno = ABTI_mem_pool_alloc(p_mem_pool_stack, &p_ythread);
109  ABTI_CHECK_ERROR(abt_errno);
110 
111  *pp_stack = (void *)(((char *)p_ythread) - stacksize);
112  *pp_ythread = (ABTI_ythread *)p_ythread;
113  return ABT_SUCCESS;
114 }
115 #endif
116 
117 ABTU_ret_err static inline int ABTI_mem_alloc_ythread_malloc_desc_stack_impl(
118  size_t stacksize, ABTI_ythread **pp_ythread, void **pp_stack)
119 {
120  /* stacksize must be a multiple of ABT_CONFIG_STATIC_CACHELINE_SIZE. */
121  size_t alloc_stacksize =
122  (stacksize + ABT_CONFIG_STATIC_CACHELINE_SIZE - 1) &
124  char *p_stack;
125  int abt_errno =
126  ABTU_malloc(alloc_stacksize + sizeof(ABTI_ythread), (void **)&p_stack);
127  ABTI_CHECK_ERROR(abt_errno);
128 
129  *pp_stack = (void *)p_stack;
130  *pp_ythread = (ABTI_ythread *)(p_stack + alloc_stacksize);
131  return ABT_SUCCESS;
132 }
133 
134 ABTU_ret_err static inline int
135 ABTI_mem_alloc_ythread_default(ABTI_local *p_local, ABTI_ythread **pp_ythread)
136 {
137  size_t stacksize = gp_ABTI_global->thread_stacksize;
138  ABTI_ythread *p_ythread;
139  void *p_stack;
140  /* If an external thread allocates a stack, we use ABTU_malloc. */
141  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
142  if (ABTI_IS_EXT_THREAD_ENABLED && p_local_xstream == NULL) {
143  int abt_errno =
144  ABTI_mem_alloc_ythread_malloc_desc_stack_impl(stacksize, &p_ythread,
145  &p_stack);
146  ABTI_CHECK_ERROR(abt_errno);
147  p_ythread->thread.type = ABTI_THREAD_TYPE_MEM_MALLOC_DESC_STACK;
148  } else {
149 #ifdef ABT_CONFIG_USE_MEM_POOL
150  int abt_errno = ABTI_mem_alloc_ythread_mempool_desc_stack_impl(
151  &p_local_xstream->mem_pool_stack, stacksize, &p_ythread, &p_stack);
152  ABTI_CHECK_ERROR(abt_errno);
153  p_ythread->thread.type = ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC_STACK;
154 #else
155  int abt_errno =
156  ABTI_mem_alloc_ythread_malloc_desc_stack_impl(stacksize, &p_ythread,
157  &p_stack);
158  ABTI_CHECK_ERROR(abt_errno);
159  p_ythread->thread.type = ABTI_THREAD_TYPE_MEM_MALLOC_DESC_STACK;
160 #endif
161  }
162  /* Initialize members of ABTI_thread_attr. */
163  p_ythread->p_stack = p_stack;
164  p_ythread->stacksize = stacksize;
165  ABTI_VALGRIND_REGISTER_STACK(p_ythread->p_stack, p_ythread->stacksize);
166  *pp_ythread = p_ythread;
167  return ABT_SUCCESS;
168 }
169 
170 #ifdef ABT_CONFIG_USE_MEM_POOL
171 ABTU_ret_err static inline int ABTI_mem_alloc_ythread_mempool_desc_stack(
172  ABTI_local *p_local, ABTI_thread_attr *p_attr, ABTI_ythread **pp_ythread)
173 {
174  size_t stacksize = gp_ABTI_global->thread_stacksize;
175  ABTI_ythread *p_ythread;
176  void *p_stack;
177  /* If an external thread allocates a stack, we use ABTU_malloc. */
178  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
179  if (ABTI_IS_EXT_THREAD_ENABLED && p_local_xstream == NULL) {
180  int abt_errno =
181  ABTI_mem_alloc_ythread_malloc_desc_stack_impl(stacksize, &p_ythread,
182  &p_stack);
183  ABTI_CHECK_ERROR(abt_errno);
184  p_ythread->thread.type = ABTI_THREAD_TYPE_MEM_MALLOC_DESC_STACK;
185  } else {
186  int abt_errno = ABTI_mem_alloc_ythread_mempool_desc_stack_impl(
187  &p_local_xstream->mem_pool_stack, stacksize, &p_ythread, &p_stack);
188  ABTI_CHECK_ERROR(abt_errno);
189  p_ythread->thread.type = ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC_STACK;
190  }
191  /* Copy members of p_attr. */
192  p_ythread->p_stack = p_stack;
193  p_ythread->stacksize = stacksize;
194  ABTI_VALGRIND_REGISTER_STACK(p_ythread->p_stack, p_ythread->stacksize);
195  *pp_ythread = p_ythread;
196  return ABT_SUCCESS;
197 }
198 #endif
199 
200 ABTU_ret_err static inline int
201 ABTI_mem_alloc_ythread_malloc_desc_stack(ABTI_thread_attr *p_attr,
202  ABTI_ythread **pp_ythread)
203 {
204  size_t stacksize = p_attr->stacksize;
205  ABTI_ythread *p_ythread;
206  void *p_stack;
207  int abt_errno =
208  ABTI_mem_alloc_ythread_malloc_desc_stack_impl(stacksize, &p_ythread,
209  &p_stack);
210  ABTI_CHECK_ERROR(abt_errno);
211 
212  /* Copy members of p_attr. */
213  p_ythread->thread.type = ABTI_THREAD_TYPE_MEM_MALLOC_DESC_STACK;
214  p_ythread->stacksize = stacksize;
215  p_ythread->p_stack = p_stack;
216  ABTI_VALGRIND_REGISTER_STACK(p_ythread->p_stack, p_ythread->stacksize);
217  *pp_ythread = p_ythread;
218  return ABT_SUCCESS;
219 }
220 
221 ABTU_ret_err static inline int ABTI_mem_alloc_ythread_mempool_desc(
222  ABTI_local *p_local, ABTI_thread_attr *p_attr, ABTI_ythread **pp_ythread)
223 {
224  ABTI_ythread *p_ythread;
225  if (sizeof(ABTI_ythread) <= ABTI_MEM_POOL_DESC_ELEM_SIZE) {
226  /* Use a descriptor pool for ABT_thread. */
227  ABTI_STATIC_ASSERT(offsetof(ABTI_ythread, thread) == 0);
228  int abt_errno =
229  ABTI_mem_alloc_nythread(p_local, (ABTI_thread **)&p_ythread);
230  ABTI_CHECK_ERROR(abt_errno);
231  } else {
232  /* Do not allocate stack, but Valgrind registration is preferred. */
233  int abt_errno = ABTU_malloc(sizeof(ABTI_ythread), (void **)&p_ythread);
234  ABTI_CHECK_ERROR(abt_errno);
235  p_ythread->thread.type = ABTI_THREAD_TYPE_MEM_MALLOC_DESC;
236  }
237  /* Copy members of p_attr. */
238  p_ythread->stacksize = p_attr->stacksize;
239  p_ythread->p_stack = p_attr->p_stack;
240  /* Note that the valgrind registration is ignored iff p_stack is NULL. */
241  ABTI_VALGRIND_REGISTER_STACK(p_ythread->p_stack, p_ythread->stacksize);
242  *pp_ythread = p_ythread;
243  return ABT_SUCCESS;
244 }
245 
246 static inline void ABTI_mem_free_thread(ABTI_local *p_local,
247  ABTI_thread *p_thread)
248 {
249  /* Return stack. */
250 #ifdef ABT_CONFIG_USE_MEM_POOL
251  if (p_thread->type & ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC_STACK) {
252  ABTI_ythread *p_ythread = ABTI_thread_get_ythread(p_thread);
253  ABTI_VALGRIND_UNREGISTER_STACK(p_ythread->p_stack);
254 
255  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
256  /* Came from a memory pool. */
257 #ifndef ABT_CONFIG_DISABLE_EXT_THREAD
258  if (p_local_xstream == NULL) {
259  /* Return a stack to the global pool. */
260  ABTI_spinlock_acquire(&gp_ABTI_global->mem_pool_stack_lock);
261  ABTI_mem_pool_free(&gp_ABTI_global->mem_pool_stack_ext, p_ythread);
262  ABTI_spinlock_release(&gp_ABTI_global->mem_pool_stack_lock);
263  return;
264  }
265 #endif
266  ABTI_mem_pool_free(&p_local_xstream->mem_pool_stack, p_ythread);
267  } else
268 #endif
269  if (p_thread->type & ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC) {
270  /* Non-yieldable thread or yieldable thread without stack. */
271 #ifdef HAVE_VALGRIND_SUPPORT
272  ABTI_ythread *p_ythread = ABTI_thread_get_ythread_or_null(p_thread);
273  if (p_ythread)
274  ABTI_VALGRIND_UNREGISTER_STACK(p_ythread->p_stack);
275 #endif
276  ABTI_mem_free_nythread(p_local, p_thread);
277  } else if (p_thread->type & ABTI_THREAD_TYPE_MEM_MALLOC_DESC_STACK) {
278  ABTI_ythread *p_ythread = ABTI_thread_get_ythread(p_thread);
279  ABTI_VALGRIND_UNREGISTER_STACK(p_ythread->p_stack);
280  ABTU_free(p_ythread->p_stack);
281  } else {
282  ABTI_ASSERT(p_thread->type & ABTI_THREAD_TYPE_MEM_MALLOC_DESC);
283  ABTI_STATIC_ASSERT(offsetof(ABTI_ythread, thread) == 0);
284 #ifdef HAVE_VALGRIND_SUPPORT
285  ABTI_ythread *p_ythread = ABTI_thread_get_ythread_or_null(p_thread);
286  if (p_ythread)
287  ABTI_VALGRIND_UNREGISTER_STACK(p_ythread->p_stack);
288 #endif
289  ABTU_free(p_thread);
290  }
291 }
292 
293 /* Generic scalable memory pools. It uses a memory pool for ABTI_thread.
294  * The last four bytes will be used to determine whether the descriptor is
295  * allocated externally (i.e., malloc()) or taken from a memory pool. */
296 #define ABTI_MEM_POOL_DESC_SIZE (ABTI_MEM_POOL_DESC_ELEM_SIZE - 4)
297 
298 ABTU_ret_err static inline int ABTI_mem_alloc_desc(ABTI_local *p_local,
299  void **pp_desc)
300 {
301 #ifndef ABT_CONFIG_USE_MEM_POOL
302  return ABTU_malloc(ABTI_MEM_POOL_DESC_SIZE, pp_desc);
303 #else
304  void *p_desc;
305  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
306  if (ABTI_IS_EXT_THREAD_ENABLED && p_local_xstream == NULL) {
307  /* For external threads */
308  int abt_errno = ABTU_malloc(ABTI_MEM_POOL_DESC_SIZE, &p_desc);
309  ABTI_CHECK_ERROR(abt_errno);
310  *(uint32_t *)(((char *)p_desc) + ABTI_MEM_POOL_DESC_SIZE) = 1;
311  *pp_desc = p_desc;
312  return ABT_SUCCESS;
313  } else {
314  /* Find the page that has an empty block */
315  int abt_errno =
316  ABTI_mem_pool_alloc(&p_local_xstream->mem_pool_desc, &p_desc);
317  ABTI_CHECK_ERROR(abt_errno);
318  /* To distinguish it from a malloc'ed case, assign non-NULL value. */
319  *(uint32_t *)(((char *)p_desc) + ABTI_MEM_POOL_DESC_SIZE) = 0;
320  *pp_desc = p_desc;
321  return ABT_SUCCESS;
322  }
323 #endif
324 }
325 
326 static inline void ABTI_mem_free_desc(ABTI_local *p_local, void *p_desc)
327 {
328 #ifndef ABT_CONFIG_USE_MEM_POOL
329  ABTU_free(p_desc);
330 #else
331  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
332 #ifndef ABT_CONFIG_DISABLE_EXT_THREAD
333  if (*(uint32_t *)(((char *)p_desc) + ABTI_MEM_POOL_DESC_SIZE)) {
334  /* This was allocated by an external thread. */
335  ABTU_free(p_desc);
336  return;
337  } else if (!p_local_xstream) {
338  /* Return a stack and a descriptor to their global pools. */
339  ABTI_spinlock_acquire(&gp_ABTI_global->mem_pool_desc_lock);
340  ABTI_mem_pool_free(&gp_ABTI_global->mem_pool_desc_ext, p_desc);
341  ABTI_spinlock_release(&gp_ABTI_global->mem_pool_desc_lock);
342  return;
343  }
344 #endif
345  ABTI_mem_pool_free(&p_local_xstream->mem_pool_desc, p_desc);
346 #endif
347 }
348 
349 #endif /* ABTI_MEM_H_INCLUDED */
#define ABT_CONFIG_STATIC_CACHELINE_SIZE
Definition: abt_config.h:57
static ABTU_ret_err int ABTU_malloc(size_t size, void **p_ptr)
Definition: abtu.h:129
ABTI_global * gp_ABTI_global
Definition: global.c:18
#define ABT_SUCCESS
Definition: abt.h:64
static void ABTU_free(void *ptr)
Definition: abtu.h:122
#define ABTU_ret_err
Definition: abtu.h:49