ARGOBOTS  4dc37e16e1b227a480715ab67dae1dcfb4d2d4e0
abti_mem.h
Go to the documentation of this file.
1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 /*
3  * See COPYRIGHT in top-level directory.
4  */
5 
6 #ifndef ABTI_MEM_H_INCLUDED
7 #define ABTI_MEM_H_INCLUDED
8 
9 /* Memory allocation */
10 
11 /* Round desc_size up to the cacheline size. The last four bytes will be
12  * used to determine whether the descriptor is allocated externally (i.e.,
13  * malloc()) or taken from a memory pool. */
14 #define ABTI_MEM_POOL_DESC_ELEM_SIZE \
15  ABTU_roundup_size(sizeof(ABTI_ythread), ABT_CONFIG_STATIC_CACHELINE_SIZE)
16 
17 enum {
23 };
24 
27  ABTI_xstream *p_local_xstream);
28 void ABTI_mem_finalize(ABTI_global *p_global);
29 void ABTI_mem_finalize_local(ABTI_xstream *p_local_xstream);
30 int ABTI_mem_check_lp_alloc(ABTI_global *p_global, int lp_alloc);
31 
32 #define ABTI_STACK_CANARY_VALUE ((uint64_t)0xbaadc0debaadc0de)
33 
34 /* Inline functions */
35 #if ABT_CONFIG_STACK_CHECK_TYPE == ABTI_STACK_CHECK_TYPE_CANARY
36 static inline void ABTI_mem_write_stack_canary(void *p_stack)
37 {
38  /* Write down stack canary. */
39  uint64_t i;
41  i += sizeof(uint64_t)) {
42  ((uint64_t *)p_stack)[i] = ABTI_STACK_CANARY_VALUE;
43  }
44 }
45 
46 static inline void ABTI_mem_check_stack_canary(void *p_stack)
47 {
48  uint64_t i;
50  i += sizeof(uint64_t)) {
51  ABTI_ASSERT(((uint64_t *)p_stack)[i] == ABTI_STACK_CANARY_VALUE);
52  }
53 }
54 #endif
55 
56 /* p_stack can be NULL. */
57 static inline void ABTI_mem_register_stack(const ABTI_global *p_global,
58  void *p_stacktop, size_t stacksize,
59  ABT_bool mprotect_if_needed)
60 {
61  void *p_stack = (void *)(((char *)p_stacktop) - stacksize);
62  if (mprotect_if_needed) {
65  if (p_stack) {
66  int abt_errno =
68  p_global->sys_page_size),
69  p_global->sys_page_size, ABT_TRUE);
70  if (p_global->stack_guard_kind ==
72  ABTI_ASSERT(abt_errno == ABT_SUCCESS);
73  }
74  }
75  } else {
76 #if ABT_CONFIG_STACK_CHECK_TYPE == ABTI_STACK_CHECK_TYPE_CANARY
77  if (p_stack) {
79  }
80 #endif
81  }
82  } else {
83 #if ABT_CONFIG_STACK_CHECK_TYPE == ABTI_STACK_CHECK_TYPE_CANARY
84  if (!(p_global->stack_guard_kind == ABTI_STACK_GUARD_MPROTECT ||
86  p_stack) {
88  }
89 #endif
90  }
91  ABTI_VALGRIND_REGISTER_STACK(p_stack, stacksize);
92 }
93 
94 static inline void ABTI_mem_unregister_stack(const ABTI_global *p_global,
95  void *p_stacktop, size_t stacksize,
96  ABT_bool mprotect_if_needed)
97 {
98  void *p_stack = (void *)(((char *)p_stacktop) - stacksize);
99  if (mprotect_if_needed) {
100  if (p_global->stack_guard_kind == ABTI_STACK_GUARD_MPROTECT ||
102  if (p_stack) {
103  int abt_errno =
105  p_global->sys_page_size),
106  p_global->sys_page_size, ABT_FALSE);
107  /* This should not fail since otherwise we cannot free this
108  * memory. */
109  ABTI_ASSERT(abt_errno == ABT_SUCCESS);
110  }
111  } else {
112 #if ABT_CONFIG_STACK_CHECK_TYPE == ABTI_STACK_CHECK_TYPE_CANARY
113  if (p_stack) {
115  }
116 #endif
117  }
118  } else {
119 #if ABT_CONFIG_STACK_CHECK_TYPE == ABTI_STACK_CHECK_TYPE_CANARY
120  if (!(p_global->stack_guard_kind == ABTI_STACK_GUARD_MPROTECT ||
122  p_stack) {
124  }
125 #endif
126  }
128 }
129 
131  ABTI_thread **pp_thread)
132 {
134  ABTI_thread *p_thread;
135 #ifdef ABT_CONFIG_USE_MEM_POOL
136  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
137  if (!ABTI_IS_EXT_THREAD_ENABLED || p_local_xstream) {
138  /* It's not called on an external thread. Use a memory pool. */
139  int abt_errno = ABTI_mem_pool_alloc(&p_local_xstream->mem_pool_desc,
140  (void **)&p_thread);
141  ABTI_CHECK_ERROR(abt_errno);
142  p_thread->type = ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC;
143  } else
144 #endif
145  {
146  int abt_errno =
147  ABTU_malloc(ABTI_MEM_POOL_DESC_ELEM_SIZE, (void **)&p_thread);
148  ABTI_CHECK_ERROR(abt_errno);
149  p_thread->type = ABTI_THREAD_TYPE_MEM_MALLOC_DESC;
150  }
151  *pp_thread = p_thread;
152  return ABT_SUCCESS;
153 }
154 
156  ABTI_local *p_local,
157  ABTI_thread *p_thread)
158 {
159  /* Return a descriptor. */
160 #ifdef ABT_CONFIG_USE_MEM_POOL
161  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
162 #ifdef ABT_CONFIG_DISABLE_EXT_THREAD
163  /* Came from a memory pool. */
164  ABTI_mem_pool_free(&p_local_xstream->mem_pool_desc, p_thread);
165 #else
166  if (p_local_xstream) {
167  /* Came from a memory pool. */
168  ABTI_mem_pool_free(&p_local_xstream->mem_pool_desc, p_thread);
169  } else {
170  /* Return a stack to the global pool. */
172  ABTI_mem_pool_free(&p_global->mem_pool_desc_ext, p_thread);
174  }
175 #endif
176 #else /* !ABT_CONFIG_USE_MEM_POOL */
177  /* If a memory pool is disabled, this function should not be called. */
178  ABTI_ASSERT(0);
179 #endif
180 }
181 
182 ABTU_ret_err static inline int
184  ABTI_ythread **pp_ythread)
185 {
187  ABTI_ythread *p_ythread;
188 #ifdef ABT_CONFIG_USE_MEM_POOL
189  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
190  if (!ABTI_IS_EXT_THREAD_ENABLED || p_local_xstream) {
191  /* It's not called on an external thread. Use a memory pool. */
192  int abt_errno = ABTI_mem_pool_alloc(&p_local_xstream->mem_pool_desc,
193  (void **)&p_ythread);
194  ABTI_CHECK_ERROR(abt_errno);
195  p_ythread->thread.type =
196  use_lazy_stack
199  } else
200 #endif
201  {
202  int abt_errno =
203  ABTU_malloc(ABTI_MEM_POOL_DESC_ELEM_SIZE, (void **)&p_ythread);
204  ABTI_CHECK_ERROR(abt_errno);
205  p_ythread->thread.type =
208  }
209  *pp_ythread = p_ythread;
210  return ABT_SUCCESS;
211 }
212 
214  ABTI_global *p_global, ABTI_local *p_local, ABTI_ythread *p_ythread)
215 {
216  /* Return a descriptor. */
217 #ifdef ABT_CONFIG_USE_MEM_POOL
218  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
219 #ifdef ABT_CONFIG_DISABLE_EXT_THREAD
220  /* Came from a memory pool. */
221  ABTI_mem_pool_free(&p_local_xstream->mem_pool_desc, p_ythread);
222 #else
223  if (p_local_xstream) {
224  /* Came from a memory pool. */
225  ABTI_mem_pool_free(&p_local_xstream->mem_pool_desc, p_ythread);
226  } else {
227  /* Return a stack to the global pool. */
229  ABTI_mem_pool_free(&p_global->mem_pool_desc_ext, p_ythread);
231  }
232 #endif
233 #else /* !ABT_CONFIG_USE_MEM_POOL */
234  /* If a memory pool is disabled, this function should not be called. */
235  ABTI_ASSERT(0);
236 #endif
237 }
238 
239 #ifdef ABT_CONFIG_USE_MEM_POOL
240 ABTU_ret_err static inline int ABTI_mem_alloc_ythread_mempool_desc_stack_impl(
241  ABTI_mem_pool_local_pool *p_mem_pool_stack, size_t stacksize,
242  ABTI_ythread **pp_ythread, void **pp_stacktop)
243 {
244  /* stacksize must be a multiple of ABT_CONFIG_STATIC_CACHELINE_SIZE. */
245  ABTI_ASSERT((stacksize & (ABT_CONFIG_STATIC_CACHELINE_SIZE - 1)) == 0);
246  void *p_ythread;
247  int abt_errno = ABTI_mem_pool_alloc(p_mem_pool_stack, &p_ythread);
248  ABTI_CHECK_ERROR(abt_errno);
249 
250  *pp_stacktop = (void *)p_ythread;
251  *pp_ythread = (ABTI_ythread *)p_ythread;
252  return ABT_SUCCESS;
253 }
254 #endif
255 
257  size_t stacksize, ABTI_ythread **pp_ythread, void **pp_stacktop)
258 {
259  /* stacksize must be a multiple of ABT_CONFIG_STATIC_CACHELINE_SIZE. */
260  size_t alloc_stacksize =
262  char *p_stack;
263  int abt_errno =
264  ABTU_malloc(alloc_stacksize + sizeof(ABTI_ythread), (void **)&p_stack);
265  ABTI_CHECK_ERROR(abt_errno);
266 
267  *pp_stacktop = (void *)(p_stack + alloc_stacksize);
268  *pp_ythread = (ABTI_ythread *)(p_stack + alloc_stacksize);
269  return ABT_SUCCESS;
270 }
271 
272 ABTU_ret_err static inline int
274  ABTI_local *p_local, size_t stacksize,
275  ABTI_ythread **pp_ythread)
276 {
277  ABTI_UB_ASSERT(stacksize == p_global->thread_stacksize);
278  ABTI_ythread *p_ythread;
279 #ifdef ABT_CONFIG_USE_MEM_POOL
280 #ifdef ABT_CONFIG_DISABLE_LAZY_STACK_ALLOC
281  const ABT_bool use_lazy_stack = ABT_FALSE;
282 #else
283  const ABT_bool use_lazy_stack = ABT_TRUE;
284 #endif
285  if (use_lazy_stack) {
286  /* Only allocate a descriptor here. */
287  int abt_errno =
288  ABTI_mem_alloc_ythread_desc_impl(p_local, ABT_TRUE, &p_ythread);
289  ABTI_CHECK_ERROR(abt_errno);
290  /* Initialize the context. */
291  ABTD_ythread_context_init_lazy(&p_ythread->ctx, stacksize);
292  *pp_ythread = p_ythread;
293  return ABT_SUCCESS;
294  } else {
295  void *p_stacktop;
296  /* Allocate a ULT stack and a descriptor together. */
297  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
298  if (!ABTI_IS_EXT_THREAD_ENABLED || p_local_xstream) {
299  int abt_errno = ABTI_mem_alloc_ythread_mempool_desc_stack_impl(
300  &p_local_xstream->mem_pool_stack, stacksize, &p_ythread,
301  &p_stacktop);
302  ABTI_CHECK_ERROR(abt_errno);
304  ABTI_mem_register_stack(p_global, p_stacktop, stacksize, ABT_FALSE);
305  } else {
306  /* If an external thread allocates a stack, we use ABTU_malloc. */
307  int abt_errno =
309  &p_ythread,
310  &p_stacktop);
311  ABTI_CHECK_ERROR(abt_errno);
313  ABTI_mem_register_stack(p_global, p_stacktop, stacksize, ABT_TRUE);
314  }
315  /* Initialize the context. */
316  ABTD_ythread_context_init(&p_ythread->ctx, p_stacktop, stacksize);
317  *pp_ythread = p_ythread;
318  return ABT_SUCCESS;
319  }
320 #else
321  void *p_stacktop;
322  int abt_errno =
323  ABTI_mem_alloc_ythread_malloc_desc_stack_impl(stacksize, &p_ythread,
324  &p_stacktop);
325  ABTI_CHECK_ERROR(abt_errno);
327  ABTI_mem_register_stack(p_global, p_stacktop, stacksize, ABT_TRUE);
328  /* Initialize the context. */
329  ABTD_ythread_context_init(&p_ythread->ctx, p_stacktop, stacksize);
330  *pp_ythread = p_ythread;
331  return ABT_SUCCESS;
332 #endif
333 }
334 
335 ABTU_ret_err static inline int
337  ABTI_ythread **pp_ythread)
338 {
339  size_t stacksize = p_global->thread_stacksize;
340  return ABTI_mem_alloc_ythread_mempool_desc_stack(p_global, p_local,
341  stacksize, pp_ythread);
342 }
343 
345  ABTI_global *p_global, size_t stacksize, ABTI_ythread **pp_ythread)
346 {
347  ABTI_ythread *p_ythread;
348  void *p_stacktop;
349  int abt_errno =
350  ABTI_mem_alloc_ythread_malloc_desc_stack_impl(stacksize, &p_ythread,
351  &p_stacktop);
352  ABTI_CHECK_ERROR(abt_errno);
353 
354  /* Initialize the context. */
356  ABTD_ythread_context_init(&p_ythread->ctx, p_stacktop, stacksize);
357  ABTI_mem_register_stack(p_global, p_stacktop, stacksize, ABT_TRUE);
358  *pp_ythread = p_ythread;
359  return ABT_SUCCESS;
360 }
361 
362 ABTU_ret_err static inline int
364  size_t stacksize, void *p_stacktop,
365  ABTI_ythread **pp_ythread)
366 {
367  ABTI_ythread *p_ythread;
368 
369  /* Use a descriptor pool for ABT_ythread. */
371  ABTI_STATIC_ASSERT(offsetof(ABTI_ythread, thread) == 0);
372  int abt_errno =
373  ABTI_mem_alloc_nythread(p_local, (ABTI_thread **)&p_ythread);
374  ABTI_CHECK_ERROR(abt_errno);
375  /* Initialize the context. */
376  ABTD_ythread_context_init(&p_ythread->ctx, p_stacktop, stacksize);
377  ABTI_mem_register_stack(p_global, p_stacktop, stacksize, ABT_TRUE);
378  *pp_ythread = p_ythread;
379  return ABT_SUCCESS;
380 }
381 
382 static inline void ABTI_mem_free_thread(ABTI_global *p_global,
383  ABTI_local *p_local,
384  ABTI_thread *p_thread)
385 {
386  /* Return stack. */
387 #ifdef ABT_CONFIG_USE_MEM_POOL
389  ABTI_ythread *p_ythread = ABTI_thread_get_ythread(p_thread);
390  ABTI_mem_unregister_stack(p_global,
392  &p_ythread->ctx),
394  &p_ythread->ctx),
395  ABT_FALSE);
396 
397  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
398  /* Came from a memory pool. */
399 #ifndef ABT_CONFIG_DISABLE_EXT_THREAD
400  if (p_local_xstream == NULL) {
401  /* Return a stack to the global pool. */
403  ABTI_mem_pool_free(&p_global->mem_pool_stack_ext, p_ythread);
405  return;
406  }
407 #endif
408  ABTI_mem_pool_free(&p_local_xstream->mem_pool_stack, p_ythread);
409  } else
410 #endif
411  if (p_thread->type &
413  ABTI_ythread *p_ythread = ABTI_thread_get_ythread(p_thread);
414  /* If it is a lazy stack ULT, it should not have a stack. */
416  ABTI_mem_free_ythread_desc_mempool_impl(p_global, p_local, p_ythread);
417  } else if (p_thread->type & ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC) {
418  /* Non-yieldable thread or yieldable thread without stack. */
419  ABTI_ythread *p_ythread = ABTI_thread_get_ythread_or_null(p_thread);
420  if (p_ythread) {
421  ABTI_mem_unregister_stack(p_global,
423  &p_ythread->ctx),
425  &p_ythread->ctx),
426  ABT_TRUE);
427  ABTI_mem_free_ythread_desc_mempool_impl(p_global, p_local,
428  p_ythread);
429  } else {
430  ABTI_mem_free_nythread_mempool_impl(p_global, p_local, p_thread);
431  }
432  } else if (p_thread->type & ABTI_THREAD_TYPE_MEM_MALLOC_DESC_STACK) {
433  ABTI_ythread *p_ythread = ABTI_thread_get_ythread(p_thread);
434  void *p_stacktop = ABTD_ythread_context_get_stacktop(&p_ythread->ctx);
435  size_t stacksize = ABTD_ythread_context_get_stacksize(&p_ythread->ctx);
436  ABTI_mem_unregister_stack(p_global, p_stacktop, stacksize, ABT_TRUE);
437  void *p_stack = (void *)(((char *)p_stacktop) - stacksize);
438  ABTU_free(p_stack);
439  } else if (p_thread->type &
441  ABTI_ythread *p_ythread = ABTI_thread_get_ythread(p_thread);
442  /* If it is a lazy stack ULT, it should not have a stack. */
444  ABTU_free(p_ythread);
445  } else {
447  ABTI_STATIC_ASSERT(offsetof(ABTI_ythread, thread) == 0);
448  ABTI_ythread *p_ythread = ABTI_thread_get_ythread_or_null(p_thread);
449  if (p_ythread)
450  ABTI_mem_unregister_stack(p_global,
452  &p_ythread->ctx),
454  &p_ythread->ctx),
455  ABT_TRUE);
456  ABTU_free(p_thread);
457  }
458 }
459 
460 ABTU_ret_err static inline int
462  ABTI_ythread *p_ythread)
463 {
464 #ifdef ABT_CONFIG_USE_MEM_POOL
465  ABTI_UB_ASSERT(p_ythread->thread.type &
468  void *p_stacktop;
469  int abt_errno =
470  ABTI_mem_pool_alloc(&p_local_xstream->mem_pool_stack, &p_stacktop);
471  ABTI_CHECK_ERROR(abt_errno);
472  ABTD_ythread_context_lazy_set_stack(&p_ythread->ctx, p_stacktop);
473  return ABT_SUCCESS;
474 #else
475  /* This function should not be called. */
476  ABTI_ASSERT(0);
477  return 0;
478 #endif
479 }
480 
481 static inline void
483  ABTI_ythread *p_ythread)
484 {
485 #ifdef ABT_CONFIG_USE_MEM_POOL
486  ABTI_UB_ASSERT(p_ythread->thread.type &
489  void *p_stacktop = ABTD_ythread_context_get_stacktop(&p_ythread->ctx);
491  ABTI_mem_pool_free(&p_local_xstream->mem_pool_stack, p_stacktop);
492 #else
493  /* This function should not be called. */
494  ABTI_ASSERT(0);
495 #endif
496 }
497 
498 /* Generic scalable memory pools. It uses a memory pool for ABTI_thread.
499  * The last four bytes will be used to determine whether the descriptor is
500  * allocated externally (i.e., malloc()) or taken from a memory pool. */
501 #define ABTI_MEM_POOL_DESC_SIZE (ABTI_MEM_POOL_DESC_ELEM_SIZE - 4)
502 
503 ABTU_ret_err static inline int ABTI_mem_alloc_desc(ABTI_local *p_local,
504  void **pp_desc)
505 {
506 #ifndef ABT_CONFIG_USE_MEM_POOL
507  return ABTU_malloc(ABTI_MEM_POOL_DESC_SIZE, pp_desc);
508 #else
509  void *p_desc;
510  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
511  if (ABTI_IS_EXT_THREAD_ENABLED && p_local_xstream == NULL) {
512  /* For external threads */
513  int abt_errno = ABTU_malloc(ABTI_MEM_POOL_DESC_SIZE, &p_desc);
514  ABTI_CHECK_ERROR(abt_errno);
515  *(uint32_t *)(((char *)p_desc) + ABTI_MEM_POOL_DESC_SIZE) = 1;
516  *pp_desc = p_desc;
517  return ABT_SUCCESS;
518  } else {
519  /* Find the page that has an empty block */
520  int abt_errno =
521  ABTI_mem_pool_alloc(&p_local_xstream->mem_pool_desc, &p_desc);
522  ABTI_CHECK_ERROR(abt_errno);
523  /* To distinguish it from a malloc'ed case, assign non-NULL value. */
524  *(uint32_t *)(((char *)p_desc) + ABTI_MEM_POOL_DESC_SIZE) = 0;
525  *pp_desc = p_desc;
526  return ABT_SUCCESS;
527  }
528 #endif
529 }
530 
531 static inline void ABTI_mem_free_desc(ABTI_global *p_global,
532  ABTI_local *p_local, void *p_desc)
533 {
534 #ifndef ABT_CONFIG_USE_MEM_POOL
535  ABTU_free(p_desc);
536 #else
537  ABTI_xstream *p_local_xstream = ABTI_local_get_xstream_or_null(p_local);
538 #ifndef ABT_CONFIG_DISABLE_EXT_THREAD
539  if (*(uint32_t *)(((char *)p_desc) + ABTI_MEM_POOL_DESC_SIZE)) {
540  /* This was allocated by an external thread. */
541  ABTU_free(p_desc);
542  return;
543  } else if (!p_local_xstream) {
544  /* Return a stack and a descriptor to their global pools. */
546  ABTI_mem_pool_free(&p_global->mem_pool_desc_ext, p_desc);
548  return;
549  }
550 #endif
551  ABTI_mem_pool_free(&p_local_xstream->mem_pool_desc, p_desc);
552 #endif
553 }
554 
555 #endif /* ABTI_MEM_H_INCLUDED */
ABTI_THREAD_TYPE_MEM_MALLOC_DESC_MEMPOOL_LAZY_STACK
#define ABTI_THREAD_TYPE_MEM_MALLOC_DESC_MEMPOOL_LAZY_STACK
Definition: abti.h:112
ABTI_global::mem_pool_desc_ext
ABTI_mem_pool_local_pool mem_pool_desc_ext
Definition: abti.h:263
ABTI_mem_free_thread
static void ABTI_mem_free_thread(ABTI_global *p_global, ABTI_local *p_local, ABTI_thread *p_thread)
Definition: abti_mem.h:382
ABT_bool
int ABT_bool
Boolean type.
Definition: abt.h:1043
ABTI_mem_unregister_stack
static void ABTI_mem_unregister_stack(const ABTI_global *p_global, void *p_stacktop, size_t stacksize, ABT_bool mprotect_if_needed)
Definition: abti_mem.h:94
ABTD_ythread_context_get_stacktop
static void * ABTD_ythread_context_get_stacktop(ABTD_ythread_context *p_ctx)
Definition: abtd_fcontext.h:118
ABTI_MEM_LP_MMAP_HP_THP
@ ABTI_MEM_LP_MMAP_HP_THP
Definition: abti_mem.h:21
ABTI_mem_alloc_ythread_desc_impl
static ABTU_ret_err int ABTI_mem_alloc_ythread_desc_impl(ABTI_local *p_local, ABT_bool use_lazy_stack, ABTI_ythread **pp_ythread)
Definition: abti_mem.h:183
ABTI_STACK_GUARD_MPROTECT_STRICT
@ ABTI_STACK_GUARD_MPROTECT_STRICT
Definition: abti.h:79
ABTU_roundup_ptr
static void * ABTU_roundup_ptr(void *ptr, size_t multiple)
Definition: abtu.h:105
ABTI_mem_pool_local_pool
Definition: abti_mem_pool.h:101
ABTU_roundup_size
static size_t ABTU_roundup_size(size_t val, size_t multiple)
Definition: abtu.h:95
ABTI_thread::type
ABTI_thread_type type
Definition: abti.h:426
ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC
#define ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC
Definition: abti.h:94
ABTI_mem_register_stack
static void ABTI_mem_register_stack(const ABTI_global *p_global, void *p_stacktop, size_t stacksize, ABT_bool mprotect_if_needed)
Definition: abti_mem.h:57
ABTI_CHECK_ERROR
#define ABTI_CHECK_ERROR(abt_errno)
Definition: abti_error.h:136
ABTI_mem_alloc_nythread
static ABTU_ret_err int ABTI_mem_alloc_nythread(ABTI_local *p_local, ABTI_thread **pp_thread)
Definition: abti_mem.h:130
ABTI_MEM_LP_MALLOC
@ ABTI_MEM_LP_MALLOC
Definition: abti_mem.h:18
ABTI_thread_get_ythread
static ABTI_ythread * ABTI_thread_get_ythread(ABTI_thread *p_thread)
Definition: abti_thread.h:52
ABTI_xstream::mem_pool_stack
ABTI_mem_pool_local_pool mem_pool_stack
Definition: abti.h:314
ABTI_MEM_POOL_DESC_SIZE
#define ABTI_MEM_POOL_DESC_SIZE
Definition: abti_mem.h:501
ABTI_mem_alloc_desc
static ABTU_ret_err int ABTI_mem_alloc_desc(ABTI_local *p_local, void **pp_desc)
Definition: abti_mem.h:503
ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC_STACK
#define ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC_STACK
Definition: abti.h:100
ABTI_mem_alloc_ythread_mempool_desc_stack
static ABTU_ret_err int ABTI_mem_alloc_ythread_mempool_desc_stack(ABTI_global *p_global, ABTI_local *p_local, size_t stacksize, ABTI_ythread **pp_ythread)
Definition: abti_mem.h:273
ABTU_mprotect
ABTU_ret_err int ABTU_mprotect(void *addr, size_t size, ABT_bool protect)
Definition: mprotect.c:9
ABTI_thread
Definition: abti.h:422
ABTI_mem_free_ythread_desc_mempool_impl
static void ABTI_mem_free_ythread_desc_mempool_impl(ABTI_global *p_global, ABTI_local *p_local, ABTI_ythread *p_ythread)
Definition: abti_mem.h:213
ABTI_xstream
Definition: abti.h:294
ABTI_global::stack_guard_kind
ABTI_stack_guard stack_guard_kind
Definition: abti.h:266
ABTI_mem_init
ABTU_ret_err int ABTI_mem_init(ABTI_global *p_global)
Definition: malloc.c:195
ABTI_VALGRIND_REGISTER_STACK
#define ABTI_VALGRIND_REGISTER_STACK(p_stack, size)
Definition: abti_valgrind.h:32
ABTI_MEM_LP_THP
@ ABTI_MEM_LP_THP
Definition: abti_mem.h:22
ABTI_mem_pool_alloc
static ABTU_ret_err int ABTI_mem_pool_alloc(ABTI_mem_pool_local_pool *p_local_pool, void **p_mem)
Definition: abti_mem_pool.h:128
ABTD_ythread_context_lazy_unset_stack
static void ABTD_ythread_context_lazy_unset_stack(ABTD_ythread_context *p_ctx)
Definition: abtd_fcontext.h:106
ABTI_global::sys_page_size
size_t sys_page_size
Definition: abti.h:246
ABTI_thread_get_ythread_or_null
static ABTI_ythread * ABTI_thread_get_ythread_or_null(ABTI_thread *p_thread)
Definition: abti_thread.h:59
ABTI_global::thread_stacksize
size_t thread_stacksize
Definition: abti.h:237
ABTD_spinlock_acquire
static void ABTD_spinlock_acquire(ABTD_spinlock *p_lock)
Definition: abtd_spinlock.h:28
ABTU_malloc
static ABTU_ret_err int ABTU_malloc(size_t size, void **p_ptr)
Definition: abtu.h:235
ABTI_MEM_LP_MMAP_HP_RP
@ ABTI_MEM_LP_MMAP_HP_RP
Definition: abti_mem.h:20
ABTI_mem_finalize
void ABTI_mem_finalize(ABTI_global *p_global)
Definition: malloc.c:206
ABTI_mem_free_nythread_mempool_impl
static void ABTI_mem_free_nythread_mempool_impl(ABTI_global *p_global, ABTI_local *p_local, ABTI_thread *p_thread)
Definition: abti_mem.h:155
ABTU_roundup_uint64
static uint64_t ABTU_roundup_uint64(uint64_t val, uint64_t multiple)
Definition: abtu.h:85
ABTI_global::mem_pool_stack_ext
ABTI_mem_pool_local_pool mem_pool_stack_ext
Definition: abti.h:261
ABTI_ythread::ctx
ABTD_ythread_context ctx
Definition: abti.h:458
ABTI_ASSERT
#define ABTI_ASSERT(cond)
Definition: abti_error.h:12
ABTD_ythread_context_lazy_set_stack
static void ABTD_ythread_context_lazy_set_stack(ABTD_ythread_context *p_ctx, void *p_stacktop)
Definition: abtd_fcontext.h:99
ABT_CONFIG_STATIC_CACHELINE_SIZE
#define ABT_CONFIG_STATIC_CACHELINE_SIZE
Definition: abt_config.h:81
ABTI_STATIC_ASSERT
#define ABTI_STATIC_ASSERT(cond)
Definition: abti_error.h:35
ABT_SUCCESS
#define ABT_SUCCESS
Error code: the routine returns successfully.
Definition: abt.h:92
ABTU_ret_err
#define ABTU_ret_err
Definition: abtu.h:155
ABTI_xstream::mem_pool_desc
ABTI_mem_pool_local_pool mem_pool_desc
Definition: abti.h:315
ABTI_mem_alloc_ythread_malloc_desc_stack
static ABTU_ret_err int ABTI_mem_alloc_ythread_malloc_desc_stack(ABTI_global *p_global, size_t stacksize, ABTI_ythread **pp_ythread)
Definition: abti_mem.h:344
ABTI_mem_alloc_ythread_mempool_desc
static ABTU_ret_err int ABTI_mem_alloc_ythread_mempool_desc(ABTI_global *p_global, ABTI_local *p_local, size_t stacksize, void *p_stacktop, ABTI_ythread **pp_ythread)
Definition: abti_mem.h:363
ABTI_local_get_xstream_or_null
static ABTI_xstream * ABTI_local_get_xstream_or_null(ABTI_local *p_local)
Definition: abti_local.h:77
ABTI_STACK_GUARD_MPROTECT
@ ABTI_STACK_GUARD_MPROTECT
Definition: abti.h:78
ABTI_mem_init_local
ABTU_ret_err int ABTI_mem_init_local(ABTI_global *p_global, ABTI_xstream *p_local_xstream)
Definition: malloc.c:200
ABTI_IS_EXT_THREAD_ENABLED
#define ABTI_IS_EXT_THREAD_ENABLED
Definition: abti.h:28
ABTI_STACK_CANARY_VALUE
#define ABTI_STACK_CANARY_VALUE
Definition: abti_mem.h:32
ABT_TRUE
#define ABT_TRUE
True constant for ABT_bool.
Definition: abt.h:784
ABT_FALSE
#define ABT_FALSE
False constant for ABT_bool.
Definition: abt.h:786
ABTI_ythread
Definition: abti.h:456
ABTD_ythread_context_init
static void ABTD_ythread_context_init(ABTD_ythread_context *p_ctx, void *p_stacktop, size_t stacksize)
Definition: abtd_fcontext.h:80
ABTI_UB_ASSERT
#define ABTI_UB_ASSERT(cond)
Definition: abti_error.h:19
ABTU_free
static void ABTU_free(void *ptr)
Definition: abtu.h:228
ABTD_spinlock_release
static void ABTD_spinlock_release(ABTD_spinlock *p_lock)
Definition: abtd_spinlock.h:42
ABTD_ythread_context_init_lazy
static void ABTD_ythread_context_init_lazy(ABTD_ythread_context *p_ctx, size_t stacksize)
Definition: abtd_fcontext.h:89
ABTI_mem_free_ythread_mempool_stack
static void ABTI_mem_free_ythread_mempool_stack(ABTI_xstream *p_local_xstream, ABTI_ythread *p_ythread)
Definition: abti_mem.h:482
ABTI_global::mem_pool_desc_lock
ABTD_spinlock mem_pool_desc_lock
Definition: abti.h:262
ABTI_global::mem_pool_stack_lock
ABTD_spinlock mem_pool_stack_lock
Definition: abti.h:260
ABTI_ythread::thread
ABTI_thread thread
Definition: abti.h:457
ABTI_local
struct ABTI_local ABTI_local
Definition: abti.h:132
ABTI_VALGRIND_UNREGISTER_STACK
#define ABTI_VALGRIND_UNREGISTER_STACK(p_stack)
Definition: abti_valgrind.h:35
ABT_CONFIG_STACK_CHECK_CANARY_SIZE
#define ABT_CONFIG_STACK_CHECK_CANARY_SIZE
Definition: abt_config.h:75
ABTI_global
Definition: abti.h:223
ABTI_mem_pool_free
static void ABTI_mem_pool_free(ABTI_mem_pool_local_pool *p_local_pool, void *mem)
Definition: abti_mem_pool.h:174
ABTI_mem_check_lp_alloc
int ABTI_mem_check_lp_alloc(ABTI_global *p_global, int lp_alloc)
ABTI_mem_free_desc
static void ABTI_mem_free_desc(ABTI_global *p_global, ABTI_local *p_local, void *p_desc)
Definition: abti_mem.h:531
ABTD_ythread_context_get_stacksize
static size_t ABTD_ythread_context_get_stacksize(ABTD_ythread_context *p_ctx)
Definition: abtd_fcontext.h:130
ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC_MEMPOOL_LAZY_STACK
#define ABTI_THREAD_TYPE_MEM_MEMPOOL_DESC_MEMPOOL_LAZY_STACK
Definition: abti.h:107
ABTI_mem_alloc_ythread_malloc_desc_stack_impl
static ABTU_ret_err int ABTI_mem_alloc_ythread_malloc_desc_stack_impl(size_t stacksize, ABTI_ythread **pp_ythread, void **pp_stacktop)
Definition: abti_mem.h:256
ABTI_mem_finalize_local
void ABTI_mem_finalize_local(ABTI_xstream *p_local_xstream)
Definition: malloc.c:210
ABTI_MEM_POOL_DESC_ELEM_SIZE
#define ABTI_MEM_POOL_DESC_ELEM_SIZE
Definition: abti_mem.h:14
ABTI_mem_write_stack_canary
static void ABTI_mem_write_stack_canary(void *p_stack)
Definition: abti_mem.h:36
ABTI_MEM_LP_MMAP_RP
@ ABTI_MEM_LP_MMAP_RP
Definition: abti_mem.h:19
ABTD_ythread_context_has_stack
static ABT_bool ABTD_ythread_context_has_stack(const ABTD_ythread_context *p_ctx)
Definition: abtd_fcontext.h:124
ABTI_mem_check_stack_canary
static void ABTI_mem_check_stack_canary(void *p_stack)
Definition: abti_mem.h:46
ABTI_THREAD_TYPE_MEM_MALLOC_DESC_STACK
#define ABTI_THREAD_TYPE_MEM_MALLOC_DESC_STACK
Definition: abti.h:103
ABTI_mem_alloc_ythread_mempool_stack
static ABTU_ret_err int ABTI_mem_alloc_ythread_mempool_stack(ABTI_xstream *p_local_xstream, ABTI_ythread *p_ythread)
Definition: abti_mem.h:461
ABTI_THREAD_TYPE_MEM_MALLOC_DESC
#define ABTI_THREAD_TYPE_MEM_MALLOC_DESC
Definition: abti.h:97
ABTI_mem_alloc_ythread_default
static ABTU_ret_err int ABTI_mem_alloc_ythread_default(ABTI_global *p_global, ABTI_local *p_local, ABTI_ythread **pp_ythread)
Definition: abti_mem.h:336