ARGOBOTS  36a41b57298ad4e16b5ada8c95c81f00dfed4d3c
abti_mem_pool.h
Go to the documentation of this file.
1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 /*
3  * See COPYRIGHT in top-level directory.
4  */
5 
6 #ifndef ABTI_MEM_POOL_H_INCLUDED
7 #define ABTI_MEM_POOL_H_INCLUDED
8 
9 #define ABT_MEM_POOL_MAX_LOCAL_BUCKETS 2
10 #define ABT_MEM_POOL_NUM_RETURN_BUCKETS 1
11 #define ABT_MEM_POOL_NUM_TAKE_BUCKETS 1
12 
14  /* This is used when it is in ABTI_mem_pool_global_pool */
16  /* This is used when it is in ABTI_mem_pool_local_pool */
17  size_t num_headers;
19 
20 typedef struct ABTI_mem_pool_header {
24 
25 typedef struct ABTI_mem_pool_page {
28  void *mem;
29  size_t page_size;
31  void *p_mem_extra;
34 
36  ABT_bool enabled; /* Use page protection or not. */
37  ABT_bool check_error; /* Check error. */
38  size_t offset; /* Page protection offset. */
39  size_t page_size; /* Protection page size. */
40  size_t alignment; /* Alignment of protected page. It should be a multiple
41  of the system page size. */
43 
44 /*
45  * To efficiently take/return multiple headers per bucket, headers are linked as
46  * follows in the global pool (bucket_lifo).
47  *
48  * header (p_next)> header (p_next)> header ... (num_headers_per_bucket)
49  * | (connected via lifo_elem)
50  * V
51  * header (p_next)> header (p_next)> header ... (num_headers_per_bucket)
52  * | (connected via lifo_elem)
53  * V
54  * header (p_next)> header (p_next)> header ... (num_headers_per_bucket)
55  * .
56  * .
57  */
58 typedef struct ABTI_mem_pool_global_pool {
59  size_t header_size; /* Size of header. This size includes a protected
60  * page. */
61  size_t page_size; /* Size of page (mem of ABTI_mem_pool_page) */
62  size_t alignment_hint; /* Alignment hint for page */
63  size_t header_offset; /* Offset of ABTI_mem_pool_header from the top
64  * of the memory segment; i.e., the pool returns
65  * p_header_memory_top + offset. */
66  size_t num_headers_per_bucket; /* Number of headers per bucket. */
67  uint32_t
68  num_lp_type_requests; /* Number of requests for large page allocation.
69  */
71  lp_type_requests[4]; /* Requests for large page allocation */
74  ABTI_sync_lifo bucket_lifo; /* LIFO of available buckets. */
76  ABTI_sync_lifo mem_page_lifo; /* LIFO of non-empty pages. */
78  ABTD_atomic_ptr p_mem_page_empty; /* List of empty pages. */
80  /* List of the remaining headers that are not enough to create one
81  * complete bucket. This is protected by a spinlock. The number of
82  * headers is stored in partial_bucket.bucket_info.num_headers. */
83  ABTD_spinlock partial_bucket_lock;
86 
87 /*
88  * To efficiently take/return multiple headers per bucket, headers are stored as
89  * follows in the local pool.
90  *
91  * buckets[0]:
92  * = header (p_next)> header (p_next)> header ... (num_headers_per_bucket)
93  * buckets[1]:
94  * = header (p_next)> header (p_next)> header ... (num_headers_per_bucket)
95  * .
96  * .
97  * buckets[bucket_index]:
98  * = header (p_next)> header (p_next)> header ...
99  * (buckets[bucket_index]->bucket_info.num_headers)
100  */
101 typedef struct ABTI_mem_pool_local_pool {
103  size_t num_headers_per_bucket; /* Cached value to reduce dereference. It
104  must be equal to
105  p_global_pool->num_headers_per_bucket. */
106  size_t bucket_index;
109 
111  ABTI_mem_pool_global_pool *p_global_pool, size_t num_headers_per_bucket,
112  size_t header_size, size_t header_offset, size_t page_size,
113  const ABTU_MEM_LARGEPAGE_TYPE *lp_type_requests,
114  uint32_t num_lp_type_requests, size_t alignment_hint,
115  ABTI_mem_pool_global_pool_mprotect_config *p_mprotect_config);
117  ABTI_mem_pool_global_pool *p_global_pool);
118 ABTU_ret_err int
120  ABTI_mem_pool_global_pool *p_global_pool);
123  ABTI_mem_pool_header **p_bucket);
125  ABTI_mem_pool_header *bucket);
126 
127 ABTU_ret_err static inline int
128 ABTI_mem_pool_alloc(ABTI_mem_pool_local_pool *p_local_pool, void **p_mem)
129 {
130  size_t bucket_index = p_local_pool->bucket_index;
131  ABTI_mem_pool_header *cur_bucket = p_local_pool->buckets[bucket_index];
132  size_t num_headers_in_cur_bucket = cur_bucket->bucket_info.num_headers;
133  /* At least one header is available in the current bucket, so it must be
134  * larger than 0. */
135  ABTI_ASSERT(num_headers_in_cur_bucket >= 1);
136  if (num_headers_in_cur_bucket == 1) {
137  /*cur_bucket will be empty after allocation. */
138  if (bucket_index == 0) {
139  /* cur_bucket is the last header in this pool.
140  * Let's get some buckets from the global pool. */
141  size_t i;
142  for (i = 0; i < ABT_MEM_POOL_NUM_TAKE_BUCKETS; i++) {
143  int abt_errno =
145  &p_local_pool->buckets[i]);
146  if (ABTI_IS_ERROR_CHECK_ENABLED && abt_errno != ABT_SUCCESS) {
147  /* Return buckets that have been already taken. */
148 #if ABT_MEM_POOL_NUM_TAKE_BUCKETS > 1
149  size_t j;
150  for (j = 0; j < i; j++) {
152  p_local_pool->buckets[j]);
153  }
154 #endif
155  return abt_errno;
156  }
157  }
158  p_local_pool->bucket_index = ABT_MEM_POOL_NUM_TAKE_BUCKETS - 1;
159  } else {
160  p_local_pool->bucket_index = bucket_index - 1;
161  }
162  /* Now buckets[bucket_index] is full of headers. */
163  } else {
164  /* Let's return the header in the bucket. */
165  ABTI_mem_pool_header *p_next = cur_bucket->p_next;
166  p_next->bucket_info.num_headers = num_headers_in_cur_bucket - 1;
167  p_local_pool->buckets[bucket_index] = p_next;
168  }
169  /* At least one header is available in the current bucket. */
170  *p_mem = (void *)cur_bucket;
171  return ABT_SUCCESS;
172 }
173 
174 static inline void ABTI_mem_pool_free(ABTI_mem_pool_local_pool *p_local_pool,
175  void *mem)
176 {
177  /* At least one header is available in the current bucket. */
178  size_t bucket_index = p_local_pool->bucket_index;
179  ABTI_mem_pool_header *p_freed_header = (ABTI_mem_pool_header *)mem;
180  ABTI_mem_pool_header *cur_bucket = p_local_pool->buckets[bucket_index];
181  if (cur_bucket->bucket_info.num_headers ==
182  p_local_pool->num_headers_per_bucket) {
183  /* cur_bucket is full. */
184  if (++bucket_index == ABT_MEM_POOL_MAX_LOCAL_BUCKETS) {
185  size_t i;
186  /* All buckets are full, so let's return some old buckets. */
187  for (i = 0; i < ABT_MEM_POOL_NUM_RETURN_BUCKETS; i++) {
189  p_local_pool->buckets[i]);
190  }
193  p_local_pool->buckets[i - ABT_MEM_POOL_NUM_RETURN_BUCKETS] =
194  p_local_pool->buckets[i];
195  }
196  bucket_index = ABT_MEM_POOL_MAX_LOCAL_BUCKETS -
198  }
199  p_local_pool->bucket_index = bucket_index;
200  p_freed_header->p_next = NULL;
201  p_freed_header->bucket_info.num_headers = 1;
202  } else {
203  p_freed_header->p_next = cur_bucket;
204  p_freed_header->bucket_info.num_headers =
205  cur_bucket->bucket_info.num_headers + 1;
206  }
207  p_local_pool->buckets[bucket_index] = p_freed_header;
208  /* At least one header is available in the current bucket. */
209 }
210 
211 #endif /* ABTI_MEM_POOL_H_INCLUDED */
ABTI_mem_pool_destroy_local_pool
void ABTI_mem_pool_destroy_local_pool(ABTI_mem_pool_local_pool *p_local_pool)
Definition: mem_pool.c:201
ABT_bool
int ABT_bool
Boolean type.
Definition: abt.h:1043
ABT_MEM_POOL_NUM_RETURN_BUCKETS
#define ABT_MEM_POOL_NUM_RETURN_BUCKETS
Definition: abti_mem_pool.h:10
ABTI_mem_pool_header::bucket_info
ABTI_mem_pool_header_bucket_info bucket_info
Definition: abti_mem_pool.h:22
ABTI_mem_pool_header
struct ABTI_mem_pool_header ABTI_mem_pool_header
ABTI_mem_pool_global_pool::num_lp_type_requests
uint32_t num_lp_type_requests
Definition: abti_mem_pool.h:68
ABTI_mem_pool_local_pool
Definition: abti_mem_pool.h:101
ABTI_mem_pool_global_pool_mprotect_config::offset
size_t offset
Definition: abti_mem_pool.h:38
ABTI_mem_pool_local_pool::bucket_index
size_t bucket_index
Definition: abti_mem_pool.h:106
ABTI_sync_lifo_element
Definition: abti_sync_lifo.h:16
ABT_MEM_POOL_MAX_LOCAL_BUCKETS
#define ABT_MEM_POOL_MAX_LOCAL_BUCKETS
Definition: abti_mem_pool.h:9
ABTI_mem_pool_global_pool_mprotect_config
struct ABTI_mem_pool_global_pool_mprotect_config ABTI_mem_pool_global_pool_mprotect_config
ABTI_mem_pool_page::p_mem_extra
void * p_mem_extra
Definition: abti_mem_pool.h:31
ABTI_mem_pool_return_bucket
void ABTI_mem_pool_return_bucket(ABTI_mem_pool_global_pool *p_global_pool, ABTI_mem_pool_header *bucket)
Definition: mem_pool.c:374
ABTI_IS_ERROR_CHECK_ENABLED
#define ABTI_IS_ERROR_CHECK_ENABLED
Definition: abti.h:20
ABTI_mem_pool_header_bucket_info
union ABTI_mem_pool_header_bucket_info ABTI_mem_pool_header_bucket_info
ABT_MEM_POOL_NUM_TAKE_BUCKETS
#define ABT_MEM_POOL_NUM_TAKE_BUCKETS
Definition: abti_mem_pool.h:11
ABTI_mem_pool_page::p_next_empty_page
struct ABTI_mem_pool_page * p_next_empty_page
Definition: abti_mem_pool.h:27
ABTD_spinlock
Definition: abtd_spinlock.h:9
ABTI_mem_pool_local_pool::p_global_pool
ABTI_mem_pool_global_pool * p_global_pool
Definition: abti_mem_pool.h:102
ABTI_mem_pool_alloc
static ABTU_ret_err int ABTI_mem_pool_alloc(ABTI_mem_pool_local_pool *p_local_pool, void **p_mem)
Definition: abti_mem_pool.h:128
ABTI_mem_pool_global_pool_mprotect_config::alignment
size_t alignment
Definition: abti_mem_pool.h:40
ABTI_mem_pool_local_pool
struct ABTI_mem_pool_local_pool ABTI_mem_pool_local_pool
ABTI_mem_pool_global_pool::page_size
size_t page_size
Definition: abti_mem_pool.h:61
ABTI_mem_pool_global_pool::num_headers_per_bucket
size_t num_headers_per_bucket
Definition: abti_mem_pool.h:66
ABTI_mem_pool_global_pool::ABTU_align_member_var
ABTU_align_member_var(ABT_CONFIG_STATIC_CACHELINE_SIZE) ABTI_sync_lifo bucket_lifo
ABTD_atomic_ptr
Definition: abtd_atomic.h:39
ABTI_mem_pool_global_pool::alignment_hint
size_t alignment_hint
Definition: abti_mem_pool.h:62
ABTI_mem_pool_header_bucket_info::num_headers
size_t num_headers
Definition: abti_mem_pool.h:17
ABTI_mem_pool_global_pool::header_offset
size_t header_offset
Definition: abti_mem_pool.h:63
ABTI_mem_pool_page::page_size
size_t page_size
Definition: abti_mem_pool.h:29
ABTI_ASSERT
#define ABTI_ASSERT(cond)
Definition: abti_error.h:12
ABTI_mem_pool_header
Definition: abti_mem_pool.h:20
ABTU_MEM_LARGEPAGE_TYPE
ABTU_MEM_LARGEPAGE_TYPE
Definition: abtu.h:308
ABT_CONFIG_STATIC_CACHELINE_SIZE
#define ABT_CONFIG_STATIC_CACHELINE_SIZE
Definition: abt_config.h:81
ABTI_mem_pool_init_local_pool
ABTU_ret_err int ABTI_mem_pool_init_local_pool(ABTI_mem_pool_local_pool *p_local_pool, ABTI_mem_pool_global_pool *p_global_pool)
Definition: mem_pool.c:186
ABT_SUCCESS
#define ABT_SUCCESS
Error code: the routine returns successfully.
Definition: abt.h:92
ABTU_ret_err
#define ABTU_ret_err
Definition: abtu.h:155
ABTI_mem_pool_global_pool::mprotect_config
ABTI_mem_pool_global_pool_mprotect_config mprotect_config
Definition: abti_mem_pool.h:72
ABTI_mem_pool_global_pool
struct ABTI_mem_pool_global_pool ABTI_mem_pool_global_pool
ABTI_mem_pool_global_pool::lp_type_requests
ABTU_MEM_LARGEPAGE_TYPE lp_type_requests[4]
Definition: abti_mem_pool.h:71
ABTI_mem_pool_take_bucket
int ABTI_mem_pool_take_bucket(ABTI_mem_pool_global_pool *p_global_pool, ABTI_mem_pool_header **p_bucket)
Definition: mem_pool.c:222
ABTI_mem_pool_global_pool
Definition: abti_mem_pool.h:58
ABTI_mem_pool_page::mem
void * mem
Definition: abti_mem_pool.h:28
ABTI_mem_pool_header_bucket_info::lifo_elem
ABTI_sync_lifo_element lifo_elem
Definition: abti_mem_pool.h:15
ABTI_sync_lifo
ABTI_sync_lifo
Definition: abti_sync_lifo.h:27
ABTI_mem_pool_page::lp_type
ABTU_MEM_LARGEPAGE_TYPE lp_type
Definition: abti_mem_pool.h:30
ABTI_mem_pool_global_pool_mprotect_config
Definition: abti_mem_pool.h:35
ABTI_mem_pool_global_pool::header_size
size_t header_size
Definition: abti_mem_pool.h:59
ABTI_mem_pool_global_pool_mprotect_config::page_size
size_t page_size
Definition: abti_mem_pool.h:39
ABTI_mem_pool_global_pool_mprotect_config::enabled
ABT_bool enabled
Definition: abti_mem_pool.h:36
ABTI_mem_pool_page
Definition: abti_mem_pool.h:25
ABTI_mem_pool_global_pool::partial_bucket
ABTI_mem_pool_header * partial_bucket
Definition: abti_mem_pool.h:84
ABTI_mem_pool_init_global_pool
void ABTI_mem_pool_init_global_pool(ABTI_mem_pool_global_pool *p_global_pool, size_t num_headers_per_bucket, size_t header_size, size_t header_offset, size_t page_size, const ABTU_MEM_LARGEPAGE_TYPE *lp_type_requests, uint32_t num_lp_type_requests, size_t alignment_hint, ABTI_mem_pool_global_pool_mprotect_config *p_mprotect_config)
Definition: mem_pool.c:89
ABTI_mem_pool_header::p_next
struct ABTI_mem_pool_header * p_next
Definition: abti_mem_pool.h:21
ABTI_mem_pool_header_bucket_info
Definition: abti_mem_pool.h:13
ABTI_mem_pool_destroy_global_pool
void ABTI_mem_pool_destroy_global_pool(ABTI_mem_pool_global_pool *p_global_pool)
Definition: mem_pool.c:142
ABTI_mem_pool_free
static void ABTI_mem_pool_free(ABTI_mem_pool_local_pool *p_local_pool, void *mem)
Definition: abti_mem_pool.h:174
ABTI_mem_pool_page::lifo_elem
ABTI_sync_lifo_element lifo_elem
Definition: abti_mem_pool.h:26
ABTI_mem_pool_page::mem_extra_size
size_t mem_extra_size
Definition: abti_mem_pool.h:32
ABTI_mem_pool_page
struct ABTI_mem_pool_page ABTI_mem_pool_page
ABTI_mem_pool_local_pool::num_headers_per_bucket
size_t num_headers_per_bucket
Definition: abti_mem_pool.h:103
ABTI_mem_pool_global_pool_mprotect_config::check_error
ABT_bool check_error
Definition: abti_mem_pool.h:37
ABTI_mem_pool_local_pool::buckets
ABTI_mem_pool_header * buckets[ABT_MEM_POOL_MAX_LOCAL_BUCKETS]
Definition: abti_mem_pool.h:107