ARGOBOTS  4dc37e16e1b227a480715ab67dae1dcfb4d2d4e0
abti_mem_pool.h
Go to the documentation of this file.
1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 /*
3  * See COPYRIGHT in top-level directory.
4  */
5 
6 #ifndef ABTI_MEM_POOL_H_INCLUDED
7 #define ABTI_MEM_POOL_H_INCLUDED
8 
9 #define ABT_MEM_POOL_MAX_LOCAL_BUCKETS 2
10 #define ABT_MEM_POOL_NUM_RETURN_BUCKETS 1
11 #define ABT_MEM_POOL_NUM_TAKE_BUCKETS 1
12 
13 typedef union ABTI_mem_pool_header_bucket_info {
14  /* This is used when it is in ABTI_mem_pool_global_pool */
15  ABTI_sync_lifo_element lifo_elem;
16  /* This is used when it is in ABTI_mem_pool_local_pool */
17  size_t num_headers;
18 } ABTI_mem_pool_header_bucket_info;
19 
20 typedef struct ABTI_mem_pool_header {
21  struct ABTI_mem_pool_header *p_next;
22  ABTI_mem_pool_header_bucket_info bucket_info;
23 } ABTI_mem_pool_header;
24 
25 typedef struct ABTI_mem_pool_page {
26  ABTI_sync_lifo_element lifo_elem;
27  struct ABTI_mem_pool_page *p_next_empty_page;
28  void *mem;
29  size_t page_size;
31  void *p_mem_extra;
32  size_t mem_extra_size;
33 } ABTI_mem_pool_page;
34 
35 typedef struct ABTI_mem_pool_global_pool_mprotect_config {
36  ABT_bool enabled; /* Use page protection or not. */
37  ABT_bool check_error; /* Check error. */
38  size_t offset; /* Page protection offset. */
39  size_t page_size; /* Protection page size. */
40  size_t alignment; /* Alignment of protected page. It should be a multiple
41  of the system page size. */
42 } ABTI_mem_pool_global_pool_mprotect_config;
43 
44 /*
45  * To efficiently take/return multiple headers per bucket, headers are linked as
46  * follows in the global pool (bucket_lifo).
47  *
48  * header (p_next)> header (p_next)> header ... (num_headers_per_bucket)
49  * | (connected via lifo_elem)
50  * V
51  * header (p_next)> header (p_next)> header ... (num_headers_per_bucket)
52  * | (connected via lifo_elem)
53  * V
54  * header (p_next)> header (p_next)> header ... (num_headers_per_bucket)
55  * .
56  * .
57  */
58 typedef struct ABTI_mem_pool_global_pool {
59  size_t header_size; /* Size of header. This size includes a protected
60  * page. */
61  size_t page_size; /* Size of page (mem of ABTI_mem_pool_page) */
62  size_t alignment_hint; /* Alignment hint for page */
63  size_t header_offset; /* Offset of ABTI_mem_pool_header from the top
64  * of the memory segment; i.e., the pool returns
65  * p_header_memory_top + offset. */
66  size_t num_headers_per_bucket; /* Number of headers per bucket. */
67  uint32_t
68  num_lp_type_requests; /* Number of requests for large page allocation.
69  */
71  lp_type_requests[4]; /* Requests for large page allocation */
72  ABTI_mem_pool_global_pool_mprotect_config mprotect_config;
74  ABTI_sync_lifo bucket_lifo; /* LIFO of available buckets. */
76  ABTI_sync_lifo mem_page_lifo; /* LIFO of non-empty pages. */
78  ABTD_atomic_ptr p_mem_page_empty; /* List of empty pages. */
80  /* List of the remaining headers that are not enough to create one
81  * complete bucket. This is protected by a spinlock. The number of
82  * headers is stored in partial_bucket.bucket_info.num_headers. */
83  ABTD_spinlock partial_bucket_lock;
84  ABTI_mem_pool_header *partial_bucket;
85 } ABTI_mem_pool_global_pool;
86 
87 /*
88  * To efficiently take/return multiple headers per bucket, headers are stored as
89  * follows in the local pool.
90  *
91  * buckets[0]:
92  * = header (p_next)> header (p_next)> header ... (num_headers_per_bucket)
93  * buckets[1]:
94  * = header (p_next)> header (p_next)> header ... (num_headers_per_bucket)
95  * .
96  * .
97  * buckets[bucket_index]:
98  * = header (p_next)> header (p_next)> header ...
99  * (buckets[bucket_index]->bucket_info.num_headers)
100  */
101 typedef struct ABTI_mem_pool_local_pool {
102  ABTI_mem_pool_global_pool *p_global_pool;
103  size_t num_headers_per_bucket; /* Cached value to reduce dereference. It
104  must be equal to
105  p_global_pool->num_headers_per_bucket. */
106  size_t bucket_index;
107  ABTI_mem_pool_header *buckets[ABT_MEM_POOL_MAX_LOCAL_BUCKETS];
108 } ABTI_mem_pool_local_pool;
109 
110 void ABTI_mem_pool_init_global_pool(
111  ABTI_mem_pool_global_pool *p_global_pool, size_t num_headers_per_bucket,
112  size_t header_size, size_t header_offset, size_t page_size,
113  const ABTU_MEM_LARGEPAGE_TYPE *lp_type_requests,
114  uint32_t num_lp_type_requests, size_t alignment_hint,
115  ABTI_mem_pool_global_pool_mprotect_config *p_mprotect_config);
116 void ABTI_mem_pool_destroy_global_pool(
117  ABTI_mem_pool_global_pool *p_global_pool);
118 ABTU_ret_err int
119 ABTI_mem_pool_init_local_pool(ABTI_mem_pool_local_pool *p_local_pool,
120  ABTI_mem_pool_global_pool *p_global_pool);
121 void ABTI_mem_pool_destroy_local_pool(ABTI_mem_pool_local_pool *p_local_pool);
122 int ABTI_mem_pool_take_bucket(ABTI_mem_pool_global_pool *p_global_pool,
123  ABTI_mem_pool_header **p_bucket);
124 void ABTI_mem_pool_return_bucket(ABTI_mem_pool_global_pool *p_global_pool,
125  ABTI_mem_pool_header *bucket);
126 
127 ABTU_ret_err static inline int
128 ABTI_mem_pool_alloc(ABTI_mem_pool_local_pool *p_local_pool, void **p_mem)
129 {
130  size_t bucket_index = p_local_pool->bucket_index;
131  ABTI_mem_pool_header *cur_bucket = p_local_pool->buckets[bucket_index];
132  size_t num_headers_in_cur_bucket = cur_bucket->bucket_info.num_headers;
133  /* At least one header is available in the current bucket, so it must be
134  * larger than 0. */
135  ABTI_ASSERT(num_headers_in_cur_bucket >= 1);
136  if (num_headers_in_cur_bucket == 1) {
137  /*cur_bucket will be empty after allocation. */
138  if (bucket_index == 0) {
139  /* cur_bucket is the last header in this pool.
140  * Let's get some buckets from the global pool. */
141  size_t i;
142  for (i = 0; i < ABT_MEM_POOL_NUM_TAKE_BUCKETS; i++) {
143  int abt_errno =
144  ABTI_mem_pool_take_bucket(p_local_pool->p_global_pool,
145  &p_local_pool->buckets[i]);
146  if (ABTI_IS_ERROR_CHECK_ENABLED && abt_errno != ABT_SUCCESS) {
147  /* Return buckets that have been already taken. */
148 #if ABT_MEM_POOL_NUM_TAKE_BUCKETS > 1
149  size_t j;
150  for (j = 0; j < i; j++) {
151  ABTI_mem_pool_return_bucket(p_local_pool->p_global_pool,
152  p_local_pool->buckets[j]);
153  }
154 #endif
155  return abt_errno;
156  }
157  }
158  p_local_pool->bucket_index = ABT_MEM_POOL_NUM_TAKE_BUCKETS - 1;
159  } else {
160  p_local_pool->bucket_index = bucket_index - 1;
161  }
162  /* Now buckets[bucket_index] is full of headers. */
163  } else {
164  /* Let's return the header in the bucket. */
165  ABTI_mem_pool_header *p_next = cur_bucket->p_next;
166  p_next->bucket_info.num_headers = num_headers_in_cur_bucket - 1;
167  p_local_pool->buckets[bucket_index] = p_next;
168  }
169  /* At least one header is available in the current bucket. */
170  *p_mem = (void *)cur_bucket;
171  return ABT_SUCCESS;
172 }
173 
174 static inline void ABTI_mem_pool_free(ABTI_mem_pool_local_pool *p_local_pool,
175  void *mem)
176 {
177  /* At least one header is available in the current bucket. */
178  size_t bucket_index = p_local_pool->bucket_index;
179  ABTI_mem_pool_header *p_freed_header = (ABTI_mem_pool_header *)mem;
180  ABTI_mem_pool_header *cur_bucket = p_local_pool->buckets[bucket_index];
181  if (cur_bucket->bucket_info.num_headers ==
182  p_local_pool->num_headers_per_bucket) {
183  /* cur_bucket is full. */
184  if (++bucket_index == ABT_MEM_POOL_MAX_LOCAL_BUCKETS) {
185  size_t i;
186  /* All buckets are full, so let's return some old buckets. */
187  for (i = 0; i < ABT_MEM_POOL_NUM_RETURN_BUCKETS; i++) {
188  ABTI_mem_pool_return_bucket(p_local_pool->p_global_pool,
189  p_local_pool->buckets[i]);
190  }
193  p_local_pool->buckets[i - ABT_MEM_POOL_NUM_RETURN_BUCKETS] =
194  p_local_pool->buckets[i];
195  }
196  bucket_index = ABT_MEM_POOL_MAX_LOCAL_BUCKETS -
198  }
199  p_local_pool->bucket_index = bucket_index;
200  p_freed_header->p_next = NULL;
201  p_freed_header->bucket_info.num_headers = 1;
202  } else {
203  p_freed_header->p_next = cur_bucket;
204  p_freed_header->bucket_info.num_headers =
205  cur_bucket->bucket_info.num_headers + 1;
206  }
207  p_local_pool->buckets[bucket_index] = p_freed_header;
208  /* At least one header is available in the current bucket. */
209 }
210 
211 #endif /* ABTI_MEM_POOL_H_INCLUDED */
ABT_bool
int ABT_bool
Boolean type.
Definition: abt.h:1043
ABT_MEM_POOL_NUM_RETURN_BUCKETS
#define ABT_MEM_POOL_NUM_RETURN_BUCKETS
Definition: abti_mem_pool.h:10
ABT_MEM_POOL_MAX_LOCAL_BUCKETS
#define ABT_MEM_POOL_MAX_LOCAL_BUCKETS
Definition: abti_mem_pool.h:9
ABT_MEM_POOL_NUM_TAKE_BUCKETS
#define ABT_MEM_POOL_NUM_TAKE_BUCKETS
Definition: abti_mem_pool.h:11
ABTU_MEM_LARGEPAGE_TYPE
ABTU_MEM_LARGEPAGE_TYPE
Definition: abtu.h:308
ABT_CONFIG_STATIC_CACHELINE_SIZE
#define ABT_CONFIG_STATIC_CACHELINE_SIZE
Definition: abt_config.h:81
ABT_SUCCESS
#define ABT_SUCCESS
Error code: the routine returns successfully.
Definition: abt.h:92
ABTU_ret_err
#define ABTU_ret_err
Definition: abtu.h:155
ABTU_align_member_var
#define ABTU_align_member_var(size)
Definition: abtu.h:170