ARGOBOTS  1059a7c2eb7e3f99f736a9c3a4f6ea476ac1b804
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Macros Groups
abti_key.h
Go to the documentation of this file.
1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 /*
3  * See COPYRIGHT in top-level directory.
4  */
5 
6 #ifndef ABTI_KEY_H_INCLUDED
7 #define ABTI_KEY_H_INCLUDED
8 
9 /* Inlined functions for thread-specific data key */
10 
11 static inline ABTI_key *ABTI_key_get_ptr(ABT_key key)
12 {
13 #ifndef ABT_CONFIG_DISABLE_ERROR_CHECK
14  ABTI_key *p_key;
15  if (key == ABT_KEY_NULL) {
16  p_key = NULL;
17  } else {
18  p_key = (ABTI_key *)key;
19  }
20  return p_key;
21 #else
22  return (ABTI_key *)key;
23 #endif
24 }
25 
26 static inline ABT_key ABTI_key_get_handle(ABTI_key *p_key)
27 {
28 #ifndef ABT_CONFIG_DISABLE_ERROR_CHECK
29  ABT_key h_key;
30  if (p_key == NULL) {
31  h_key = ABT_KEY_NULL;
32  } else {
33  h_key = (ABT_key)p_key;
34  }
35  return h_key;
36 #else
37  return (ABT_key)p_key;
38 #endif
39 }
40 
41 /* Static initializer for ABTI_key */
42 #define ABTI_KEY_STATIC_INITIALIZER(f_destructor, id) \
43  { \
44  f_destructor, id \
45  }
46 
47 #define ABTI_KEY_ID_STACKABLE_SCHED 0
48 #define ABTI_KEY_ID_MIGRATION 1
49 #define ABTI_KEY_ID_END_ 2
50 
51 typedef struct ABTI_ktable_mem_header {
55 
56 #define ABTI_KTABLE_DESC_SIZE \
57  (ABTI_MEM_POOL_DESC_SIZE - sizeof(ABTI_ktable_mem_header))
58 
59 #define ABTI_KTABLE_LOCKED ((ABTI_ktable *)0x1)
60 static inline int ABTI_ktable_is_valid(ABTI_ktable *p_ktable)
61 {
62  /* Only 0x0 and 0x1 (=ABTI_KTABLE_LOCKED) are special */
63  return (((uintptr_t)(void *)p_ktable) & (~((uintptr_t)(void *)0x1))) !=
64  ((uintptr_t)(void *)0x0);
65 }
66 
67 ABTU_ret_err static inline int ABTI_ktable_create(ABTI_local *p_local,
68  ABTI_ktable **pp_ktable)
69 {
70  ABTI_ktable *p_ktable;
71  int key_table_size = gp_ABTI_global->key_table_size;
72  /* size must be a power of 2. */
73  ABTI_ASSERT((key_table_size & (key_table_size - 1)) == 0);
74  /* max alignment must be a power of 2. */
76  size_t ktable_size =
77  (offsetof(ABTI_ktable, p_elems) +
78  sizeof(ABTD_atomic_ptr) * key_table_size + ABTU_MAX_ALIGNMENT - 1) &
79  (~(ABTU_MAX_ALIGNMENT - 1));
80  /* Since only one ES can access the memory pool on creation, this uses an
81  * unsafe memory pool without taking a lock. */
82  if (ABTU_likely(ktable_size <= ABTI_KTABLE_DESC_SIZE)) {
83  /* Use memory pool. */
84  void *p_mem;
85  int abt_errno = ABTI_mem_alloc_desc(p_local, &p_mem);
86  ABTI_CHECK_ERROR(abt_errno);
88  p_ktable =
89  (ABTI_ktable *)(((char *)p_mem) + sizeof(ABTI_ktable_mem_header));
90  p_header->p_next = NULL;
91  p_header->is_from_mempool = ABT_TRUE;
92  p_ktable->p_used_mem = p_mem;
93  p_ktable->p_extra_mem = (void *)(((char *)p_ktable) + ktable_size);
94  p_ktable->extra_mem_size = ABTI_KTABLE_DESC_SIZE - ktable_size;
95  } else {
96  /* Use malloc() */
97  void *p_mem;
98  int abt_errno =
99  ABTU_malloc(ktable_size + sizeof(ABTI_ktable_mem_header), &p_mem);
100  ABTI_CHECK_ERROR(abt_errno);
101  ABTI_ktable_mem_header *p_header = (ABTI_ktable_mem_header *)p_mem;
102  p_ktable =
103  (ABTI_ktable *)(((char *)p_mem) + sizeof(ABTI_ktable_mem_header));
104  p_header->p_next = NULL;
105  p_header->is_from_mempool = ABT_FALSE;
106  p_ktable->p_used_mem = p_mem;
107  p_ktable->p_extra_mem = NULL;
108  p_ktable->extra_mem_size = 0;
109  }
110  p_ktable->size = key_table_size;
111  ABTI_spinlock_clear(&p_ktable->lock);
112  memset(p_ktable->p_elems, 0, sizeof(ABTD_atomic_ptr) * key_table_size);
113  *pp_ktable = p_ktable;
114  return ABT_SUCCESS;
115 }
116 
117 ABTU_ret_err static inline int ABTI_ktable_alloc_elem(ABTI_local *p_local,
118  ABTI_ktable *p_ktable,
119  size_t size,
120  void **pp_mem)
121 {
122  ABTI_ASSERT((size & (ABTU_MAX_ALIGNMENT - 1)) == 0);
123  size_t extra_mem_size = p_ktable->extra_mem_size;
124  if (size <= extra_mem_size) {
125  /* Use the extra memory. */
126  void *p_mem = p_ktable->p_extra_mem;
127  p_ktable->p_extra_mem = (void *)(((char *)p_mem) + size);
128  p_ktable->extra_mem_size = extra_mem_size - size;
129  *pp_mem = p_mem;
130  return ABT_SUCCESS;
131  } else if (ABTU_likely(size <= ABTI_KTABLE_DESC_SIZE)) {
132  /* Use memory pool. */
133  void *p_mem;
134  int abt_errno = ABTI_mem_alloc_desc(p_local, &p_mem);
135  ABTI_CHECK_ERROR(abt_errno);
136  ABTI_ktable_mem_header *p_header = (ABTI_ktable_mem_header *)p_mem;
137  p_header->p_next = (ABTI_ktable_mem_header *)p_ktable->p_used_mem;
138  p_header->is_from_mempool = ABT_TRUE;
139  p_ktable->p_used_mem = (void *)p_header;
140  p_mem = (void *)(((char *)p_mem) + sizeof(ABTI_ktable_mem_header));
141  p_ktable->p_extra_mem = (void *)(((char *)p_mem) + size);
142  p_ktable->extra_mem_size = ABTI_KTABLE_DESC_SIZE - size;
143  *pp_mem = p_mem;
144  return ABT_SUCCESS;
145  } else {
146  /* Use malloc() */
147  void *p_mem;
148  int abt_errno =
149  ABTU_malloc(size + sizeof(ABTI_ktable_mem_header), &p_mem);
150  ABTI_CHECK_ERROR(abt_errno);
151  ABTI_ktable_mem_header *p_header = (ABTI_ktable_mem_header *)p_mem;
152  p_header->p_next = (ABTI_ktable_mem_header *)p_ktable->p_used_mem;
153  p_header->is_from_mempool = ABT_FALSE;
154  p_ktable->p_used_mem = (void *)p_header;
155  p_mem = (void *)(((char *)p_mem) + sizeof(ABTI_ktable_mem_header));
156  *pp_mem = p_mem;
157  return ABT_SUCCESS;
158  }
159 }
160 
161 static inline uint32_t ABTI_ktable_get_idx(ABTI_key *p_key, int size)
162 {
163  return p_key->id & (size - 1);
164 }
165 
166 ABTU_ret_err static inline int
168  ABTI_key *p_key, void *value, ABT_bool is_safe)
169 {
170  uint32_t idx;
171  ABTD_atomic_ptr *pp_elem;
172  ABTI_ktelem *p_elem;
173 
174  /* Look for the same key */
175  idx = ABTI_ktable_get_idx(p_key, p_ktable->size);
176  pp_elem = &p_ktable->p_elems[idx];
177  p_elem = (ABTI_ktelem *)ABTD_atomic_acquire_load_ptr(pp_elem);
178  uint32_t key_id = p_key->id;
179  while (p_elem) {
180  if (p_elem->key_id == key_id) {
181  p_elem->value = value;
182  return ABT_SUCCESS;
183  }
184  pp_elem = &p_elem->p_next;
185  p_elem = (ABTI_ktelem *)ABTD_atomic_acquire_load_ptr(pp_elem);
186  }
187 
188  /* The table does not have the same key */
189  if (is_safe)
190  ABTI_spinlock_acquire(&p_ktable->lock);
191  /* The linked list might have been extended. */
192  p_elem = (ABTI_ktelem *)ABTD_atomic_acquire_load_ptr(pp_elem);
193  while (p_elem) {
194  if (p_elem->key_id == key_id) {
195  if (is_safe)
196  ABTI_spinlock_release(&p_ktable->lock);
197  p_elem->value = value;
198  return ABT_SUCCESS;
199  }
200  pp_elem = &p_elem->p_next;
201  p_elem = (ABTI_ktelem *)ABTD_atomic_acquire_load_ptr(pp_elem);
202  }
203  /* Now the pp_elem points to the tail of the list. Add a new element. */
205  size_t ktelem_size = (sizeof(ABTI_ktelem) + ABTU_MAX_ALIGNMENT - 1) &
206  (~(ABTU_MAX_ALIGNMENT - 1));
207  int abt_errno = ABTI_ktable_alloc_elem(p_local, p_ktable, ktelem_size,
208  (void **)&p_elem);
209  if (ABTI_IS_ERROR_CHECK_ENABLED && abt_errno != ABT_SUCCESS) {
210  if (is_safe)
211  ABTI_spinlock_release(&p_ktable->lock);
212  return abt_errno;
213  }
214  p_elem->f_destructor = p_key->f_destructor;
215  p_elem->key_id = p_key->id;
216  p_elem->value = value;
217  ABTD_atomic_relaxed_store_ptr(&p_elem->p_next, NULL);
218  ABTD_atomic_release_store_ptr(pp_elem, p_elem);
219  if (is_safe)
220  ABTI_spinlock_release(&p_ktable->lock);
221  return ABT_SUCCESS;
222 }
223 
224 ABTU_ret_err static inline int ABTI_ktable_set(ABTI_local *p_local,
225  ABTD_atomic_ptr *pp_ktable,
226  ABTI_key *p_key, void *value)
227 {
228  int abt_errno;
229  ABTI_ktable *p_ktable = ABTD_atomic_acquire_load_ptr(pp_ktable);
230  if (ABTU_unlikely(!ABTI_ktable_is_valid(p_ktable))) {
231  /* Spinlock pp_ktable */
232  while (1) {
233  if (ABTD_atomic_bool_cas_weak_ptr(pp_ktable, NULL,
235  /* The lock was acquired, so let's allocate this table. */
236  abt_errno = ABTI_ktable_create(p_local, &p_ktable);
237  ABTI_CHECK_ERROR(abt_errno);
238 
239  /* Write down the value. The lock is released here. */
240  ABTD_atomic_release_store_ptr(pp_ktable, p_ktable);
241  break;
242  } else {
243  /* Failed to take a lock. Check if the value to see if it should
244  * try to take a lock again. */
245  p_ktable = ABTD_atomic_acquire_load_ptr(pp_ktable);
246  if (p_ktable == NULL) {
247  /* Try once more. */
248  continue;
249  }
250  /* It has been locked by another. */
251  while (p_ktable == ABTI_KTABLE_LOCKED) {
253  p_ktable = ABTD_atomic_acquire_load_ptr(pp_ktable);
254  }
255  /* p_ktable has been allocated by another. */
256  break;
257  }
258  }
259  }
260  abt_errno = ABTI_ktable_set_impl(p_local, p_ktable, p_key, value, ABT_TRUE);
261  ABTI_CHECK_ERROR(abt_errno);
262  return ABT_SUCCESS;
263 }
264 
265 ABTU_ret_err static inline int ABTI_ktable_set_unsafe(ABTI_local *p_local,
266  ABTI_ktable **pp_ktable,
267  ABTI_key *p_key,
268  void *value)
269 {
270  int abt_errno;
271  ABTI_ktable *p_ktable = *pp_ktable;
272  if (!p_ktable) {
273  abt_errno = ABTI_ktable_create(p_local, &p_ktable);
274  ABTI_CHECK_ERROR(abt_errno);
275  *pp_ktable = p_ktable;
276  }
277  abt_errno =
278  ABTI_ktable_set_impl(p_local, p_ktable, p_key, value, ABT_FALSE);
279  ABTI_CHECK_ERROR(abt_errno);
280  return ABT_SUCCESS;
281 }
282 
283 static inline void *ABTI_ktable_get(ABTD_atomic_ptr *pp_ktable, ABTI_key *p_key)
284 {
285  ABTI_ktable *p_ktable = ABTD_atomic_acquire_load_ptr(pp_ktable);
286  if (ABTI_ktable_is_valid(p_ktable)) {
287  uint32_t idx;
288  ABTI_ktelem *p_elem;
289 
290  idx = ABTI_ktable_get_idx(p_key, p_ktable->size);
292  &p_ktable->p_elems[idx]);
293  uint32_t key_id = p_key->id;
294  while (p_elem) {
295  if (p_elem->key_id == key_id) {
296  return p_elem->value;
297  }
298  p_elem =
300  }
301  }
302  return NULL;
303 }
304 
305 #endif /* ABTI_KEY_H_INCLUDED */
static void * ABTI_ktable_get(ABTD_atomic_ptr *pp_ktable, ABTI_key *p_key)
Definition: abti_key.h:283
#define ABTU_MAX_ALIGNMENT
Definition: abtu.h:43
ABTI_spinlock lock
Definition: abti.h:369
int key_table_size
Definition: abti.h:181
void * value
Definition: abti.h:363
static void * ABTD_atomic_acquire_load_ptr(const ABTD_atomic_ptr *ptr)
Definition: abtd_atomic.h:848
static ABTU_ret_err int ABTI_ktable_set_impl(ABTI_local *p_local, ABTI_ktable *p_ktable, ABTI_key *p_key, void *value, ABT_bool is_safe)
Definition: abti_key.h:167
#define ABTU_likely(cond)
Definition: abtu.h:17
int size
Definition: abti.h:368
static ABTU_ret_err int ABTI_ktable_alloc_elem(ABTI_local *p_local, ABTI_ktable *p_ktable, size_t size, void **pp_mem)
Definition: abti_key.h:117
static void ABTD_atomic_relaxed_store_ptr(ABTD_atomic_ptr *ptr, void *val)
Definition: abtd_atomic.h:914
struct ABTI_ktelem ABTI_ktelem
Definition: abti.h:119
struct ABTI_local ABTI_local
Definition: abti.h:101
#define ABTU_unlikely(cond)
Definition: abtu.h:18
static uint32_t ABTI_ktable_get_idx(ABTI_key *p_key, int size)
Definition: abti_key.h:161
static ABTI_key * ABTI_key_get_ptr(ABT_key key)
Definition: abti_key.h:11
#define ABT_KEY_NULL
Definition: abt.h:418
int ABT_bool
Definition: abt.h:373
void * p_extra_mem
Definition: abti.h:371
Definition: abti.h:354
static ABT_key ABTI_key_get_handle(ABTI_key *p_key)
Definition: abti_key.h:26
static void ABTI_spinlock_clear(ABTI_spinlock *p_lock)
Definition: abti_spinlock.h:18
static void ABTD_atomic_release_store_ptr(ABTD_atomic_ptr *ptr, void *val)
Definition: abtd_atomic.h:983
uint32_t id
Definition: abti.h:356
static int ABTD_atomic_bool_cas_weak_ptr(ABTD_atomic_ptr *ptr, void *oldv, void *newv)
Definition: abtd_atomic.h:328
void * p_used_mem
Definition: abti.h:370
struct ABT_key_opaque * ABT_key
Definition: abt.h:355
static ABTU_ret_err int ABTU_malloc(size_t size, void **p_ptr)
Definition: abtu.h:129
#define ABT_FALSE
Definition: abt.h:285
uint32_t key_id
Definition: abti.h:362
void(* f_destructor)(void *value)
Definition: abti.h:361
struct ABTI_ktable_mem_header * p_next
Definition: abti_key.h:52
static void ABTI_spinlock_release(ABTI_spinlock *p_lock)
Definition: abti_spinlock.h:31
ABTD_atomic_ptr p_next
Definition: abti.h:364
ABTI_global * gp_ABTI_global
Definition: global.c:18
#define ABT_SUCCESS
Definition: abt.h:64
#define ABT_TRUE
Definition: abt.h:284
struct ABTI_ktable_mem_header ABTI_ktable_mem_header
static ABTU_ret_err int ABTI_ktable_set(ABTI_local *p_local, ABTD_atomic_ptr *pp_ktable, ABTI_key *p_key, void *value)
Definition: abti_key.h:224
static ABTU_ret_err int ABTI_ktable_set_unsafe(ABTI_local *p_local, ABTI_ktable **pp_ktable, ABTI_key *p_key, void *value)
Definition: abti_key.h:265
#define ABTI_KTABLE_LOCKED
Definition: abti_key.h:59
static void ABTI_spinlock_acquire(ABTI_spinlock *p_lock)
Definition: abti_spinlock.h:23
ABTD_atomic_ptr p_elems[1]
Definition: abti.h:373
void(* f_destructor)(void *value)
Definition: abti.h:355
static ABTU_ret_err int ABTI_mem_alloc_desc(ABTI_local *p_local, void **pp_desc)
Definition: abti_mem.h:298
#define ABTI_ASSERT(cond)
Definition: abti_error.h:12
ABT_bool is_from_mempool
Definition: abti_key.h:53
static void ABTD_atomic_pause(void)
Definition: abtd_atomic.h:1091
#define ABTI_KTABLE_DESC_SIZE
Definition: abti_key.h:56
struct ABTD_atomic_ptr ABTD_atomic_ptr
size_t extra_mem_size
Definition: abti.h:372
#define ABTI_CHECK_ERROR(abt_errno)
Definition: abti_error.h:127
static ABTU_ret_err int ABTI_ktable_create(ABTI_local *p_local, ABTI_ktable **pp_ktable)
Definition: abti_key.h:67
#define ABTI_IS_ERROR_CHECK_ENABLED
Definition: abti.h:20
static int ABTI_ktable_is_valid(ABTI_ktable *p_ktable)
Definition: abti_key.h:60
#define ABTI_STATIC_ASSERT(cond)
Definition: abti_error.h:19
#define ABTU_ret_err
Definition: abtu.h:49