Gitly


1 /*
2  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a copy
5  * of this software and associated documentation files (the "Software"), to deal
6  * in the Software without restriction, including without limitation the rights
7  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8  * copies of the Software, and to permit persons to whom the Software is
9  * furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20  * SOFTWARE.
21  */
22 
23 /* char_fetch_compare_and_swap */
24 #if defined(AO_HAVE_char_fetch_compare_and_swap) \
25     && defined(AO_HAVE_nop_full) \
26     && !defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
27   AO_INLINE unsigned/**/char
28   AO_char_fetch_compare_and_swap_acquire(volatile unsigned/**/char *addr,
29                                           unsigned/**/char old_val, unsigned/**/char new_val)
30   {
31     unsigned/**/char result = AO_char_fetch_compare_and_swap(addr, old_val, new_val);
32     AO_nop_full();
33     return result;
34   }
35 # define AO_HAVE_char_fetch_compare_and_swap_acquire
36 #endif
37 #if defined(AO_HAVE_char_fetch_compare_and_swap) \
38     && defined(AO_HAVE_nop_full) \
39     && !defined(AO_HAVE_char_fetch_compare_and_swap_release)
40 # define AO_char_fetch_compare_and_swap_release(addr, old_val, new_val) \
41                 (AO_nop_full(), \
42                  AO_char_fetch_compare_and_swap(addr, old_val, new_val))
43 # define AO_HAVE_char_fetch_compare_and_swap_release
44 #endif
45 #if defined(AO_HAVE_char_fetch_compare_and_swap_full)
46 # if !defined(AO_HAVE_char_fetch_compare_and_swap_release)
47 #   define AO_char_fetch_compare_and_swap_release(addr, old_val, new_val) \
48                 AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
49 #   define AO_HAVE_char_fetch_compare_and_swap_release
50 # endif
51 # if !defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
52 #   define AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
53                 AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
54 #   define AO_HAVE_char_fetch_compare_and_swap_acquire
55 # endif
56 # if !defined(AO_HAVE_char_fetch_compare_and_swap_write)
57 #   define AO_char_fetch_compare_and_swap_write(addr, old_val, new_val) \
58                 AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
59 #   define AO_HAVE_char_fetch_compare_and_swap_write
60 # endif
61 # if !defined(AO_HAVE_char_fetch_compare_and_swap_read)
62 #   define AO_char_fetch_compare_and_swap_read(addr, old_val, new_val) \
63                 AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
64 #   define AO_HAVE_char_fetch_compare_and_swap_read
65 # endif
66 #endif /* AO_HAVE_char_fetch_compare_and_swap_full */
67 
68 #if !defined(AO_HAVE_char_fetch_compare_and_swap) \
69     && defined(AO_HAVE_char_fetch_compare_and_swap_release)
70 # define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
71             AO_char_fetch_compare_and_swap_release(addr, old_val, new_val)
72 # define AO_HAVE_char_fetch_compare_and_swap
73 #endif
74 #if !defined(AO_HAVE_char_fetch_compare_and_swap) \
75     && defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
76 # define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
77             AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val)
78 # define AO_HAVE_char_fetch_compare_and_swap
79 #endif
80 #if !defined(AO_HAVE_char_fetch_compare_and_swap) \
81     && defined(AO_HAVE_char_fetch_compare_and_swap_write)
82 # define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
83                 AO_char_fetch_compare_and_swap_write(addr, old_val, new_val)
84 # define AO_HAVE_char_fetch_compare_and_swap
85 #endif
86 #if !defined(AO_HAVE_char_fetch_compare_and_swap) \
87     && defined(AO_HAVE_char_fetch_compare_and_swap_read)
88 # define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
89                 AO_char_fetch_compare_and_swap_read(addr, old_val, new_val)
90 # define AO_HAVE_char_fetch_compare_and_swap
91 #endif
92 
93 #if defined(AO_HAVE_char_fetch_compare_and_swap_acquire) \
94     && defined(AO_HAVE_nop_full) \
95     && !defined(AO_HAVE_char_fetch_compare_and_swap_full)
96 # define AO_char_fetch_compare_and_swap_full(addr, old_val, new_val) \
97             (AO_nop_full(), \
98              AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val))
99 # define AO_HAVE_char_fetch_compare_and_swap_full
100 #endif
101 
102 #if !defined(AO_HAVE_char_fetch_compare_and_swap_release_write) \
103     && defined(AO_HAVE_char_fetch_compare_and_swap_write)
104 # define AO_char_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
105                 AO_char_fetch_compare_and_swap_write(addr, old_val, new_val)
106 # define AO_HAVE_char_fetch_compare_and_swap_release_write
107 #endif
108 #if !defined(AO_HAVE_char_fetch_compare_and_swap_release_write) \
109     && defined(AO_HAVE_char_fetch_compare_and_swap_release)
110 # define AO_char_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
111             AO_char_fetch_compare_and_swap_release(addr, old_val, new_val)
112 # define AO_HAVE_char_fetch_compare_and_swap_release_write
113 #endif
114 #if !defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read) \
115     && defined(AO_HAVE_char_fetch_compare_and_swap_read)
116 # define AO_char_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
117                 AO_char_fetch_compare_and_swap_read(addr, old_val, new_val)
118 # define AO_HAVE_char_fetch_compare_and_swap_acquire_read
119 #endif
120 #if !defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read) \
121     && defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
122 # define AO_char_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
123             AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val)
124 # define AO_HAVE_char_fetch_compare_and_swap_acquire_read
125 #endif
126 
127 #ifdef AO_NO_DD_ORDERING
128 # if defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read)
129 #   define AO_char_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
130         AO_char_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
131 #   define AO_HAVE_char_fetch_compare_and_swap_dd_acquire_read
132 # endif
133 #else
134 # if defined(AO_HAVE_char_fetch_compare_and_swap)
135 #   define AO_char_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
136                 AO_char_fetch_compare_and_swap(addr, old_val, new_val)
137 #   define AO_HAVE_char_fetch_compare_and_swap_dd_acquire_read
138 # endif
139 #endif /* !AO_NO_DD_ORDERING */
140 
141 /* char_compare_and_swap */
142 #if defined(AO_HAVE_char_compare_and_swap) && defined(AO_HAVE_nop_full) \
143     && !defined(AO_HAVE_char_compare_and_swap_acquire)
144   AO_INLINE int
145   AO_char_compare_and_swap_acquire(volatile unsigned/**/char *addr, unsigned/**/char old,
146                                     unsigned/**/char new_val)
147   {
148     int result = AO_char_compare_and_swap(addr, old, new_val);
149     AO_nop_full();
150     return result;
151   }
152 # define AO_HAVE_char_compare_and_swap_acquire
153 #endif
154 #if defined(AO_HAVE_char_compare_and_swap) && defined(AO_HAVE_nop_full) \
155     && !defined(AO_HAVE_char_compare_and_swap_release)
156 # define AO_char_compare_and_swap_release(addr, old, new_val) \
157                 (AO_nop_full(), AO_char_compare_and_swap(addr, old, new_val))
158 # define AO_HAVE_char_compare_and_swap_release
159 #endif
160 #if defined(AO_HAVE_char_compare_and_swap_full)
161 # if !defined(AO_HAVE_char_compare_and_swap_release)
162 #   define AO_char_compare_and_swap_release(addr, old, new_val) \
163                 AO_char_compare_and_swap_full(addr, old, new_val)
164 #   define AO_HAVE_char_compare_and_swap_release
165 # endif
166 # if !defined(AO_HAVE_char_compare_and_swap_acquire)
167 #   define AO_char_compare_and_swap_acquire(addr, old, new_val) \
168                 AO_char_compare_and_swap_full(addr, old, new_val)
169 #   define AO_HAVE_char_compare_and_swap_acquire
170 # endif
171 # if !defined(AO_HAVE_char_compare_and_swap_write)
172 #   define AO_char_compare_and_swap_write(addr, old, new_val) \
173                 AO_char_compare_and_swap_full(addr, old, new_val)
174 #   define AO_HAVE_char_compare_and_swap_write
175 # endif
176 # if !defined(AO_HAVE_char_compare_and_swap_read)
177 #   define AO_char_compare_and_swap_read(addr, old, new_val) \
178                 AO_char_compare_and_swap_full(addr, old, new_val)
179 #   define AO_HAVE_char_compare_and_swap_read
180 # endif
181 #endif /* AO_HAVE_char_compare_and_swap_full */
182 
183 #if !defined(AO_HAVE_char_compare_and_swap) \
184     && defined(AO_HAVE_char_compare_and_swap_release)
185 # define AO_char_compare_and_swap(addr, old, new_val) \
186                 AO_char_compare_and_swap_release(addr, old, new_val)
187 # define AO_HAVE_char_compare_and_swap
188 #endif
189 #if !defined(AO_HAVE_char_compare_and_swap) \
190     && defined(AO_HAVE_char_compare_and_swap_acquire)
191 # define AO_char_compare_and_swap(addr, old, new_val) \
192                 AO_char_compare_and_swap_acquire(addr, old, new_val)
193 # define AO_HAVE_char_compare_and_swap
194 #endif
195 #if !defined(AO_HAVE_char_compare_and_swap) \
196     && defined(AO_HAVE_char_compare_and_swap_write)
197 # define AO_char_compare_and_swap(addr, old, new_val) \
198                 AO_char_compare_and_swap_write(addr, old, new_val)
199 # define AO_HAVE_char_compare_and_swap
200 #endif
201 #if !defined(AO_HAVE_char_compare_and_swap) \
202     && defined(AO_HAVE_char_compare_and_swap_read)
203 # define AO_char_compare_and_swap(addr, old, new_val) \
204                 AO_char_compare_and_swap_read(addr, old, new_val)
205 # define AO_HAVE_char_compare_and_swap
206 #endif
207 
208 #if defined(AO_HAVE_char_compare_and_swap_acquire) \
209     && defined(AO_HAVE_nop_full) \
210     && !defined(AO_HAVE_char_compare_and_swap_full)
211 # define AO_char_compare_and_swap_full(addr, old, new_val) \
212                 (AO_nop_full(), \
213                  AO_char_compare_and_swap_acquire(addr, old, new_val))
214 # define AO_HAVE_char_compare_and_swap_full
215 #endif
216 
217 #if !defined(AO_HAVE_char_compare_and_swap_release_write) \
218     && defined(AO_HAVE_char_compare_and_swap_write)
219 # define AO_char_compare_and_swap_release_write(addr, old, new_val) \
220                 AO_char_compare_and_swap_write(addr, old, new_val)
221 # define AO_HAVE_char_compare_and_swap_release_write
222 #endif
223 #if !defined(AO_HAVE_char_compare_and_swap_release_write) \
224     && defined(AO_HAVE_char_compare_and_swap_release)
225 # define AO_char_compare_and_swap_release_write(addr, old, new_val) \
226                 AO_char_compare_and_swap_release(addr, old, new_val)
227 # define AO_HAVE_char_compare_and_swap_release_write
228 #endif
229 #if !defined(AO_HAVE_char_compare_and_swap_acquire_read) \
230     && defined(AO_HAVE_char_compare_and_swap_read)
231 # define AO_char_compare_and_swap_acquire_read(addr, old, new_val) \
232                 AO_char_compare_and_swap_read(addr, old, new_val)
233 # define AO_HAVE_char_compare_and_swap_acquire_read
234 #endif
235 #if !defined(AO_HAVE_char_compare_and_swap_acquire_read) \
236     && defined(AO_HAVE_char_compare_and_swap_acquire)
237 # define AO_char_compare_and_swap_acquire_read(addr, old, new_val) \
238                 AO_char_compare_and_swap_acquire(addr, old, new_val)
239 # define AO_HAVE_char_compare_and_swap_acquire_read
240 #endif
241 
242 #ifdef AO_NO_DD_ORDERING
243 # if defined(AO_HAVE_char_compare_and_swap_acquire_read)
244 #   define AO_char_compare_and_swap_dd_acquire_read(addr, old, new_val) \
245                 AO_char_compare_and_swap_acquire_read(addr, old, new_val)
246 #   define AO_HAVE_char_compare_and_swap_dd_acquire_read
247 # endif
248 #else
249 # if defined(AO_HAVE_char_compare_and_swap)
250 #   define AO_char_compare_and_swap_dd_acquire_read(addr, old, new_val) \
251                 AO_char_compare_and_swap(addr, old, new_val)
252 #   define AO_HAVE_char_compare_and_swap_dd_acquire_read
253 # endif
254 #endif /* !AO_NO_DD_ORDERING */
255 
256 /* char_load */
257 #if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_acquire)
258 # define AO_char_load_acquire(addr) AO_char_load_full(addr)
259 # define AO_HAVE_char_load_acquire
260 #endif
261 
262 #if defined(AO_HAVE_char_load_acquire) && !defined(AO_HAVE_char_load)
263 # define AO_char_load(addr) AO_char_load_acquire(addr)
264 # define AO_HAVE_char_load
265 #endif
266 
267 #if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_read)
268 # define AO_char_load_read(addr) AO_char_load_full(addr)
269 # define AO_HAVE_char_load_read
270 #endif
271 
272 #if !defined(AO_HAVE_char_load_acquire_read) \
273     && defined(AO_HAVE_char_load_acquire)
274 # define AO_char_load_acquire_read(addr) AO_char_load_acquire(addr)
275 # define AO_HAVE_char_load_acquire_read
276 #endif
277 
278 #if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_full) \
279     && !defined(AO_HAVE_char_load_acquire)
280   AO_INLINE unsigned/**/char
281   AO_char_load_acquire(const volatile unsigned/**/char *addr)
282   {
283     unsigned/**/char result = AO_char_load(addr);
284 
285     /* Acquire barrier would be useless, since the load could be delayed    */
286     /* beyond it.                                                           */
287     AO_nop_full();
288     return result;
289   }
290 # define AO_HAVE_char_load_acquire
291 #endif
292 
293 #if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_read) \
294     && !defined(AO_HAVE_char_load_read)
295   AO_INLINE unsigned/**/char
296   AO_char_load_read(const volatile unsigned/**/char *addr)
297   {
298     unsigned/**/char result = AO_char_load(addr);
299 
300     AO_nop_read();
301     return result;
302   }
303 # define AO_HAVE_char_load_read
304 #endif
305 
306 #if defined(AO_HAVE_char_load_acquire) && defined(AO_HAVE_nop_full) \
307     && !defined(AO_HAVE_char_load_full)
308 # define AO_char_load_full(addr) (AO_nop_full(), AO_char_load_acquire(addr))
309 # define AO_HAVE_char_load_full
310 #endif
311 
312 #if defined(AO_HAVE_char_compare_and_swap_read) \
313     && !defined(AO_HAVE_char_load_read)
314 # define AO_char_CAS_BASED_LOAD_READ
315   AO_ATTR_NO_SANITIZE_THREAD
316   AO_INLINE unsigned/**/char
317   AO_char_load_read(const volatile unsigned/**/char *addr)
318   {
319     unsigned/**/char result;
320 
321     do {
322       result = *(const unsigned/**/char *)addr;
323     } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_read(
324                                                 (volatile unsigned/**/char *)addr,
325                                                 result, result)));
326     return result;
327   }
328 # define AO_HAVE_char_load_read
329 #endif
330 
331 #if !defined(AO_HAVE_char_load_acquire_read) \
332     && defined(AO_HAVE_char_load_read)
333 # define AO_char_load_acquire_read(addr) AO_char_load_read(addr)
334 # define AO_HAVE_char_load_acquire_read
335 #endif
336 
337 #if defined(AO_HAVE_char_load_acquire_read) && !defined(AO_HAVE_char_load) \
338     && (!defined(AO_char_CAS_BASED_LOAD_READ) \
339         || !defined(AO_HAVE_char_compare_and_swap))
340 # define AO_char_load(addr) AO_char_load_acquire_read(addr)
341 # define AO_HAVE_char_load
342 #endif
343 
344 #if defined(AO_HAVE_char_compare_and_swap_full) \
345     && !defined(AO_HAVE_char_load_full)
346   AO_ATTR_NO_SANITIZE_THREAD
347   AO_INLINE unsigned/**/char
348   AO_char_load_full(const volatile unsigned/**/char *addr)
349   {
350     unsigned/**/char result;
351 
352     do {
353       result = *(const unsigned/**/char *)addr;
354     } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(
355                                                 (volatile unsigned/**/char *)addr,
356                                                 result, result)));
357     return result;
358   }
359 # define AO_HAVE_char_load_full
360 #endif
361 
362 #if defined(AO_HAVE_char_compare_and_swap_acquire) \
363     && !defined(AO_HAVE_char_load_acquire)
364   AO_ATTR_NO_SANITIZE_THREAD
365   AO_INLINE unsigned/**/char
366   AO_char_load_acquire(const volatile unsigned/**/char *addr)
367   {
368     unsigned/**/char result;
369 
370     do {
371       result = *(const unsigned/**/char *)addr;
372     } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(
373                                                 (volatile unsigned/**/char *)addr,
374                                                 result, result)));
375     return result;
376   }
377 # define AO_HAVE_char_load_acquire
378 #endif
379 
380 #if defined(AO_HAVE_char_compare_and_swap) && !defined(AO_HAVE_char_load)
381   AO_ATTR_NO_SANITIZE_THREAD
382   AO_INLINE unsigned/**/char
383   AO_char_load(const volatile unsigned/**/char *addr)
384   {
385     unsigned/**/char result;
386 
387     do {
388       result = *(const unsigned/**/char *)addr;
389     } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(
390                                                 (volatile unsigned/**/char *)addr,
391                                                 result, result)));
392     return result;
393   }
394 # define AO_HAVE_char_load
395 #endif
396 
397 #ifdef AO_NO_DD_ORDERING
398 # if defined(AO_HAVE_char_load_acquire_read)
399 #   define AO_char_load_dd_acquire_read(addr) \
400                                 AO_char_load_acquire_read(addr)
401 #   define AO_HAVE_char_load_dd_acquire_read
402 # endif
403 #else
404 # if defined(AO_HAVE_char_load)
405 #   define AO_char_load_dd_acquire_read(addr) AO_char_load(addr)
406 #   define AO_HAVE_char_load_dd_acquire_read
407 # endif
408 #endif /* !AO_NO_DD_ORDERING */
409 
410 /* char_store */
411 #if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_release)
412 # define AO_char_store_release(addr, val) AO_char_store_full(addr, val)
413 # define AO_HAVE_char_store_release
414 #endif
415 
416 #if defined(AO_HAVE_char_store_release) && !defined(AO_HAVE_char_store)
417 # define AO_char_store(addr, val) AO_char_store_release(addr, val)
418 # define AO_HAVE_char_store
419 #endif
420 
421 #if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_write)
422 # define AO_char_store_write(addr, val) AO_char_store_full(addr, val)
423 # define AO_HAVE_char_store_write
424 #endif
425 
426 #if defined(AO_HAVE_char_store_release) \
427     && !defined(AO_HAVE_char_store_release_write)
428 # define AO_char_store_release_write(addr, val) \
429                                 AO_char_store_release(addr, val)
430 # define AO_HAVE_char_store_release_write
431 #endif
432 
433 #if defined(AO_HAVE_char_store_write) && !defined(AO_HAVE_char_store)
434 # define AO_char_store(addr, val) AO_char_store_write(addr, val)
435 # define AO_HAVE_char_store
436 #endif
437 
438 #if defined(AO_HAVE_char_store) && defined(AO_HAVE_nop_full) \
439     && !defined(AO_HAVE_char_store_release)
440 # define AO_char_store_release(addr, val) \
441                                 (AO_nop_full(), AO_char_store(addr, val))
442 # define AO_HAVE_char_store_release
443 #endif
444 
445 #if defined(AO_HAVE_char_store) && defined(AO_HAVE_nop_write) \
446     && !defined(AO_HAVE_char_store_write)
447 # define AO_char_store_write(addr, val) \
448                                 (AO_nop_write(), AO_char_store(addr, val))
449 # define AO_HAVE_char_store_write
450 #endif
451 
452 #if defined(AO_HAVE_char_compare_and_swap_write) \
453     && !defined(AO_HAVE_char_store_write)
454   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
455   AO_INLINE void
456   AO_char_store_write(volatile unsigned/**/char *addr, unsigned/**/char new_val)
457   {
458     unsigned/**/char old_val;
459 
460     do {
461       old_val = *(unsigned/**/char *)addr;
462     } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_write(addr, old_val,
463                                                               new_val)));
464   }
465 # define AO_HAVE_char_store_write
466 #endif
467 
468 #if defined(AO_HAVE_char_store_write) \
469     && !defined(AO_HAVE_char_store_release_write)
470 # define AO_char_store_release_write(addr, val) \
471                                 AO_char_store_write(addr, val)
472 # define AO_HAVE_char_store_release_write
473 #endif
474 
475 #if defined(AO_HAVE_char_store_release) && defined(AO_HAVE_nop_full) \
476     && !defined(AO_HAVE_char_store_full)
477 # define AO_char_store_full(addr, val) \
478                                 (AO_char_store_release(addr, val), \
479                                  AO_nop_full())
480 # define AO_HAVE_char_store_full
481 #endif
482 
483 #if defined(AO_HAVE_char_compare_and_swap) && !defined(AO_HAVE_char_store)
484   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
485   AO_INLINE void
486   AO_char_store(volatile unsigned/**/char *addr, unsigned/**/char new_val)
487   {
488     unsigned/**/char old_val;
489 
490     do {
491       old_val = *(unsigned/**/char *)addr;
492     } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(addr,
493                                                         old_val, new_val)));
494   }
495 # define AO_HAVE_char_store
496 #endif
497 
498 #if defined(AO_HAVE_char_compare_and_swap_release) \
499     && !defined(AO_HAVE_char_store_release)
500   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
501   AO_INLINE void
502   AO_char_store_release(volatile unsigned/**/char *addr, unsigned/**/char new_val)
503   {
504     unsigned/**/char old_val;
505 
506     do {
507       old_val = *(unsigned/**/char *)addr;
508     } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_release(addr, old_val,
509                                                                 new_val)));
510   }
511 # define AO_HAVE_char_store_release
512 #endif
513 
514 #if defined(AO_HAVE_char_compare_and_swap_full) \
515     && !defined(AO_HAVE_char_store_full)
516   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
517   AO_INLINE void
518   AO_char_store_full(volatile unsigned/**/char *addr, unsigned/**/char new_val)
519   {
520     unsigned/**/char old_val;
521 
522     do {
523       old_val = *(unsigned/**/char *)addr;
524     } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old_val,
525                                                              new_val)));
526   }
527 # define AO_HAVE_char_store_full
528 #endif
529 /*
530  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
531  *
532  * Permission is hereby granted, free of charge, to any person obtaining a copy
533  * of this software and associated documentation files (the "Software"), to deal
534  * in the Software without restriction, including without limitation the rights
535  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
536  * copies of the Software, and to permit persons to whom the Software is
537  * furnished to do so, subject to the following conditions:
538  *
539  * The above copyright notice and this permission notice shall be included in
540  * all copies or substantial portions of the Software.
541  *
542  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
543  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
544  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
545  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
546  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
547  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
548  * SOFTWARE.
549  */
550 
551 /* short_fetch_compare_and_swap */
552 #if defined(AO_HAVE_short_fetch_compare_and_swap) \
553     && defined(AO_HAVE_nop_full) \
554     && !defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
555   AO_INLINE unsigned/**/short
556   AO_short_fetch_compare_and_swap_acquire(volatile unsigned/**/short *addr,
557                                           unsigned/**/short old_val, unsigned/**/short new_val)
558   {
559     unsigned/**/short result = AO_short_fetch_compare_and_swap(addr, old_val, new_val);
560     AO_nop_full();
561     return result;
562   }
563 # define AO_HAVE_short_fetch_compare_and_swap_acquire
564 #endif
565 #if defined(AO_HAVE_short_fetch_compare_and_swap) \
566     && defined(AO_HAVE_nop_full) \
567     && !defined(AO_HAVE_short_fetch_compare_and_swap_release)
568 # define AO_short_fetch_compare_and_swap_release(addr, old_val, new_val) \
569                 (AO_nop_full(), \
570                  AO_short_fetch_compare_and_swap(addr, old_val, new_val))
571 # define AO_HAVE_short_fetch_compare_and_swap_release
572 #endif
573 #if defined(AO_HAVE_short_fetch_compare_and_swap_full)
574 # if !defined(AO_HAVE_short_fetch_compare_and_swap_release)
575 #   define AO_short_fetch_compare_and_swap_release(addr, old_val, new_val) \
576                 AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
577 #   define AO_HAVE_short_fetch_compare_and_swap_release
578 # endif
579 # if !defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
580 #   define AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
581                 AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
582 #   define AO_HAVE_short_fetch_compare_and_swap_acquire
583 # endif
584 # if !defined(AO_HAVE_short_fetch_compare_and_swap_write)
585 #   define AO_short_fetch_compare_and_swap_write(addr, old_val, new_val) \
586                 AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
587 #   define AO_HAVE_short_fetch_compare_and_swap_write
588 # endif
589 # if !defined(AO_HAVE_short_fetch_compare_and_swap_read)
590 #   define AO_short_fetch_compare_and_swap_read(addr, old_val, new_val) \
591                 AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
592 #   define AO_HAVE_short_fetch_compare_and_swap_read
593 # endif
594 #endif /* AO_HAVE_short_fetch_compare_and_swap_full */
595 
596 #if !defined(AO_HAVE_short_fetch_compare_and_swap) \
597     && defined(AO_HAVE_short_fetch_compare_and_swap_release)
598 # define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
599             AO_short_fetch_compare_and_swap_release(addr, old_val, new_val)
600 # define AO_HAVE_short_fetch_compare_and_swap
601 #endif
602 #if !defined(AO_HAVE_short_fetch_compare_and_swap) \
603     && defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
604 # define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
605             AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val)
606 # define AO_HAVE_short_fetch_compare_and_swap
607 #endif
608 #if !defined(AO_HAVE_short_fetch_compare_and_swap) \
609     && defined(AO_HAVE_short_fetch_compare_and_swap_write)
610 # define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
611                 AO_short_fetch_compare_and_swap_write(addr, old_val, new_val)
612 # define AO_HAVE_short_fetch_compare_and_swap
613 #endif
614 #if !defined(AO_HAVE_short_fetch_compare_and_swap) \
615     && defined(AO_HAVE_short_fetch_compare_and_swap_read)
616 # define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
617                 AO_short_fetch_compare_and_swap_read(addr, old_val, new_val)
618 # define AO_HAVE_short_fetch_compare_and_swap
619 #endif
620 
621 #if defined(AO_HAVE_short_fetch_compare_and_swap_acquire) \
622     && defined(AO_HAVE_nop_full) \
623     && !defined(AO_HAVE_short_fetch_compare_and_swap_full)
624 # define AO_short_fetch_compare_and_swap_full(addr, old_val, new_val) \
625             (AO_nop_full(), \
626              AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val))
627 # define AO_HAVE_short_fetch_compare_and_swap_full
628 #endif
629 
630 #if !defined(AO_HAVE_short_fetch_compare_and_swap_release_write) \
631     && defined(AO_HAVE_short_fetch_compare_and_swap_write)
632 # define AO_short_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
633                 AO_short_fetch_compare_and_swap_write(addr, old_val, new_val)
634 # define AO_HAVE_short_fetch_compare_and_swap_release_write
635 #endif
636 #if !defined(AO_HAVE_short_fetch_compare_and_swap_release_write) \
637     && defined(AO_HAVE_short_fetch_compare_and_swap_release)
638 # define AO_short_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
639             AO_short_fetch_compare_and_swap_release(addr, old_val, new_val)
640 # define AO_HAVE_short_fetch_compare_and_swap_release_write
641 #endif
642 #if !defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read) \
643     && defined(AO_HAVE_short_fetch_compare_and_swap_read)
644 # define AO_short_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
645                 AO_short_fetch_compare_and_swap_read(addr, old_val, new_val)
646 # define AO_HAVE_short_fetch_compare_and_swap_acquire_read
647 #endif
648 #if !defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read) \
649     && defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
650 # define AO_short_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
651             AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val)
652 # define AO_HAVE_short_fetch_compare_and_swap_acquire_read
653 #endif
654 
655 #ifdef AO_NO_DD_ORDERING
656 # if defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read)
657 #   define AO_short_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
658         AO_short_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
659 #   define AO_HAVE_short_fetch_compare_and_swap_dd_acquire_read
660 # endif
661 #else
662 # if defined(AO_HAVE_short_fetch_compare_and_swap)
663 #   define AO_short_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
664                 AO_short_fetch_compare_and_swap(addr, old_val, new_val)
665 #   define AO_HAVE_short_fetch_compare_and_swap_dd_acquire_read
666 # endif
667 #endif /* !AO_NO_DD_ORDERING */
668 
669 /* short_compare_and_swap */
670 #if defined(AO_HAVE_short_compare_and_swap) && defined(AO_HAVE_nop_full) \
671     && !defined(AO_HAVE_short_compare_and_swap_acquire)
672   AO_INLINE int
673   AO_short_compare_and_swap_acquire(volatile unsigned/**/short *addr, unsigned/**/short old,
674                                     unsigned/**/short new_val)
675   {
676     int result = AO_short_compare_and_swap(addr, old, new_val);
677     AO_nop_full();
678     return result;
679   }
680 # define AO_HAVE_short_compare_and_swap_acquire
681 #endif
682 #if defined(AO_HAVE_short_compare_and_swap) && defined(AO_HAVE_nop_full) \
683     && !defined(AO_HAVE_short_compare_and_swap_release)
684 # define AO_short_compare_and_swap_release(addr, old, new_val) \
685                 (AO_nop_full(), AO_short_compare_and_swap(addr, old, new_val))
686 # define AO_HAVE_short_compare_and_swap_release
687 #endif
688 #if defined(AO_HAVE_short_compare_and_swap_full)
689 # if !defined(AO_HAVE_short_compare_and_swap_release)
690 #   define AO_short_compare_and_swap_release(addr, old, new_val) \
691                 AO_short_compare_and_swap_full(addr, old, new_val)
692 #   define AO_HAVE_short_compare_and_swap_release
693 # endif
694 # if !defined(AO_HAVE_short_compare_and_swap_acquire)
695 #   define AO_short_compare_and_swap_acquire(addr, old, new_val) \
696                 AO_short_compare_and_swap_full(addr, old, new_val)
697 #   define AO_HAVE_short_compare_and_swap_acquire
698 # endif
699 # if !defined(AO_HAVE_short_compare_and_swap_write)
700 #   define AO_short_compare_and_swap_write(addr, old, new_val) \
701                 AO_short_compare_and_swap_full(addr, old, new_val)
702 #   define AO_HAVE_short_compare_and_swap_write
703 # endif
704 # if !defined(AO_HAVE_short_compare_and_swap_read)
705 #   define AO_short_compare_and_swap_read(addr, old, new_val) \
706                 AO_short_compare_and_swap_full(addr, old, new_val)
707 #   define AO_HAVE_short_compare_and_swap_read
708 # endif
709 #endif /* AO_HAVE_short_compare_and_swap_full */
710 
711 #if !defined(AO_HAVE_short_compare_and_swap) \
712     && defined(AO_HAVE_short_compare_and_swap_release)
713 # define AO_short_compare_and_swap(addr, old, new_val) \
714                 AO_short_compare_and_swap_release(addr, old, new_val)
715 # define AO_HAVE_short_compare_and_swap
716 #endif
717 #if !defined(AO_HAVE_short_compare_and_swap) \
718     && defined(AO_HAVE_short_compare_and_swap_acquire)
719 # define AO_short_compare_and_swap(addr, old, new_val) \
720                 AO_short_compare_and_swap_acquire(addr, old, new_val)
721 # define AO_HAVE_short_compare_and_swap
722 #endif
723 #if !defined(AO_HAVE_short_compare_and_swap) \
724     && defined(AO_HAVE_short_compare_and_swap_write)
725 # define AO_short_compare_and_swap(addr, old, new_val) \
726                 AO_short_compare_and_swap_write(addr, old, new_val)
727 # define AO_HAVE_short_compare_and_swap
728 #endif
729 #if !defined(AO_HAVE_short_compare_and_swap) \
730     && defined(AO_HAVE_short_compare_and_swap_read)
731 # define AO_short_compare_and_swap(addr, old, new_val) \
732                 AO_short_compare_and_swap_read(addr, old, new_val)
733 # define AO_HAVE_short_compare_and_swap
734 #endif
735 
736 #if defined(AO_HAVE_short_compare_and_swap_acquire) \
737     && defined(AO_HAVE_nop_full) \
738     && !defined(AO_HAVE_short_compare_and_swap_full)
739 # define AO_short_compare_and_swap_full(addr, old, new_val) \
740                 (AO_nop_full(), \
741                  AO_short_compare_and_swap_acquire(addr, old, new_val))
742 # define AO_HAVE_short_compare_and_swap_full
743 #endif
744 
745 #if !defined(AO_HAVE_short_compare_and_swap_release_write) \
746     && defined(AO_HAVE_short_compare_and_swap_write)
747 # define AO_short_compare_and_swap_release_write(addr, old, new_val) \
748                 AO_short_compare_and_swap_write(addr, old, new_val)
749 # define AO_HAVE_short_compare_and_swap_release_write
750 #endif
751 #if !defined(AO_HAVE_short_compare_and_swap_release_write) \
752     && defined(AO_HAVE_short_compare_and_swap_release)
753 # define AO_short_compare_and_swap_release_write(addr, old, new_val) \
754                 AO_short_compare_and_swap_release(addr, old, new_val)
755 # define AO_HAVE_short_compare_and_swap_release_write
756 #endif
757 #if !defined(AO_HAVE_short_compare_and_swap_acquire_read) \
758     && defined(AO_HAVE_short_compare_and_swap_read)
759 # define AO_short_compare_and_swap_acquire_read(addr, old, new_val) \
760                 AO_short_compare_and_swap_read(addr, old, new_val)
761 # define AO_HAVE_short_compare_and_swap_acquire_read
762 #endif
763 #if !defined(AO_HAVE_short_compare_and_swap_acquire_read) \
764     && defined(AO_HAVE_short_compare_and_swap_acquire)
765 # define AO_short_compare_and_swap_acquire_read(addr, old, new_val) \
766                 AO_short_compare_and_swap_acquire(addr, old, new_val)
767 # define AO_HAVE_short_compare_and_swap_acquire_read
768 #endif
769 
770 #ifdef AO_NO_DD_ORDERING
771 # if defined(AO_HAVE_short_compare_and_swap_acquire_read)
772 #   define AO_short_compare_and_swap_dd_acquire_read(addr, old, new_val) \
773                 AO_short_compare_and_swap_acquire_read(addr, old, new_val)
774 #   define AO_HAVE_short_compare_and_swap_dd_acquire_read
775 # endif
776 #else
777 # if defined(AO_HAVE_short_compare_and_swap)
778 #   define AO_short_compare_and_swap_dd_acquire_read(addr, old, new_val) \
779                 AO_short_compare_and_swap(addr, old, new_val)
780 #   define AO_HAVE_short_compare_and_swap_dd_acquire_read
781 # endif
782 #endif /* !AO_NO_DD_ORDERING */
783 
784 /* short_load */
785 #if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_acquire)
786 # define AO_short_load_acquire(addr) AO_short_load_full(addr)
787 # define AO_HAVE_short_load_acquire
788 #endif
789 
790 #if defined(AO_HAVE_short_load_acquire) && !defined(AO_HAVE_short_load)
791 # define AO_short_load(addr) AO_short_load_acquire(addr)
792 # define AO_HAVE_short_load
793 #endif
794 
795 #if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_read)
796 # define AO_short_load_read(addr) AO_short_load_full(addr)
797 # define AO_HAVE_short_load_read
798 #endif
799 
800 #if !defined(AO_HAVE_short_load_acquire_read) \
801     && defined(AO_HAVE_short_load_acquire)
802 # define AO_short_load_acquire_read(addr) AO_short_load_acquire(addr)
803 # define AO_HAVE_short_load_acquire_read
804 #endif
805 
806 #if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_full) \
807     && !defined(AO_HAVE_short_load_acquire)
808   AO_INLINE unsigned/**/short
809   AO_short_load_acquire(const volatile unsigned/**/short *addr)
810   {
811     unsigned/**/short result = AO_short_load(addr);
812 
813     /* Acquire barrier would be useless, since the load could be delayed    */
814     /* beyond it.                                                           */
815     AO_nop_full();
816     return result;
817   }
818 # define AO_HAVE_short_load_acquire
819 #endif
820 
821 #if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_read) \
822     && !defined(AO_HAVE_short_load_read)
823   AO_INLINE unsigned/**/short
824   AO_short_load_read(const volatile unsigned/**/short *addr)
825   {
826     unsigned/**/short result = AO_short_load(addr);
827 
828     AO_nop_read();
829     return result;
830   }
831 # define AO_HAVE_short_load_read
832 #endif
833 
834 #if defined(AO_HAVE_short_load_acquire) && defined(AO_HAVE_nop_full) \
835     && !defined(AO_HAVE_short_load_full)
836 # define AO_short_load_full(addr) (AO_nop_full(), AO_short_load_acquire(addr))
837 # define AO_HAVE_short_load_full
838 #endif
839 
840 #if defined(AO_HAVE_short_compare_and_swap_read) \
841     && !defined(AO_HAVE_short_load_read)
842 # define AO_short_CAS_BASED_LOAD_READ
843   AO_ATTR_NO_SANITIZE_THREAD
844   AO_INLINE unsigned/**/short
845   AO_short_load_read(const volatile unsigned/**/short *addr)
846   {
847     unsigned/**/short result;
848 
849     do {
850       result = *(const unsigned/**/short *)addr;
851     } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_read(
852                                                 (volatile unsigned/**/short *)addr,
853                                                 result, result)));
854     return result;
855   }
856 # define AO_HAVE_short_load_read
857 #endif
858 
859 #if !defined(AO_HAVE_short_load_acquire_read) \
860     && defined(AO_HAVE_short_load_read)
861 # define AO_short_load_acquire_read(addr) AO_short_load_read(addr)
862 # define AO_HAVE_short_load_acquire_read
863 #endif
864 
865 #if defined(AO_HAVE_short_load_acquire_read) && !defined(AO_HAVE_short_load) \
866     && (!defined(AO_short_CAS_BASED_LOAD_READ) \
867         || !defined(AO_HAVE_short_compare_and_swap))
868 # define AO_short_load(addr) AO_short_load_acquire_read(addr)
869 # define AO_HAVE_short_load
870 #endif
871 
872 #if defined(AO_HAVE_short_compare_and_swap_full) \
873     && !defined(AO_HAVE_short_load_full)
874   AO_ATTR_NO_SANITIZE_THREAD
875   AO_INLINE unsigned/**/short
876   AO_short_load_full(const volatile unsigned/**/short *addr)
877   {
878     unsigned/**/short result;
879 
880     do {
881       result = *(const unsigned/**/short *)addr;
882     } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(
883                                                 (volatile unsigned/**/short *)addr,
884                                                 result, result)));
885     return result;
886   }
887 # define AO_HAVE_short_load_full
888 #endif
889 
890 #if defined(AO_HAVE_short_compare_and_swap_acquire) \
891     && !defined(AO_HAVE_short_load_acquire)
892   AO_ATTR_NO_SANITIZE_THREAD
893   AO_INLINE unsigned/**/short
894   AO_short_load_acquire(const volatile unsigned/**/short *addr)
895   {
896     unsigned/**/short result;
897 
898     do {
899       result = *(const unsigned/**/short *)addr;
900     } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(
901                                                 (volatile unsigned/**/short *)addr,
902                                                 result, result)));
903     return result;
904   }
905 # define AO_HAVE_short_load_acquire
906 #endif
907 
908 #if defined(AO_HAVE_short_compare_and_swap) && !defined(AO_HAVE_short_load)
909   AO_ATTR_NO_SANITIZE_THREAD
910   AO_INLINE unsigned/**/short
911   AO_short_load(const volatile unsigned/**/short *addr)
912   {
913     unsigned/**/short result;
914 
915     do {
916       result = *(const unsigned/**/short *)addr;
917     } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(
918                                                 (volatile unsigned/**/short *)addr,
919                                                 result, result)));
920     return result;
921   }
922 # define AO_HAVE_short_load
923 #endif
924 
925 #ifdef AO_NO_DD_ORDERING
926 # if defined(AO_HAVE_short_load_acquire_read)
927 #   define AO_short_load_dd_acquire_read(addr) \
928                                 AO_short_load_acquire_read(addr)
929 #   define AO_HAVE_short_load_dd_acquire_read
930 # endif
931 #else
932 # if defined(AO_HAVE_short_load)
933 #   define AO_short_load_dd_acquire_read(addr) AO_short_load(addr)
934 #   define AO_HAVE_short_load_dd_acquire_read
935 # endif
936 #endif /* !AO_NO_DD_ORDERING */
937 
938 /* short_store */
939 #if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_release)
940 # define AO_short_store_release(addr, val) AO_short_store_full(addr, val)
941 # define AO_HAVE_short_store_release
942 #endif
943 
944 #if defined(AO_HAVE_short_store_release) && !defined(AO_HAVE_short_store)
945 # define AO_short_store(addr, val) AO_short_store_release(addr, val)
946 # define AO_HAVE_short_store
947 #endif
948 
949 #if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_write)
950 # define AO_short_store_write(addr, val) AO_short_store_full(addr, val)
951 # define AO_HAVE_short_store_write
952 #endif
953 
954 #if defined(AO_HAVE_short_store_release) \
955     && !defined(AO_HAVE_short_store_release_write)
956 # define AO_short_store_release_write(addr, val) \
957                                 AO_short_store_release(addr, val)
958 # define AO_HAVE_short_store_release_write
959 #endif
960 
961 #if defined(AO_HAVE_short_store_write) && !defined(AO_HAVE_short_store)
962 # define AO_short_store(addr, val) AO_short_store_write(addr, val)
963 # define AO_HAVE_short_store
964 #endif
965 
966 #if defined(AO_HAVE_short_store) && defined(AO_HAVE_nop_full) \
967     && !defined(AO_HAVE_short_store_release)
968 # define AO_short_store_release(addr, val) \
969                                 (AO_nop_full(), AO_short_store(addr, val))
970 # define AO_HAVE_short_store_release
971 #endif
972 
973 #if defined(AO_HAVE_short_store) && defined(AO_HAVE_nop_write) \
974     && !defined(AO_HAVE_short_store_write)
975 # define AO_short_store_write(addr, val) \
976                                 (AO_nop_write(), AO_short_store(addr, val))
977 # define AO_HAVE_short_store_write
978 #endif
979 
980 #if defined(AO_HAVE_short_compare_and_swap_write) \
981     && !defined(AO_HAVE_short_store_write)
982   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
983   AO_INLINE void
984   AO_short_store_write(volatile unsigned/**/short *addr, unsigned/**/short new_val)
985   {
986     unsigned/**/short old_val;
987 
988     do {
989       old_val = *(unsigned/**/short *)addr;
990     } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_write(addr, old_val,
991                                                               new_val)));
992   }
993 # define AO_HAVE_short_store_write
994 #endif
995 
996 #if defined(AO_HAVE_short_store_write) \
997     && !defined(AO_HAVE_short_store_release_write)
998 # define AO_short_store_release_write(addr, val) \
999                                 AO_short_store_write(addr, val)
1000 # define AO_HAVE_short_store_release_write
1001 #endif
1002 
1003 #if defined(AO_HAVE_short_store_release) && defined(AO_HAVE_nop_full) \
1004     && !defined(AO_HAVE_short_store_full)
1005 # define AO_short_store_full(addr, val) \
1006                                 (AO_short_store_release(addr, val), \
1007                                  AO_nop_full())
1008 # define AO_HAVE_short_store_full
1009 #endif
1010 
1011 #if defined(AO_HAVE_short_compare_and_swap) && !defined(AO_HAVE_short_store)
1012   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1013   AO_INLINE void
1014   AO_short_store(volatile unsigned/**/short *addr, unsigned/**/short new_val)
1015   {
1016     unsigned/**/short old_val;
1017 
1018     do {
1019       old_val = *(unsigned/**/short *)addr;
1020     } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(addr,
1021                                                         old_val, new_val)));
1022   }
1023 # define AO_HAVE_short_store
1024 #endif
1025 
1026 #if defined(AO_HAVE_short_compare_and_swap_release) \
1027     && !defined(AO_HAVE_short_store_release)
1028   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1029   AO_INLINE void
1030   AO_short_store_release(volatile unsigned/**/short *addr, unsigned/**/short new_val)
1031   {
1032     unsigned/**/short old_val;
1033 
1034     do {
1035       old_val = *(unsigned/**/short *)addr;
1036     } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_release(addr, old_val,
1037                                                                 new_val)));
1038   }
1039 # define AO_HAVE_short_store_release
1040 #endif
1041 
1042 #if defined(AO_HAVE_short_compare_and_swap_full) \
1043     && !defined(AO_HAVE_short_store_full)
1044   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1045   AO_INLINE void
1046   AO_short_store_full(volatile unsigned/**/short *addr, unsigned/**/short new_val)
1047   {
1048     unsigned/**/short old_val;
1049 
1050     do {
1051       old_val = *(unsigned/**/short *)addr;
1052     } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old_val,
1053                                                              new_val)));
1054   }
1055 # define AO_HAVE_short_store_full
1056 #endif
1057 /*
1058  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
1059  *
1060  * Permission is hereby granted, free of charge, to any person obtaining a copy
1061  * of this software and associated documentation files (the "Software"), to deal
1062  * in the Software without restriction, including without limitation the rights
1063  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1064  * copies of the Software, and to permit persons to whom the Software is
1065  * furnished to do so, subject to the following conditions:
1066  *
1067  * The above copyright notice and this permission notice shall be included in
1068  * all copies or substantial portions of the Software.
1069  *
1070  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1071  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1072  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
1073  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1074  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
1075  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
1076  * SOFTWARE.
1077  */
1078 
1079 /* int_fetch_compare_and_swap */
1080 #if defined(AO_HAVE_int_fetch_compare_and_swap) \
1081     && defined(AO_HAVE_nop_full) \
1082     && !defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1083   AO_INLINE unsigned
1084   AO_int_fetch_compare_and_swap_acquire(volatile unsigned *addr,
1085                                           unsigned old_val, unsigned new_val)
1086   {
1087     unsigned result = AO_int_fetch_compare_and_swap(addr, old_val, new_val);
1088     AO_nop_full();
1089     return result;
1090   }
1091 # define AO_HAVE_int_fetch_compare_and_swap_acquire
1092 #endif
1093 #if defined(AO_HAVE_int_fetch_compare_and_swap) \
1094     && defined(AO_HAVE_nop_full) \
1095     && !defined(AO_HAVE_int_fetch_compare_and_swap_release)
1096 # define AO_int_fetch_compare_and_swap_release(addr, old_val, new_val) \
1097                 (AO_nop_full(), \
1098                  AO_int_fetch_compare_and_swap(addr, old_val, new_val))
1099 # define AO_HAVE_int_fetch_compare_and_swap_release
1100 #endif
1101 #if defined(AO_HAVE_int_fetch_compare_and_swap_full)
1102 # if !defined(AO_HAVE_int_fetch_compare_and_swap_release)
1103 #   define AO_int_fetch_compare_and_swap_release(addr, old_val, new_val) \
1104                 AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1105 #   define AO_HAVE_int_fetch_compare_and_swap_release
1106 # endif
1107 # if !defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1108 #   define AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
1109                 AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1110 #   define AO_HAVE_int_fetch_compare_and_swap_acquire
1111 # endif
1112 # if !defined(AO_HAVE_int_fetch_compare_and_swap_write)
1113 #   define AO_int_fetch_compare_and_swap_write(addr, old_val, new_val) \
1114                 AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1115 #   define AO_HAVE_int_fetch_compare_and_swap_write
1116 # endif
1117 # if !defined(AO_HAVE_int_fetch_compare_and_swap_read)
1118 #   define AO_int_fetch_compare_and_swap_read(addr, old_val, new_val) \
1119                 AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1120 #   define AO_HAVE_int_fetch_compare_and_swap_read
1121 # endif
1122 #endif /* AO_HAVE_int_fetch_compare_and_swap_full */
1123 
1124 #if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1125     && defined(AO_HAVE_int_fetch_compare_and_swap_release)
1126 # define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1127             AO_int_fetch_compare_and_swap_release(addr, old_val, new_val)
1128 # define AO_HAVE_int_fetch_compare_and_swap
1129 #endif
1130 #if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1131     && defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1132 # define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1133             AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1134 # define AO_HAVE_int_fetch_compare_and_swap
1135 #endif
1136 #if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1137     && defined(AO_HAVE_int_fetch_compare_and_swap_write)
1138 # define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1139                 AO_int_fetch_compare_and_swap_write(addr, old_val, new_val)
1140 # define AO_HAVE_int_fetch_compare_and_swap
1141 #endif
1142 #if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1143     && defined(AO_HAVE_int_fetch_compare_and_swap_read)
1144 # define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1145                 AO_int_fetch_compare_and_swap_read(addr, old_val, new_val)
1146 # define AO_HAVE_int_fetch_compare_and_swap
1147 #endif
1148 
1149 #if defined(AO_HAVE_int_fetch_compare_and_swap_acquire) \
1150     && defined(AO_HAVE_nop_full) \
1151     && !defined(AO_HAVE_int_fetch_compare_and_swap_full)
1152 # define AO_int_fetch_compare_and_swap_full(addr, old_val, new_val) \
1153             (AO_nop_full(), \
1154              AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val))
1155 # define AO_HAVE_int_fetch_compare_and_swap_full
1156 #endif
1157 
1158 #if !defined(AO_HAVE_int_fetch_compare_and_swap_release_write) \
1159     && defined(AO_HAVE_int_fetch_compare_and_swap_write)
1160 # define AO_int_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1161                 AO_int_fetch_compare_and_swap_write(addr, old_val, new_val)
1162 # define AO_HAVE_int_fetch_compare_and_swap_release_write
1163 #endif
1164 #if !defined(AO_HAVE_int_fetch_compare_and_swap_release_write) \
1165     && defined(AO_HAVE_int_fetch_compare_and_swap_release)
1166 # define AO_int_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1167             AO_int_fetch_compare_and_swap_release(addr, old_val, new_val)
1168 # define AO_HAVE_int_fetch_compare_and_swap_release_write
1169 #endif
1170 #if !defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read) \
1171     && defined(AO_HAVE_int_fetch_compare_and_swap_read)
1172 # define AO_int_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1173                 AO_int_fetch_compare_and_swap_read(addr, old_val, new_val)
1174 # define AO_HAVE_int_fetch_compare_and_swap_acquire_read
1175 #endif
1176 #if !defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read) \
1177     && defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1178 # define AO_int_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1179             AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1180 # define AO_HAVE_int_fetch_compare_and_swap_acquire_read
1181 #endif
1182 
1183 #ifdef AO_NO_DD_ORDERING
1184 # if defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read)
1185 #   define AO_int_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1186         AO_int_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
1187 #   define AO_HAVE_int_fetch_compare_and_swap_dd_acquire_read
1188 # endif
1189 #else
1190 # if defined(AO_HAVE_int_fetch_compare_and_swap)
1191 #   define AO_int_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1192                 AO_int_fetch_compare_and_swap(addr, old_val, new_val)
1193 #   define AO_HAVE_int_fetch_compare_and_swap_dd_acquire_read
1194 # endif
1195 #endif /* !AO_NO_DD_ORDERING */
1196 
1197 /* int_compare_and_swap */
1198 #if defined(AO_HAVE_int_compare_and_swap) && defined(AO_HAVE_nop_full) \
1199     && !defined(AO_HAVE_int_compare_and_swap_acquire)
1200   AO_INLINE int
1201   AO_int_compare_and_swap_acquire(volatile unsigned *addr, unsigned old,
1202                                     unsigned new_val)
1203   {
1204     int result = AO_int_compare_and_swap(addr, old, new_val);
1205     AO_nop_full();
1206     return result;
1207   }
1208 # define AO_HAVE_int_compare_and_swap_acquire
1209 #endif
1210 #if defined(AO_HAVE_int_compare_and_swap) && defined(AO_HAVE_nop_full) \
1211     && !defined(AO_HAVE_int_compare_and_swap_release)
1212 # define AO_int_compare_and_swap_release(addr, old, new_val) \
1213                 (AO_nop_full(), AO_int_compare_and_swap(addr, old, new_val))
1214 # define AO_HAVE_int_compare_and_swap_release
1215 #endif
1216 #if defined(AO_HAVE_int_compare_and_swap_full)
1217 # if !defined(AO_HAVE_int_compare_and_swap_release)
1218 #   define AO_int_compare_and_swap_release(addr, old, new_val) \
1219                 AO_int_compare_and_swap_full(addr, old, new_val)
1220 #   define AO_HAVE_int_compare_and_swap_release
1221 # endif
1222 # if !defined(AO_HAVE_int_compare_and_swap_acquire)
1223 #   define AO_int_compare_and_swap_acquire(addr, old, new_val) \
1224                 AO_int_compare_and_swap_full(addr, old, new_val)
1225 #   define AO_HAVE_int_compare_and_swap_acquire
1226 # endif
1227 # if !defined(AO_HAVE_int_compare_and_swap_write)
1228 #   define AO_int_compare_and_swap_write(addr, old, new_val) \
1229                 AO_int_compare_and_swap_full(addr, old, new_val)
1230 #   define AO_HAVE_int_compare_and_swap_write
1231 # endif
1232 # if !defined(AO_HAVE_int_compare_and_swap_read)
1233 #   define AO_int_compare_and_swap_read(addr, old, new_val) \
1234                 AO_int_compare_and_swap_full(addr, old, new_val)
1235 #   define AO_HAVE_int_compare_and_swap_read
1236 # endif
1237 #endif /* AO_HAVE_int_compare_and_swap_full */
1238 
1239 #if !defined(AO_HAVE_int_compare_and_swap) \
1240     && defined(AO_HAVE_int_compare_and_swap_release)
1241 # define AO_int_compare_and_swap(addr, old, new_val) \
1242                 AO_int_compare_and_swap_release(addr, old, new_val)
1243 # define AO_HAVE_int_compare_and_swap
1244 #endif
1245 #if !defined(AO_HAVE_int_compare_and_swap) \
1246     && defined(AO_HAVE_int_compare_and_swap_acquire)
1247 # define AO_int_compare_and_swap(addr, old, new_val) \
1248                 AO_int_compare_and_swap_acquire(addr, old, new_val)
1249 # define AO_HAVE_int_compare_and_swap
1250 #endif
1251 #if !defined(AO_HAVE_int_compare_and_swap) \
1252     && defined(AO_HAVE_int_compare_and_swap_write)
1253 # define AO_int_compare_and_swap(addr, old, new_val) \
1254                 AO_int_compare_and_swap_write(addr, old, new_val)
1255 # define AO_HAVE_int_compare_and_swap
1256 #endif
1257 #if !defined(AO_HAVE_int_compare_and_swap) \
1258     && defined(AO_HAVE_int_compare_and_swap_read)
1259 # define AO_int_compare_and_swap(addr, old, new_val) \
1260                 AO_int_compare_and_swap_read(addr, old, new_val)
1261 # define AO_HAVE_int_compare_and_swap
1262 #endif
1263 
1264 #if defined(AO_HAVE_int_compare_and_swap_acquire) \
1265     && defined(AO_HAVE_nop_full) \
1266     && !defined(AO_HAVE_int_compare_and_swap_full)
1267 # define AO_int_compare_and_swap_full(addr, old, new_val) \
1268                 (AO_nop_full(), \
1269                  AO_int_compare_and_swap_acquire(addr, old, new_val))
1270 # define AO_HAVE_int_compare_and_swap_full
1271 #endif
1272 
1273 #if !defined(AO_HAVE_int_compare_and_swap_release_write) \
1274     && defined(AO_HAVE_int_compare_and_swap_write)
1275 # define AO_int_compare_and_swap_release_write(addr, old, new_val) \
1276                 AO_int_compare_and_swap_write(addr, old, new_val)
1277 # define AO_HAVE_int_compare_and_swap_release_write
1278 #endif
1279 #if !defined(AO_HAVE_int_compare_and_swap_release_write) \
1280     && defined(AO_HAVE_int_compare_and_swap_release)
1281 # define AO_int_compare_and_swap_release_write(addr, old, new_val) \
1282                 AO_int_compare_and_swap_release(addr, old, new_val)
1283 # define AO_HAVE_int_compare_and_swap_release_write
1284 #endif
1285 #if !defined(AO_HAVE_int_compare_and_swap_acquire_read) \
1286     && defined(AO_HAVE_int_compare_and_swap_read)
1287 # define AO_int_compare_and_swap_acquire_read(addr, old, new_val) \
1288                 AO_int_compare_and_swap_read(addr, old, new_val)
1289 # define AO_HAVE_int_compare_and_swap_acquire_read
1290 #endif
1291 #if !defined(AO_HAVE_int_compare_and_swap_acquire_read) \
1292     && defined(AO_HAVE_int_compare_and_swap_acquire)
1293 # define AO_int_compare_and_swap_acquire_read(addr, old, new_val) \
1294                 AO_int_compare_and_swap_acquire(addr, old, new_val)
1295 # define AO_HAVE_int_compare_and_swap_acquire_read
1296 #endif
1297 
1298 #ifdef AO_NO_DD_ORDERING
1299 # if defined(AO_HAVE_int_compare_and_swap_acquire_read)
1300 #   define AO_int_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1301                 AO_int_compare_and_swap_acquire_read(addr, old, new_val)
1302 #   define AO_HAVE_int_compare_and_swap_dd_acquire_read
1303 # endif
1304 #else
1305 # if defined(AO_HAVE_int_compare_and_swap)
1306 #   define AO_int_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1307                 AO_int_compare_and_swap(addr, old, new_val)
1308 #   define AO_HAVE_int_compare_and_swap_dd_acquire_read
1309 # endif
1310 #endif /* !AO_NO_DD_ORDERING */
1311 
1312 /* int_load */
1313 #if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_acquire)
1314 # define AO_int_load_acquire(addr) AO_int_load_full(addr)
1315 # define AO_HAVE_int_load_acquire
1316 #endif
1317 
1318 #if defined(AO_HAVE_int_load_acquire) && !defined(AO_HAVE_int_load)
1319 # define AO_int_load(addr) AO_int_load_acquire(addr)
1320 # define AO_HAVE_int_load
1321 #endif
1322 
1323 #if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_read)
1324 # define AO_int_load_read(addr) AO_int_load_full(addr)
1325 # define AO_HAVE_int_load_read
1326 #endif
1327 
1328 #if !defined(AO_HAVE_int_load_acquire_read) \
1329     && defined(AO_HAVE_int_load_acquire)
1330 # define AO_int_load_acquire_read(addr) AO_int_load_acquire(addr)
1331 # define AO_HAVE_int_load_acquire_read
1332 #endif
1333 
1334 #if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_full) \
1335     && !defined(AO_HAVE_int_load_acquire)
1336   AO_INLINE unsigned
1337   AO_int_load_acquire(const volatile unsigned *addr)
1338   {
1339     unsigned result = AO_int_load(addr);
1340 
1341     /* Acquire barrier would be useless, since the load could be delayed    */
1342     /* beyond it.                                                           */
1343     AO_nop_full();
1344     return result;
1345   }
1346 # define AO_HAVE_int_load_acquire
1347 #endif
1348 
1349 #if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_read) \
1350     && !defined(AO_HAVE_int_load_read)
1351   AO_INLINE unsigned
1352   AO_int_load_read(const volatile unsigned *addr)
1353   {
1354     unsigned result = AO_int_load(addr);
1355 
1356     AO_nop_read();
1357     return result;
1358   }
1359 # define AO_HAVE_int_load_read
1360 #endif
1361 
1362 #if defined(AO_HAVE_int_load_acquire) && defined(AO_HAVE_nop_full) \
1363     && !defined(AO_HAVE_int_load_full)
1364 # define AO_int_load_full(addr) (AO_nop_full(), AO_int_load_acquire(addr))
1365 # define AO_HAVE_int_load_full
1366 #endif
1367 
1368 #if defined(AO_HAVE_int_compare_and_swap_read) \
1369     && !defined(AO_HAVE_int_load_read)
1370 # define AO_int_CAS_BASED_LOAD_READ
1371   AO_ATTR_NO_SANITIZE_THREAD
1372   AO_INLINE unsigned
1373   AO_int_load_read(const volatile unsigned *addr)
1374   {
1375     unsigned result;
1376 
1377     do {
1378       result = *(const unsigned *)addr;
1379     } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_read(
1380                                                 (volatile unsigned *)addr,
1381                                                 result, result)));
1382     return result;
1383   }
1384 # define AO_HAVE_int_load_read
1385 #endif
1386 
1387 #if !defined(AO_HAVE_int_load_acquire_read) \
1388     && defined(AO_HAVE_int_load_read)
1389 # define AO_int_load_acquire_read(addr) AO_int_load_read(addr)
1390 # define AO_HAVE_int_load_acquire_read
1391 #endif
1392 
1393 #if defined(AO_HAVE_int_load_acquire_read) && !defined(AO_HAVE_int_load) \
1394     && (!defined(AO_int_CAS_BASED_LOAD_READ) \
1395         || !defined(AO_HAVE_int_compare_and_swap))
1396 # define AO_int_load(addr) AO_int_load_acquire_read(addr)
1397 # define AO_HAVE_int_load
1398 #endif
1399 
1400 #if defined(AO_HAVE_int_compare_and_swap_full) \
1401     && !defined(AO_HAVE_int_load_full)
1402   AO_ATTR_NO_SANITIZE_THREAD
1403   AO_INLINE unsigned
1404   AO_int_load_full(const volatile unsigned *addr)
1405   {
1406     unsigned result;
1407 
1408     do {
1409       result = *(const unsigned *)addr;
1410     } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(
1411                                                 (volatile unsigned *)addr,
1412                                                 result, result)));
1413     return result;
1414   }
1415 # define AO_HAVE_int_load_full
1416 #endif
1417 
1418 #if defined(AO_HAVE_int_compare_and_swap_acquire) \
1419     && !defined(AO_HAVE_int_load_acquire)
1420   AO_ATTR_NO_SANITIZE_THREAD
1421   AO_INLINE unsigned
1422   AO_int_load_acquire(const volatile unsigned *addr)
1423   {
1424     unsigned result;
1425 
1426     do {
1427       result = *(const unsigned *)addr;
1428     } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(
1429                                                 (volatile unsigned *)addr,
1430                                                 result, result)));
1431     return result;
1432   }
1433 # define AO_HAVE_int_load_acquire
1434 #endif
1435 
1436 #if defined(AO_HAVE_int_compare_and_swap) && !defined(AO_HAVE_int_load)
1437   AO_ATTR_NO_SANITIZE_THREAD
1438   AO_INLINE unsigned
1439   AO_int_load(const volatile unsigned *addr)
1440   {
1441     unsigned result;
1442 
1443     do {
1444       result = *(const unsigned *)addr;
1445     } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(
1446                                                 (volatile unsigned *)addr,
1447                                                 result, result)));
1448     return result;
1449   }
1450 # define AO_HAVE_int_load
1451 #endif
1452 
1453 #ifdef AO_NO_DD_ORDERING
1454 # if defined(AO_HAVE_int_load_acquire_read)
1455 #   define AO_int_load_dd_acquire_read(addr) \
1456                                 AO_int_load_acquire_read(addr)
1457 #   define AO_HAVE_int_load_dd_acquire_read
1458 # endif
1459 #else
1460 # if defined(AO_HAVE_int_load)
1461 #   define AO_int_load_dd_acquire_read(addr) AO_int_load(addr)
1462 #   define AO_HAVE_int_load_dd_acquire_read
1463 # endif
1464 #endif /* !AO_NO_DD_ORDERING */
1465 
1466 /* int_store */
1467 #if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_release)
1468 # define AO_int_store_release(addr, val) AO_int_store_full(addr, val)
1469 # define AO_HAVE_int_store_release
1470 #endif
1471 
1472 #if defined(AO_HAVE_int_store_release) && !defined(AO_HAVE_int_store)
1473 # define AO_int_store(addr, val) AO_int_store_release(addr, val)
1474 # define AO_HAVE_int_store
1475 #endif
1476 
1477 #if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_write)
1478 # define AO_int_store_write(addr, val) AO_int_store_full(addr, val)
1479 # define AO_HAVE_int_store_write
1480 #endif
1481 
1482 #if defined(AO_HAVE_int_store_release) \
1483     && !defined(AO_HAVE_int_store_release_write)
1484 # define AO_int_store_release_write(addr, val) \
1485                                 AO_int_store_release(addr, val)
1486 # define AO_HAVE_int_store_release_write
1487 #endif
1488 
1489 #if defined(AO_HAVE_int_store_write) && !defined(AO_HAVE_int_store)
1490 # define AO_int_store(addr, val) AO_int_store_write(addr, val)
1491 # define AO_HAVE_int_store
1492 #endif
1493 
1494 #if defined(AO_HAVE_int_store) && defined(AO_HAVE_nop_full) \
1495     && !defined(AO_HAVE_int_store_release)
1496 # define AO_int_store_release(addr, val) \
1497                                 (AO_nop_full(), AO_int_store(addr, val))
1498 # define AO_HAVE_int_store_release
1499 #endif
1500 
1501 #if defined(AO_HAVE_int_store) && defined(AO_HAVE_nop_write) \
1502     && !defined(AO_HAVE_int_store_write)
1503 # define AO_int_store_write(addr, val) \
1504                                 (AO_nop_write(), AO_int_store(addr, val))
1505 # define AO_HAVE_int_store_write
1506 #endif
1507 
1508 #if defined(AO_HAVE_int_compare_and_swap_write) \
1509     && !defined(AO_HAVE_int_store_write)
1510   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1511   AO_INLINE void
1512   AO_int_store_write(volatile unsigned *addr, unsigned new_val)
1513   {
1514     unsigned old_val;
1515 
1516     do {
1517       old_val = *(unsigned *)addr;
1518     } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_write(addr, old_val,
1519                                                               new_val)));
1520   }
1521 # define AO_HAVE_int_store_write
1522 #endif
1523 
1524 #if defined(AO_HAVE_int_store_write) \
1525     && !defined(AO_HAVE_int_store_release_write)
1526 # define AO_int_store_release_write(addr, val) \
1527                                 AO_int_store_write(addr, val)
1528 # define AO_HAVE_int_store_release_write
1529 #endif
1530 
1531 #if defined(AO_HAVE_int_store_release) && defined(AO_HAVE_nop_full) \
1532     && !defined(AO_HAVE_int_store_full)
1533 # define AO_int_store_full(addr, val) \
1534                                 (AO_int_store_release(addr, val), \
1535                                  AO_nop_full())
1536 # define AO_HAVE_int_store_full
1537 #endif
1538 
1539 #if defined(AO_HAVE_int_compare_and_swap) && !defined(AO_HAVE_int_store)
1540   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1541   AO_INLINE void
1542   AO_int_store(volatile unsigned *addr, unsigned new_val)
1543   {
1544     unsigned old_val;
1545 
1546     do {
1547       old_val = *(unsigned *)addr;
1548     } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(addr,
1549                                                         old_val, new_val)));
1550   }
1551 # define AO_HAVE_int_store
1552 #endif
1553 
1554 #if defined(AO_HAVE_int_compare_and_swap_release) \
1555     && !defined(AO_HAVE_int_store_release)
1556   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1557   AO_INLINE void
1558   AO_int_store_release(volatile unsigned *addr, unsigned new_val)
1559   {
1560     unsigned old_val;
1561 
1562     do {
1563       old_val = *(unsigned *)addr;
1564     } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_release(addr, old_val,
1565                                                                 new_val)));
1566   }
1567 # define AO_HAVE_int_store_release
1568 #endif
1569 
1570 #if defined(AO_HAVE_int_compare_and_swap_full) \
1571     && !defined(AO_HAVE_int_store_full)
1572   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1573   AO_INLINE void
1574   AO_int_store_full(volatile unsigned *addr, unsigned new_val)
1575   {
1576     unsigned old_val;
1577 
1578     do {
1579       old_val = *(unsigned *)addr;
1580     } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old_val,
1581                                                              new_val)));
1582   }
1583 # define AO_HAVE_int_store_full
1584 #endif
1585 /*
1586  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
1587  *
1588  * Permission is hereby granted, free of charge, to any person obtaining a copy
1589  * of this software and associated documentation files (the "Software"), to deal
1590  * in the Software without restriction, including without limitation the rights
1591  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1592  * copies of the Software, and to permit persons to whom the Software is
1593  * furnished to do so, subject to the following conditions:
1594  *
1595  * The above copyright notice and this permission notice shall be included in
1596  * all copies or substantial portions of the Software.
1597  *
1598  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1599  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1600  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
1601  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1602  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
1603  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
1604  * SOFTWARE.
1605  */
1606 
1607 /* fetch_compare_and_swap */
1608 #if defined(AO_HAVE_fetch_compare_and_swap) \
1609     && defined(AO_HAVE_nop_full) \
1610     && !defined(AO_HAVE_fetch_compare_and_swap_acquire)
1611   AO_INLINE AO_t
1612   AO_fetch_compare_and_swap_acquire(volatile AO_t *addr,
1613                                           AO_t old_val, AO_t new_val)
1614   {
1615     AO_t result = AO_fetch_compare_and_swap(addr, old_val, new_val);
1616     AO_nop_full();
1617     return result;
1618   }
1619 # define AO_HAVE_fetch_compare_and_swap_acquire
1620 #endif
1621 #if defined(AO_HAVE_fetch_compare_and_swap) \
1622     && defined(AO_HAVE_nop_full) \
1623     && !defined(AO_HAVE_fetch_compare_and_swap_release)
1624 # define AO_fetch_compare_and_swap_release(addr, old_val, new_val) \
1625                 (AO_nop_full(), \
1626                  AO_fetch_compare_and_swap(addr, old_val, new_val))
1627 # define AO_HAVE_fetch_compare_and_swap_release
1628 #endif
1629 #if defined(AO_HAVE_fetch_compare_and_swap_full)
1630 # if !defined(AO_HAVE_fetch_compare_and_swap_release)
1631 #   define AO_fetch_compare_and_swap_release(addr, old_val, new_val) \
1632                 AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1633 #   define AO_HAVE_fetch_compare_and_swap_release
1634 # endif
1635 # if !defined(AO_HAVE_fetch_compare_and_swap_acquire)
1636 #   define AO_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
1637                 AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1638 #   define AO_HAVE_fetch_compare_and_swap_acquire
1639 # endif
1640 # if !defined(AO_HAVE_fetch_compare_and_swap_write)
1641 #   define AO_fetch_compare_and_swap_write(addr, old_val, new_val) \
1642                 AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1643 #   define AO_HAVE_fetch_compare_and_swap_write
1644 # endif
1645 # if !defined(AO_HAVE_fetch_compare_and_swap_read)
1646 #   define AO_fetch_compare_and_swap_read(addr, old_val, new_val) \
1647                 AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1648 #   define AO_HAVE_fetch_compare_and_swap_read
1649 # endif
1650 #endif /* AO_HAVE_fetch_compare_and_swap_full */
1651 
1652 #if !defined(AO_HAVE_fetch_compare_and_swap) \
1653     && defined(AO_HAVE_fetch_compare_and_swap_release)
1654 # define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1655             AO_fetch_compare_and_swap_release(addr, old_val, new_val)
1656 # define AO_HAVE_fetch_compare_and_swap
1657 #endif
1658 #if !defined(AO_HAVE_fetch_compare_and_swap) \
1659     && defined(AO_HAVE_fetch_compare_and_swap_acquire)
1660 # define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1661             AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1662 # define AO_HAVE_fetch_compare_and_swap
1663 #endif
1664 #if !defined(AO_HAVE_fetch_compare_and_swap) \
1665     && defined(AO_HAVE_fetch_compare_and_swap_write)
1666 # define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1667                 AO_fetch_compare_and_swap_write(addr, old_val, new_val)
1668 # define AO_HAVE_fetch_compare_and_swap
1669 #endif
1670 #if !defined(AO_HAVE_fetch_compare_and_swap) \
1671     && defined(AO_HAVE_fetch_compare_and_swap_read)
1672 # define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1673                 AO_fetch_compare_and_swap_read(addr, old_val, new_val)
1674 # define AO_HAVE_fetch_compare_and_swap
1675 #endif
1676 
1677 #if defined(AO_HAVE_fetch_compare_and_swap_acquire) \
1678     && defined(AO_HAVE_nop_full) \
1679     && !defined(AO_HAVE_fetch_compare_and_swap_full)
1680 # define AO_fetch_compare_and_swap_full(addr, old_val, new_val) \
1681             (AO_nop_full(), \
1682              AO_fetch_compare_and_swap_acquire(addr, old_val, new_val))
1683 # define AO_HAVE_fetch_compare_and_swap_full
1684 #endif
1685 
1686 #if !defined(AO_HAVE_fetch_compare_and_swap_release_write) \
1687     && defined(AO_HAVE_fetch_compare_and_swap_write)
1688 # define AO_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1689                 AO_fetch_compare_and_swap_write(addr, old_val, new_val)
1690 # define AO_HAVE_fetch_compare_and_swap_release_write
1691 #endif
1692 #if !defined(AO_HAVE_fetch_compare_and_swap_release_write) \
1693     && defined(AO_HAVE_fetch_compare_and_swap_release)
1694 # define AO_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1695             AO_fetch_compare_and_swap_release(addr, old_val, new_val)
1696 # define AO_HAVE_fetch_compare_and_swap_release_write
1697 #endif
1698 #if !defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \
1699     && defined(AO_HAVE_fetch_compare_and_swap_read)
1700 # define AO_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1701                 AO_fetch_compare_and_swap_read(addr, old_val, new_val)
1702 # define AO_HAVE_fetch_compare_and_swap_acquire_read
1703 #endif
1704 #if !defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \
1705     && defined(AO_HAVE_fetch_compare_and_swap_acquire)
1706 # define AO_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1707             AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1708 # define AO_HAVE_fetch_compare_and_swap_acquire_read
1709 #endif
1710 
1711 #ifdef AO_NO_DD_ORDERING
1712 # if defined(AO_HAVE_fetch_compare_and_swap_acquire_read)
1713 #   define AO_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1714         AO_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
1715 #   define AO_HAVE_fetch_compare_and_swap_dd_acquire_read
1716 # endif
1717 #else
1718 # if defined(AO_HAVE_fetch_compare_and_swap)
1719 #   define AO_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1720                 AO_fetch_compare_and_swap(addr, old_val, new_val)
1721 #   define AO_HAVE_fetch_compare_and_swap_dd_acquire_read
1722 # endif
1723 #endif /* !AO_NO_DD_ORDERING */
1724 
1725 /* compare_and_swap */
1726 #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full) \
1727     && !defined(AO_HAVE_compare_and_swap_acquire)
1728   AO_INLINE int
1729   AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old,
1730                                     AO_t new_val)
1731   {
1732     int result = AO_compare_and_swap(addr, old, new_val);
1733     AO_nop_full();
1734     return result;
1735   }
1736 # define AO_HAVE_compare_and_swap_acquire
1737 #endif
1738 #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full) \
1739     && !defined(AO_HAVE_compare_and_swap_release)
1740 # define AO_compare_and_swap_release(addr, old, new_val) \
1741                 (AO_nop_full(), AO_compare_and_swap(addr, old, new_val))
1742 # define AO_HAVE_compare_and_swap_release
1743 #endif
1744 #if defined(AO_HAVE_compare_and_swap_full)
1745 # if !defined(AO_HAVE_compare_and_swap_release)
1746 #   define AO_compare_and_swap_release(addr, old, new_val) \
1747                 AO_compare_and_swap_full(addr, old, new_val)
1748 #   define AO_HAVE_compare_and_swap_release
1749 # endif
1750 # if !defined(AO_HAVE_compare_and_swap_acquire)
1751 #   define AO_compare_and_swap_acquire(addr, old, new_val) \
1752                 AO_compare_and_swap_full(addr, old, new_val)
1753 #   define AO_HAVE_compare_and_swap_acquire
1754 # endif
1755 # if !defined(AO_HAVE_compare_and_swap_write)
1756 #   define AO_compare_and_swap_write(addr, old, new_val) \
1757                 AO_compare_and_swap_full(addr, old, new_val)
1758 #   define AO_HAVE_compare_and_swap_write
1759 # endif
1760 # if !defined(AO_HAVE_compare_and_swap_read)
1761 #   define AO_compare_and_swap_read(addr, old, new_val) \
1762                 AO_compare_and_swap_full(addr, old, new_val)
1763 #   define AO_HAVE_compare_and_swap_read
1764 # endif
1765 #endif /* AO_HAVE_compare_and_swap_full */
1766 
1767 #if !defined(AO_HAVE_compare_and_swap) \
1768     && defined(AO_HAVE_compare_and_swap_release)
1769 # define AO_compare_and_swap(addr, old, new_val) \
1770                 AO_compare_and_swap_release(addr, old, new_val)
1771 # define AO_HAVE_compare_and_swap
1772 #endif
1773 #if !defined(AO_HAVE_compare_and_swap) \
1774     && defined(AO_HAVE_compare_and_swap_acquire)
1775 # define AO_compare_and_swap(addr, old, new_val) \
1776                 AO_compare_and_swap_acquire(addr, old, new_val)
1777 # define AO_HAVE_compare_and_swap
1778 #endif
1779 #if !defined(AO_HAVE_compare_and_swap) \
1780     && defined(AO_HAVE_compare_and_swap_write)
1781 # define AO_compare_and_swap(addr, old, new_val) \
1782                 AO_compare_and_swap_write(addr, old, new_val)
1783 # define AO_HAVE_compare_and_swap
1784 #endif
1785 #if !defined(AO_HAVE_compare_and_swap) \
1786     && defined(AO_HAVE_compare_and_swap_read)
1787 # define AO_compare_and_swap(addr, old, new_val) \
1788                 AO_compare_and_swap_read(addr, old, new_val)
1789 # define AO_HAVE_compare_and_swap
1790 #endif
1791 
1792 #if defined(AO_HAVE_compare_and_swap_acquire) \
1793     && defined(AO_HAVE_nop_full) \
1794     && !defined(AO_HAVE_compare_and_swap_full)
1795 # define AO_compare_and_swap_full(addr, old, new_val) \
1796                 (AO_nop_full(), \
1797                  AO_compare_and_swap_acquire(addr, old, new_val))
1798 # define AO_HAVE_compare_and_swap_full
1799 #endif
1800 
1801 #if !defined(AO_HAVE_compare_and_swap_release_write) \
1802     && defined(AO_HAVE_compare_and_swap_write)
1803 # define AO_compare_and_swap_release_write(addr, old, new_val) \
1804                 AO_compare_and_swap_write(addr, old, new_val)
1805 # define AO_HAVE_compare_and_swap_release_write
1806 #endif
1807 #if !defined(AO_HAVE_compare_and_swap_release_write) \
1808     && defined(AO_HAVE_compare_and_swap_release)
1809 # define AO_compare_and_swap_release_write(addr, old, new_val) \
1810                 AO_compare_and_swap_release(addr, old, new_val)
1811 # define AO_HAVE_compare_and_swap_release_write
1812 #endif
1813 #if !defined(AO_HAVE_compare_and_swap_acquire_read) \
1814     && defined(AO_HAVE_compare_and_swap_read)
1815 # define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1816                 AO_compare_and_swap_read(addr, old, new_val)
1817 # define AO_HAVE_compare_and_swap_acquire_read
1818 #endif
1819 #if !defined(AO_HAVE_compare_and_swap_acquire_read) \
1820     && defined(AO_HAVE_compare_and_swap_acquire)
1821 # define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1822                 AO_compare_and_swap_acquire(addr, old, new_val)
1823 # define AO_HAVE_compare_and_swap_acquire_read
1824 #endif
1825 
1826 #ifdef AO_NO_DD_ORDERING
1827 # if defined(AO_HAVE_compare_and_swap_acquire_read)
1828 #   define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1829                 AO_compare_and_swap_acquire_read(addr, old, new_val)
1830 #   define AO_HAVE_compare_and_swap_dd_acquire_read
1831 # endif
1832 #else
1833 # if defined(AO_HAVE_compare_and_swap)
1834 #   define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1835                 AO_compare_and_swap(addr, old, new_val)
1836 #   define AO_HAVE_compare_and_swap_dd_acquire_read
1837 # endif
1838 #endif /* !AO_NO_DD_ORDERING */
1839 
1840 /* load */
1841 #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
1842 # define AO_load_acquire(addr) AO_load_full(addr)
1843 # define AO_HAVE_load_acquire
1844 #endif
1845 
1846 #if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
1847 # define AO_load(addr) AO_load_acquire(addr)
1848 # define AO_HAVE_load
1849 #endif
1850 
1851 #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
1852 # define AO_load_read(addr) AO_load_full(addr)
1853 # define AO_HAVE_load_read
1854 #endif
1855 
1856 #if !defined(AO_HAVE_load_acquire_read) \
1857     && defined(AO_HAVE_load_acquire)
1858 # define AO_load_acquire_read(addr) AO_load_acquire(addr)
1859 # define AO_HAVE_load_acquire_read
1860 #endif
1861 
1862 #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) \
1863     && !defined(AO_HAVE_load_acquire)
1864   AO_INLINE AO_t
1865   AO_load_acquire(const volatile AO_t *addr)
1866   {
1867     AO_t result = AO_load(addr);
1868 
1869     /* Acquire barrier would be useless, since the load could be delayed    */
1870     /* beyond it.                                                           */
1871     AO_nop_full();
1872     return result;
1873   }
1874 # define AO_HAVE_load_acquire
1875 #endif
1876 
1877 #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) \
1878     && !defined(AO_HAVE_load_read)
1879   AO_INLINE AO_t
1880   AO_load_read(const volatile AO_t *addr)
1881   {
1882     AO_t result = AO_load(addr);
1883 
1884     AO_nop_read();
1885     return result;
1886   }
1887 # define AO_HAVE_load_read
1888 #endif
1889 
1890 #if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) \
1891     && !defined(AO_HAVE_load_full)
1892 # define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
1893 # define AO_HAVE_load_full
1894 #endif
1895 
1896 #if defined(AO_HAVE_compare_and_swap_read) \
1897     && !defined(AO_HAVE_load_read)
1898 # define AO_CAS_BASED_LOAD_READ
1899   AO_ATTR_NO_SANITIZE_THREAD
1900   AO_INLINE AO_t
1901   AO_load_read(const volatile AO_t *addr)
1902   {
1903     AO_t result;
1904 
1905     do {
1906       result = *(const AO_t *)addr;
1907     } while (AO_EXPECT_FALSE(!AO_compare_and_swap_read(
1908                                                 (volatile AO_t *)addr,
1909                                                 result, result)));
1910     return result;
1911   }
1912 # define AO_HAVE_load_read
1913 #endif
1914 
1915 #if !defined(AO_HAVE_load_acquire_read) \
1916     && defined(AO_HAVE_load_read)
1917 # define AO_load_acquire_read(addr) AO_load_read(addr)
1918 # define AO_HAVE_load_acquire_read
1919 #endif
1920 
1921 #if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load) \
1922     && (!defined(AO_CAS_BASED_LOAD_READ) \
1923         || !defined(AO_HAVE_compare_and_swap))
1924 # define AO_load(addr) AO_load_acquire_read(addr)
1925 # define AO_HAVE_load
1926 #endif
1927 
1928 #if defined(AO_HAVE_compare_and_swap_full) \
1929     && !defined(AO_HAVE_load_full)
1930   AO_ATTR_NO_SANITIZE_THREAD
1931   AO_INLINE AO_t
1932   AO_load_full(const volatile AO_t *addr)
1933   {
1934     AO_t result;
1935 
1936     do {
1937       result = *(const AO_t *)addr;
1938     } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(
1939                                                 (volatile AO_t *)addr,
1940                                                 result, result)));
1941     return result;
1942   }
1943 # define AO_HAVE_load_full
1944 #endif
1945 
1946 #if defined(AO_HAVE_compare_and_swap_acquire) \
1947     && !defined(AO_HAVE_load_acquire)
1948   AO_ATTR_NO_SANITIZE_THREAD
1949   AO_INLINE AO_t
1950   AO_load_acquire(const volatile AO_t *addr)
1951   {
1952     AO_t result;
1953 
1954     do {
1955       result = *(const AO_t *)addr;
1956     } while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(
1957                                                 (volatile AO_t *)addr,
1958                                                 result, result)));
1959     return result;
1960   }
1961 # define AO_HAVE_load_acquire
1962 #endif
1963 
1964 #if defined(AO_HAVE_compare_and_swap) && !defined(AO_HAVE_load)
1965   AO_ATTR_NO_SANITIZE_THREAD
1966   AO_INLINE AO_t
1967   AO_load(const volatile AO_t *addr)
1968   {
1969     AO_t result;
1970 
1971     do {
1972       result = *(const AO_t *)addr;
1973     } while (AO_EXPECT_FALSE(!AO_compare_and_swap(
1974                                                 (volatile AO_t *)addr,
1975                                                 result, result)));
1976     return result;
1977   }
1978 # define AO_HAVE_load
1979 #endif
1980 
1981 #ifdef AO_NO_DD_ORDERING
1982 # if defined(AO_HAVE_load_acquire_read)
1983 #   define AO_load_dd_acquire_read(addr) \
1984                                 AO_load_acquire_read(addr)
1985 #   define AO_HAVE_load_dd_acquire_read
1986 # endif
1987 #else
1988 # if defined(AO_HAVE_load)
1989 #   define AO_load_dd_acquire_read(addr) AO_load(addr)
1990 #   define AO_HAVE_load_dd_acquire_read
1991 # endif
1992 #endif /* !AO_NO_DD_ORDERING */
1993 
1994 /* store */
1995 #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
1996 # define AO_store_release(addr, val) AO_store_full(addr, val)
1997 # define AO_HAVE_store_release
1998 #endif
1999 
2000 #if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
2001 # define AO_store(addr, val) AO_store_release(addr, val)
2002 # define AO_HAVE_store
2003 #endif
2004 
2005 #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
2006 # define AO_store_write(addr, val) AO_store_full(addr, val)
2007 # define AO_HAVE_store_write
2008 #endif
2009 
2010 #if defined(AO_HAVE_store_release) \
2011     && !defined(AO_HAVE_store_release_write)
2012 # define AO_store_release_write(addr, val) \
2013                                 AO_store_release(addr, val)
2014 # define AO_HAVE_store_release_write
2015 #endif
2016 
2017 #if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
2018 # define AO_store(addr, val) AO_store_write(addr, val)
2019 # define AO_HAVE_store
2020 #endif
2021 
2022 #if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) \
2023     && !defined(AO_HAVE_store_release)
2024 # define AO_store_release(addr, val) \
2025                                 (AO_nop_full(), AO_store(addr, val))
2026 # define AO_HAVE_store_release
2027 #endif
2028 
2029 #if defined(AO_HAVE_store) && defined(AO_HAVE_nop_write) \
2030     && !defined(AO_HAVE_store_write)
2031 # define AO_store_write(addr, val) \
2032                                 (AO_nop_write(), AO_store(addr, val))
2033 # define AO_HAVE_store_write
2034 #endif
2035 
2036 #if defined(AO_HAVE_compare_and_swap_write) \
2037     && !defined(AO_HAVE_store_write)
2038   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2039   AO_INLINE void
2040   AO_store_write(volatile AO_t *addr, AO_t new_val)
2041   {
2042     AO_t old_val;
2043 
2044     do {
2045       old_val = *(AO_t *)addr;
2046     } while (AO_EXPECT_FALSE(!AO_compare_and_swap_write(addr, old_val,
2047                                                               new_val)));
2048   }
2049 # define AO_HAVE_store_write
2050 #endif
2051 
2052 #if defined(AO_HAVE_store_write) \
2053     && !defined(AO_HAVE_store_release_write)
2054 # define AO_store_release_write(addr, val) \
2055                                 AO_store_write(addr, val)
2056 # define AO_HAVE_store_release_write
2057 #endif
2058 
2059 #if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) \
2060     && !defined(AO_HAVE_store_full)
2061 # define AO_store_full(addr, val) \
2062                                 (AO_store_release(addr, val), \
2063                                  AO_nop_full())
2064 # define AO_HAVE_store_full
2065 #endif
2066 
2067 #if defined(AO_HAVE_compare_and_swap) && !defined(AO_HAVE_store)
2068   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2069   AO_INLINE void
2070   AO_store(volatile AO_t *addr, AO_t new_val)
2071   {
2072     AO_t old_val;
2073 
2074     do {
2075       old_val = *(AO_t *)addr;
2076     } while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr,
2077                                                         old_val, new_val)));
2078   }
2079 # define AO_HAVE_store
2080 #endif
2081 
2082 #if defined(AO_HAVE_compare_and_swap_release) \
2083     && !defined(AO_HAVE_store_release)
2084   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2085   AO_INLINE void
2086   AO_store_release(volatile AO_t *addr, AO_t new_val)
2087   {
2088     AO_t old_val;
2089 
2090     do {
2091       old_val = *(AO_t *)addr;
2092     } while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old_val,
2093                                                                 new_val)));
2094   }
2095 # define AO_HAVE_store_release
2096 #endif
2097 
2098 #if defined(AO_HAVE_compare_and_swap_full) \
2099     && !defined(AO_HAVE_store_full)
2100   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2101   AO_INLINE void
2102   AO_store_full(volatile AO_t *addr, AO_t new_val)
2103   {
2104     AO_t old_val;
2105 
2106     do {
2107       old_val = *(AO_t *)addr;
2108     } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old_val,
2109                                                              new_val)));
2110   }
2111 # define AO_HAVE_store_full
2112 #endif
2113 /*
2114  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
2115  *
2116  * Permission is hereby granted, free of charge, to any person obtaining a copy
2117  * of this software and associated documentation files (the "Software"), to deal
2118  * in the Software without restriction, including without limitation the rights
2119  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
2120  * copies of the Software, and to permit persons to whom the Software is
2121  * furnished to do so, subject to the following conditions:
2122  *
2123  * The above copyright notice and this permission notice shall be included in
2124  * all copies or substantial portions of the Software.
2125  *
2126  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
2127  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
2128  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
2129  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2130  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2131  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2132  * SOFTWARE.
2133  */
2134 
2135 /* double_fetch_compare_and_swap */
2136 #if defined(AO_HAVE_double_fetch_compare_and_swap) \
2137     && defined(AO_HAVE_nop_full) \
2138     && !defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2139   AO_INLINE AO_double_t
2140   AO_double_fetch_compare_and_swap_acquire(volatile AO_double_t *addr,
2141                                           AO_double_t old_val, AO_double_t new_val)
2142   {
2143     AO_double_t result = AO_double_fetch_compare_and_swap(addr, old_val, new_val);
2144     AO_nop_full();
2145     return result;
2146   }
2147 # define AO_HAVE_double_fetch_compare_and_swap_acquire
2148 #endif
2149 #if defined(AO_HAVE_double_fetch_compare_and_swap) \
2150     && defined(AO_HAVE_nop_full) \
2151     && !defined(AO_HAVE_double_fetch_compare_and_swap_release)
2152 # define AO_double_fetch_compare_and_swap_release(addr, old_val, new_val) \
2153                 (AO_nop_full(), \
2154                  AO_double_fetch_compare_and_swap(addr, old_val, new_val))
2155 # define AO_HAVE_double_fetch_compare_and_swap_release
2156 #endif
2157 #if defined(AO_HAVE_double_fetch_compare_and_swap_full)
2158 # if !defined(AO_HAVE_double_fetch_compare_and_swap_release)
2159 #   define AO_double_fetch_compare_and_swap_release(addr, old_val, new_val) \
2160                 AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2161 #   define AO_HAVE_double_fetch_compare_and_swap_release
2162 # endif
2163 # if !defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2164 #   define AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
2165                 AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2166 #   define AO_HAVE_double_fetch_compare_and_swap_acquire
2167 # endif
2168 # if !defined(AO_HAVE_double_fetch_compare_and_swap_write)
2169 #   define AO_double_fetch_compare_and_swap_write(addr, old_val, new_val) \
2170                 AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2171 #   define AO_HAVE_double_fetch_compare_and_swap_write
2172 # endif
2173 # if !defined(AO_HAVE_double_fetch_compare_and_swap_read)
2174 #   define AO_double_fetch_compare_and_swap_read(addr, old_val, new_val) \
2175                 AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2176 #   define AO_HAVE_double_fetch_compare_and_swap_read
2177 # endif
2178 #endif /* AO_HAVE_double_fetch_compare_and_swap_full */
2179 
2180 #if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2181     && defined(AO_HAVE_double_fetch_compare_and_swap_release)
2182 # define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2183             AO_double_fetch_compare_and_swap_release(addr, old_val, new_val)
2184 # define AO_HAVE_double_fetch_compare_and_swap
2185 #endif
2186 #if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2187     && defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2188 # define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2189             AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val)
2190 # define AO_HAVE_double_fetch_compare_and_swap
2191 #endif
2192 #if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2193     && defined(AO_HAVE_double_fetch_compare_and_swap_write)
2194 # define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2195                 AO_double_fetch_compare_and_swap_write(addr, old_val, new_val)
2196 # define AO_HAVE_double_fetch_compare_and_swap
2197 #endif
2198 #if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2199     && defined(AO_HAVE_double_fetch_compare_and_swap_read)
2200 # define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2201                 AO_double_fetch_compare_and_swap_read(addr, old_val, new_val)
2202 # define AO_HAVE_double_fetch_compare_and_swap
2203 #endif
2204 
2205 #if defined(AO_HAVE_double_fetch_compare_and_swap_acquire) \
2206     && defined(AO_HAVE_nop_full) \
2207     && !defined(AO_HAVE_double_fetch_compare_and_swap_full)
2208 # define AO_double_fetch_compare_and_swap_full(addr, old_val, new_val) \
2209             (AO_nop_full(), \
2210              AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val))
2211 # define AO_HAVE_double_fetch_compare_and_swap_full
2212 #endif
2213 
2214 #if !defined(AO_HAVE_double_fetch_compare_and_swap_release_write) \
2215     && defined(AO_HAVE_double_fetch_compare_and_swap_write)
2216 # define AO_double_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
2217                 AO_double_fetch_compare_and_swap_write(addr, old_val, new_val)
2218 # define AO_HAVE_double_fetch_compare_and_swap_release_write
2219 #endif
2220 #if !defined(AO_HAVE_double_fetch_compare_and_swap_release_write) \
2221     && defined(AO_HAVE_double_fetch_compare_and_swap_release)
2222 # define AO_double_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
2223             AO_double_fetch_compare_and_swap_release(addr, old_val, new_val)
2224 # define AO_HAVE_double_fetch_compare_and_swap_release_write
2225 #endif
2226 #if !defined(AO_HAVE_double_fetch_compare_and_swap_acquire_read) \
2227     && defined(AO_HAVE_double_fetch_compare_and_swap_read)
2228 # define AO_double_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
2229                 AO_double_fetch_compare_and_swap_read(addr, old_val, new_val)
2230 # define AO_HAVE_double_fetch_compare_and_swap_acquire_read
2231 #endif
2232 #if !defined(AO_HAVE_double_fetch_compare_and_swap_acquire_read) \
2233     && defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2234 # define AO_double_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
2235             AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val)
2236 # define AO_HAVE_double_fetch_compare_and_swap_acquire_read
2237 #endif
2238 
2239 #ifdef AO_NO_DD_ORDERING
2240 # if defined(AO_HAVE_double_fetch_compare_and_swap_acquire_read)
2241 #   define AO_double_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
2242         AO_double_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
2243 #   define AO_HAVE_double_fetch_compare_and_swap_dd_acquire_read
2244 # endif
2245 #else
2246 # if defined(AO_HAVE_double_fetch_compare_and_swap)
2247 #   define AO_double_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
2248                 AO_double_fetch_compare_and_swap(addr, old_val, new_val)
2249 #   define AO_HAVE_double_fetch_compare_and_swap_dd_acquire_read
2250 # endif
2251 #endif /* !AO_NO_DD_ORDERING */
2252 
2253 /* double_compare_and_swap */
2254 #if defined(AO_HAVE_double_compare_and_swap) && defined(AO_HAVE_nop_full) \
2255     && !defined(AO_HAVE_double_compare_and_swap_acquire)
2256   AO_INLINE int
2257   AO_double_compare_and_swap_acquire(volatile AO_double_t *addr, AO_double_t old,
2258                                     AO_double_t new_val)
2259   {
2260     int result = AO_double_compare_and_swap(addr, old, new_val);
2261     AO_nop_full();
2262     return result;
2263   }
2264 # define AO_HAVE_double_compare_and_swap_acquire
2265 #endif
2266 #if defined(AO_HAVE_double_compare_and_swap) && defined(AO_HAVE_nop_full) \
2267     && !defined(AO_HAVE_double_compare_and_swap_release)
2268 # define AO_double_compare_and_swap_release(addr, old, new_val) \
2269                 (AO_nop_full(), AO_double_compare_and_swap(addr, old, new_val))
2270 # define AO_HAVE_double_compare_and_swap_release
2271 #endif
2272 #if defined(AO_HAVE_double_compare_and_swap_full)
2273 # if !defined(AO_HAVE_double_compare_and_swap_release)
2274 #   define AO_double_compare_and_swap_release(addr, old, new_val) \
2275                 AO_double_compare_and_swap_full(addr, old, new_val)
2276 #   define AO_HAVE_double_compare_and_swap_release
2277 # endif
2278 # if !defined(AO_HAVE_double_compare_and_swap_acquire)
2279 #   define AO_double_compare_and_swap_acquire(addr, old, new_val) \
2280                 AO_double_compare_and_swap_full(addr, old, new_val)
2281 #   define AO_HAVE_double_compare_and_swap_acquire
2282 # endif
2283 # if !defined(AO_HAVE_double_compare_and_swap_write)
2284 #   define AO_double_compare_and_swap_write(addr, old, new_val) \
2285                 AO_double_compare_and_swap_full(addr, old, new_val)
2286 #   define AO_HAVE_double_compare_and_swap_write
2287 # endif
2288 # if !defined(AO_HAVE_double_compare_and_swap_read)
2289 #   define AO_double_compare_and_swap_read(addr, old, new_val) \
2290                 AO_double_compare_and_swap_full(addr, old, new_val)
2291 #   define AO_HAVE_double_compare_and_swap_read
2292 # endif
2293 #endif /* AO_HAVE_double_compare_and_swap_full */
2294 
2295 #if !defined(AO_HAVE_double_compare_and_swap) \
2296     && defined(AO_HAVE_double_compare_and_swap_release)
2297 # define AO_double_compare_and_swap(addr, old, new_val) \
2298                 AO_double_compare_and_swap_release(addr, old, new_val)
2299 # define AO_HAVE_double_compare_and_swap
2300 #endif
2301 #if !defined(AO_HAVE_double_compare_and_swap) \
2302     && defined(AO_HAVE_double_compare_and_swap_acquire)
2303 # define AO_double_compare_and_swap(addr, old, new_val) \
2304                 AO_double_compare_and_swap_acquire(addr, old, new_val)
2305 # define AO_HAVE_double_compare_and_swap
2306 #endif
2307 #if !defined(AO_HAVE_double_compare_and_swap) \
2308     && defined(AO_HAVE_double_compare_and_swap_write)
2309 # define AO_double_compare_and_swap(addr, old, new_val) \
2310                 AO_double_compare_and_swap_write(addr, old, new_val)
2311 # define AO_HAVE_double_compare_and_swap
2312 #endif
2313 #if !defined(AO_HAVE_double_compare_and_swap) \
2314     && defined(AO_HAVE_double_compare_and_swap_read)
2315 # define AO_double_compare_and_swap(addr, old, new_val) \
2316                 AO_double_compare_and_swap_read(addr, old, new_val)
2317 # define AO_HAVE_double_compare_and_swap
2318 #endif
2319 
2320 #if defined(AO_HAVE_double_compare_and_swap_acquire) \
2321     && defined(AO_HAVE_nop_full) \
2322     && !defined(AO_HAVE_double_compare_and_swap_full)
2323 # define AO_double_compare_and_swap_full(addr, old, new_val) \
2324                 (AO_nop_full(), \
2325                  AO_double_compare_and_swap_acquire(addr, old, new_val))
2326 # define AO_HAVE_double_compare_and_swap_full
2327 #endif
2328 
2329 #if !defined(AO_HAVE_double_compare_and_swap_release_write) \
2330     && defined(AO_HAVE_double_compare_and_swap_write)
2331 # define AO_double_compare_and_swap_release_write(addr, old, new_val) \
2332                 AO_double_compare_and_swap_write(addr, old, new_val)
2333 # define AO_HAVE_double_compare_and_swap_release_write
2334 #endif
2335 #if !defined(AO_HAVE_double_compare_and_swap_release_write) \
2336     && defined(AO_HAVE_double_compare_and_swap_release)
2337 # define AO_double_compare_and_swap_release_write(addr, old, new_val) \
2338                 AO_double_compare_and_swap_release(addr, old, new_val)
2339 # define AO_HAVE_double_compare_and_swap_release_write
2340 #endif
2341 #if !defined(AO_HAVE_double_compare_and_swap_acquire_read) \
2342     && defined(AO_HAVE_double_compare_and_swap_read)
2343 # define AO_double_compare_and_swap_acquire_read(addr, old, new_val) \
2344                 AO_double_compare_and_swap_read(addr, old, new_val)
2345 # define AO_HAVE_double_compare_and_swap_acquire_read
2346 #endif
2347 #if !defined(AO_HAVE_double_compare_and_swap_acquire_read) \
2348     && defined(AO_HAVE_double_compare_and_swap_acquire)
2349 # define AO_double_compare_and_swap_acquire_read(addr, old, new_val) \
2350                 AO_double_compare_and_swap_acquire(addr, old, new_val)
2351 # define AO_HAVE_double_compare_and_swap_acquire_read
2352 #endif
2353 
2354 #ifdef AO_NO_DD_ORDERING
2355 # if defined(AO_HAVE_double_compare_and_swap_acquire_read)
2356 #   define AO_double_compare_and_swap_dd_acquire_read(addr, old, new_val) \
2357                 AO_double_compare_and_swap_acquire_read(addr, old, new_val)
2358 #   define AO_HAVE_double_compare_and_swap_dd_acquire_read
2359 # endif
2360 #else
2361 # if defined(AO_HAVE_double_compare_and_swap)
2362 #   define AO_double_compare_and_swap_dd_acquire_read(addr, old, new_val) \
2363                 AO_double_compare_and_swap(addr, old, new_val)
2364 #   define AO_HAVE_double_compare_and_swap_dd_acquire_read
2365 # endif
2366 #endif /* !AO_NO_DD_ORDERING */
2367 
2368 /* double_load */
2369 #if defined(AO_HAVE_double_load_full) && !defined(AO_HAVE_double_load_acquire)
2370 # define AO_double_load_acquire(addr) AO_double_load_full(addr)
2371 # define AO_HAVE_double_load_acquire
2372 #endif
2373 
2374 #if defined(AO_HAVE_double_load_acquire) && !defined(AO_HAVE_double_load)
2375 # define AO_double_load(addr) AO_double_load_acquire(addr)
2376 # define AO_HAVE_double_load
2377 #endif
2378 
2379 #if defined(AO_HAVE_double_load_full) && !defined(AO_HAVE_double_load_read)
2380 # define AO_double_load_read(addr) AO_double_load_full(addr)
2381 # define AO_HAVE_double_load_read
2382 #endif
2383 
2384 #if !defined(AO_HAVE_double_load_acquire_read) \
2385     && defined(AO_HAVE_double_load_acquire)
2386 # define AO_double_load_acquire_read(addr) AO_double_load_acquire(addr)
2387 # define AO_HAVE_double_load_acquire_read
2388 #endif
2389 
2390 #if defined(AO_HAVE_double_load) && defined(AO_HAVE_nop_full) \
2391     && !defined(AO_HAVE_double_load_acquire)
2392   AO_INLINE AO_double_t
2393   AO_double_load_acquire(const volatile AO_double_t *addr)
2394   {
2395     AO_double_t result = AO_double_load(addr);
2396 
2397     /* Acquire barrier would be useless, since the load could be delayed    */
2398     /* beyond it.                                                           */
2399     AO_nop_full();
2400     return result;
2401   }
2402 # define AO_HAVE_double_load_acquire
2403 #endif
2404 
2405 #if defined(AO_HAVE_double_load) && defined(AO_HAVE_nop_read) \
2406     && !defined(AO_HAVE_double_load_read)
2407   AO_INLINE AO_double_t
2408   AO_double_load_read(const volatile AO_double_t *addr)
2409   {
2410     AO_double_t result = AO_double_load(addr);
2411 
2412     AO_nop_read();
2413     return result;
2414   }
2415 # define AO_HAVE_double_load_read
2416 #endif
2417 
2418 #if defined(AO_HAVE_double_load_acquire) && defined(AO_HAVE_nop_full) \
2419     && !defined(AO_HAVE_double_load_full)
2420 # define AO_double_load_full(addr) (AO_nop_full(), AO_double_load_acquire(addr))
2421 # define AO_HAVE_double_load_full
2422 #endif
2423 
2424 #if defined(AO_HAVE_double_compare_and_swap_read) \
2425     && !defined(AO_HAVE_double_load_read)
2426 # define AO_double_CAS_BASED_LOAD_READ
2427   AO_ATTR_NO_SANITIZE_THREAD
2428   AO_INLINE AO_double_t
2429   AO_double_load_read(const volatile AO_double_t *addr)
2430   {
2431     AO_double_t result;
2432 
2433     do {
2434       result = *(const AO_double_t *)addr;
2435     } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_read(
2436                                                 (volatile AO_double_t *)addr,
2437                                                 result, result)));
2438     return result;
2439   }
2440 # define AO_HAVE_double_load_read
2441 #endif
2442 
2443 #if !defined(AO_HAVE_double_load_acquire_read) \
2444     && defined(AO_HAVE_double_load_read)
2445 # define AO_double_load_acquire_read(addr) AO_double_load_read(addr)
2446 # define AO_HAVE_double_load_acquire_read
2447 #endif
2448 
2449 #if defined(AO_HAVE_double_load_acquire_read) && !defined(AO_HAVE_double_load) \
2450     && (!defined(AO_double_CAS_BASED_LOAD_READ) \
2451         || !defined(AO_HAVE_double_compare_and_swap))
2452 # define AO_double_load(addr) AO_double_load_acquire_read(addr)
2453 # define AO_HAVE_double_load
2454 #endif
2455 
2456 #if defined(AO_HAVE_double_compare_and_swap_full) \
2457     && !defined(AO_HAVE_double_load_full)
2458   AO_ATTR_NO_SANITIZE_THREAD
2459   AO_INLINE AO_double_t
2460   AO_double_load_full(const volatile AO_double_t *addr)
2461   {
2462     AO_double_t result;
2463 
2464     do {
2465       result = *(const AO_double_t *)addr;
2466     } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_full(
2467                                                 (volatile AO_double_t *)addr,
2468                                                 result, result)));
2469     return result;
2470   }
2471 # define AO_HAVE_double_load_full
2472 #endif
2473 
2474 #if defined(AO_HAVE_double_compare_and_swap_acquire) \
2475     && !defined(AO_HAVE_double_load_acquire)
2476   AO_ATTR_NO_SANITIZE_THREAD
2477   AO_INLINE AO_double_t
2478   AO_double_load_acquire(const volatile AO_double_t *addr)
2479   {
2480     AO_double_t result;
2481 
2482     do {
2483       result = *(const AO_double_t *)addr;
2484     } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_acquire(
2485                                                 (volatile AO_double_t *)addr,
2486                                                 result, result)));
2487     return result;
2488   }
2489 # define AO_HAVE_double_load_acquire
2490 #endif
2491 
2492 #if defined(AO_HAVE_double_compare_and_swap) && !defined(AO_HAVE_double_load)
2493   AO_ATTR_NO_SANITIZE_THREAD
2494   AO_INLINE AO_double_t
2495   AO_double_load(const volatile AO_double_t *addr)
2496   {
2497     AO_double_t result;
2498 
2499     do {
2500       result = *(const AO_double_t *)addr;
2501     } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap(
2502                                                 (volatile AO_double_t *)addr,
2503                                                 result, result)));
2504     return result;
2505   }
2506 # define AO_HAVE_double_load
2507 #endif
2508 
2509 #ifdef AO_NO_DD_ORDERING
2510 # if defined(AO_HAVE_double_load_acquire_read)
2511 #   define AO_double_load_dd_acquire_read(addr) \
2512                                 AO_double_load_acquire_read(addr)
2513 #   define AO_HAVE_double_load_dd_acquire_read
2514 # endif
2515 #else
2516 # if defined(AO_HAVE_double_load)
2517 #   define AO_double_load_dd_acquire_read(addr) AO_double_load(addr)
2518 #   define AO_HAVE_double_load_dd_acquire_read
2519 # endif
2520 #endif /* !AO_NO_DD_ORDERING */
2521 
2522 /* double_store */
2523 #if defined(AO_HAVE_double_store_full) && !defined(AO_HAVE_double_store_release)
2524 # define AO_double_store_release(addr, val) AO_double_store_full(addr, val)
2525 # define AO_HAVE_double_store_release
2526 #endif
2527 
2528 #if defined(AO_HAVE_double_store_release) && !defined(AO_HAVE_double_store)
2529 # define AO_double_store(addr, val) AO_double_store_release(addr, val)
2530 # define AO_HAVE_double_store
2531 #endif
2532 
2533 #if defined(AO_HAVE_double_store_full) && !defined(AO_HAVE_double_store_write)
2534 # define AO_double_store_write(addr, val) AO_double_store_full(addr, val)
2535 # define AO_HAVE_double_store_write
2536 #endif
2537 
2538 #if defined(AO_HAVE_double_store_release) \
2539     && !defined(AO_HAVE_double_store_release_write)
2540 # define AO_double_store_release_write(addr, val) \
2541                                 AO_double_store_release(addr, val)
2542 # define AO_HAVE_double_store_release_write
2543 #endif
2544 
2545 #if defined(AO_HAVE_double_store_write) && !defined(AO_HAVE_double_store)
2546 # define AO_double_store(addr, val) AO_double_store_write(addr, val)
2547 # define AO_HAVE_double_store
2548 #endif
2549 
2550 #if defined(AO_HAVE_double_store) && defined(AO_HAVE_nop_full) \
2551     && !defined(AO_HAVE_double_store_release)
2552 # define AO_double_store_release(addr, val) \
2553                                 (AO_nop_full(), AO_double_store(addr, val))
2554 # define AO_HAVE_double_store_release
2555 #endif
2556 
2557 #if defined(AO_HAVE_double_store) && defined(AO_HAVE_nop_write) \
2558     && !defined(AO_HAVE_double_store_write)
2559 # define AO_double_store_write(addr, val) \
2560                                 (AO_nop_write(), AO_double_store(addr, val))
2561 # define AO_HAVE_double_store_write
2562 #endif
2563 
2564 #if defined(AO_HAVE_double_compare_and_swap_write) \
2565     && !defined(AO_HAVE_double_store_write)
2566   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2567   AO_INLINE void
2568   AO_double_store_write(volatile AO_double_t *addr, AO_double_t new_val)
2569   {
2570     AO_double_t old_val;
2571 
2572     do {
2573       old_val = *(AO_double_t *)addr;
2574     } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_write(addr, old_val,
2575                                                               new_val)));
2576   }
2577 # define AO_HAVE_double_store_write
2578 #endif
2579 
2580 #if defined(AO_HAVE_double_store_write) \
2581     && !defined(AO_HAVE_double_store_release_write)
2582 # define AO_double_store_release_write(addr, val) \
2583                                 AO_double_store_write(addr, val)
2584 # define AO_HAVE_double_store_release_write
2585 #endif
2586 
2587 #if defined(AO_HAVE_double_store_release) && defined(AO_HAVE_nop_full) \
2588     && !defined(AO_HAVE_double_store_full)
2589 # define AO_double_store_full(addr, val) \
2590                                 (AO_double_store_release(addr, val), \
2591                                  AO_nop_full())
2592 # define AO_HAVE_double_store_full
2593 #endif
2594 
2595 #if defined(AO_HAVE_double_compare_and_swap) && !defined(AO_HAVE_double_store)
2596   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2597   AO_INLINE void
2598   AO_double_store(volatile AO_double_t *addr, AO_double_t new_val)
2599   {
2600     AO_double_t old_val;
2601 
2602     do {
2603       old_val = *(AO_double_t *)addr;
2604     } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap(addr,
2605                                                         old_val, new_val)));
2606   }
2607 # define AO_HAVE_double_store
2608 #endif
2609 
2610 #if defined(AO_HAVE_double_compare_and_swap_release) \
2611     && !defined(AO_HAVE_double_store_release)
2612   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2613   AO_INLINE void
2614   AO_double_store_release(volatile AO_double_t *addr, AO_double_t new_val)
2615   {
2616     AO_double_t old_val;
2617 
2618     do {
2619       old_val = *(AO_double_t *)addr;
2620     } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_release(addr, old_val,
2621                                                                 new_val)));
2622   }
2623 # define AO_HAVE_double_store_release
2624 #endif
2625 
2626 #if defined(AO_HAVE_double_compare_and_swap_full) \
2627     && !defined(AO_HAVE_double_store_full)
2628   AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2629   AO_INLINE void
2630   AO_double_store_full(volatile AO_double_t *addr, AO_double_t new_val)
2631   {
2632     AO_double_t old_val;
2633 
2634     do {
2635       old_val = *(AO_double_t *)addr;
2636     } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_full(addr, old_val,
2637                                                              new_val)));
2638   }
2639 # define AO_HAVE_double_store_full
2640 #endif
2641