Gitly


1 /*
2  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a copy
5  * of this software and associated documentation files (the "Software"), to deal
6  * in the Software without restriction, including without limitation the rights
7  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8  * copies of the Software, and to permit persons to whom the Software is
9  * furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20  * SOFTWARE.
21  */
22 
23 /* char_compare_and_swap (based on fetch_compare_and_swap) */
24 #if defined(AO_HAVE_char_fetch_compare_and_swap_full) \
25     && !defined(AO_HAVE_char_compare_and_swap_full)
26   AO_INLINE int
27   AO_char_compare_and_swap_full(volatile unsigned/**/char *addr, unsigned/**/char old_val,
28                                  unsigned/**/char new_val)
29   {
30     return AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
31              == old_val;
32   }
33 # define AO_HAVE_char_compare_and_swap_full
34 #endif
35 
36 #if defined(AO_HAVE_char_fetch_compare_and_swap_acquire) \
37     && !defined(AO_HAVE_char_compare_and_swap_acquire)
38   AO_INLINE int
39   AO_char_compare_and_swap_acquire(volatile unsigned/**/char *addr, unsigned/**/char old_val,
40                                     unsigned/**/char new_val)
41   {
42     return AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val)
43              == old_val;
44   }
45 # define AO_HAVE_char_compare_and_swap_acquire
46 #endif
47 
48 #if defined(AO_HAVE_char_fetch_compare_and_swap_release) \
49     && !defined(AO_HAVE_char_compare_and_swap_release)
50   AO_INLINE int
51   AO_char_compare_and_swap_release(volatile unsigned/**/char *addr, unsigned/**/char old_val,
52                                     unsigned/**/char new_val)
53   {
54     return AO_char_fetch_compare_and_swap_release(addr, old_val, new_val)
55              == old_val;
56   }
57 # define AO_HAVE_char_compare_and_swap_release
58 #endif
59 
60 #if defined(AO_HAVE_char_fetch_compare_and_swap_write) \
61     && !defined(AO_HAVE_char_compare_and_swap_write)
62   AO_INLINE int
63   AO_char_compare_and_swap_write(volatile unsigned/**/char *addr, unsigned/**/char old_val,
64                                   unsigned/**/char new_val)
65   {
66     return AO_char_fetch_compare_and_swap_write(addr, old_val, new_val)
67              == old_val;
68   }
69 # define AO_HAVE_char_compare_and_swap_write
70 #endif
71 
72 #if defined(AO_HAVE_char_fetch_compare_and_swap_read) \
73     && !defined(AO_HAVE_char_compare_and_swap_read)
74   AO_INLINE int
75   AO_char_compare_and_swap_read(volatile unsigned/**/char *addr, unsigned/**/char old_val,
76                                  unsigned/**/char new_val)
77   {
78     return AO_char_fetch_compare_and_swap_read(addr, old_val, new_val)
79              == old_val;
80   }
81 # define AO_HAVE_char_compare_and_swap_read
82 #endif
83 
84 #if defined(AO_HAVE_char_fetch_compare_and_swap) \
85     && !defined(AO_HAVE_char_compare_and_swap)
86   AO_INLINE int
87   AO_char_compare_and_swap(volatile unsigned/**/char *addr, unsigned/**/char old_val,
88                             unsigned/**/char new_val)
89   {
90     return AO_char_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
91   }
92 # define AO_HAVE_char_compare_and_swap
93 #endif
94 
95 #if defined(AO_HAVE_char_fetch_compare_and_swap_release_write) \
96     && !defined(AO_HAVE_char_compare_and_swap_release_write)
97   AO_INLINE int
98   AO_char_compare_and_swap_release_write(volatile unsigned/**/char *addr,
99                                           unsigned/**/char old_val, unsigned/**/char new_val)
100   {
101     return AO_char_fetch_compare_and_swap_release_write(addr, old_val,
102                                                          new_val) == old_val;
103   }
104 # define AO_HAVE_char_compare_and_swap_release_write
105 #endif
106 
107 #if defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read) \
108     && !defined(AO_HAVE_char_compare_and_swap_acquire_read)
109   AO_INLINE int
110   AO_char_compare_and_swap_acquire_read(volatile unsigned/**/char *addr,
111                                          unsigned/**/char old_val, unsigned/**/char new_val)
112   {
113     return AO_char_fetch_compare_and_swap_acquire_read(addr, old_val,
114                                                         new_val) == old_val;
115   }
116 # define AO_HAVE_char_compare_and_swap_acquire_read
117 #endif
118 
119 #if defined(AO_HAVE_char_fetch_compare_and_swap_dd_acquire_read) \
120     && !defined(AO_HAVE_char_compare_and_swap_dd_acquire_read)
121   AO_INLINE int
122   AO_char_compare_and_swap_dd_acquire_read(volatile unsigned/**/char *addr,
123                                             unsigned/**/char old_val, unsigned/**/char new_val)
124   {
125     return AO_char_fetch_compare_and_swap_dd_acquire_read(addr, old_val,
126                                                            new_val) == old_val;
127   }
128 # define AO_HAVE_char_compare_and_swap_dd_acquire_read
129 #endif
130 
131 /* char_fetch_and_add */
132 /* We first try to implement fetch_and_add variants in terms of the     */
133 /* corresponding compare_and_swap variants to minimize adding barriers. */
134 #if defined(AO_HAVE_char_compare_and_swap_full) \
135     && !defined(AO_HAVE_char_fetch_and_add_full)
136   AO_ATTR_NO_SANITIZE_THREAD
137   AO_INLINE unsigned/**/char
138   AO_char_fetch_and_add_full(volatile unsigned/**/char *addr, unsigned/**/char incr)
139   {
140     unsigned/**/char old;
141 
142     do
143       {
144         old = *(unsigned/**/char *)addr;
145       }
146     while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
147                                                            old + incr)));
148     return old;
149   }
150 # define AO_HAVE_char_fetch_and_add_full
151 #endif
152 
153 #if defined(AO_HAVE_char_compare_and_swap_acquire) \
154     && !defined(AO_HAVE_char_fetch_and_add_acquire)
155   AO_ATTR_NO_SANITIZE_THREAD
156   AO_INLINE unsigned/**/char
157   AO_char_fetch_and_add_acquire(volatile unsigned/**/char *addr, unsigned/**/char incr)
158   {
159     unsigned/**/char old;
160 
161     do
162       {
163         old = *(unsigned/**/char *)addr;
164       }
165     while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(addr, old,
166                                                               old + incr)));
167     return old;
168   }
169 # define AO_HAVE_char_fetch_and_add_acquire
170 #endif
171 
172 #if defined(AO_HAVE_char_compare_and_swap_release) \
173     && !defined(AO_HAVE_char_fetch_and_add_release)
174   AO_ATTR_NO_SANITIZE_THREAD
175   AO_INLINE unsigned/**/char
176   AO_char_fetch_and_add_release(volatile unsigned/**/char *addr, unsigned/**/char incr)
177   {
178     unsigned/**/char old;
179 
180     do
181       {
182         old = *(unsigned/**/char *)addr;
183       }
184     while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_release(addr, old,
185                                                               old + incr)));
186     return old;
187   }
188 # define AO_HAVE_char_fetch_and_add_release
189 #endif
190 
191 #if defined(AO_HAVE_char_compare_and_swap) \
192     && !defined(AO_HAVE_char_fetch_and_add)
193   AO_ATTR_NO_SANITIZE_THREAD
194   AO_INLINE unsigned/**/char
195   AO_char_fetch_and_add(volatile unsigned/**/char *addr, unsigned/**/char incr)
196   {
197     unsigned/**/char old;
198 
199     do
200       {
201         old = *(unsigned/**/char *)addr;
202       }
203     while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(addr, old,
204                                                       old + incr)));
205     return old;
206   }
207 # define AO_HAVE_char_fetch_and_add
208 #endif
209 
210 #if defined(AO_HAVE_char_fetch_and_add_full)
211 # if !defined(AO_HAVE_char_fetch_and_add_release)
212 #   define AO_char_fetch_and_add_release(addr, val) \
213                                 AO_char_fetch_and_add_full(addr, val)
214 #   define AO_HAVE_char_fetch_and_add_release
215 # endif
216 # if !defined(AO_HAVE_char_fetch_and_add_acquire)
217 #   define AO_char_fetch_and_add_acquire(addr, val) \
218                                 AO_char_fetch_and_add_full(addr, val)
219 #   define AO_HAVE_char_fetch_and_add_acquire
220 # endif
221 # if !defined(AO_HAVE_char_fetch_and_add_write)
222 #   define AO_char_fetch_and_add_write(addr, val) \
223                                 AO_char_fetch_and_add_full(addr, val)
224 #   define AO_HAVE_char_fetch_and_add_write
225 # endif
226 # if !defined(AO_HAVE_char_fetch_and_add_read)
227 #   define AO_char_fetch_and_add_read(addr, val) \
228                                 AO_char_fetch_and_add_full(addr, val)
229 #   define AO_HAVE_char_fetch_and_add_read
230 # endif
231 #endif /* AO_HAVE_char_fetch_and_add_full */
232 
233 #if defined(AO_HAVE_char_fetch_and_add) && defined(AO_HAVE_nop_full) \
234     && !defined(AO_HAVE_char_fetch_and_add_acquire)
235   AO_INLINE unsigned/**/char
236   AO_char_fetch_and_add_acquire(volatile unsigned/**/char *addr, unsigned/**/char incr)
237   {
238     unsigned/**/char result = AO_char_fetch_and_add(addr, incr);
239     AO_nop_full();
240     return result;
241   }
242 # define AO_HAVE_char_fetch_and_add_acquire
243 #endif
244 #if defined(AO_HAVE_char_fetch_and_add) && defined(AO_HAVE_nop_full) \
245     && !defined(AO_HAVE_char_fetch_and_add_release)
246 # define AO_char_fetch_and_add_release(addr, incr) \
247                 (AO_nop_full(), AO_char_fetch_and_add(addr, incr))
248 # define AO_HAVE_char_fetch_and_add_release
249 #endif
250 
251 #if !defined(AO_HAVE_char_fetch_and_add) \
252     && defined(AO_HAVE_char_fetch_and_add_release)
253 # define AO_char_fetch_and_add(addr, val) \
254                                 AO_char_fetch_and_add_release(addr, val)
255 # define AO_HAVE_char_fetch_and_add
256 #endif
257 #if !defined(AO_HAVE_char_fetch_and_add) \
258     && defined(AO_HAVE_char_fetch_and_add_acquire)
259 # define AO_char_fetch_and_add(addr, val) \
260                                 AO_char_fetch_and_add_acquire(addr, val)
261 # define AO_HAVE_char_fetch_and_add
262 #endif
263 #if !defined(AO_HAVE_char_fetch_and_add) \
264     && defined(AO_HAVE_char_fetch_and_add_write)
265 # define AO_char_fetch_and_add(addr, val) \
266                                 AO_char_fetch_and_add_write(addr, val)
267 # define AO_HAVE_char_fetch_and_add
268 #endif
269 #if !defined(AO_HAVE_char_fetch_and_add) \
270     && defined(AO_HAVE_char_fetch_and_add_read)
271 # define AO_char_fetch_and_add(addr, val) \
272                                 AO_char_fetch_and_add_read(addr, val)
273 # define AO_HAVE_char_fetch_and_add
274 #endif
275 
276 #if defined(AO_HAVE_char_fetch_and_add_acquire) \
277     && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_char_fetch_and_add_full)
278 # define AO_char_fetch_and_add_full(addr, val) \
279                 (AO_nop_full(), AO_char_fetch_and_add_acquire(addr, val))
280 # define AO_HAVE_char_fetch_and_add_full
281 #endif
282 
283 #if !defined(AO_HAVE_char_fetch_and_add_release_write) \
284     && defined(AO_HAVE_char_fetch_and_add_write)
285 # define AO_char_fetch_and_add_release_write(addr, val) \
286                                 AO_char_fetch_and_add_write(addr, val)
287 # define AO_HAVE_char_fetch_and_add_release_write
288 #endif
289 #if !defined(AO_HAVE_char_fetch_and_add_release_write) \
290     && defined(AO_HAVE_char_fetch_and_add_release)
291 # define AO_char_fetch_and_add_release_write(addr, val) \
292                                 AO_char_fetch_and_add_release(addr, val)
293 # define AO_HAVE_char_fetch_and_add_release_write
294 #endif
295 
296 #if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \
297     && defined(AO_HAVE_char_fetch_and_add_read)
298 # define AO_char_fetch_and_add_acquire_read(addr, val) \
299                                 AO_char_fetch_and_add_read(addr, val)
300 # define AO_HAVE_char_fetch_and_add_acquire_read
301 #endif
302 #if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \
303     && defined(AO_HAVE_char_fetch_and_add_acquire)
304 # define AO_char_fetch_and_add_acquire_read(addr, val) \
305                                 AO_char_fetch_and_add_acquire(addr, val)
306 # define AO_HAVE_char_fetch_and_add_acquire_read
307 #endif
308 
309 #ifdef AO_NO_DD_ORDERING
310 # if defined(AO_HAVE_char_fetch_and_add_acquire_read)
311 #   define AO_char_fetch_and_add_dd_acquire_read(addr, val) \
312                                 AO_char_fetch_and_add_acquire_read(addr, val)
313 #   define AO_HAVE_char_fetch_and_add_dd_acquire_read
314 # endif
315 #else
316 # if defined(AO_HAVE_char_fetch_and_add)
317 #   define AO_char_fetch_and_add_dd_acquire_read(addr, val) \
318                                 AO_char_fetch_and_add(addr, val)
319 #   define AO_HAVE_char_fetch_and_add_dd_acquire_read
320 # endif
321 #endif /* !AO_NO_DD_ORDERING */
322 
323 /* char_fetch_and_add1 */
324 #if defined(AO_HAVE_char_fetch_and_add_full) \
325     && !defined(AO_HAVE_char_fetch_and_add1_full)
326 # define AO_char_fetch_and_add1_full(addr) \
327                                 AO_char_fetch_and_add_full(addr, 1)
328 # define AO_HAVE_char_fetch_and_add1_full
329 #endif
330 #if defined(AO_HAVE_char_fetch_and_add_release) \
331     && !defined(AO_HAVE_char_fetch_and_add1_release)
332 # define AO_char_fetch_and_add1_release(addr) \
333                                 AO_char_fetch_and_add_release(addr, 1)
334 # define AO_HAVE_char_fetch_and_add1_release
335 #endif
336 #if defined(AO_HAVE_char_fetch_and_add_acquire) \
337     && !defined(AO_HAVE_char_fetch_and_add1_acquire)
338 # define AO_char_fetch_and_add1_acquire(addr) \
339                                 AO_char_fetch_and_add_acquire(addr, 1)
340 # define AO_HAVE_char_fetch_and_add1_acquire
341 #endif
342 #if defined(AO_HAVE_char_fetch_and_add_write) \
343     && !defined(AO_HAVE_char_fetch_and_add1_write)
344 # define AO_char_fetch_and_add1_write(addr) \
345                                 AO_char_fetch_and_add_write(addr, 1)
346 # define AO_HAVE_char_fetch_and_add1_write
347 #endif
348 #if defined(AO_HAVE_char_fetch_and_add_read) \
349     && !defined(AO_HAVE_char_fetch_and_add1_read)
350 # define AO_char_fetch_and_add1_read(addr) \
351                                 AO_char_fetch_and_add_read(addr, 1)
352 # define AO_HAVE_char_fetch_and_add1_read
353 #endif
354 #if defined(AO_HAVE_char_fetch_and_add_release_write) \
355     && !defined(AO_HAVE_char_fetch_and_add1_release_write)
356 # define AO_char_fetch_and_add1_release_write(addr) \
357                                 AO_char_fetch_and_add_release_write(addr, 1)
358 # define AO_HAVE_char_fetch_and_add1_release_write
359 #endif
360 #if defined(AO_HAVE_char_fetch_and_add_acquire_read) \
361     && !defined(AO_HAVE_char_fetch_and_add1_acquire_read)
362 # define AO_char_fetch_and_add1_acquire_read(addr) \
363                                 AO_char_fetch_and_add_acquire_read(addr, 1)
364 # define AO_HAVE_char_fetch_and_add1_acquire_read
365 #endif
366 #if defined(AO_HAVE_char_fetch_and_add) \
367     && !defined(AO_HAVE_char_fetch_and_add1)
368 # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add(addr, 1)
369 # define AO_HAVE_char_fetch_and_add1
370 #endif
371 
372 #if defined(AO_HAVE_char_fetch_and_add1_full)
373 # if !defined(AO_HAVE_char_fetch_and_add1_release)
374 #   define AO_char_fetch_and_add1_release(addr) \
375                                 AO_char_fetch_and_add1_full(addr)
376 #   define AO_HAVE_char_fetch_and_add1_release
377 # endif
378 # if !defined(AO_HAVE_char_fetch_and_add1_acquire)
379 #   define AO_char_fetch_and_add1_acquire(addr) \
380                                 AO_char_fetch_and_add1_full(addr)
381 #   define AO_HAVE_char_fetch_and_add1_acquire
382 # endif
383 # if !defined(AO_HAVE_char_fetch_and_add1_write)
384 #   define AO_char_fetch_and_add1_write(addr) \
385                                 AO_char_fetch_and_add1_full(addr)
386 #   define AO_HAVE_char_fetch_and_add1_write
387 # endif
388 # if !defined(AO_HAVE_char_fetch_and_add1_read)
389 #   define AO_char_fetch_and_add1_read(addr) \
390                                 AO_char_fetch_and_add1_full(addr)
391 #   define AO_HAVE_char_fetch_and_add1_read
392 # endif
393 #endif /* AO_HAVE_char_fetch_and_add1_full */
394 
395 #if !defined(AO_HAVE_char_fetch_and_add1) \
396     && defined(AO_HAVE_char_fetch_and_add1_release)
397 # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_release(addr)
398 # define AO_HAVE_char_fetch_and_add1
399 #endif
400 #if !defined(AO_HAVE_char_fetch_and_add1) \
401     && defined(AO_HAVE_char_fetch_and_add1_acquire)
402 # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_acquire(addr)
403 # define AO_HAVE_char_fetch_and_add1
404 #endif
405 #if !defined(AO_HAVE_char_fetch_and_add1) \
406     && defined(AO_HAVE_char_fetch_and_add1_write)
407 # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_write(addr)
408 # define AO_HAVE_char_fetch_and_add1
409 #endif
410 #if !defined(AO_HAVE_char_fetch_and_add1) \
411     && defined(AO_HAVE_char_fetch_and_add1_read)
412 # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_read(addr)
413 # define AO_HAVE_char_fetch_and_add1
414 #endif
415 
416 #if defined(AO_HAVE_char_fetch_and_add1_acquire) \
417     && defined(AO_HAVE_nop_full) \
418     && !defined(AO_HAVE_char_fetch_and_add1_full)
419 # define AO_char_fetch_and_add1_full(addr) \
420                         (AO_nop_full(), AO_char_fetch_and_add1_acquire(addr))
421 # define AO_HAVE_char_fetch_and_add1_full
422 #endif
423 
424 #if !defined(AO_HAVE_char_fetch_and_add1_release_write) \
425     && defined(AO_HAVE_char_fetch_and_add1_write)
426 # define AO_char_fetch_and_add1_release_write(addr) \
427                                 AO_char_fetch_and_add1_write(addr)
428 # define AO_HAVE_char_fetch_and_add1_release_write
429 #endif
430 #if !defined(AO_HAVE_char_fetch_and_add1_release_write) \
431     && defined(AO_HAVE_char_fetch_and_add1_release)
432 # define AO_char_fetch_and_add1_release_write(addr) \
433                                 AO_char_fetch_and_add1_release(addr)
434 # define AO_HAVE_char_fetch_and_add1_release_write
435 #endif
436 #if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \
437     && defined(AO_HAVE_char_fetch_and_add1_read)
438 # define AO_char_fetch_and_add1_acquire_read(addr) \
439                                 AO_char_fetch_and_add1_read(addr)
440 # define AO_HAVE_char_fetch_and_add1_acquire_read
441 #endif
442 #if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \
443     && defined(AO_HAVE_char_fetch_and_add1_acquire)
444 # define AO_char_fetch_and_add1_acquire_read(addr) \
445                                 AO_char_fetch_and_add1_acquire(addr)
446 # define AO_HAVE_char_fetch_and_add1_acquire_read
447 #endif
448 
449 #ifdef AO_NO_DD_ORDERING
450 # if defined(AO_HAVE_char_fetch_and_add1_acquire_read)
451 #   define AO_char_fetch_and_add1_dd_acquire_read(addr) \
452                                 AO_char_fetch_and_add1_acquire_read(addr)
453 #   define AO_HAVE_char_fetch_and_add1_dd_acquire_read
454 # endif
455 #else
456 # if defined(AO_HAVE_char_fetch_and_add1)
457 #   define AO_char_fetch_and_add1_dd_acquire_read(addr) \
458                                 AO_char_fetch_and_add1(addr)
459 #   define AO_HAVE_char_fetch_and_add1_dd_acquire_read
460 # endif
461 #endif /* !AO_NO_DD_ORDERING */
462 
463 /* char_fetch_and_sub1 */
464 #if defined(AO_HAVE_char_fetch_and_add_full) \
465     && !defined(AO_HAVE_char_fetch_and_sub1_full)
466 # define AO_char_fetch_and_sub1_full(addr) \
467                 AO_char_fetch_and_add_full(addr, (unsigned/**/char)(-1))
468 # define AO_HAVE_char_fetch_and_sub1_full
469 #endif
470 #if defined(AO_HAVE_char_fetch_and_add_release) \
471     && !defined(AO_HAVE_char_fetch_and_sub1_release)
472 # define AO_char_fetch_and_sub1_release(addr) \
473                 AO_char_fetch_and_add_release(addr, (unsigned/**/char)(-1))
474 # define AO_HAVE_char_fetch_and_sub1_release
475 #endif
476 #if defined(AO_HAVE_char_fetch_and_add_acquire) \
477     && !defined(AO_HAVE_char_fetch_and_sub1_acquire)
478 # define AO_char_fetch_and_sub1_acquire(addr) \
479                 AO_char_fetch_and_add_acquire(addr, (unsigned/**/char)(-1))
480 # define AO_HAVE_char_fetch_and_sub1_acquire
481 #endif
482 #if defined(AO_HAVE_char_fetch_and_add_write) \
483     && !defined(AO_HAVE_char_fetch_and_sub1_write)
484 # define AO_char_fetch_and_sub1_write(addr) \
485                 AO_char_fetch_and_add_write(addr, (unsigned/**/char)(-1))
486 # define AO_HAVE_char_fetch_and_sub1_write
487 #endif
488 #if defined(AO_HAVE_char_fetch_and_add_read) \
489     && !defined(AO_HAVE_char_fetch_and_sub1_read)
490 # define AO_char_fetch_and_sub1_read(addr) \
491                 AO_char_fetch_and_add_read(addr, (unsigned/**/char)(-1))
492 # define AO_HAVE_char_fetch_and_sub1_read
493 #endif
494 #if defined(AO_HAVE_char_fetch_and_add_release_write) \
495     && !defined(AO_HAVE_char_fetch_and_sub1_release_write)
496 # define AO_char_fetch_and_sub1_release_write(addr) \
497                 AO_char_fetch_and_add_release_write(addr, (unsigned/**/char)(-1))
498 # define AO_HAVE_char_fetch_and_sub1_release_write
499 #endif
500 #if defined(AO_HAVE_char_fetch_and_add_acquire_read) \
501     && !defined(AO_HAVE_char_fetch_and_sub1_acquire_read)
502 # define AO_char_fetch_and_sub1_acquire_read(addr) \
503                 AO_char_fetch_and_add_acquire_read(addr, (unsigned/**/char)(-1))
504 # define AO_HAVE_char_fetch_and_sub1_acquire_read
505 #endif
506 #if defined(AO_HAVE_char_fetch_and_add) \
507     && !defined(AO_HAVE_char_fetch_and_sub1)
508 # define AO_char_fetch_and_sub1(addr) \
509                 AO_char_fetch_and_add(addr, (unsigned/**/char)(-1))
510 # define AO_HAVE_char_fetch_and_sub1
511 #endif
512 
513 #if defined(AO_HAVE_char_fetch_and_sub1_full)
514 # if !defined(AO_HAVE_char_fetch_and_sub1_release)
515 #   define AO_char_fetch_and_sub1_release(addr) \
516                                 AO_char_fetch_and_sub1_full(addr)
517 #   define AO_HAVE_char_fetch_and_sub1_release
518 # endif
519 # if !defined(AO_HAVE_char_fetch_and_sub1_acquire)
520 #   define AO_char_fetch_and_sub1_acquire(addr) \
521                                 AO_char_fetch_and_sub1_full(addr)
522 #   define AO_HAVE_char_fetch_and_sub1_acquire
523 # endif
524 # if !defined(AO_HAVE_char_fetch_and_sub1_write)
525 #   define AO_char_fetch_and_sub1_write(addr) \
526                                 AO_char_fetch_and_sub1_full(addr)
527 #   define AO_HAVE_char_fetch_and_sub1_write
528 # endif
529 # if !defined(AO_HAVE_char_fetch_and_sub1_read)
530 #   define AO_char_fetch_and_sub1_read(addr) \
531                                 AO_char_fetch_and_sub1_full(addr)
532 #   define AO_HAVE_char_fetch_and_sub1_read
533 # endif
534 #endif /* AO_HAVE_char_fetch_and_sub1_full */
535 
536 #if !defined(AO_HAVE_char_fetch_and_sub1) \
537     && defined(AO_HAVE_char_fetch_and_sub1_release)
538 # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_release(addr)
539 # define AO_HAVE_char_fetch_and_sub1
540 #endif
541 #if !defined(AO_HAVE_char_fetch_and_sub1) \
542     && defined(AO_HAVE_char_fetch_and_sub1_acquire)
543 # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_acquire(addr)
544 # define AO_HAVE_char_fetch_and_sub1
545 #endif
546 #if !defined(AO_HAVE_char_fetch_and_sub1) \
547     && defined(AO_HAVE_char_fetch_and_sub1_write)
548 # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_write(addr)
549 # define AO_HAVE_char_fetch_and_sub1
550 #endif
551 #if !defined(AO_HAVE_char_fetch_and_sub1) \
552     && defined(AO_HAVE_char_fetch_and_sub1_read)
553 # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_read(addr)
554 # define AO_HAVE_char_fetch_and_sub1
555 #endif
556 
557 #if defined(AO_HAVE_char_fetch_and_sub1_acquire) \
558     && defined(AO_HAVE_nop_full) \
559     && !defined(AO_HAVE_char_fetch_and_sub1_full)
560 # define AO_char_fetch_and_sub1_full(addr) \
561                         (AO_nop_full(), AO_char_fetch_and_sub1_acquire(addr))
562 # define AO_HAVE_char_fetch_and_sub1_full
563 #endif
564 
565 #if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \
566     && defined(AO_HAVE_char_fetch_and_sub1_write)
567 # define AO_char_fetch_and_sub1_release_write(addr) \
568                                 AO_char_fetch_and_sub1_write(addr)
569 # define AO_HAVE_char_fetch_and_sub1_release_write
570 #endif
571 #if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \
572     && defined(AO_HAVE_char_fetch_and_sub1_release)
573 # define AO_char_fetch_and_sub1_release_write(addr) \
574                                 AO_char_fetch_and_sub1_release(addr)
575 # define AO_HAVE_char_fetch_and_sub1_release_write
576 #endif
577 #if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \
578     && defined(AO_HAVE_char_fetch_and_sub1_read)
579 # define AO_char_fetch_and_sub1_acquire_read(addr) \
580                                 AO_char_fetch_and_sub1_read(addr)
581 # define AO_HAVE_char_fetch_and_sub1_acquire_read
582 #endif
583 #if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \
584     && defined(AO_HAVE_char_fetch_and_sub1_acquire)
585 # define AO_char_fetch_and_sub1_acquire_read(addr) \
586                                 AO_char_fetch_and_sub1_acquire(addr)
587 # define AO_HAVE_char_fetch_and_sub1_acquire_read
588 #endif
589 
590 #ifdef AO_NO_DD_ORDERING
591 # if defined(AO_HAVE_char_fetch_and_sub1_acquire_read)
592 #   define AO_char_fetch_and_sub1_dd_acquire_read(addr) \
593                                 AO_char_fetch_and_sub1_acquire_read(addr)
594 #   define AO_HAVE_char_fetch_and_sub1_dd_acquire_read
595 # endif
596 #else
597 # if defined(AO_HAVE_char_fetch_and_sub1)
598 #   define AO_char_fetch_and_sub1_dd_acquire_read(addr) \
599                                 AO_char_fetch_and_sub1(addr)
600 #   define AO_HAVE_char_fetch_and_sub1_dd_acquire_read
601 # endif
602 #endif /* !AO_NO_DD_ORDERING */
603 
604 /* char_and */
605 #if defined(AO_HAVE_char_compare_and_swap_full) \
606     && !defined(AO_HAVE_char_and_full)
607   AO_ATTR_NO_SANITIZE_THREAD
608   AO_INLINE void
609   AO_char_and_full(volatile unsigned/**/char *addr, unsigned/**/char value)
610   {
611     unsigned/**/char old;
612 
613     do
614       {
615         old = *(unsigned/**/char *)addr;
616       }
617     while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
618                                                            old & value)));
619   }
620 # define AO_HAVE_char_and_full
621 #endif
622 
623 #if defined(AO_HAVE_char_and_full)
624 # if !defined(AO_HAVE_char_and_release)
625 #   define AO_char_and_release(addr, val) AO_char_and_full(addr, val)
626 #   define AO_HAVE_char_and_release
627 # endif
628 # if !defined(AO_HAVE_char_and_acquire)
629 #   define AO_char_and_acquire(addr, val) AO_char_and_full(addr, val)
630 #   define AO_HAVE_char_and_acquire
631 # endif
632 # if !defined(AO_HAVE_char_and_write)
633 #   define AO_char_and_write(addr, val) AO_char_and_full(addr, val)
634 #   define AO_HAVE_char_and_write
635 # endif
636 # if !defined(AO_HAVE_char_and_read)
637 #   define AO_char_and_read(addr, val) AO_char_and_full(addr, val)
638 #   define AO_HAVE_char_and_read
639 # endif
640 #endif /* AO_HAVE_char_and_full */
641 
642 #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_release)
643 # define AO_char_and(addr, val) AO_char_and_release(addr, val)
644 # define AO_HAVE_char_and
645 #endif
646 #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_acquire)
647 # define AO_char_and(addr, val) AO_char_and_acquire(addr, val)
648 # define AO_HAVE_char_and
649 #endif
650 #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_write)
651 # define AO_char_and(addr, val) AO_char_and_write(addr, val)
652 # define AO_HAVE_char_and
653 #endif
654 #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_read)
655 # define AO_char_and(addr, val) AO_char_and_read(addr, val)
656 # define AO_HAVE_char_and
657 #endif
658 
659 #if defined(AO_HAVE_char_and_acquire) && defined(AO_HAVE_nop_full) \
660     && !defined(AO_HAVE_char_and_full)
661 # define AO_char_and_full(addr, val) \
662                         (AO_nop_full(), AO_char_and_acquire(addr, val))
663 # define AO_HAVE_char_and_full
664 #endif
665 
666 #if !defined(AO_HAVE_char_and_release_write) \
667     && defined(AO_HAVE_char_and_write)
668 # define AO_char_and_release_write(addr, val) AO_char_and_write(addr, val)
669 # define AO_HAVE_char_and_release_write
670 #endif
671 #if !defined(AO_HAVE_char_and_release_write) \
672     && defined(AO_HAVE_char_and_release)
673 # define AO_char_and_release_write(addr, val) AO_char_and_release(addr, val)
674 # define AO_HAVE_char_and_release_write
675 #endif
676 #if !defined(AO_HAVE_char_and_acquire_read) \
677     && defined(AO_HAVE_char_and_read)
678 # define AO_char_and_acquire_read(addr, val) AO_char_and_read(addr, val)
679 # define AO_HAVE_char_and_acquire_read
680 #endif
681 #if !defined(AO_HAVE_char_and_acquire_read) \
682     && defined(AO_HAVE_char_and_acquire)
683 # define AO_char_and_acquire_read(addr, val) AO_char_and_acquire(addr, val)
684 # define AO_HAVE_char_and_acquire_read
685 #endif
686 
687 /* char_or */
688 #if defined(AO_HAVE_char_compare_and_swap_full) \
689     && !defined(AO_HAVE_char_or_full)
690   AO_ATTR_NO_SANITIZE_THREAD
691   AO_INLINE void
692   AO_char_or_full(volatile unsigned/**/char *addr, unsigned/**/char value)
693   {
694     unsigned/**/char old;
695 
696     do
697       {
698         old = *(unsigned/**/char *)addr;
699       }
700     while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
701                                                            old | value)));
702   }
703 # define AO_HAVE_char_or_full
704 #endif
705 
706 #if defined(AO_HAVE_char_or_full)
707 # if !defined(AO_HAVE_char_or_release)
708 #   define AO_char_or_release(addr, val) AO_char_or_full(addr, val)
709 #   define AO_HAVE_char_or_release
710 # endif
711 # if !defined(AO_HAVE_char_or_acquire)
712 #   define AO_char_or_acquire(addr, val) AO_char_or_full(addr, val)
713 #   define AO_HAVE_char_or_acquire
714 # endif
715 # if !defined(AO_HAVE_char_or_write)
716 #   define AO_char_or_write(addr, val) AO_char_or_full(addr, val)
717 #   define AO_HAVE_char_or_write
718 # endif
719 # if !defined(AO_HAVE_char_or_read)
720 #   define AO_char_or_read(addr, val) AO_char_or_full(addr, val)
721 #   define AO_HAVE_char_or_read
722 # endif
723 #endif /* AO_HAVE_char_or_full */
724 
725 #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_release)
726 # define AO_char_or(addr, val) AO_char_or_release(addr, val)
727 # define AO_HAVE_char_or
728 #endif
729 #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_acquire)
730 # define AO_char_or(addr, val) AO_char_or_acquire(addr, val)
731 # define AO_HAVE_char_or
732 #endif
733 #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_write)
734 # define AO_char_or(addr, val) AO_char_or_write(addr, val)
735 # define AO_HAVE_char_or
736 #endif
737 #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_read)
738 # define AO_char_or(addr, val) AO_char_or_read(addr, val)
739 # define AO_HAVE_char_or
740 #endif
741 
742 #if defined(AO_HAVE_char_or_acquire) && defined(AO_HAVE_nop_full) \
743     && !defined(AO_HAVE_char_or_full)
744 # define AO_char_or_full(addr, val) \
745                         (AO_nop_full(), AO_char_or_acquire(addr, val))
746 # define AO_HAVE_char_or_full
747 #endif
748 
749 #if !defined(AO_HAVE_char_or_release_write) \
750     && defined(AO_HAVE_char_or_write)
751 # define AO_char_or_release_write(addr, val) AO_char_or_write(addr, val)
752 # define AO_HAVE_char_or_release_write
753 #endif
754 #if !defined(AO_HAVE_char_or_release_write) \
755     && defined(AO_HAVE_char_or_release)
756 # define AO_char_or_release_write(addr, val) AO_char_or_release(addr, val)
757 # define AO_HAVE_char_or_release_write
758 #endif
759 #if !defined(AO_HAVE_char_or_acquire_read) && defined(AO_HAVE_char_or_read)
760 # define AO_char_or_acquire_read(addr, val) AO_char_or_read(addr, val)
761 # define AO_HAVE_char_or_acquire_read
762 #endif
763 #if !defined(AO_HAVE_char_or_acquire_read) \
764     && defined(AO_HAVE_char_or_acquire)
765 # define AO_char_or_acquire_read(addr, val) AO_char_or_acquire(addr, val)
766 # define AO_HAVE_char_or_acquire_read
767 #endif
768 
769 /* char_xor */
770 #if defined(AO_HAVE_char_compare_and_swap_full) \
771     && !defined(AO_HAVE_char_xor_full)
772   AO_ATTR_NO_SANITIZE_THREAD
773   AO_INLINE void
774   AO_char_xor_full(volatile unsigned/**/char *addr, unsigned/**/char value)
775   {
776     unsigned/**/char old;
777 
778     do
779       {
780         old = *(unsigned/**/char *)addr;
781       }
782     while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
783                                                            old ^ value)));
784   }
785 # define AO_HAVE_char_xor_full
786 #endif
787 
788 #if defined(AO_HAVE_char_xor_full)
789 # if !defined(AO_HAVE_char_xor_release)
790 #   define AO_char_xor_release(addr, val) AO_char_xor_full(addr, val)
791 #   define AO_HAVE_char_xor_release
792 # endif
793 # if !defined(AO_HAVE_char_xor_acquire)
794 #   define AO_char_xor_acquire(addr, val) AO_char_xor_full(addr, val)
795 #   define AO_HAVE_char_xor_acquire
796 # endif
797 # if !defined(AO_HAVE_char_xor_write)
798 #   define AO_char_xor_write(addr, val) AO_char_xor_full(addr, val)
799 #   define AO_HAVE_char_xor_write
800 # endif
801 # if !defined(AO_HAVE_char_xor_read)
802 #   define AO_char_xor_read(addr, val) AO_char_xor_full(addr, val)
803 #   define AO_HAVE_char_xor_read
804 # endif
805 #endif /* AO_HAVE_char_xor_full */
806 
807 #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_release)
808 # define AO_char_xor(addr, val) AO_char_xor_release(addr, val)
809 # define AO_HAVE_char_xor
810 #endif
811 #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_acquire)
812 # define AO_char_xor(addr, val) AO_char_xor_acquire(addr, val)
813 # define AO_HAVE_char_xor
814 #endif
815 #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_write)
816 # define AO_char_xor(addr, val) AO_char_xor_write(addr, val)
817 # define AO_HAVE_char_xor
818 #endif
819 #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_read)
820 # define AO_char_xor(addr, val) AO_char_xor_read(addr, val)
821 # define AO_HAVE_char_xor
822 #endif
823 
824 #if defined(AO_HAVE_char_xor_acquire) && defined(AO_HAVE_nop_full) \
825     && !defined(AO_HAVE_char_xor_full)
826 # define AO_char_xor_full(addr, val) \
827                         (AO_nop_full(), AO_char_xor_acquire(addr, val))
828 # define AO_HAVE_char_xor_full
829 #endif
830 
831 #if !defined(AO_HAVE_char_xor_release_write) \
832     && defined(AO_HAVE_char_xor_write)
833 # define AO_char_xor_release_write(addr, val) AO_char_xor_write(addr, val)
834 # define AO_HAVE_char_xor_release_write
835 #endif
836 #if !defined(AO_HAVE_char_xor_release_write) \
837     && defined(AO_HAVE_char_xor_release)
838 # define AO_char_xor_release_write(addr, val) AO_char_xor_release(addr, val)
839 # define AO_HAVE_char_xor_release_write
840 #endif
841 #if !defined(AO_HAVE_char_xor_acquire_read) \
842     && defined(AO_HAVE_char_xor_read)
843 # define AO_char_xor_acquire_read(addr, val) AO_char_xor_read(addr, val)
844 # define AO_HAVE_char_xor_acquire_read
845 #endif
846 #if !defined(AO_HAVE_char_xor_acquire_read) \
847     && defined(AO_HAVE_char_xor_acquire)
848 # define AO_char_xor_acquire_read(addr, val) AO_char_xor_acquire(addr, val)
849 # define AO_HAVE_char_xor_acquire_read
850 #endif
851 
852 /* char_and/or/xor_dd_acquire_read are meaningless.    */
853 /*
854  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
855  *
856  * Permission is hereby granted, free of charge, to any person obtaining a copy
857  * of this software and associated documentation files (the "Software"), to deal
858  * in the Software without restriction, including without limitation the rights
859  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
860  * copies of the Software, and to permit persons to whom the Software is
861  * furnished to do so, subject to the following conditions:
862  *
863  * The above copyright notice and this permission notice shall be included in
864  * all copies or substantial portions of the Software.
865  *
866  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
867  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
868  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
869  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
870  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
871  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
872  * SOFTWARE.
873  */
874 
875 /* short_compare_and_swap (based on fetch_compare_and_swap) */
876 #if defined(AO_HAVE_short_fetch_compare_and_swap_full) \
877     && !defined(AO_HAVE_short_compare_and_swap_full)
878   AO_INLINE int
879   AO_short_compare_and_swap_full(volatile unsigned/**/short *addr, unsigned/**/short old_val,
880                                  unsigned/**/short new_val)
881   {
882     return AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
883              == old_val;
884   }
885 # define AO_HAVE_short_compare_and_swap_full
886 #endif
887 
888 #if defined(AO_HAVE_short_fetch_compare_and_swap_acquire) \
889     && !defined(AO_HAVE_short_compare_and_swap_acquire)
890   AO_INLINE int
891   AO_short_compare_and_swap_acquire(volatile unsigned/**/short *addr, unsigned/**/short old_val,
892                                     unsigned/**/short new_val)
893   {
894     return AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val)
895              == old_val;
896   }
897 # define AO_HAVE_short_compare_and_swap_acquire
898 #endif
899 
900 #if defined(AO_HAVE_short_fetch_compare_and_swap_release) \
901     && !defined(AO_HAVE_short_compare_and_swap_release)
902   AO_INLINE int
903   AO_short_compare_and_swap_release(volatile unsigned/**/short *addr, unsigned/**/short old_val,
904                                     unsigned/**/short new_val)
905   {
906     return AO_short_fetch_compare_and_swap_release(addr, old_val, new_val)
907              == old_val;
908   }
909 # define AO_HAVE_short_compare_and_swap_release
910 #endif
911 
912 #if defined(AO_HAVE_short_fetch_compare_and_swap_write) \
913     && !defined(AO_HAVE_short_compare_and_swap_write)
914   AO_INLINE int
915   AO_short_compare_and_swap_write(volatile unsigned/**/short *addr, unsigned/**/short old_val,
916                                   unsigned/**/short new_val)
917   {
918     return AO_short_fetch_compare_and_swap_write(addr, old_val, new_val)
919              == old_val;
920   }
921 # define AO_HAVE_short_compare_and_swap_write
922 #endif
923 
924 #if defined(AO_HAVE_short_fetch_compare_and_swap_read) \
925     && !defined(AO_HAVE_short_compare_and_swap_read)
926   AO_INLINE int
927   AO_short_compare_and_swap_read(volatile unsigned/**/short *addr, unsigned/**/short old_val,
928                                  unsigned/**/short new_val)
929   {
930     return AO_short_fetch_compare_and_swap_read(addr, old_val, new_val)
931              == old_val;
932   }
933 # define AO_HAVE_short_compare_and_swap_read
934 #endif
935 
936 #if defined(AO_HAVE_short_fetch_compare_and_swap) \
937     && !defined(AO_HAVE_short_compare_and_swap)
938   AO_INLINE int
939   AO_short_compare_and_swap(volatile unsigned/**/short *addr, unsigned/**/short old_val,
940                             unsigned/**/short new_val)
941   {
942     return AO_short_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
943   }
944 # define AO_HAVE_short_compare_and_swap
945 #endif
946 
947 #if defined(AO_HAVE_short_fetch_compare_and_swap_release_write) \
948     && !defined(AO_HAVE_short_compare_and_swap_release_write)
949   AO_INLINE int
950   AO_short_compare_and_swap_release_write(volatile unsigned/**/short *addr,
951                                           unsigned/**/short old_val, unsigned/**/short new_val)
952   {
953     return AO_short_fetch_compare_and_swap_release_write(addr, old_val,
954                                                          new_val) == old_val;
955   }
956 # define AO_HAVE_short_compare_and_swap_release_write
957 #endif
958 
959 #if defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read) \
960     && !defined(AO_HAVE_short_compare_and_swap_acquire_read)
961   AO_INLINE int
962   AO_short_compare_and_swap_acquire_read(volatile unsigned/**/short *addr,
963                                          unsigned/**/short old_val, unsigned/**/short new_val)
964   {
965     return AO_short_fetch_compare_and_swap_acquire_read(addr, old_val,
966                                                         new_val) == old_val;
967   }
968 # define AO_HAVE_short_compare_and_swap_acquire_read
969 #endif
970 
971 #if defined(AO_HAVE_short_fetch_compare_and_swap_dd_acquire_read) \
972     && !defined(AO_HAVE_short_compare_and_swap_dd_acquire_read)
973   AO_INLINE int
974   AO_short_compare_and_swap_dd_acquire_read(volatile unsigned/**/short *addr,
975                                             unsigned/**/short old_val, unsigned/**/short new_val)
976   {
977     return AO_short_fetch_compare_and_swap_dd_acquire_read(addr, old_val,
978                                                            new_val) == old_val;
979   }
980 # define AO_HAVE_short_compare_and_swap_dd_acquire_read
981 #endif
982 
983 /* short_fetch_and_add */
984 /* We first try to implement fetch_and_add variants in terms of the     */
985 /* corresponding compare_and_swap variants to minimize adding barriers. */
986 #if defined(AO_HAVE_short_compare_and_swap_full) \
987     && !defined(AO_HAVE_short_fetch_and_add_full)
988   AO_ATTR_NO_SANITIZE_THREAD
989   AO_INLINE unsigned/**/short
990   AO_short_fetch_and_add_full(volatile unsigned/**/short *addr, unsigned/**/short incr)
991   {
992     unsigned/**/short old;
993 
994     do
995       {
996         old = *(unsigned/**/short *)addr;
997       }
998     while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
999                                                            old + incr)));
1000     return old;
1001   }
1002 # define AO_HAVE_short_fetch_and_add_full
1003 #endif
1004 
1005 #if defined(AO_HAVE_short_compare_and_swap_acquire) \
1006     && !defined(AO_HAVE_short_fetch_and_add_acquire)
1007   AO_ATTR_NO_SANITIZE_THREAD
1008   AO_INLINE unsigned/**/short
1009   AO_short_fetch_and_add_acquire(volatile unsigned/**/short *addr, unsigned/**/short incr)
1010   {
1011     unsigned/**/short old;
1012 
1013     do
1014       {
1015         old = *(unsigned/**/short *)addr;
1016       }
1017     while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(addr, old,
1018                                                               old + incr)));
1019     return old;
1020   }
1021 # define AO_HAVE_short_fetch_and_add_acquire
1022 #endif
1023 
1024 #if defined(AO_HAVE_short_compare_and_swap_release) \
1025     && !defined(AO_HAVE_short_fetch_and_add_release)
1026   AO_ATTR_NO_SANITIZE_THREAD
1027   AO_INLINE unsigned/**/short
1028   AO_short_fetch_and_add_release(volatile unsigned/**/short *addr, unsigned/**/short incr)
1029   {
1030     unsigned/**/short old;
1031 
1032     do
1033       {
1034         old = *(unsigned/**/short *)addr;
1035       }
1036     while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_release(addr, old,
1037                                                               old + incr)));
1038     return old;
1039   }
1040 # define AO_HAVE_short_fetch_and_add_release
1041 #endif
1042 
1043 #if defined(AO_HAVE_short_compare_and_swap) \
1044     && !defined(AO_HAVE_short_fetch_and_add)
1045   AO_ATTR_NO_SANITIZE_THREAD
1046   AO_INLINE unsigned/**/short
1047   AO_short_fetch_and_add(volatile unsigned/**/short *addr, unsigned/**/short incr)
1048   {
1049     unsigned/**/short old;
1050 
1051     do
1052       {
1053         old = *(unsigned/**/short *)addr;
1054       }
1055     while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(addr, old,
1056                                                       old + incr)));
1057     return old;
1058   }
1059 # define AO_HAVE_short_fetch_and_add
1060 #endif
1061 
1062 #if defined(AO_HAVE_short_fetch_and_add_full)
1063 # if !defined(AO_HAVE_short_fetch_and_add_release)
1064 #   define AO_short_fetch_and_add_release(addr, val) \
1065                                 AO_short_fetch_and_add_full(addr, val)
1066 #   define AO_HAVE_short_fetch_and_add_release
1067 # endif
1068 # if !defined(AO_HAVE_short_fetch_and_add_acquire)
1069 #   define AO_short_fetch_and_add_acquire(addr, val) \
1070                                 AO_short_fetch_and_add_full(addr, val)
1071 #   define AO_HAVE_short_fetch_and_add_acquire
1072 # endif
1073 # if !defined(AO_HAVE_short_fetch_and_add_write)
1074 #   define AO_short_fetch_and_add_write(addr, val) \
1075                                 AO_short_fetch_and_add_full(addr, val)
1076 #   define AO_HAVE_short_fetch_and_add_write
1077 # endif
1078 # if !defined(AO_HAVE_short_fetch_and_add_read)
1079 #   define AO_short_fetch_and_add_read(addr, val) \
1080                                 AO_short_fetch_and_add_full(addr, val)
1081 #   define AO_HAVE_short_fetch_and_add_read
1082 # endif
1083 #endif /* AO_HAVE_short_fetch_and_add_full */
1084 
1085 #if defined(AO_HAVE_short_fetch_and_add) && defined(AO_HAVE_nop_full) \
1086     && !defined(AO_HAVE_short_fetch_and_add_acquire)
1087   AO_INLINE unsigned/**/short
1088   AO_short_fetch_and_add_acquire(volatile unsigned/**/short *addr, unsigned/**/short incr)
1089   {
1090     unsigned/**/short result = AO_short_fetch_and_add(addr, incr);
1091     AO_nop_full();
1092     return result;
1093   }
1094 # define AO_HAVE_short_fetch_and_add_acquire
1095 #endif
1096 #if defined(AO_HAVE_short_fetch_and_add) && defined(AO_HAVE_nop_full) \
1097     && !defined(AO_HAVE_short_fetch_and_add_release)
1098 # define AO_short_fetch_and_add_release(addr, incr) \
1099                 (AO_nop_full(), AO_short_fetch_and_add(addr, incr))
1100 # define AO_HAVE_short_fetch_and_add_release
1101 #endif
1102 
1103 #if !defined(AO_HAVE_short_fetch_and_add) \
1104     && defined(AO_HAVE_short_fetch_and_add_release)
1105 # define AO_short_fetch_and_add(addr, val) \
1106                                 AO_short_fetch_and_add_release(addr, val)
1107 # define AO_HAVE_short_fetch_and_add
1108 #endif
1109 #if !defined(AO_HAVE_short_fetch_and_add) \
1110     && defined(AO_HAVE_short_fetch_and_add_acquire)
1111 # define AO_short_fetch_and_add(addr, val) \
1112                                 AO_short_fetch_and_add_acquire(addr, val)
1113 # define AO_HAVE_short_fetch_and_add
1114 #endif
1115 #if !defined(AO_HAVE_short_fetch_and_add) \
1116     && defined(AO_HAVE_short_fetch_and_add_write)
1117 # define AO_short_fetch_and_add(addr, val) \
1118                                 AO_short_fetch_and_add_write(addr, val)
1119 # define AO_HAVE_short_fetch_and_add
1120 #endif
1121 #if !defined(AO_HAVE_short_fetch_and_add) \
1122     && defined(AO_HAVE_short_fetch_and_add_read)
1123 # define AO_short_fetch_and_add(addr, val) \
1124                                 AO_short_fetch_and_add_read(addr, val)
1125 # define AO_HAVE_short_fetch_and_add
1126 #endif
1127 
1128 #if defined(AO_HAVE_short_fetch_and_add_acquire) \
1129     && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_short_fetch_and_add_full)
1130 # define AO_short_fetch_and_add_full(addr, val) \
1131                 (AO_nop_full(), AO_short_fetch_and_add_acquire(addr, val))
1132 # define AO_HAVE_short_fetch_and_add_full
1133 #endif
1134 
1135 #if !defined(AO_HAVE_short_fetch_and_add_release_write) \
1136     && defined(AO_HAVE_short_fetch_and_add_write)
1137 # define AO_short_fetch_and_add_release_write(addr, val) \
1138                                 AO_short_fetch_and_add_write(addr, val)
1139 # define AO_HAVE_short_fetch_and_add_release_write
1140 #endif
1141 #if !defined(AO_HAVE_short_fetch_and_add_release_write) \
1142     && defined(AO_HAVE_short_fetch_and_add_release)
1143 # define AO_short_fetch_and_add_release_write(addr, val) \
1144                                 AO_short_fetch_and_add_release(addr, val)
1145 # define AO_HAVE_short_fetch_and_add_release_write
1146 #endif
1147 
1148 #if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \
1149     && defined(AO_HAVE_short_fetch_and_add_read)
1150 # define AO_short_fetch_and_add_acquire_read(addr, val) \
1151                                 AO_short_fetch_and_add_read(addr, val)
1152 # define AO_HAVE_short_fetch_and_add_acquire_read
1153 #endif
1154 #if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \
1155     && defined(AO_HAVE_short_fetch_and_add_acquire)
1156 # define AO_short_fetch_and_add_acquire_read(addr, val) \
1157                                 AO_short_fetch_and_add_acquire(addr, val)
1158 # define AO_HAVE_short_fetch_and_add_acquire_read
1159 #endif
1160 
1161 #ifdef AO_NO_DD_ORDERING
1162 # if defined(AO_HAVE_short_fetch_and_add_acquire_read)
1163 #   define AO_short_fetch_and_add_dd_acquire_read(addr, val) \
1164                                 AO_short_fetch_and_add_acquire_read(addr, val)
1165 #   define AO_HAVE_short_fetch_and_add_dd_acquire_read
1166 # endif
1167 #else
1168 # if defined(AO_HAVE_short_fetch_and_add)
1169 #   define AO_short_fetch_and_add_dd_acquire_read(addr, val) \
1170                                 AO_short_fetch_and_add(addr, val)
1171 #   define AO_HAVE_short_fetch_and_add_dd_acquire_read
1172 # endif
1173 #endif /* !AO_NO_DD_ORDERING */
1174 
1175 /* short_fetch_and_add1 */
1176 #if defined(AO_HAVE_short_fetch_and_add_full) \
1177     && !defined(AO_HAVE_short_fetch_and_add1_full)
1178 # define AO_short_fetch_and_add1_full(addr) \
1179                                 AO_short_fetch_and_add_full(addr, 1)
1180 # define AO_HAVE_short_fetch_and_add1_full
1181 #endif
1182 #if defined(AO_HAVE_short_fetch_and_add_release) \
1183     && !defined(AO_HAVE_short_fetch_and_add1_release)
1184 # define AO_short_fetch_and_add1_release(addr) \
1185                                 AO_short_fetch_and_add_release(addr, 1)
1186 # define AO_HAVE_short_fetch_and_add1_release
1187 #endif
1188 #if defined(AO_HAVE_short_fetch_and_add_acquire) \
1189     && !defined(AO_HAVE_short_fetch_and_add1_acquire)
1190 # define AO_short_fetch_and_add1_acquire(addr) \
1191                                 AO_short_fetch_and_add_acquire(addr, 1)
1192 # define AO_HAVE_short_fetch_and_add1_acquire
1193 #endif
1194 #if defined(AO_HAVE_short_fetch_and_add_write) \
1195     && !defined(AO_HAVE_short_fetch_and_add1_write)
1196 # define AO_short_fetch_and_add1_write(addr) \
1197                                 AO_short_fetch_and_add_write(addr, 1)
1198 # define AO_HAVE_short_fetch_and_add1_write
1199 #endif
1200 #if defined(AO_HAVE_short_fetch_and_add_read) \
1201     && !defined(AO_HAVE_short_fetch_and_add1_read)
1202 # define AO_short_fetch_and_add1_read(addr) \
1203                                 AO_short_fetch_and_add_read(addr, 1)
1204 # define AO_HAVE_short_fetch_and_add1_read
1205 #endif
1206 #if defined(AO_HAVE_short_fetch_and_add_release_write) \
1207     && !defined(AO_HAVE_short_fetch_and_add1_release_write)
1208 # define AO_short_fetch_and_add1_release_write(addr) \
1209                                 AO_short_fetch_and_add_release_write(addr, 1)
1210 # define AO_HAVE_short_fetch_and_add1_release_write
1211 #endif
1212 #if defined(AO_HAVE_short_fetch_and_add_acquire_read) \
1213     && !defined(AO_HAVE_short_fetch_and_add1_acquire_read)
1214 # define AO_short_fetch_and_add1_acquire_read(addr) \
1215                                 AO_short_fetch_and_add_acquire_read(addr, 1)
1216 # define AO_HAVE_short_fetch_and_add1_acquire_read
1217 #endif
1218 #if defined(AO_HAVE_short_fetch_and_add) \
1219     && !defined(AO_HAVE_short_fetch_and_add1)
1220 # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add(addr, 1)
1221 # define AO_HAVE_short_fetch_and_add1
1222 #endif
1223 
1224 #if defined(AO_HAVE_short_fetch_and_add1_full)
1225 # if !defined(AO_HAVE_short_fetch_and_add1_release)
1226 #   define AO_short_fetch_and_add1_release(addr) \
1227                                 AO_short_fetch_and_add1_full(addr)
1228 #   define AO_HAVE_short_fetch_and_add1_release
1229 # endif
1230 # if !defined(AO_HAVE_short_fetch_and_add1_acquire)
1231 #   define AO_short_fetch_and_add1_acquire(addr) \
1232                                 AO_short_fetch_and_add1_full(addr)
1233 #   define AO_HAVE_short_fetch_and_add1_acquire
1234 # endif
1235 # if !defined(AO_HAVE_short_fetch_and_add1_write)
1236 #   define AO_short_fetch_and_add1_write(addr) \
1237                                 AO_short_fetch_and_add1_full(addr)
1238 #   define AO_HAVE_short_fetch_and_add1_write
1239 # endif
1240 # if !defined(AO_HAVE_short_fetch_and_add1_read)
1241 #   define AO_short_fetch_and_add1_read(addr) \
1242                                 AO_short_fetch_and_add1_full(addr)
1243 #   define AO_HAVE_short_fetch_and_add1_read
1244 # endif
1245 #endif /* AO_HAVE_short_fetch_and_add1_full */
1246 
1247 #if !defined(AO_HAVE_short_fetch_and_add1) \
1248     && defined(AO_HAVE_short_fetch_and_add1_release)
1249 # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_release(addr)
1250 # define AO_HAVE_short_fetch_and_add1
1251 #endif
1252 #if !defined(AO_HAVE_short_fetch_and_add1) \
1253     && defined(AO_HAVE_short_fetch_and_add1_acquire)
1254 # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_acquire(addr)
1255 # define AO_HAVE_short_fetch_and_add1
1256 #endif
1257 #if !defined(AO_HAVE_short_fetch_and_add1) \
1258     && defined(AO_HAVE_short_fetch_and_add1_write)
1259 # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_write(addr)
1260 # define AO_HAVE_short_fetch_and_add1
1261 #endif
1262 #if !defined(AO_HAVE_short_fetch_and_add1) \
1263     && defined(AO_HAVE_short_fetch_and_add1_read)
1264 # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_read(addr)
1265 # define AO_HAVE_short_fetch_and_add1
1266 #endif
1267 
1268 #if defined(AO_HAVE_short_fetch_and_add1_acquire) \
1269     && defined(AO_HAVE_nop_full) \
1270     && !defined(AO_HAVE_short_fetch_and_add1_full)
1271 # define AO_short_fetch_and_add1_full(addr) \
1272                         (AO_nop_full(), AO_short_fetch_and_add1_acquire(addr))
1273 # define AO_HAVE_short_fetch_and_add1_full
1274 #endif
1275 
1276 #if !defined(AO_HAVE_short_fetch_and_add1_release_write) \
1277     && defined(AO_HAVE_short_fetch_and_add1_write)
1278 # define AO_short_fetch_and_add1_release_write(addr) \
1279                                 AO_short_fetch_and_add1_write(addr)
1280 # define AO_HAVE_short_fetch_and_add1_release_write
1281 #endif
1282 #if !defined(AO_HAVE_short_fetch_and_add1_release_write) \
1283     && defined(AO_HAVE_short_fetch_and_add1_release)
1284 # define AO_short_fetch_and_add1_release_write(addr) \
1285                                 AO_short_fetch_and_add1_release(addr)
1286 # define AO_HAVE_short_fetch_and_add1_release_write
1287 #endif
1288 #if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \
1289     && defined(AO_HAVE_short_fetch_and_add1_read)
1290 # define AO_short_fetch_and_add1_acquire_read(addr) \
1291                                 AO_short_fetch_and_add1_read(addr)
1292 # define AO_HAVE_short_fetch_and_add1_acquire_read
1293 #endif
1294 #if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \
1295     && defined(AO_HAVE_short_fetch_and_add1_acquire)
1296 # define AO_short_fetch_and_add1_acquire_read(addr) \
1297                                 AO_short_fetch_and_add1_acquire(addr)
1298 # define AO_HAVE_short_fetch_and_add1_acquire_read
1299 #endif
1300 
1301 #ifdef AO_NO_DD_ORDERING
1302 # if defined(AO_HAVE_short_fetch_and_add1_acquire_read)
1303 #   define AO_short_fetch_and_add1_dd_acquire_read(addr) \
1304                                 AO_short_fetch_and_add1_acquire_read(addr)
1305 #   define AO_HAVE_short_fetch_and_add1_dd_acquire_read
1306 # endif
1307 #else
1308 # if defined(AO_HAVE_short_fetch_and_add1)
1309 #   define AO_short_fetch_and_add1_dd_acquire_read(addr) \
1310                                 AO_short_fetch_and_add1(addr)
1311 #   define AO_HAVE_short_fetch_and_add1_dd_acquire_read
1312 # endif
1313 #endif /* !AO_NO_DD_ORDERING */
1314 
1315 /* short_fetch_and_sub1 */
1316 #if defined(AO_HAVE_short_fetch_and_add_full) \
1317     && !defined(AO_HAVE_short_fetch_and_sub1_full)
1318 # define AO_short_fetch_and_sub1_full(addr) \
1319                 AO_short_fetch_and_add_full(addr, (unsigned/**/short)(-1))
1320 # define AO_HAVE_short_fetch_and_sub1_full
1321 #endif
1322 #if defined(AO_HAVE_short_fetch_and_add_release) \
1323     && !defined(AO_HAVE_short_fetch_and_sub1_release)
1324 # define AO_short_fetch_and_sub1_release(addr) \
1325                 AO_short_fetch_and_add_release(addr, (unsigned/**/short)(-1))
1326 # define AO_HAVE_short_fetch_and_sub1_release
1327 #endif
1328 #if defined(AO_HAVE_short_fetch_and_add_acquire) \
1329     && !defined(AO_HAVE_short_fetch_and_sub1_acquire)
1330 # define AO_short_fetch_and_sub1_acquire(addr) \
1331                 AO_short_fetch_and_add_acquire(addr, (unsigned/**/short)(-1))
1332 # define AO_HAVE_short_fetch_and_sub1_acquire
1333 #endif
1334 #if defined(AO_HAVE_short_fetch_and_add_write) \
1335     && !defined(AO_HAVE_short_fetch_and_sub1_write)
1336 # define AO_short_fetch_and_sub1_write(addr) \
1337                 AO_short_fetch_and_add_write(addr, (unsigned/**/short)(-1))
1338 # define AO_HAVE_short_fetch_and_sub1_write
1339 #endif
1340 #if defined(AO_HAVE_short_fetch_and_add_read) \
1341     && !defined(AO_HAVE_short_fetch_and_sub1_read)
1342 # define AO_short_fetch_and_sub1_read(addr) \
1343                 AO_short_fetch_and_add_read(addr, (unsigned/**/short)(-1))
1344 # define AO_HAVE_short_fetch_and_sub1_read
1345 #endif
1346 #if defined(AO_HAVE_short_fetch_and_add_release_write) \
1347     && !defined(AO_HAVE_short_fetch_and_sub1_release_write)
1348 # define AO_short_fetch_and_sub1_release_write(addr) \
1349                 AO_short_fetch_and_add_release_write(addr, (unsigned/**/short)(-1))
1350 # define AO_HAVE_short_fetch_and_sub1_release_write
1351 #endif
1352 #if defined(AO_HAVE_short_fetch_and_add_acquire_read) \
1353     && !defined(AO_HAVE_short_fetch_and_sub1_acquire_read)
1354 # define AO_short_fetch_and_sub1_acquire_read(addr) \
1355                 AO_short_fetch_and_add_acquire_read(addr, (unsigned/**/short)(-1))
1356 # define AO_HAVE_short_fetch_and_sub1_acquire_read
1357 #endif
1358 #if defined(AO_HAVE_short_fetch_and_add) \
1359     && !defined(AO_HAVE_short_fetch_and_sub1)
1360 # define AO_short_fetch_and_sub1(addr) \
1361                 AO_short_fetch_and_add(addr, (unsigned/**/short)(-1))
1362 # define AO_HAVE_short_fetch_and_sub1
1363 #endif
1364 
1365 #if defined(AO_HAVE_short_fetch_and_sub1_full)
1366 # if !defined(AO_HAVE_short_fetch_and_sub1_release)
1367 #   define AO_short_fetch_and_sub1_release(addr) \
1368                                 AO_short_fetch_and_sub1_full(addr)
1369 #   define AO_HAVE_short_fetch_and_sub1_release
1370 # endif
1371 # if !defined(AO_HAVE_short_fetch_and_sub1_acquire)
1372 #   define AO_short_fetch_and_sub1_acquire(addr) \
1373                                 AO_short_fetch_and_sub1_full(addr)
1374 #   define AO_HAVE_short_fetch_and_sub1_acquire
1375 # endif
1376 # if !defined(AO_HAVE_short_fetch_and_sub1_write)
1377 #   define AO_short_fetch_and_sub1_write(addr) \
1378                                 AO_short_fetch_and_sub1_full(addr)
1379 #   define AO_HAVE_short_fetch_and_sub1_write
1380 # endif
1381 # if !defined(AO_HAVE_short_fetch_and_sub1_read)
1382 #   define AO_short_fetch_and_sub1_read(addr) \
1383                                 AO_short_fetch_and_sub1_full(addr)
1384 #   define AO_HAVE_short_fetch_and_sub1_read
1385 # endif
1386 #endif /* AO_HAVE_short_fetch_and_sub1_full */
1387 
1388 #if !defined(AO_HAVE_short_fetch_and_sub1) \
1389     && defined(AO_HAVE_short_fetch_and_sub1_release)
1390 # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_release(addr)
1391 # define AO_HAVE_short_fetch_and_sub1
1392 #endif
1393 #if !defined(AO_HAVE_short_fetch_and_sub1) \
1394     && defined(AO_HAVE_short_fetch_and_sub1_acquire)
1395 # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_acquire(addr)
1396 # define AO_HAVE_short_fetch_and_sub1
1397 #endif
1398 #if !defined(AO_HAVE_short_fetch_and_sub1) \
1399     && defined(AO_HAVE_short_fetch_and_sub1_write)
1400 # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_write(addr)
1401 # define AO_HAVE_short_fetch_and_sub1
1402 #endif
1403 #if !defined(AO_HAVE_short_fetch_and_sub1) \
1404     && defined(AO_HAVE_short_fetch_and_sub1_read)
1405 # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_read(addr)
1406 # define AO_HAVE_short_fetch_and_sub1
1407 #endif
1408 
1409 #if defined(AO_HAVE_short_fetch_and_sub1_acquire) \
1410     && defined(AO_HAVE_nop_full) \
1411     && !defined(AO_HAVE_short_fetch_and_sub1_full)
1412 # define AO_short_fetch_and_sub1_full(addr) \
1413                         (AO_nop_full(), AO_short_fetch_and_sub1_acquire(addr))
1414 # define AO_HAVE_short_fetch_and_sub1_full
1415 #endif
1416 
1417 #if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \
1418     && defined(AO_HAVE_short_fetch_and_sub1_write)
1419 # define AO_short_fetch_and_sub1_release_write(addr) \
1420                                 AO_short_fetch_and_sub1_write(addr)
1421 # define AO_HAVE_short_fetch_and_sub1_release_write
1422 #endif
1423 #if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \
1424     && defined(AO_HAVE_short_fetch_and_sub1_release)
1425 # define AO_short_fetch_and_sub1_release_write(addr) \
1426                                 AO_short_fetch_and_sub1_release(addr)
1427 # define AO_HAVE_short_fetch_and_sub1_release_write
1428 #endif
1429 #if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \
1430     && defined(AO_HAVE_short_fetch_and_sub1_read)
1431 # define AO_short_fetch_and_sub1_acquire_read(addr) \
1432                                 AO_short_fetch_and_sub1_read(addr)
1433 # define AO_HAVE_short_fetch_and_sub1_acquire_read
1434 #endif
1435 #if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \
1436     && defined(AO_HAVE_short_fetch_and_sub1_acquire)
1437 # define AO_short_fetch_and_sub1_acquire_read(addr) \
1438                                 AO_short_fetch_and_sub1_acquire(addr)
1439 # define AO_HAVE_short_fetch_and_sub1_acquire_read
1440 #endif
1441 
1442 #ifdef AO_NO_DD_ORDERING
1443 # if defined(AO_HAVE_short_fetch_and_sub1_acquire_read)
1444 #   define AO_short_fetch_and_sub1_dd_acquire_read(addr) \
1445                                 AO_short_fetch_and_sub1_acquire_read(addr)
1446 #   define AO_HAVE_short_fetch_and_sub1_dd_acquire_read
1447 # endif
1448 #else
1449 # if defined(AO_HAVE_short_fetch_and_sub1)
1450 #   define AO_short_fetch_and_sub1_dd_acquire_read(addr) \
1451                                 AO_short_fetch_and_sub1(addr)
1452 #   define AO_HAVE_short_fetch_and_sub1_dd_acquire_read
1453 # endif
1454 #endif /* !AO_NO_DD_ORDERING */
1455 
1456 /* short_and */
1457 #if defined(AO_HAVE_short_compare_and_swap_full) \
1458     && !defined(AO_HAVE_short_and_full)
1459   AO_ATTR_NO_SANITIZE_THREAD
1460   AO_INLINE void
1461   AO_short_and_full(volatile unsigned/**/short *addr, unsigned/**/short value)
1462   {
1463     unsigned/**/short old;
1464 
1465     do
1466       {
1467         old = *(unsigned/**/short *)addr;
1468       }
1469     while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
1470                                                            old & value)));
1471   }
1472 # define AO_HAVE_short_and_full
1473 #endif
1474 
1475 #if defined(AO_HAVE_short_and_full)
1476 # if !defined(AO_HAVE_short_and_release)
1477 #   define AO_short_and_release(addr, val) AO_short_and_full(addr, val)
1478 #   define AO_HAVE_short_and_release
1479 # endif
1480 # if !defined(AO_HAVE_short_and_acquire)
1481 #   define AO_short_and_acquire(addr, val) AO_short_and_full(addr, val)
1482 #   define AO_HAVE_short_and_acquire
1483 # endif
1484 # if !defined(AO_HAVE_short_and_write)
1485 #   define AO_short_and_write(addr, val) AO_short_and_full(addr, val)
1486 #   define AO_HAVE_short_and_write
1487 # endif
1488 # if !defined(AO_HAVE_short_and_read)
1489 #   define AO_short_and_read(addr, val) AO_short_and_full(addr, val)
1490 #   define AO_HAVE_short_and_read
1491 # endif
1492 #endif /* AO_HAVE_short_and_full */
1493 
1494 #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_release)
1495 # define AO_short_and(addr, val) AO_short_and_release(addr, val)
1496 # define AO_HAVE_short_and
1497 #endif
1498 #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_acquire)
1499 # define AO_short_and(addr, val) AO_short_and_acquire(addr, val)
1500 # define AO_HAVE_short_and
1501 #endif
1502 #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_write)
1503 # define AO_short_and(addr, val) AO_short_and_write(addr, val)
1504 # define AO_HAVE_short_and
1505 #endif
1506 #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_read)
1507 # define AO_short_and(addr, val) AO_short_and_read(addr, val)
1508 # define AO_HAVE_short_and
1509 #endif
1510 
1511 #if defined(AO_HAVE_short_and_acquire) && defined(AO_HAVE_nop_full) \
1512     && !defined(AO_HAVE_short_and_full)
1513 # define AO_short_and_full(addr, val) \
1514                         (AO_nop_full(), AO_short_and_acquire(addr, val))
1515 # define AO_HAVE_short_and_full
1516 #endif
1517 
1518 #if !defined(AO_HAVE_short_and_release_write) \
1519     && defined(AO_HAVE_short_and_write)
1520 # define AO_short_and_release_write(addr, val) AO_short_and_write(addr, val)
1521 # define AO_HAVE_short_and_release_write
1522 #endif
1523 #if !defined(AO_HAVE_short_and_release_write) \
1524     && defined(AO_HAVE_short_and_release)
1525 # define AO_short_and_release_write(addr, val) AO_short_and_release(addr, val)
1526 # define AO_HAVE_short_and_release_write
1527 #endif
1528 #if !defined(AO_HAVE_short_and_acquire_read) \
1529     && defined(AO_HAVE_short_and_read)
1530 # define AO_short_and_acquire_read(addr, val) AO_short_and_read(addr, val)
1531 # define AO_HAVE_short_and_acquire_read
1532 #endif
1533 #if !defined(AO_HAVE_short_and_acquire_read) \
1534     && defined(AO_HAVE_short_and_acquire)
1535 # define AO_short_and_acquire_read(addr, val) AO_short_and_acquire(addr, val)
1536 # define AO_HAVE_short_and_acquire_read
1537 #endif
1538 
1539 /* short_or */
1540 #if defined(AO_HAVE_short_compare_and_swap_full) \
1541     && !defined(AO_HAVE_short_or_full)
1542   AO_ATTR_NO_SANITIZE_THREAD
1543   AO_INLINE void
1544   AO_short_or_full(volatile unsigned/**/short *addr, unsigned/**/short value)
1545   {
1546     unsigned/**/short old;
1547 
1548     do
1549       {
1550         old = *(unsigned/**/short *)addr;
1551       }
1552     while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
1553                                                            old | value)));
1554   }
1555 # define AO_HAVE_short_or_full
1556 #endif
1557 
1558 #if defined(AO_HAVE_short_or_full)
1559 # if !defined(AO_HAVE_short_or_release)
1560 #   define AO_short_or_release(addr, val) AO_short_or_full(addr, val)
1561 #   define AO_HAVE_short_or_release
1562 # endif
1563 # if !defined(AO_HAVE_short_or_acquire)
1564 #   define AO_short_or_acquire(addr, val) AO_short_or_full(addr, val)
1565 #   define AO_HAVE_short_or_acquire
1566 # endif
1567 # if !defined(AO_HAVE_short_or_write)
1568 #   define AO_short_or_write(addr, val) AO_short_or_full(addr, val)
1569 #   define AO_HAVE_short_or_write
1570 # endif
1571 # if !defined(AO_HAVE_short_or_read)
1572 #   define AO_short_or_read(addr, val) AO_short_or_full(addr, val)
1573 #   define AO_HAVE_short_or_read
1574 # endif
1575 #endif /* AO_HAVE_short_or_full */
1576 
1577 #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_release)
1578 # define AO_short_or(addr, val) AO_short_or_release(addr, val)
1579 # define AO_HAVE_short_or
1580 #endif
1581 #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_acquire)
1582 # define AO_short_or(addr, val) AO_short_or_acquire(addr, val)
1583 # define AO_HAVE_short_or
1584 #endif
1585 #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_write)
1586 # define AO_short_or(addr, val) AO_short_or_write(addr, val)
1587 # define AO_HAVE_short_or
1588 #endif
1589 #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_read)
1590 # define AO_short_or(addr, val) AO_short_or_read(addr, val)
1591 # define AO_HAVE_short_or
1592 #endif
1593 
1594 #if defined(AO_HAVE_short_or_acquire) && defined(AO_HAVE_nop_full) \
1595     && !defined(AO_HAVE_short_or_full)
1596 # define AO_short_or_full(addr, val) \
1597                         (AO_nop_full(), AO_short_or_acquire(addr, val))
1598 # define AO_HAVE_short_or_full
1599 #endif
1600 
1601 #if !defined(AO_HAVE_short_or_release_write) \
1602     && defined(AO_HAVE_short_or_write)
1603 # define AO_short_or_release_write(addr, val) AO_short_or_write(addr, val)
1604 # define AO_HAVE_short_or_release_write
1605 #endif
1606 #if !defined(AO_HAVE_short_or_release_write) \
1607     && defined(AO_HAVE_short_or_release)
1608 # define AO_short_or_release_write(addr, val) AO_short_or_release(addr, val)
1609 # define AO_HAVE_short_or_release_write
1610 #endif
1611 #if !defined(AO_HAVE_short_or_acquire_read) && defined(AO_HAVE_short_or_read)
1612 # define AO_short_or_acquire_read(addr, val) AO_short_or_read(addr, val)
1613 # define AO_HAVE_short_or_acquire_read
1614 #endif
1615 #if !defined(AO_HAVE_short_or_acquire_read) \
1616     && defined(AO_HAVE_short_or_acquire)
1617 # define AO_short_or_acquire_read(addr, val) AO_short_or_acquire(addr, val)
1618 # define AO_HAVE_short_or_acquire_read
1619 #endif
1620 
1621 /* short_xor */
1622 #if defined(AO_HAVE_short_compare_and_swap_full) \
1623     && !defined(AO_HAVE_short_xor_full)
1624   AO_ATTR_NO_SANITIZE_THREAD
1625   AO_INLINE void
1626   AO_short_xor_full(volatile unsigned/**/short *addr, unsigned/**/short value)
1627   {
1628     unsigned/**/short old;
1629 
1630     do
1631       {
1632         old = *(unsigned/**/short *)addr;
1633       }
1634     while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
1635                                                            old ^ value)));
1636   }
1637 # define AO_HAVE_short_xor_full
1638 #endif
1639 
1640 #if defined(AO_HAVE_short_xor_full)
1641 # if !defined(AO_HAVE_short_xor_release)
1642 #   define AO_short_xor_release(addr, val) AO_short_xor_full(addr, val)
1643 #   define AO_HAVE_short_xor_release
1644 # endif
1645 # if !defined(AO_HAVE_short_xor_acquire)
1646 #   define AO_short_xor_acquire(addr, val) AO_short_xor_full(addr, val)
1647 #   define AO_HAVE_short_xor_acquire
1648 # endif
1649 # if !defined(AO_HAVE_short_xor_write)
1650 #   define AO_short_xor_write(addr, val) AO_short_xor_full(addr, val)
1651 #   define AO_HAVE_short_xor_write
1652 # endif
1653 # if !defined(AO_HAVE_short_xor_read)
1654 #   define AO_short_xor_read(addr, val) AO_short_xor_full(addr, val)
1655 #   define AO_HAVE_short_xor_read
1656 # endif
1657 #endif /* AO_HAVE_short_xor_full */
1658 
1659 #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_release)
1660 # define AO_short_xor(addr, val) AO_short_xor_release(addr, val)
1661 # define AO_HAVE_short_xor
1662 #endif
1663 #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_acquire)
1664 # define AO_short_xor(addr, val) AO_short_xor_acquire(addr, val)
1665 # define AO_HAVE_short_xor
1666 #endif
1667 #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_write)
1668 # define AO_short_xor(addr, val) AO_short_xor_write(addr, val)
1669 # define AO_HAVE_short_xor
1670 #endif
1671 #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_read)
1672 # define AO_short_xor(addr, val) AO_short_xor_read(addr, val)
1673 # define AO_HAVE_short_xor
1674 #endif
1675 
1676 #if defined(AO_HAVE_short_xor_acquire) && defined(AO_HAVE_nop_full) \
1677     && !defined(AO_HAVE_short_xor_full)
1678 # define AO_short_xor_full(addr, val) \
1679                         (AO_nop_full(), AO_short_xor_acquire(addr, val))
1680 # define AO_HAVE_short_xor_full
1681 #endif
1682 
1683 #if !defined(AO_HAVE_short_xor_release_write) \
1684     && defined(AO_HAVE_short_xor_write)
1685 # define AO_short_xor_release_write(addr, val) AO_short_xor_write(addr, val)
1686 # define AO_HAVE_short_xor_release_write
1687 #endif
1688 #if !defined(AO_HAVE_short_xor_release_write) \
1689     && defined(AO_HAVE_short_xor_release)
1690 # define AO_short_xor_release_write(addr, val) AO_short_xor_release(addr, val)
1691 # define AO_HAVE_short_xor_release_write
1692 #endif
1693 #if !defined(AO_HAVE_short_xor_acquire_read) \
1694     && defined(AO_HAVE_short_xor_read)
1695 # define AO_short_xor_acquire_read(addr, val) AO_short_xor_read(addr, val)
1696 # define AO_HAVE_short_xor_acquire_read
1697 #endif
1698 #if !defined(AO_HAVE_short_xor_acquire_read) \
1699     && defined(AO_HAVE_short_xor_acquire)
1700 # define AO_short_xor_acquire_read(addr, val) AO_short_xor_acquire(addr, val)
1701 # define AO_HAVE_short_xor_acquire_read
1702 #endif
1703 
1704 /* short_and/or/xor_dd_acquire_read are meaningless.    */
1705 /*
1706  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
1707  *
1708  * Permission is hereby granted, free of charge, to any person obtaining a copy
1709  * of this software and associated documentation files (the "Software"), to deal
1710  * in the Software without restriction, including without limitation the rights
1711  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1712  * copies of the Software, and to permit persons to whom the Software is
1713  * furnished to do so, subject to the following conditions:
1714  *
1715  * The above copyright notice and this permission notice shall be included in
1716  * all copies or substantial portions of the Software.
1717  *
1718  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1719  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1720  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
1721  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1722  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
1723  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
1724  * SOFTWARE.
1725  */
1726 
1727 /* int_compare_and_swap (based on fetch_compare_and_swap) */
1728 #if defined(AO_HAVE_int_fetch_compare_and_swap_full) \
1729     && !defined(AO_HAVE_int_compare_and_swap_full)
1730   AO_INLINE int
1731   AO_int_compare_and_swap_full(volatile unsigned *addr, unsigned old_val,
1732                                  unsigned new_val)
1733   {
1734     return AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1735              == old_val;
1736   }
1737 # define AO_HAVE_int_compare_and_swap_full
1738 #endif
1739 
1740 #if defined(AO_HAVE_int_fetch_compare_and_swap_acquire) \
1741     && !defined(AO_HAVE_int_compare_and_swap_acquire)
1742   AO_INLINE int
1743   AO_int_compare_and_swap_acquire(volatile unsigned *addr, unsigned old_val,
1744                                     unsigned new_val)
1745   {
1746     return AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1747              == old_val;
1748   }
1749 # define AO_HAVE_int_compare_and_swap_acquire
1750 #endif
1751 
1752 #if defined(AO_HAVE_int_fetch_compare_and_swap_release) \
1753     && !defined(AO_HAVE_int_compare_and_swap_release)
1754   AO_INLINE int
1755   AO_int_compare_and_swap_release(volatile unsigned *addr, unsigned old_val,
1756                                     unsigned new_val)
1757   {
1758     return AO_int_fetch_compare_and_swap_release(addr, old_val, new_val)
1759              == old_val;
1760   }
1761 # define AO_HAVE_int_compare_and_swap_release
1762 #endif
1763 
1764 #if defined(AO_HAVE_int_fetch_compare_and_swap_write) \
1765     && !defined(AO_HAVE_int_compare_and_swap_write)
1766   AO_INLINE int
1767   AO_int_compare_and_swap_write(volatile unsigned *addr, unsigned old_val,
1768                                   unsigned new_val)
1769   {
1770     return AO_int_fetch_compare_and_swap_write(addr, old_val, new_val)
1771              == old_val;
1772   }
1773 # define AO_HAVE_int_compare_and_swap_write
1774 #endif
1775 
1776 #if defined(AO_HAVE_int_fetch_compare_and_swap_read) \
1777     && !defined(AO_HAVE_int_compare_and_swap_read)
1778   AO_INLINE int
1779   AO_int_compare_and_swap_read(volatile unsigned *addr, unsigned old_val,
1780                                  unsigned new_val)
1781   {
1782     return AO_int_fetch_compare_and_swap_read(addr, old_val, new_val)
1783              == old_val;
1784   }
1785 # define AO_HAVE_int_compare_and_swap_read
1786 #endif
1787 
1788 #if defined(AO_HAVE_int_fetch_compare_and_swap) \
1789     && !defined(AO_HAVE_int_compare_and_swap)
1790   AO_INLINE int
1791   AO_int_compare_and_swap(volatile unsigned *addr, unsigned old_val,
1792                             unsigned new_val)
1793   {
1794     return AO_int_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
1795   }
1796 # define AO_HAVE_int_compare_and_swap
1797 #endif
1798 
1799 #if defined(AO_HAVE_int_fetch_compare_and_swap_release_write) \
1800     && !defined(AO_HAVE_int_compare_and_swap_release_write)
1801   AO_INLINE int
1802   AO_int_compare_and_swap_release_write(volatile unsigned *addr,
1803                                           unsigned old_val, unsigned new_val)
1804   {
1805     return AO_int_fetch_compare_and_swap_release_write(addr, old_val,
1806                                                          new_val) == old_val;
1807   }
1808 # define AO_HAVE_int_compare_and_swap_release_write
1809 #endif
1810 
1811 #if defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read) \
1812     && !defined(AO_HAVE_int_compare_and_swap_acquire_read)
1813   AO_INLINE int
1814   AO_int_compare_and_swap_acquire_read(volatile unsigned *addr,
1815                                          unsigned old_val, unsigned new_val)
1816   {
1817     return AO_int_fetch_compare_and_swap_acquire_read(addr, old_val,
1818                                                         new_val) == old_val;
1819   }
1820 # define AO_HAVE_int_compare_and_swap_acquire_read
1821 #endif
1822 
1823 #if defined(AO_HAVE_int_fetch_compare_and_swap_dd_acquire_read) \
1824     && !defined(AO_HAVE_int_compare_and_swap_dd_acquire_read)
1825   AO_INLINE int
1826   AO_int_compare_and_swap_dd_acquire_read(volatile unsigned *addr,
1827                                             unsigned old_val, unsigned new_val)
1828   {
1829     return AO_int_fetch_compare_and_swap_dd_acquire_read(addr, old_val,
1830                                                            new_val) == old_val;
1831   }
1832 # define AO_HAVE_int_compare_and_swap_dd_acquire_read
1833 #endif
1834 
1835 /* int_fetch_and_add */
1836 /* We first try to implement fetch_and_add variants in terms of the     */
1837 /* corresponding compare_and_swap variants to minimize adding barriers. */
1838 #if defined(AO_HAVE_int_compare_and_swap_full) \
1839     && !defined(AO_HAVE_int_fetch_and_add_full)
1840   AO_ATTR_NO_SANITIZE_THREAD
1841   AO_INLINE unsigned
1842   AO_int_fetch_and_add_full(volatile unsigned *addr, unsigned incr)
1843   {
1844     unsigned old;
1845 
1846     do
1847       {
1848         old = *(unsigned *)addr;
1849       }
1850     while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
1851                                                            old + incr)));
1852     return old;
1853   }
1854 # define AO_HAVE_int_fetch_and_add_full
1855 #endif
1856 
1857 #if defined(AO_HAVE_int_compare_and_swap_acquire) \
1858     && !defined(AO_HAVE_int_fetch_and_add_acquire)
1859   AO_ATTR_NO_SANITIZE_THREAD
1860   AO_INLINE unsigned
1861   AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr)
1862   {
1863     unsigned old;
1864 
1865     do
1866       {
1867         old = *(unsigned *)addr;
1868       }
1869     while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(addr, old,
1870                                                               old + incr)));
1871     return old;
1872   }
1873 # define AO_HAVE_int_fetch_and_add_acquire
1874 #endif
1875 
1876 #if defined(AO_HAVE_int_compare_and_swap_release) \
1877     && !defined(AO_HAVE_int_fetch_and_add_release)
1878   AO_ATTR_NO_SANITIZE_THREAD
1879   AO_INLINE unsigned
1880   AO_int_fetch_and_add_release(volatile unsigned *addr, unsigned incr)
1881   {
1882     unsigned old;
1883 
1884     do
1885       {
1886         old = *(unsigned *)addr;
1887       }
1888     while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_release(addr, old,
1889                                                               old + incr)));
1890     return old;
1891   }
1892 # define AO_HAVE_int_fetch_and_add_release
1893 #endif
1894 
1895 #if defined(AO_HAVE_int_compare_and_swap) \
1896     && !defined(AO_HAVE_int_fetch_and_add)
1897   AO_ATTR_NO_SANITIZE_THREAD
1898   AO_INLINE unsigned
1899   AO_int_fetch_and_add(volatile unsigned *addr, unsigned incr)
1900   {
1901     unsigned old;
1902 
1903     do
1904       {
1905         old = *(unsigned *)addr;
1906       }
1907     while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(addr, old,
1908                                                       old + incr)));
1909     return old;
1910   }
1911 # define AO_HAVE_int_fetch_and_add
1912 #endif
1913 
1914 #if defined(AO_HAVE_int_fetch_and_add_full)
1915 # if !defined(AO_HAVE_int_fetch_and_add_release)
1916 #   define AO_int_fetch_and_add_release(addr, val) \
1917                                 AO_int_fetch_and_add_full(addr, val)
1918 #   define AO_HAVE_int_fetch_and_add_release
1919 # endif
1920 # if !defined(AO_HAVE_int_fetch_and_add_acquire)
1921 #   define AO_int_fetch_and_add_acquire(addr, val) \
1922                                 AO_int_fetch_and_add_full(addr, val)
1923 #   define AO_HAVE_int_fetch_and_add_acquire
1924 # endif
1925 # if !defined(AO_HAVE_int_fetch_and_add_write)
1926 #   define AO_int_fetch_and_add_write(addr, val) \
1927                                 AO_int_fetch_and_add_full(addr, val)
1928 #   define AO_HAVE_int_fetch_and_add_write
1929 # endif
1930 # if !defined(AO_HAVE_int_fetch_and_add_read)
1931 #   define AO_int_fetch_and_add_read(addr, val) \
1932                                 AO_int_fetch_and_add_full(addr, val)
1933 #   define AO_HAVE_int_fetch_and_add_read
1934 # endif
1935 #endif /* AO_HAVE_int_fetch_and_add_full */
1936 
1937 #if defined(AO_HAVE_int_fetch_and_add) && defined(AO_HAVE_nop_full) \
1938     && !defined(AO_HAVE_int_fetch_and_add_acquire)
1939   AO_INLINE unsigned
1940   AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr)
1941   {
1942     unsigned result = AO_int_fetch_and_add(addr, incr);
1943     AO_nop_full();
1944     return result;
1945   }
1946 # define AO_HAVE_int_fetch_and_add_acquire
1947 #endif
1948 #if defined(AO_HAVE_int_fetch_and_add) && defined(AO_HAVE_nop_full) \
1949     && !defined(AO_HAVE_int_fetch_and_add_release)
1950 # define AO_int_fetch_and_add_release(addr, incr) \
1951                 (AO_nop_full(), AO_int_fetch_and_add(addr, incr))
1952 # define AO_HAVE_int_fetch_and_add_release
1953 #endif
1954 
1955 #if !defined(AO_HAVE_int_fetch_and_add) \
1956     && defined(AO_HAVE_int_fetch_and_add_release)
1957 # define AO_int_fetch_and_add(addr, val) \
1958                                 AO_int_fetch_and_add_release(addr, val)
1959 # define AO_HAVE_int_fetch_and_add
1960 #endif
1961 #if !defined(AO_HAVE_int_fetch_and_add) \
1962     && defined(AO_HAVE_int_fetch_and_add_acquire)
1963 # define AO_int_fetch_and_add(addr, val) \
1964                                 AO_int_fetch_and_add_acquire(addr, val)
1965 # define AO_HAVE_int_fetch_and_add
1966 #endif
1967 #if !defined(AO_HAVE_int_fetch_and_add) \
1968     && defined(AO_HAVE_int_fetch_and_add_write)
1969 # define AO_int_fetch_and_add(addr, val) \
1970                                 AO_int_fetch_and_add_write(addr, val)
1971 # define AO_HAVE_int_fetch_and_add
1972 #endif
1973 #if !defined(AO_HAVE_int_fetch_and_add) \
1974     && defined(AO_HAVE_int_fetch_and_add_read)
1975 # define AO_int_fetch_and_add(addr, val) \
1976                                 AO_int_fetch_and_add_read(addr, val)
1977 # define AO_HAVE_int_fetch_and_add
1978 #endif
1979 
1980 #if defined(AO_HAVE_int_fetch_and_add_acquire) \
1981     && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_int_fetch_and_add_full)
1982 # define AO_int_fetch_and_add_full(addr, val) \
1983                 (AO_nop_full(), AO_int_fetch_and_add_acquire(addr, val))
1984 # define AO_HAVE_int_fetch_and_add_full
1985 #endif
1986 
1987 #if !defined(AO_HAVE_int_fetch_and_add_release_write) \
1988     && defined(AO_HAVE_int_fetch_and_add_write)
1989 # define AO_int_fetch_and_add_release_write(addr, val) \
1990                                 AO_int_fetch_and_add_write(addr, val)
1991 # define AO_HAVE_int_fetch_and_add_release_write
1992 #endif
1993 #if !defined(AO_HAVE_int_fetch_and_add_release_write) \
1994     && defined(AO_HAVE_int_fetch_and_add_release)
1995 # define AO_int_fetch_and_add_release_write(addr, val) \
1996                                 AO_int_fetch_and_add_release(addr, val)
1997 # define AO_HAVE_int_fetch_and_add_release_write
1998 #endif
1999 
2000 #if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \
2001     && defined(AO_HAVE_int_fetch_and_add_read)
2002 # define AO_int_fetch_and_add_acquire_read(addr, val) \
2003                                 AO_int_fetch_and_add_read(addr, val)
2004 # define AO_HAVE_int_fetch_and_add_acquire_read
2005 #endif
2006 #if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \
2007     && defined(AO_HAVE_int_fetch_and_add_acquire)
2008 # define AO_int_fetch_and_add_acquire_read(addr, val) \
2009                                 AO_int_fetch_and_add_acquire(addr, val)
2010 # define AO_HAVE_int_fetch_and_add_acquire_read
2011 #endif
2012 
2013 #ifdef AO_NO_DD_ORDERING
2014 # if defined(AO_HAVE_int_fetch_and_add_acquire_read)
2015 #   define AO_int_fetch_and_add_dd_acquire_read(addr, val) \
2016                                 AO_int_fetch_and_add_acquire_read(addr, val)
2017 #   define AO_HAVE_int_fetch_and_add_dd_acquire_read
2018 # endif
2019 #else
2020 # if defined(AO_HAVE_int_fetch_and_add)
2021 #   define AO_int_fetch_and_add_dd_acquire_read(addr, val) \
2022                                 AO_int_fetch_and_add(addr, val)
2023 #   define AO_HAVE_int_fetch_and_add_dd_acquire_read
2024 # endif
2025 #endif /* !AO_NO_DD_ORDERING */
2026 
2027 /* int_fetch_and_add1 */
2028 #if defined(AO_HAVE_int_fetch_and_add_full) \
2029     && !defined(AO_HAVE_int_fetch_and_add1_full)
2030 # define AO_int_fetch_and_add1_full(addr) \
2031                                 AO_int_fetch_and_add_full(addr, 1)
2032 # define AO_HAVE_int_fetch_and_add1_full
2033 #endif
2034 #if defined(AO_HAVE_int_fetch_and_add_release) \
2035     && !defined(AO_HAVE_int_fetch_and_add1_release)
2036 # define AO_int_fetch_and_add1_release(addr) \
2037                                 AO_int_fetch_and_add_release(addr, 1)
2038 # define AO_HAVE_int_fetch_and_add1_release
2039 #endif
2040 #if defined(AO_HAVE_int_fetch_and_add_acquire) \
2041     && !defined(AO_HAVE_int_fetch_and_add1_acquire)
2042 # define AO_int_fetch_and_add1_acquire(addr) \
2043                                 AO_int_fetch_and_add_acquire(addr, 1)
2044 # define AO_HAVE_int_fetch_and_add1_acquire
2045 #endif
2046 #if defined(AO_HAVE_int_fetch_and_add_write) \
2047     && !defined(AO_HAVE_int_fetch_and_add1_write)
2048 # define AO_int_fetch_and_add1_write(addr) \
2049                                 AO_int_fetch_and_add_write(addr, 1)
2050 # define AO_HAVE_int_fetch_and_add1_write
2051 #endif
2052 #if defined(AO_HAVE_int_fetch_and_add_read) \
2053     && !defined(AO_HAVE_int_fetch_and_add1_read)
2054 # define AO_int_fetch_and_add1_read(addr) \
2055                                 AO_int_fetch_and_add_read(addr, 1)
2056 # define AO_HAVE_int_fetch_and_add1_read
2057 #endif
2058 #if defined(AO_HAVE_int_fetch_and_add_release_write) \
2059     && !defined(AO_HAVE_int_fetch_and_add1_release_write)
2060 # define AO_int_fetch_and_add1_release_write(addr) \
2061                                 AO_int_fetch_and_add_release_write(addr, 1)
2062 # define AO_HAVE_int_fetch_and_add1_release_write
2063 #endif
2064 #if defined(AO_HAVE_int_fetch_and_add_acquire_read) \
2065     && !defined(AO_HAVE_int_fetch_and_add1_acquire_read)
2066 # define AO_int_fetch_and_add1_acquire_read(addr) \
2067                                 AO_int_fetch_and_add_acquire_read(addr, 1)
2068 # define AO_HAVE_int_fetch_and_add1_acquire_read
2069 #endif
2070 #if defined(AO_HAVE_int_fetch_and_add) \
2071     && !defined(AO_HAVE_int_fetch_and_add1)
2072 # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add(addr, 1)
2073 # define AO_HAVE_int_fetch_and_add1
2074 #endif
2075 
2076 #if defined(AO_HAVE_int_fetch_and_add1_full)
2077 # if !defined(AO_HAVE_int_fetch_and_add1_release)
2078 #   define AO_int_fetch_and_add1_release(addr) \
2079                                 AO_int_fetch_and_add1_full(addr)
2080 #   define AO_HAVE_int_fetch_and_add1_release
2081 # endif
2082 # if !defined(AO_HAVE_int_fetch_and_add1_acquire)
2083 #   define AO_int_fetch_and_add1_acquire(addr) \
2084                                 AO_int_fetch_and_add1_full(addr)
2085 #   define AO_HAVE_int_fetch_and_add1_acquire
2086 # endif
2087 # if !defined(AO_HAVE_int_fetch_and_add1_write)
2088 #   define AO_int_fetch_and_add1_write(addr) \
2089                                 AO_int_fetch_and_add1_full(addr)
2090 #   define AO_HAVE_int_fetch_and_add1_write
2091 # endif
2092 # if !defined(AO_HAVE_int_fetch_and_add1_read)
2093 #   define AO_int_fetch_and_add1_read(addr) \
2094                                 AO_int_fetch_and_add1_full(addr)
2095 #   define AO_HAVE_int_fetch_and_add1_read
2096 # endif
2097 #endif /* AO_HAVE_int_fetch_and_add1_full */
2098 
2099 #if !defined(AO_HAVE_int_fetch_and_add1) \
2100     && defined(AO_HAVE_int_fetch_and_add1_release)
2101 # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_release(addr)
2102 # define AO_HAVE_int_fetch_and_add1
2103 #endif
2104 #if !defined(AO_HAVE_int_fetch_and_add1) \
2105     && defined(AO_HAVE_int_fetch_and_add1_acquire)
2106 # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_acquire(addr)
2107 # define AO_HAVE_int_fetch_and_add1
2108 #endif
2109 #if !defined(AO_HAVE_int_fetch_and_add1) \
2110     && defined(AO_HAVE_int_fetch_and_add1_write)
2111 # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_write(addr)
2112 # define AO_HAVE_int_fetch_and_add1
2113 #endif
2114 #if !defined(AO_HAVE_int_fetch_and_add1) \
2115     && defined(AO_HAVE_int_fetch_and_add1_read)
2116 # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_read(addr)
2117 # define AO_HAVE_int_fetch_and_add1
2118 #endif
2119 
2120 #if defined(AO_HAVE_int_fetch_and_add1_acquire) \
2121     && defined(AO_HAVE_nop_full) \
2122     && !defined(AO_HAVE_int_fetch_and_add1_full)
2123 # define AO_int_fetch_and_add1_full(addr) \
2124                         (AO_nop_full(), AO_int_fetch_and_add1_acquire(addr))
2125 # define AO_HAVE_int_fetch_and_add1_full
2126 #endif
2127 
2128 #if !defined(AO_HAVE_int_fetch_and_add1_release_write) \
2129     && defined(AO_HAVE_int_fetch_and_add1_write)
2130 # define AO_int_fetch_and_add1_release_write(addr) \
2131                                 AO_int_fetch_and_add1_write(addr)
2132 # define AO_HAVE_int_fetch_and_add1_release_write
2133 #endif
2134 #if !defined(AO_HAVE_int_fetch_and_add1_release_write) \
2135     && defined(AO_HAVE_int_fetch_and_add1_release)
2136 # define AO_int_fetch_and_add1_release_write(addr) \
2137                                 AO_int_fetch_and_add1_release(addr)
2138 # define AO_HAVE_int_fetch_and_add1_release_write
2139 #endif
2140 #if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \
2141     && defined(AO_HAVE_int_fetch_and_add1_read)
2142 # define AO_int_fetch_and_add1_acquire_read(addr) \
2143                                 AO_int_fetch_and_add1_read(addr)
2144 # define AO_HAVE_int_fetch_and_add1_acquire_read
2145 #endif
2146 #if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \
2147     && defined(AO_HAVE_int_fetch_and_add1_acquire)
2148 # define AO_int_fetch_and_add1_acquire_read(addr) \
2149                                 AO_int_fetch_and_add1_acquire(addr)
2150 # define AO_HAVE_int_fetch_and_add1_acquire_read
2151 #endif
2152 
2153 #ifdef AO_NO_DD_ORDERING
2154 # if defined(AO_HAVE_int_fetch_and_add1_acquire_read)
2155 #   define AO_int_fetch_and_add1_dd_acquire_read(addr) \
2156                                 AO_int_fetch_and_add1_acquire_read(addr)
2157 #   define AO_HAVE_int_fetch_and_add1_dd_acquire_read
2158 # endif
2159 #else
2160 # if defined(AO_HAVE_int_fetch_and_add1)
2161 #   define AO_int_fetch_and_add1_dd_acquire_read(addr) \
2162                                 AO_int_fetch_and_add1(addr)
2163 #   define AO_HAVE_int_fetch_and_add1_dd_acquire_read
2164 # endif
2165 #endif /* !AO_NO_DD_ORDERING */
2166 
2167 /* int_fetch_and_sub1 */
2168 #if defined(AO_HAVE_int_fetch_and_add_full) \
2169     && !defined(AO_HAVE_int_fetch_and_sub1_full)
2170 # define AO_int_fetch_and_sub1_full(addr) \
2171                 AO_int_fetch_and_add_full(addr, (unsigned)(-1))
2172 # define AO_HAVE_int_fetch_and_sub1_full
2173 #endif
2174 #if defined(AO_HAVE_int_fetch_and_add_release) \
2175     && !defined(AO_HAVE_int_fetch_and_sub1_release)
2176 # define AO_int_fetch_and_sub1_release(addr) \
2177                 AO_int_fetch_and_add_release(addr, (unsigned)(-1))
2178 # define AO_HAVE_int_fetch_and_sub1_release
2179 #endif
2180 #if defined(AO_HAVE_int_fetch_and_add_acquire) \
2181     && !defined(AO_HAVE_int_fetch_and_sub1_acquire)
2182 # define AO_int_fetch_and_sub1_acquire(addr) \
2183                 AO_int_fetch_and_add_acquire(addr, (unsigned)(-1))
2184 # define AO_HAVE_int_fetch_and_sub1_acquire
2185 #endif
2186 #if defined(AO_HAVE_int_fetch_and_add_write) \
2187     && !defined(AO_HAVE_int_fetch_and_sub1_write)
2188 # define AO_int_fetch_and_sub1_write(addr) \
2189                 AO_int_fetch_and_add_write(addr, (unsigned)(-1))
2190 # define AO_HAVE_int_fetch_and_sub1_write
2191 #endif
2192 #if defined(AO_HAVE_int_fetch_and_add_read) \
2193     && !defined(AO_HAVE_int_fetch_and_sub1_read)
2194 # define AO_int_fetch_and_sub1_read(addr) \
2195                 AO_int_fetch_and_add_read(addr, (unsigned)(-1))
2196 # define AO_HAVE_int_fetch_and_sub1_read
2197 #endif
2198 #if defined(AO_HAVE_int_fetch_and_add_release_write) \
2199     && !defined(AO_HAVE_int_fetch_and_sub1_release_write)
2200 # define AO_int_fetch_and_sub1_release_write(addr) \
2201                 AO_int_fetch_and_add_release_write(addr, (unsigned)(-1))
2202 # define AO_HAVE_int_fetch_and_sub1_release_write
2203 #endif
2204 #if defined(AO_HAVE_int_fetch_and_add_acquire_read) \
2205     && !defined(AO_HAVE_int_fetch_and_sub1_acquire_read)
2206 # define AO_int_fetch_and_sub1_acquire_read(addr) \
2207                 AO_int_fetch_and_add_acquire_read(addr, (unsigned)(-1))
2208 # define AO_HAVE_int_fetch_and_sub1_acquire_read
2209 #endif
2210 #if defined(AO_HAVE_int_fetch_and_add) \
2211     && !defined(AO_HAVE_int_fetch_and_sub1)
2212 # define AO_int_fetch_and_sub1(addr) \
2213                 AO_int_fetch_and_add(addr, (unsigned)(-1))
2214 # define AO_HAVE_int_fetch_and_sub1
2215 #endif
2216 
2217 #if defined(AO_HAVE_int_fetch_and_sub1_full)
2218 # if !defined(AO_HAVE_int_fetch_and_sub1_release)
2219 #   define AO_int_fetch_and_sub1_release(addr) \
2220                                 AO_int_fetch_and_sub1_full(addr)
2221 #   define AO_HAVE_int_fetch_and_sub1_release
2222 # endif
2223 # if !defined(AO_HAVE_int_fetch_and_sub1_acquire)
2224 #   define AO_int_fetch_and_sub1_acquire(addr) \
2225                                 AO_int_fetch_and_sub1_full(addr)
2226 #   define AO_HAVE_int_fetch_and_sub1_acquire
2227 # endif
2228 # if !defined(AO_HAVE_int_fetch_and_sub1_write)
2229 #   define AO_int_fetch_and_sub1_write(addr) \
2230                                 AO_int_fetch_and_sub1_full(addr)
2231 #   define AO_HAVE_int_fetch_and_sub1_write
2232 # endif
2233 # if !defined(AO_HAVE_int_fetch_and_sub1_read)
2234 #   define AO_int_fetch_and_sub1_read(addr) \
2235                                 AO_int_fetch_and_sub1_full(addr)
2236 #   define AO_HAVE_int_fetch_and_sub1_read
2237 # endif
2238 #endif /* AO_HAVE_int_fetch_and_sub1_full */
2239 
2240 #if !defined(AO_HAVE_int_fetch_and_sub1) \
2241     && defined(AO_HAVE_int_fetch_and_sub1_release)
2242 # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_release(addr)
2243 # define AO_HAVE_int_fetch_and_sub1
2244 #endif
2245 #if !defined(AO_HAVE_int_fetch_and_sub1) \
2246     && defined(AO_HAVE_int_fetch_and_sub1_acquire)
2247 # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_acquire(addr)
2248 # define AO_HAVE_int_fetch_and_sub1
2249 #endif
2250 #if !defined(AO_HAVE_int_fetch_and_sub1) \
2251     && defined(AO_HAVE_int_fetch_and_sub1_write)
2252 # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_write(addr)
2253 # define AO_HAVE_int_fetch_and_sub1
2254 #endif
2255 #if !defined(AO_HAVE_int_fetch_and_sub1) \
2256     && defined(AO_HAVE_int_fetch_and_sub1_read)
2257 # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_read(addr)
2258 # define AO_HAVE_int_fetch_and_sub1
2259 #endif
2260 
2261 #if defined(AO_HAVE_int_fetch_and_sub1_acquire) \
2262     && defined(AO_HAVE_nop_full) \
2263     && !defined(AO_HAVE_int_fetch_and_sub1_full)
2264 # define AO_int_fetch_and_sub1_full(addr) \
2265                         (AO_nop_full(), AO_int_fetch_and_sub1_acquire(addr))
2266 # define AO_HAVE_int_fetch_and_sub1_full
2267 #endif
2268 
2269 #if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \
2270     && defined(AO_HAVE_int_fetch_and_sub1_write)
2271 # define AO_int_fetch_and_sub1_release_write(addr) \
2272                                 AO_int_fetch_and_sub1_write(addr)
2273 # define AO_HAVE_int_fetch_and_sub1_release_write
2274 #endif
2275 #if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \
2276     && defined(AO_HAVE_int_fetch_and_sub1_release)
2277 # define AO_int_fetch_and_sub1_release_write(addr) \
2278                                 AO_int_fetch_and_sub1_release(addr)
2279 # define AO_HAVE_int_fetch_and_sub1_release_write
2280 #endif
2281 #if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \
2282     && defined(AO_HAVE_int_fetch_and_sub1_read)
2283 # define AO_int_fetch_and_sub1_acquire_read(addr) \
2284                                 AO_int_fetch_and_sub1_read(addr)
2285 # define AO_HAVE_int_fetch_and_sub1_acquire_read
2286 #endif
2287 #if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \
2288     && defined(AO_HAVE_int_fetch_and_sub1_acquire)
2289 # define AO_int_fetch_and_sub1_acquire_read(addr) \
2290                                 AO_int_fetch_and_sub1_acquire(addr)
2291 # define AO_HAVE_int_fetch_and_sub1_acquire_read
2292 #endif
2293 
2294 #ifdef AO_NO_DD_ORDERING
2295 # if defined(AO_HAVE_int_fetch_and_sub1_acquire_read)
2296 #   define AO_int_fetch_and_sub1_dd_acquire_read(addr) \
2297                                 AO_int_fetch_and_sub1_acquire_read(addr)
2298 #   define AO_HAVE_int_fetch_and_sub1_dd_acquire_read
2299 # endif
2300 #else
2301 # if defined(AO_HAVE_int_fetch_and_sub1)
2302 #   define AO_int_fetch_and_sub1_dd_acquire_read(addr) \
2303                                 AO_int_fetch_and_sub1(addr)
2304 #   define AO_HAVE_int_fetch_and_sub1_dd_acquire_read
2305 # endif
2306 #endif /* !AO_NO_DD_ORDERING */
2307 
2308 /* int_and */
2309 #if defined(AO_HAVE_int_compare_and_swap_full) \
2310     && !defined(AO_HAVE_int_and_full)
2311   AO_ATTR_NO_SANITIZE_THREAD
2312   AO_INLINE void
2313   AO_int_and_full(volatile unsigned *addr, unsigned value)
2314   {
2315     unsigned old;
2316 
2317     do
2318       {
2319         old = *(unsigned *)addr;
2320       }
2321     while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
2322                                                            old & value)));
2323   }
2324 # define AO_HAVE_int_and_full
2325 #endif
2326 
2327 #if defined(AO_HAVE_int_and_full)
2328 # if !defined(AO_HAVE_int_and_release)
2329 #   define AO_int_and_release(addr, val) AO_int_and_full(addr, val)
2330 #   define AO_HAVE_int_and_release
2331 # endif
2332 # if !defined(AO_HAVE_int_and_acquire)
2333 #   define AO_int_and_acquire(addr, val) AO_int_and_full(addr, val)
2334 #   define AO_HAVE_int_and_acquire
2335 # endif
2336 # if !defined(AO_HAVE_int_and_write)
2337 #   define AO_int_and_write(addr, val) AO_int_and_full(addr, val)
2338 #   define AO_HAVE_int_and_write
2339 # endif
2340 # if !defined(AO_HAVE_int_and_read)
2341 #   define AO_int_and_read(addr, val) AO_int_and_full(addr, val)
2342 #   define AO_HAVE_int_and_read
2343 # endif
2344 #endif /* AO_HAVE_int_and_full */
2345 
2346 #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_release)
2347 # define AO_int_and(addr, val) AO_int_and_release(addr, val)
2348 # define AO_HAVE_int_and
2349 #endif
2350 #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_acquire)
2351 # define AO_int_and(addr, val) AO_int_and_acquire(addr, val)
2352 # define AO_HAVE_int_and
2353 #endif
2354 #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_write)
2355 # define AO_int_and(addr, val) AO_int_and_write(addr, val)
2356 # define AO_HAVE_int_and
2357 #endif
2358 #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_read)
2359 # define AO_int_and(addr, val) AO_int_and_read(addr, val)
2360 # define AO_HAVE_int_and
2361 #endif
2362 
2363 #if defined(AO_HAVE_int_and_acquire) && defined(AO_HAVE_nop_full) \
2364     && !defined(AO_HAVE_int_and_full)
2365 # define AO_int_and_full(addr, val) \
2366                         (AO_nop_full(), AO_int_and_acquire(addr, val))
2367 # define AO_HAVE_int_and_full
2368 #endif
2369 
2370 #if !defined(AO_HAVE_int_and_release_write) \
2371     && defined(AO_HAVE_int_and_write)
2372 # define AO_int_and_release_write(addr, val) AO_int_and_write(addr, val)
2373 # define AO_HAVE_int_and_release_write
2374 #endif
2375 #if !defined(AO_HAVE_int_and_release_write) \
2376     && defined(AO_HAVE_int_and_release)
2377 # define AO_int_and_release_write(addr, val) AO_int_and_release(addr, val)
2378 # define AO_HAVE_int_and_release_write
2379 #endif
2380 #if !defined(AO_HAVE_int_and_acquire_read) \
2381     && defined(AO_HAVE_int_and_read)
2382 # define AO_int_and_acquire_read(addr, val) AO_int_and_read(addr, val)
2383 # define AO_HAVE_int_and_acquire_read
2384 #endif
2385 #if !defined(AO_HAVE_int_and_acquire_read) \
2386     && defined(AO_HAVE_int_and_acquire)
2387 # define AO_int_and_acquire_read(addr, val) AO_int_and_acquire(addr, val)
2388 # define AO_HAVE_int_and_acquire_read
2389 #endif
2390 
2391 /* int_or */
2392 #if defined(AO_HAVE_int_compare_and_swap_full) \
2393     && !defined(AO_HAVE_int_or_full)
2394   AO_ATTR_NO_SANITIZE_THREAD
2395   AO_INLINE void
2396   AO_int_or_full(volatile unsigned *addr, unsigned value)
2397   {
2398     unsigned old;
2399 
2400     do
2401       {
2402         old = *(unsigned *)addr;
2403       }
2404     while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
2405                                                            old | value)));
2406   }
2407 # define AO_HAVE_int_or_full
2408 #endif
2409 
2410 #if defined(AO_HAVE_int_or_full)
2411 # if !defined(AO_HAVE_int_or_release)
2412 #   define AO_int_or_release(addr, val) AO_int_or_full(addr, val)
2413 #   define AO_HAVE_int_or_release
2414 # endif
2415 # if !defined(AO_HAVE_int_or_acquire)
2416 #   define AO_int_or_acquire(addr, val) AO_int_or_full(addr, val)
2417 #   define AO_HAVE_int_or_acquire
2418 # endif
2419 # if !defined(AO_HAVE_int_or_write)
2420 #   define AO_int_or_write(addr, val) AO_int_or_full(addr, val)
2421 #   define AO_HAVE_int_or_write
2422 # endif
2423 # if !defined(AO_HAVE_int_or_read)
2424 #   define AO_int_or_read(addr, val) AO_int_or_full(addr, val)
2425 #   define AO_HAVE_int_or_read
2426 # endif
2427 #endif /* AO_HAVE_int_or_full */
2428 
2429 #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_release)
2430 # define AO_int_or(addr, val) AO_int_or_release(addr, val)
2431 # define AO_HAVE_int_or
2432 #endif
2433 #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_acquire)
2434 # define AO_int_or(addr, val) AO_int_or_acquire(addr, val)
2435 # define AO_HAVE_int_or
2436 #endif
2437 #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_write)
2438 # define AO_int_or(addr, val) AO_int_or_write(addr, val)
2439 # define AO_HAVE_int_or
2440 #endif
2441 #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_read)
2442 # define AO_int_or(addr, val) AO_int_or_read(addr, val)
2443 # define AO_HAVE_int_or
2444 #endif
2445 
2446 #if defined(AO_HAVE_int_or_acquire) && defined(AO_HAVE_nop_full) \
2447     && !defined(AO_HAVE_int_or_full)
2448 # define AO_int_or_full(addr, val) \
2449                         (AO_nop_full(), AO_int_or_acquire(addr, val))
2450 # define AO_HAVE_int_or_full
2451 #endif
2452 
2453 #if !defined(AO_HAVE_int_or_release_write) \
2454     && defined(AO_HAVE_int_or_write)
2455 # define AO_int_or_release_write(addr, val) AO_int_or_write(addr, val)
2456 # define AO_HAVE_int_or_release_write
2457 #endif
2458 #if !defined(AO_HAVE_int_or_release_write) \
2459     && defined(AO_HAVE_int_or_release)
2460 # define AO_int_or_release_write(addr, val) AO_int_or_release(addr, val)
2461 # define AO_HAVE_int_or_release_write
2462 #endif
2463 #if !defined(AO_HAVE_int_or_acquire_read) && defined(AO_HAVE_int_or_read)
2464 # define AO_int_or_acquire_read(addr, val) AO_int_or_read(addr, val)
2465 # define AO_HAVE_int_or_acquire_read
2466 #endif
2467 #if !defined(AO_HAVE_int_or_acquire_read) \
2468     && defined(AO_HAVE_int_or_acquire)
2469 # define AO_int_or_acquire_read(addr, val) AO_int_or_acquire(addr, val)
2470 # define AO_HAVE_int_or_acquire_read
2471 #endif
2472 
2473 /* int_xor */
2474 #if defined(AO_HAVE_int_compare_and_swap_full) \
2475     && !defined(AO_HAVE_int_xor_full)
2476   AO_ATTR_NO_SANITIZE_THREAD
2477   AO_INLINE void
2478   AO_int_xor_full(volatile unsigned *addr, unsigned value)
2479   {
2480     unsigned old;
2481 
2482     do
2483       {
2484         old = *(unsigned *)addr;
2485       }
2486     while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
2487                                                            old ^ value)));
2488   }
2489 # define AO_HAVE_int_xor_full
2490 #endif
2491 
2492 #if defined(AO_HAVE_int_xor_full)
2493 # if !defined(AO_HAVE_int_xor_release)
2494 #   define AO_int_xor_release(addr, val) AO_int_xor_full(addr, val)
2495 #   define AO_HAVE_int_xor_release
2496 # endif
2497 # if !defined(AO_HAVE_int_xor_acquire)
2498 #   define AO_int_xor_acquire(addr, val) AO_int_xor_full(addr, val)
2499 #   define AO_HAVE_int_xor_acquire
2500 # endif
2501 # if !defined(AO_HAVE_int_xor_write)
2502 #   define AO_int_xor_write(addr, val) AO_int_xor_full(addr, val)
2503 #   define AO_HAVE_int_xor_write
2504 # endif
2505 # if !defined(AO_HAVE_int_xor_read)
2506 #   define AO_int_xor_read(addr, val) AO_int_xor_full(addr, val)
2507 #   define AO_HAVE_int_xor_read
2508 # endif
2509 #endif /* AO_HAVE_int_xor_full */
2510 
2511 #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_release)
2512 # define AO_int_xor(addr, val) AO_int_xor_release(addr, val)
2513 # define AO_HAVE_int_xor
2514 #endif
2515 #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_acquire)
2516 # define AO_int_xor(addr, val) AO_int_xor_acquire(addr, val)
2517 # define AO_HAVE_int_xor
2518 #endif
2519 #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_write)
2520 # define AO_int_xor(addr, val) AO_int_xor_write(addr, val)
2521 # define AO_HAVE_int_xor
2522 #endif
2523 #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_read)
2524 # define AO_int_xor(addr, val) AO_int_xor_read(addr, val)
2525 # define AO_HAVE_int_xor
2526 #endif
2527 
2528 #if defined(AO_HAVE_int_xor_acquire) && defined(AO_HAVE_nop_full) \
2529     && !defined(AO_HAVE_int_xor_full)
2530 # define AO_int_xor_full(addr, val) \
2531                         (AO_nop_full(), AO_int_xor_acquire(addr, val))
2532 # define AO_HAVE_int_xor_full
2533 #endif
2534 
2535 #if !defined(AO_HAVE_int_xor_release_write) \
2536     && defined(AO_HAVE_int_xor_write)
2537 # define AO_int_xor_release_write(addr, val) AO_int_xor_write(addr, val)
2538 # define AO_HAVE_int_xor_release_write
2539 #endif
2540 #if !defined(AO_HAVE_int_xor_release_write) \
2541     && defined(AO_HAVE_int_xor_release)
2542 # define AO_int_xor_release_write(addr, val) AO_int_xor_release(addr, val)
2543 # define AO_HAVE_int_xor_release_write
2544 #endif
2545 #if !defined(AO_HAVE_int_xor_acquire_read) \
2546     && defined(AO_HAVE_int_xor_read)
2547 # define AO_int_xor_acquire_read(addr, val) AO_int_xor_read(addr, val)
2548 # define AO_HAVE_int_xor_acquire_read
2549 #endif
2550 #if !defined(AO_HAVE_int_xor_acquire_read) \
2551     && defined(AO_HAVE_int_xor_acquire)
2552 # define AO_int_xor_acquire_read(addr, val) AO_int_xor_acquire(addr, val)
2553 # define AO_HAVE_int_xor_acquire_read
2554 #endif
2555 
2556 /* int_and/or/xor_dd_acquire_read are meaningless.    */
2557 /*
2558  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
2559  *
2560  * Permission is hereby granted, free of charge, to any person obtaining a copy
2561  * of this software and associated documentation files (the "Software"), to deal
2562  * in the Software without restriction, including without limitation the rights
2563  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
2564  * copies of the Software, and to permit persons to whom the Software is
2565  * furnished to do so, subject to the following conditions:
2566  *
2567  * The above copyright notice and this permission notice shall be included in
2568  * all copies or substantial portions of the Software.
2569  *
2570  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
2571  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
2572  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
2573  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2574  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2575  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2576  * SOFTWARE.
2577  */
2578 
2579 /* compare_and_swap (based on fetch_compare_and_swap) */
2580 #if defined(AO_HAVE_fetch_compare_and_swap_full) \
2581     && !defined(AO_HAVE_compare_and_swap_full)
2582   AO_INLINE int
2583   AO_compare_and_swap_full(volatile AO_t *addr, AO_t old_val,
2584                                  AO_t new_val)
2585   {
2586     return AO_fetch_compare_and_swap_full(addr, old_val, new_val)
2587              == old_val;
2588   }
2589 # define AO_HAVE_compare_and_swap_full
2590 #endif
2591 
2592 #if defined(AO_HAVE_fetch_compare_and_swap_acquire) \
2593     && !defined(AO_HAVE_compare_and_swap_acquire)
2594   AO_INLINE int
2595   AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old_val,
2596                                     AO_t new_val)
2597   {
2598     return AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
2599              == old_val;
2600   }
2601 # define AO_HAVE_compare_and_swap_acquire
2602 #endif
2603 
2604 #if defined(AO_HAVE_fetch_compare_and_swap_release) \
2605     && !defined(AO_HAVE_compare_and_swap_release)
2606   AO_INLINE int
2607   AO_compare_and_swap_release(volatile AO_t *addr, AO_t old_val,
2608                                     AO_t new_val)
2609   {
2610     return AO_fetch_compare_and_swap_release(addr, old_val, new_val)
2611              == old_val;
2612   }
2613 # define AO_HAVE_compare_and_swap_release
2614 #endif
2615 
2616 #if defined(AO_HAVE_fetch_compare_and_swap_write) \
2617     && !defined(AO_HAVE_compare_and_swap_write)
2618   AO_INLINE int
2619   AO_compare_and_swap_write(volatile AO_t *addr, AO_t old_val,
2620                                   AO_t new_val)
2621   {
2622     return AO_fetch_compare_and_swap_write(addr, old_val, new_val)
2623              == old_val;
2624   }
2625 # define AO_HAVE_compare_and_swap_write
2626 #endif
2627 
2628 #if defined(AO_HAVE_fetch_compare_and_swap_read) \
2629     && !defined(AO_HAVE_compare_and_swap_read)
2630   AO_INLINE int
2631   AO_compare_and_swap_read(volatile AO_t *addr, AO_t old_val,
2632                                  AO_t new_val)
2633   {
2634     return AO_fetch_compare_and_swap_read(addr, old_val, new_val)
2635              == old_val;
2636   }
2637 # define AO_HAVE_compare_and_swap_read
2638 #endif
2639 
2640 #if defined(AO_HAVE_fetch_compare_and_swap) \
2641     && !defined(AO_HAVE_compare_and_swap)
2642   AO_INLINE int
2643   AO_compare_and_swap(volatile AO_t *addr, AO_t old_val,
2644                             AO_t new_val)
2645   {
2646     return AO_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
2647   }
2648 # define AO_HAVE_compare_and_swap
2649 #endif
2650 
2651 #if defined(AO_HAVE_fetch_compare_and_swap_release_write) \
2652     && !defined(AO_HAVE_compare_and_swap_release_write)
2653   AO_INLINE int
2654   AO_compare_and_swap_release_write(volatile AO_t *addr,
2655                                           AO_t old_val, AO_t new_val)
2656   {
2657     return AO_fetch_compare_and_swap_release_write(addr, old_val,
2658                                                          new_val) == old_val;
2659   }
2660 # define AO_HAVE_compare_and_swap_release_write
2661 #endif
2662 
2663 #if defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \
2664     && !defined(AO_HAVE_compare_and_swap_acquire_read)
2665   AO_INLINE int
2666   AO_compare_and_swap_acquire_read(volatile AO_t *addr,
2667                                          AO_t old_val, AO_t new_val)
2668   {
2669     return AO_fetch_compare_and_swap_acquire_read(addr, old_val,
2670                                                         new_val) == old_val;
2671   }
2672 # define AO_HAVE_compare_and_swap_acquire_read
2673 #endif
2674 
2675 #if defined(AO_HAVE_fetch_compare_and_swap_dd_acquire_read) \
2676     && !defined(AO_HAVE_compare_and_swap_dd_acquire_read)
2677   AO_INLINE int
2678   AO_compare_and_swap_dd_acquire_read(volatile AO_t *addr,
2679                                             AO_t old_val, AO_t new_val)
2680   {
2681     return AO_fetch_compare_and_swap_dd_acquire_read(addr, old_val,
2682                                                            new_val) == old_val;
2683   }
2684 # define AO_HAVE_compare_and_swap_dd_acquire_read
2685 #endif
2686 
2687 /* fetch_and_add */
2688 /* We first try to implement fetch_and_add variants in terms of the     */
2689 /* corresponding compare_and_swap variants to minimize adding barriers. */
2690 #if defined(AO_HAVE_compare_and_swap_full) \
2691     && !defined(AO_HAVE_fetch_and_add_full)
2692   AO_ATTR_NO_SANITIZE_THREAD
2693   AO_INLINE AO_t
2694   AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
2695   {
2696     AO_t old;
2697 
2698     do
2699       {
2700         old = *(AO_t *)addr;
2701       }
2702     while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
2703                                                            old + incr)));
2704     return old;
2705   }
2706 # define AO_HAVE_fetch_and_add_full
2707 #endif
2708 
2709 #if defined(AO_HAVE_compare_and_swap_acquire) \
2710     && !defined(AO_HAVE_fetch_and_add_acquire)
2711   AO_ATTR_NO_SANITIZE_THREAD
2712   AO_INLINE AO_t
2713   AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
2714   {
2715     AO_t old;
2716 
2717     do
2718       {
2719         old = *(AO_t *)addr;
2720       }
2721     while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(addr, old,
2722                                                               old + incr)));
2723     return old;
2724   }
2725 # define AO_HAVE_fetch_and_add_acquire
2726 #endif
2727 
2728 #if defined(AO_HAVE_compare_and_swap_release) \
2729     && !defined(AO_HAVE_fetch_and_add_release)
2730   AO_ATTR_NO_SANITIZE_THREAD
2731   AO_INLINE AO_t
2732   AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
2733   {
2734     AO_t old;
2735 
2736     do
2737       {
2738         old = *(AO_t *)addr;
2739       }
2740     while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old,
2741                                                               old + incr)));
2742     return old;
2743   }
2744 # define AO_HAVE_fetch_and_add_release
2745 #endif
2746 
2747 #if defined(AO_HAVE_compare_and_swap) \
2748     && !defined(AO_HAVE_fetch_and_add)
2749   AO_ATTR_NO_SANITIZE_THREAD
2750   AO_INLINE AO_t
2751   AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
2752   {
2753     AO_t old;
2754 
2755     do
2756       {
2757         old = *(AO_t *)addr;
2758       }
2759     while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr, old,
2760                                                       old + incr)));
2761     return old;
2762   }
2763 # define AO_HAVE_fetch_and_add
2764 #endif
2765 
2766 #if defined(AO_HAVE_fetch_and_add_full)
2767 # if !defined(AO_HAVE_fetch_and_add_release)
2768 #   define AO_fetch_and_add_release(addr, val) \
2769                                 AO_fetch_and_add_full(addr, val)
2770 #   define AO_HAVE_fetch_and_add_release
2771 # endif
2772 # if !defined(AO_HAVE_fetch_and_add_acquire)
2773 #   define AO_fetch_and_add_acquire(addr, val) \
2774                                 AO_fetch_and_add_full(addr, val)
2775 #   define AO_HAVE_fetch_and_add_acquire
2776 # endif
2777 # if !defined(AO_HAVE_fetch_and_add_write)
2778 #   define AO_fetch_and_add_write(addr, val) \
2779                                 AO_fetch_and_add_full(addr, val)
2780 #   define AO_HAVE_fetch_and_add_write
2781 # endif
2782 # if !defined(AO_HAVE_fetch_and_add_read)
2783 #   define AO_fetch_and_add_read(addr, val) \
2784                                 AO_fetch_and_add_full(addr, val)
2785 #   define AO_HAVE_fetch_and_add_read
2786 # endif
2787 #endif /* AO_HAVE_fetch_and_add_full */
2788 
2789 #if defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_nop_full) \
2790     && !defined(AO_HAVE_fetch_and_add_acquire)
2791   AO_INLINE AO_t
2792   AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
2793   {
2794     AO_t result = AO_fetch_and_add(addr, incr);
2795     AO_nop_full();
2796     return result;
2797   }
2798 # define AO_HAVE_fetch_and_add_acquire
2799 #endif
2800 #if defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_nop_full) \
2801     && !defined(AO_HAVE_fetch_and_add_release)
2802 # define AO_fetch_and_add_release(addr, incr) \
2803                 (AO_nop_full(), AO_fetch_and_add(addr, incr))
2804 # define AO_HAVE_fetch_and_add_release
2805 #endif
2806 
2807 #if !defined(AO_HAVE_fetch_and_add) \
2808     && defined(AO_HAVE_fetch_and_add_release)
2809 # define AO_fetch_and_add(addr, val) \
2810                                 AO_fetch_and_add_release(addr, val)
2811 # define AO_HAVE_fetch_and_add
2812 #endif
2813 #if !defined(AO_HAVE_fetch_and_add) \
2814     && defined(AO_HAVE_fetch_and_add_acquire)
2815 # define AO_fetch_and_add(addr, val) \
2816                                 AO_fetch_and_add_acquire(addr, val)
2817 # define AO_HAVE_fetch_and_add
2818 #endif
2819 #if !defined(AO_HAVE_fetch_and_add) \
2820     && defined(AO_HAVE_fetch_and_add_write)
2821 # define AO_fetch_and_add(addr, val) \
2822                                 AO_fetch_and_add_write(addr, val)
2823 # define AO_HAVE_fetch_and_add
2824 #endif
2825 #if !defined(AO_HAVE_fetch_and_add) \
2826     && defined(AO_HAVE_fetch_and_add_read)
2827 # define AO_fetch_and_add(addr, val) \
2828                                 AO_fetch_and_add_read(addr, val)
2829 # define AO_HAVE_fetch_and_add
2830 #endif
2831 
2832 #if defined(AO_HAVE_fetch_and_add_acquire) \
2833     && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_fetch_and_add_full)
2834 # define AO_fetch_and_add_full(addr, val) \
2835                 (AO_nop_full(), AO_fetch_and_add_acquire(addr, val))
2836 # define AO_HAVE_fetch_and_add_full
2837 #endif
2838 
2839 #if !defined(AO_HAVE_fetch_and_add_release_write) \
2840     && defined(AO_HAVE_fetch_and_add_write)
2841 # define AO_fetch_and_add_release_write(addr, val) \
2842                                 AO_fetch_and_add_write(addr, val)
2843 # define AO_HAVE_fetch_and_add_release_write
2844 #endif
2845 #if !defined(AO_HAVE_fetch_and_add_release_write) \
2846     && defined(AO_HAVE_fetch_and_add_release)
2847 # define AO_fetch_and_add_release_write(addr, val) \
2848                                 AO_fetch_and_add_release(addr, val)
2849 # define AO_HAVE_fetch_and_add_release_write
2850 #endif
2851 
2852 #if !defined(AO_HAVE_fetch_and_add_acquire_read) \
2853     && defined(AO_HAVE_fetch_and_add_read)
2854 # define AO_fetch_and_add_acquire_read(addr, val) \
2855                                 AO_fetch_and_add_read(addr, val)
2856 # define AO_HAVE_fetch_and_add_acquire_read
2857 #endif
2858 #if !defined(AO_HAVE_fetch_and_add_acquire_read) \
2859     && defined(AO_HAVE_fetch_and_add_acquire)
2860 # define AO_fetch_and_add_acquire_read(addr, val) \
2861                                 AO_fetch_and_add_acquire(addr, val)
2862 # define AO_HAVE_fetch_and_add_acquire_read
2863 #endif
2864 
2865 #ifdef AO_NO_DD_ORDERING
2866 # if defined(AO_HAVE_fetch_and_add_acquire_read)
2867 #   define AO_fetch_and_add_dd_acquire_read(addr, val) \
2868                                 AO_fetch_and_add_acquire_read(addr, val)
2869 #   define AO_HAVE_fetch_and_add_dd_acquire_read
2870 # endif
2871 #else
2872 # if defined(AO_HAVE_fetch_and_add)
2873 #   define AO_fetch_and_add_dd_acquire_read(addr, val) \
2874                                 AO_fetch_and_add(addr, val)
2875 #   define AO_HAVE_fetch_and_add_dd_acquire_read
2876 # endif
2877 #endif /* !AO_NO_DD_ORDERING */
2878 
2879 /* fetch_and_add1 */
2880 #if defined(AO_HAVE_fetch_and_add_full) \
2881     && !defined(AO_HAVE_fetch_and_add1_full)
2882 # define AO_fetch_and_add1_full(addr) \
2883                                 AO_fetch_and_add_full(addr, 1)
2884 # define AO_HAVE_fetch_and_add1_full
2885 #endif
2886 #if defined(AO_HAVE_fetch_and_add_release) \
2887     && !defined(AO_HAVE_fetch_and_add1_release)
2888 # define AO_fetch_and_add1_release(addr) \
2889                                 AO_fetch_and_add_release(addr, 1)
2890 # define AO_HAVE_fetch_and_add1_release
2891 #endif
2892 #if defined(AO_HAVE_fetch_and_add_acquire) \
2893     && !defined(AO_HAVE_fetch_and_add1_acquire)
2894 # define AO_fetch_and_add1_acquire(addr) \
2895                                 AO_fetch_and_add_acquire(addr, 1)
2896 # define AO_HAVE_fetch_and_add1_acquire
2897 #endif
2898 #if defined(AO_HAVE_fetch_and_add_write) \
2899     && !defined(AO_HAVE_fetch_and_add1_write)
2900 # define AO_fetch_and_add1_write(addr) \
2901                                 AO_fetch_and_add_write(addr, 1)
2902 # define AO_HAVE_fetch_and_add1_write
2903 #endif
2904 #if defined(AO_HAVE_fetch_and_add_read) \
2905     && !defined(AO_HAVE_fetch_and_add1_read)
2906 # define AO_fetch_and_add1_read(addr) \
2907                                 AO_fetch_and_add_read(addr, 1)
2908 # define AO_HAVE_fetch_and_add1_read
2909 #endif
2910 #if defined(AO_HAVE_fetch_and_add_release_write) \
2911     && !defined(AO_HAVE_fetch_and_add1_release_write)
2912 # define AO_fetch_and_add1_release_write(addr) \
2913                                 AO_fetch_and_add_release_write(addr, 1)
2914 # define AO_HAVE_fetch_and_add1_release_write
2915 #endif
2916 #if defined(AO_HAVE_fetch_and_add_acquire_read) \
2917     && !defined(AO_HAVE_fetch_and_add1_acquire_read)
2918 # define AO_fetch_and_add1_acquire_read(addr) \
2919                                 AO_fetch_and_add_acquire_read(addr, 1)
2920 # define AO_HAVE_fetch_and_add1_acquire_read
2921 #endif
2922 #if defined(AO_HAVE_fetch_and_add) \
2923     && !defined(AO_HAVE_fetch_and_add1)
2924 # define AO_fetch_and_add1(addr) AO_fetch_and_add(addr, 1)
2925 # define AO_HAVE_fetch_and_add1
2926 #endif
2927 
2928 #if defined(AO_HAVE_fetch_and_add1_full)
2929 # if !defined(AO_HAVE_fetch_and_add1_release)
2930 #   define AO_fetch_and_add1_release(addr) \
2931                                 AO_fetch_and_add1_full(addr)
2932 #   define AO_HAVE_fetch_and_add1_release
2933 # endif
2934 # if !defined(AO_HAVE_fetch_and_add1_acquire)
2935 #   define AO_fetch_and_add1_acquire(addr) \
2936                                 AO_fetch_and_add1_full(addr)
2937 #   define AO_HAVE_fetch_and_add1_acquire
2938 # endif
2939 # if !defined(AO_HAVE_fetch_and_add1_write)
2940 #   define AO_fetch_and_add1_write(addr) \
2941                                 AO_fetch_and_add1_full(addr)
2942 #   define AO_HAVE_fetch_and_add1_write
2943 # endif
2944 # if !defined(AO_HAVE_fetch_and_add1_read)
2945 #   define AO_fetch_and_add1_read(addr) \
2946                                 AO_fetch_and_add1_full(addr)
2947 #   define AO_HAVE_fetch_and_add1_read
2948 # endif
2949 #endif /* AO_HAVE_fetch_and_add1_full */
2950 
2951 #if !defined(AO_HAVE_fetch_and_add1) \
2952     && defined(AO_HAVE_fetch_and_add1_release)
2953 # define AO_fetch_and_add1(addr) AO_fetch_and_add1_release(addr)
2954 # define AO_HAVE_fetch_and_add1
2955 #endif
2956 #if !defined(AO_HAVE_fetch_and_add1) \
2957     && defined(AO_HAVE_fetch_and_add1_acquire)
2958 # define AO_fetch_and_add1(addr) AO_fetch_and_add1_acquire(addr)
2959 # define AO_HAVE_fetch_and_add1
2960 #endif
2961 #if !defined(AO_HAVE_fetch_and_add1) \
2962     && defined(AO_HAVE_fetch_and_add1_write)
2963 # define AO_fetch_and_add1(addr) AO_fetch_and_add1_write(addr)
2964 # define AO_HAVE_fetch_and_add1
2965 #endif
2966 #if !defined(AO_HAVE_fetch_and_add1) \
2967     && defined(AO_HAVE_fetch_and_add1_read)
2968 # define AO_fetch_and_add1(addr) AO_fetch_and_add1_read(addr)
2969 # define AO_HAVE_fetch_and_add1
2970 #endif
2971 
2972 #if defined(AO_HAVE_fetch_and_add1_acquire) \
2973     && defined(AO_HAVE_nop_full) \
2974     && !defined(AO_HAVE_fetch_and_add1_full)
2975 # define AO_fetch_and_add1_full(addr) \
2976                         (AO_nop_full(), AO_fetch_and_add1_acquire(addr))
2977 # define AO_HAVE_fetch_and_add1_full
2978 #endif
2979 
2980 #if !defined(AO_HAVE_fetch_and_add1_release_write) \
2981     && defined(AO_HAVE_fetch_and_add1_write)
2982 # define AO_fetch_and_add1_release_write(addr) \
2983                                 AO_fetch_and_add1_write(addr)
2984 # define AO_HAVE_fetch_and_add1_release_write
2985 #endif
2986 #if !defined(AO_HAVE_fetch_and_add1_release_write) \
2987     && defined(AO_HAVE_fetch_and_add1_release)
2988 # define AO_fetch_and_add1_release_write(addr) \
2989                                 AO_fetch_and_add1_release(addr)
2990 # define AO_HAVE_fetch_and_add1_release_write
2991 #endif
2992 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
2993     && defined(AO_HAVE_fetch_and_add1_read)
2994 # define AO_fetch_and_add1_acquire_read(addr) \
2995                                 AO_fetch_and_add1_read(addr)
2996 # define AO_HAVE_fetch_and_add1_acquire_read
2997 #endif
2998 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
2999     && defined(AO_HAVE_fetch_and_add1_acquire)
3000 # define AO_fetch_and_add1_acquire_read(addr) \
3001                                 AO_fetch_and_add1_acquire(addr)
3002 # define AO_HAVE_fetch_and_add1_acquire_read
3003 #endif
3004 
3005 #ifdef AO_NO_DD_ORDERING
3006 # if defined(AO_HAVE_fetch_and_add1_acquire_read)
3007 #   define AO_fetch_and_add1_dd_acquire_read(addr) \
3008                                 AO_fetch_and_add1_acquire_read(addr)
3009 #   define AO_HAVE_fetch_and_add1_dd_acquire_read
3010 # endif
3011 #else
3012 # if defined(AO_HAVE_fetch_and_add1)
3013 #   define AO_fetch_and_add1_dd_acquire_read(addr) \
3014                                 AO_fetch_and_add1(addr)
3015 #   define AO_HAVE_fetch_and_add1_dd_acquire_read
3016 # endif
3017 #endif /* !AO_NO_DD_ORDERING */
3018 
3019 /* fetch_and_sub1 */
3020 #if defined(AO_HAVE_fetch_and_add_full) \
3021     && !defined(AO_HAVE_fetch_and_sub1_full)
3022 # define AO_fetch_and_sub1_full(addr) \
3023                 AO_fetch_and_add_full(addr, (AO_t)(-1))
3024 # define AO_HAVE_fetch_and_sub1_full
3025 #endif
3026 #if defined(AO_HAVE_fetch_and_add_release) \
3027     && !defined(AO_HAVE_fetch_and_sub1_release)
3028 # define AO_fetch_and_sub1_release(addr) \
3029                 AO_fetch_and_add_release(addr, (AO_t)(-1))
3030 # define AO_HAVE_fetch_and_sub1_release
3031 #endif
3032 #if defined(AO_HAVE_fetch_and_add_acquire) \
3033     && !defined(AO_HAVE_fetch_and_sub1_acquire)
3034 # define AO_fetch_and_sub1_acquire(addr) \
3035                 AO_fetch_and_add_acquire(addr, (AO_t)(-1))
3036 # define AO_HAVE_fetch_and_sub1_acquire
3037 #endif
3038 #if defined(AO_HAVE_fetch_and_add_write) \
3039     && !defined(AO_HAVE_fetch_and_sub1_write)
3040 # define AO_fetch_and_sub1_write(addr) \
3041                 AO_fetch_and_add_write(addr, (AO_t)(-1))
3042 # define AO_HAVE_fetch_and_sub1_write
3043 #endif
3044 #if defined(AO_HAVE_fetch_and_add_read) \
3045     && !defined(AO_HAVE_fetch_and_sub1_read)
3046 # define AO_fetch_and_sub1_read(addr) \
3047                 AO_fetch_and_add_read(addr, (AO_t)(-1))
3048 # define AO_HAVE_fetch_and_sub1_read
3049 #endif
3050 #if defined(AO_HAVE_fetch_and_add_release_write) \
3051     && !defined(AO_HAVE_fetch_and_sub1_release_write)
3052 # define AO_fetch_and_sub1_release_write(addr) \
3053                 AO_fetch_and_add_release_write(addr, (AO_t)(-1))
3054 # define AO_HAVE_fetch_and_sub1_release_write
3055 #endif
3056 #if defined(AO_HAVE_fetch_and_add_acquire_read) \
3057     && !defined(AO_HAVE_fetch_and_sub1_acquire_read)
3058 # define AO_fetch_and_sub1_acquire_read(addr) \
3059                 AO_fetch_and_add_acquire_read(addr, (AO_t)(-1))
3060 # define AO_HAVE_fetch_and_sub1_acquire_read
3061 #endif
3062 #if defined(AO_HAVE_fetch_and_add) \
3063     && !defined(AO_HAVE_fetch_and_sub1)
3064 # define AO_fetch_and_sub1(addr) \
3065                 AO_fetch_and_add(addr, (AO_t)(-1))
3066 # define AO_HAVE_fetch_and_sub1
3067 #endif
3068 
3069 #if defined(AO_HAVE_fetch_and_sub1_full)
3070 # if !defined(AO_HAVE_fetch_and_sub1_release)
3071 #   define AO_fetch_and_sub1_release(addr) \
3072                                 AO_fetch_and_sub1_full(addr)
3073 #   define AO_HAVE_fetch_and_sub1_release
3074 # endif
3075 # if !defined(AO_HAVE_fetch_and_sub1_acquire)
3076 #   define AO_fetch_and_sub1_acquire(addr) \
3077                                 AO_fetch_and_sub1_full(addr)
3078 #   define AO_HAVE_fetch_and_sub1_acquire
3079 # endif
3080 # if !defined(AO_HAVE_fetch_and_sub1_write)
3081 #   define AO_fetch_and_sub1_write(addr) \
3082                                 AO_fetch_and_sub1_full(addr)
3083 #   define AO_HAVE_fetch_and_sub1_write
3084 # endif
3085 # if !defined(AO_HAVE_fetch_and_sub1_read)
3086 #   define AO_fetch_and_sub1_read(addr) \
3087                                 AO_fetch_and_sub1_full(addr)
3088 #   define AO_HAVE_fetch_and_sub1_read
3089 # endif
3090 #endif /* AO_HAVE_fetch_and_sub1_full */
3091 
3092 #if !defined(AO_HAVE_fetch_and_sub1) \
3093     && defined(AO_HAVE_fetch_and_sub1_release)
3094 # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_release(addr)
3095 # define AO_HAVE_fetch_and_sub1
3096 #endif
3097 #if !defined(AO_HAVE_fetch_and_sub1) \
3098     && defined(AO_HAVE_fetch_and_sub1_acquire)
3099 # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_acquire(addr)
3100 # define AO_HAVE_fetch_and_sub1
3101 #endif
3102 #if !defined(AO_HAVE_fetch_and_sub1) \
3103     && defined(AO_HAVE_fetch_and_sub1_write)
3104 # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_write(addr)
3105 # define AO_HAVE_fetch_and_sub1
3106 #endif
3107 #if !defined(AO_HAVE_fetch_and_sub1) \
3108     && defined(AO_HAVE_fetch_and_sub1_read)
3109 # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_read(addr)
3110 # define AO_HAVE_fetch_and_sub1
3111 #endif
3112 
3113 #if defined(AO_HAVE_fetch_and_sub1_acquire) \
3114     && defined(AO_HAVE_nop_full) \
3115     && !defined(AO_HAVE_fetch_and_sub1_full)
3116 # define AO_fetch_and_sub1_full(addr) \
3117                         (AO_nop_full(), AO_fetch_and_sub1_acquire(addr))
3118 # define AO_HAVE_fetch_and_sub1_full
3119 #endif
3120 
3121 #if !defined(AO_HAVE_fetch_and_sub1_release_write) \
3122     && defined(AO_HAVE_fetch_and_sub1_write)
3123 # define AO_fetch_and_sub1_release_write(addr) \
3124                                 AO_fetch_and_sub1_write(addr)
3125 # define AO_HAVE_fetch_and_sub1_release_write
3126 #endif
3127 #if !defined(AO_HAVE_fetch_and_sub1_release_write) \
3128     && defined(AO_HAVE_fetch_and_sub1_release)
3129 # define AO_fetch_and_sub1_release_write(addr) \
3130                                 AO_fetch_and_sub1_release(addr)
3131 # define AO_HAVE_fetch_and_sub1_release_write
3132 #endif
3133 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
3134     && defined(AO_HAVE_fetch_and_sub1_read)
3135 # define AO_fetch_and_sub1_acquire_read(addr) \
3136                                 AO_fetch_and_sub1_read(addr)
3137 # define AO_HAVE_fetch_and_sub1_acquire_read
3138 #endif
3139 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
3140     && defined(AO_HAVE_fetch_and_sub1_acquire)
3141 # define AO_fetch_and_sub1_acquire_read(addr) \
3142                                 AO_fetch_and_sub1_acquire(addr)
3143 # define AO_HAVE_fetch_and_sub1_acquire_read
3144 #endif
3145 
3146 #ifdef AO_NO_DD_ORDERING
3147 # if defined(AO_HAVE_fetch_and_sub1_acquire_read)
3148 #   define AO_fetch_and_sub1_dd_acquire_read(addr) \
3149                                 AO_fetch_and_sub1_acquire_read(addr)
3150 #   define AO_HAVE_fetch_and_sub1_dd_acquire_read
3151 # endif
3152 #else
3153 # if defined(AO_HAVE_fetch_and_sub1)
3154 #   define AO_fetch_and_sub1_dd_acquire_read(addr) \
3155                                 AO_fetch_and_sub1(addr)
3156 #   define AO_HAVE_fetch_and_sub1_dd_acquire_read
3157 # endif
3158 #endif /* !AO_NO_DD_ORDERING */
3159 
3160 /* and */
3161 #if defined(AO_HAVE_compare_and_swap_full) \
3162     && !defined(AO_HAVE_and_full)
3163   AO_ATTR_NO_SANITIZE_THREAD
3164   AO_INLINE void
3165   AO_and_full(volatile AO_t *addr, AO_t value)
3166   {
3167     AO_t old;
3168 
3169     do
3170       {
3171         old = *(AO_t *)addr;
3172       }
3173     while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
3174                                                            old & value)));
3175   }
3176 # define AO_HAVE_and_full
3177 #endif
3178 
3179 #if defined(AO_HAVE_and_full)
3180 # if !defined(AO_HAVE_and_release)
3181 #   define AO_and_release(addr, val) AO_and_full(addr, val)
3182 #   define AO_HAVE_and_release
3183 # endif
3184 # if !defined(AO_HAVE_and_acquire)
3185 #   define AO_and_acquire(addr, val) AO_and_full(addr, val)
3186 #   define AO_HAVE_and_acquire
3187 # endif
3188 # if !defined(AO_HAVE_and_write)
3189 #   define AO_and_write(addr, val) AO_and_full(addr, val)
3190 #   define AO_HAVE_and_write
3191 # endif
3192 # if !defined(AO_HAVE_and_read)
3193 #   define AO_and_read(addr, val) AO_and_full(addr, val)
3194 #   define AO_HAVE_and_read
3195 # endif
3196 #endif /* AO_HAVE_and_full */
3197 
3198 #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_release)
3199 # define AO_and(addr, val) AO_and_release(addr, val)
3200 # define AO_HAVE_and
3201 #endif
3202 #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_acquire)
3203 # define AO_and(addr, val) AO_and_acquire(addr, val)
3204 # define AO_HAVE_and
3205 #endif
3206 #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_write)
3207 # define AO_and(addr, val) AO_and_write(addr, val)
3208 # define AO_HAVE_and
3209 #endif
3210 #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_read)
3211 # define AO_and(addr, val) AO_and_read(addr, val)
3212 # define AO_HAVE_and
3213 #endif
3214 
3215 #if defined(AO_HAVE_and_acquire) && defined(AO_HAVE_nop_full) \
3216     && !defined(AO_HAVE_and_full)
3217 # define AO_and_full(addr, val) \
3218                         (AO_nop_full(), AO_and_acquire(addr, val))
3219 # define AO_HAVE_and_full
3220 #endif
3221 
3222 #if !defined(AO_HAVE_and_release_write) \
3223     && defined(AO_HAVE_and_write)
3224 # define AO_and_release_write(addr, val) AO_and_write(addr, val)
3225 # define AO_HAVE_and_release_write
3226 #endif
3227 #if !defined(AO_HAVE_and_release_write) \
3228     && defined(AO_HAVE_and_release)
3229 # define AO_and_release_write(addr, val) AO_and_release(addr, val)
3230 # define AO_HAVE_and_release_write
3231 #endif
3232 #if !defined(AO_HAVE_and_acquire_read) \
3233     && defined(AO_HAVE_and_read)
3234 # define AO_and_acquire_read(addr, val) AO_and_read(addr, val)
3235 # define AO_HAVE_and_acquire_read
3236 #endif
3237 #if !defined(AO_HAVE_and_acquire_read) \
3238     && defined(AO_HAVE_and_acquire)
3239 # define AO_and_acquire_read(addr, val) AO_and_acquire(addr, val)
3240 # define AO_HAVE_and_acquire_read
3241 #endif
3242 
3243 /* or */
3244 #if defined(AO_HAVE_compare_and_swap_full) \
3245     && !defined(AO_HAVE_or_full)
3246   AO_ATTR_NO_SANITIZE_THREAD
3247   AO_INLINE void
3248   AO_or_full(volatile AO_t *addr, AO_t value)
3249   {
3250     AO_t old;
3251 
3252     do
3253       {
3254         old = *(AO_t *)addr;
3255       }
3256     while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
3257                                                            old | value)));
3258   }
3259 # define AO_HAVE_or_full
3260 #endif
3261 
3262 #if defined(AO_HAVE_or_full)
3263 # if !defined(AO_HAVE_or_release)
3264 #   define AO_or_release(addr, val) AO_or_full(addr, val)
3265 #   define AO_HAVE_or_release
3266 # endif
3267 # if !defined(AO_HAVE_or_acquire)
3268 #   define AO_or_acquire(addr, val) AO_or_full(addr, val)
3269 #   define AO_HAVE_or_acquire
3270 # endif
3271 # if !defined(AO_HAVE_or_write)
3272 #   define AO_or_write(addr, val) AO_or_full(addr, val)
3273 #   define AO_HAVE_or_write
3274 # endif
3275 # if !defined(AO_HAVE_or_read)
3276 #   define AO_or_read(addr, val) AO_or_full(addr, val)
3277 #   define AO_HAVE_or_read
3278 # endif
3279 #endif /* AO_HAVE_or_full */
3280 
3281 #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_release)
3282 # define AO_or(addr, val) AO_or_release(addr, val)
3283 # define AO_HAVE_or
3284 #endif
3285 #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_acquire)
3286 # define AO_or(addr, val) AO_or_acquire(addr, val)
3287 # define AO_HAVE_or
3288 #endif
3289 #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_write)
3290 # define AO_or(addr, val) AO_or_write(addr, val)
3291 # define AO_HAVE_or
3292 #endif
3293 #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_read)
3294 # define AO_or(addr, val) AO_or_read(addr, val)
3295 # define AO_HAVE_or
3296 #endif
3297 
3298 #if defined(AO_HAVE_or_acquire) && defined(AO_HAVE_nop_full) \
3299     && !defined(AO_HAVE_or_full)
3300 # define AO_or_full(addr, val) \
3301                         (AO_nop_full(), AO_or_acquire(addr, val))
3302 # define AO_HAVE_or_full
3303 #endif
3304 
3305 #if !defined(AO_HAVE_or_release_write) \
3306     && defined(AO_HAVE_or_write)
3307 # define AO_or_release_write(addr, val) AO_or_write(addr, val)
3308 # define AO_HAVE_or_release_write
3309 #endif
3310 #if !defined(AO_HAVE_or_release_write) \
3311     && defined(AO_HAVE_or_release)
3312 # define AO_or_release_write(addr, val) AO_or_release(addr, val)
3313 # define AO_HAVE_or_release_write
3314 #endif
3315 #if !defined(AO_HAVE_or_acquire_read) && defined(AO_HAVE_or_read)
3316 # define AO_or_acquire_read(addr, val) AO_or_read(addr, val)
3317 # define AO_HAVE_or_acquire_read
3318 #endif
3319 #if !defined(AO_HAVE_or_acquire_read) \
3320     && defined(AO_HAVE_or_acquire)
3321 # define AO_or_acquire_read(addr, val) AO_or_acquire(addr, val)
3322 # define AO_HAVE_or_acquire_read
3323 #endif
3324 
3325 /* xor */
3326 #if defined(AO_HAVE_compare_and_swap_full) \
3327     && !defined(AO_HAVE_xor_full)
3328   AO_ATTR_NO_SANITIZE_THREAD
3329   AO_INLINE void
3330   AO_xor_full(volatile AO_t *addr, AO_t value)
3331   {
3332     AO_t old;
3333 
3334     do
3335       {
3336         old = *(AO_t *)addr;
3337       }
3338     while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
3339                                                            old ^ value)));
3340   }
3341 # define AO_HAVE_xor_full
3342 #endif
3343 
3344 #if defined(AO_HAVE_xor_full)
3345 # if !defined(AO_HAVE_xor_release)
3346 #   define AO_xor_release(addr, val) AO_xor_full(addr, val)
3347 #   define AO_HAVE_xor_release
3348 # endif
3349 # if !defined(AO_HAVE_xor_acquire)
3350 #   define AO_xor_acquire(addr, val) AO_xor_full(addr, val)
3351 #   define AO_HAVE_xor_acquire
3352 # endif
3353 # if !defined(AO_HAVE_xor_write)
3354 #   define AO_xor_write(addr, val) AO_xor_full(addr, val)
3355 #   define AO_HAVE_xor_write
3356 # endif
3357 # if !defined(AO_HAVE_xor_read)
3358 #   define AO_xor_read(addr, val) AO_xor_full(addr, val)
3359 #   define AO_HAVE_xor_read
3360 # endif
3361 #endif /* AO_HAVE_xor_full */
3362 
3363 #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_release)
3364 # define AO_xor(addr, val) AO_xor_release(addr, val)
3365 # define AO_HAVE_xor
3366 #endif
3367 #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_acquire)
3368 # define AO_xor(addr, val) AO_xor_acquire(addr, val)
3369 # define AO_HAVE_xor
3370 #endif
3371 #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_write)
3372 # define AO_xor(addr, val) AO_xor_write(addr, val)
3373 # define AO_HAVE_xor
3374 #endif
3375 #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_read)
3376 # define AO_xor(addr, val) AO_xor_read(addr, val)
3377 # define AO_HAVE_xor
3378 #endif
3379 
3380 #if defined(AO_HAVE_xor_acquire) && defined(AO_HAVE_nop_full) \
3381     && !defined(AO_HAVE_xor_full)
3382 # define AO_xor_full(addr, val) \
3383                         (AO_nop_full(), AO_xor_acquire(addr, val))
3384 # define AO_HAVE_xor_full
3385 #endif
3386 
3387 #if !defined(AO_HAVE_xor_release_write) \
3388     && defined(AO_HAVE_xor_write)
3389 # define AO_xor_release_write(addr, val) AO_xor_write(addr, val)
3390 # define AO_HAVE_xor_release_write
3391 #endif
3392 #if !defined(AO_HAVE_xor_release_write) \
3393     && defined(AO_HAVE_xor_release)
3394 # define AO_xor_release_write(addr, val) AO_xor_release(addr, val)
3395 # define AO_HAVE_xor_release_write
3396 #endif
3397 #if !defined(AO_HAVE_xor_acquire_read) \
3398     && defined(AO_HAVE_xor_read)
3399 # define AO_xor_acquire_read(addr, val) AO_xor_read(addr, val)
3400 # define AO_HAVE_xor_acquire_read
3401 #endif
3402 #if !defined(AO_HAVE_xor_acquire_read) \
3403     && defined(AO_HAVE_xor_acquire)
3404 # define AO_xor_acquire_read(addr, val) AO_xor_acquire(addr, val)
3405 # define AO_HAVE_xor_acquire_read
3406 #endif
3407 
3408 /* and/or/xor_dd_acquire_read are meaningless.    */
3409