Gitly


1 /*
2  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a copy
5  * of this software and associated documentation files (the "Software"), to deal
6  * in the Software without restriction, including without limitation the rights
7  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8  * copies of the Software, and to permit persons to whom the Software is
9  * furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20  * SOFTWARE.
21  */
22 
23 /* XSIZE_compare_and_swap (based on fetch_compare_and_swap) */
24 #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_full) \
25     && !defined(AO_HAVE_XSIZE_compare_and_swap_full)
26   AO_INLINE int
27   AO_XSIZE_compare_and_swap_full(volatile XCTYPE *addr, XCTYPE old_val,
28                                  XCTYPE new_val)
29   {
30     return AO_XSIZE_fetch_compare_and_swap_full(addr, old_val, new_val)
31              == old_val;
32   }
33 # define AO_HAVE_XSIZE_compare_and_swap_full
34 #endif
35 
36 #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_acquire) \
37     && !defined(AO_HAVE_XSIZE_compare_and_swap_acquire)
38   AO_INLINE int
39   AO_XSIZE_compare_and_swap_acquire(volatile XCTYPE *addr, XCTYPE old_val,
40                                     XCTYPE new_val)
41   {
42     return AO_XSIZE_fetch_compare_and_swap_acquire(addr, old_val, new_val)
43              == old_val;
44   }
45 # define AO_HAVE_XSIZE_compare_and_swap_acquire
46 #endif
47 
48 #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_release) \
49     && !defined(AO_HAVE_XSIZE_compare_and_swap_release)
50   AO_INLINE int
51   AO_XSIZE_compare_and_swap_release(volatile XCTYPE *addr, XCTYPE old_val,
52                                     XCTYPE new_val)
53   {
54     return AO_XSIZE_fetch_compare_and_swap_release(addr, old_val, new_val)
55              == old_val;
56   }
57 # define AO_HAVE_XSIZE_compare_and_swap_release
58 #endif
59 
60 #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_write) \
61     && !defined(AO_HAVE_XSIZE_compare_and_swap_write)
62   AO_INLINE int
63   AO_XSIZE_compare_and_swap_write(volatile XCTYPE *addr, XCTYPE old_val,
64                                   XCTYPE new_val)
65   {
66     return AO_XSIZE_fetch_compare_and_swap_write(addr, old_val, new_val)
67              == old_val;
68   }
69 # define AO_HAVE_XSIZE_compare_and_swap_write
70 #endif
71 
72 #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_read) \
73     && !defined(AO_HAVE_XSIZE_compare_and_swap_read)
74   AO_INLINE int
75   AO_XSIZE_compare_and_swap_read(volatile XCTYPE *addr, XCTYPE old_val,
76                                  XCTYPE new_val)
77   {
78     return AO_XSIZE_fetch_compare_and_swap_read(addr, old_val, new_val)
79              == old_val;
80   }
81 # define AO_HAVE_XSIZE_compare_and_swap_read
82 #endif
83 
84 #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap) \
85     && !defined(AO_HAVE_XSIZE_compare_and_swap)
86   AO_INLINE int
87   AO_XSIZE_compare_and_swap(volatile XCTYPE *addr, XCTYPE old_val,
88                             XCTYPE new_val)
89   {
90     return AO_XSIZE_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
91   }
92 # define AO_HAVE_XSIZE_compare_and_swap
93 #endif
94 
95 #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_release_write) \
96     && !defined(AO_HAVE_XSIZE_compare_and_swap_release_write)
97   AO_INLINE int
98   AO_XSIZE_compare_and_swap_release_write(volatile XCTYPE *addr,
99                                           XCTYPE old_val, XCTYPE new_val)
100   {
101     return AO_XSIZE_fetch_compare_and_swap_release_write(addr, old_val,
102                                                          new_val) == old_val;
103   }
104 # define AO_HAVE_XSIZE_compare_and_swap_release_write
105 #endif
106 
107 #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_acquire_read) \
108     && !defined(AO_HAVE_XSIZE_compare_and_swap_acquire_read)
109   AO_INLINE int
110   AO_XSIZE_compare_and_swap_acquire_read(volatile XCTYPE *addr,
111                                          XCTYPE old_val, XCTYPE new_val)
112   {
113     return AO_XSIZE_fetch_compare_and_swap_acquire_read(addr, old_val,
114                                                         new_val) == old_val;
115   }
116 # define AO_HAVE_XSIZE_compare_and_swap_acquire_read
117 #endif
118 
119 #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_dd_acquire_read) \
120     && !defined(AO_HAVE_XSIZE_compare_and_swap_dd_acquire_read)
121   AO_INLINE int
122   AO_XSIZE_compare_and_swap_dd_acquire_read(volatile XCTYPE *addr,
123                                             XCTYPE old_val, XCTYPE new_val)
124   {
125     return AO_XSIZE_fetch_compare_and_swap_dd_acquire_read(addr, old_val,
126                                                            new_val) == old_val;
127   }
128 # define AO_HAVE_XSIZE_compare_and_swap_dd_acquire_read
129 #endif
130 
131 /* XSIZE_fetch_and_add */
132 /* We first try to implement fetch_and_add variants in terms of the     */
133 /* corresponding compare_and_swap variants to minimize adding barriers. */
134 #if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
135     && !defined(AO_HAVE_XSIZE_fetch_and_add_full)
136   AO_ATTR_NO_SANITIZE_THREAD
137   AO_INLINE XCTYPE
138   AO_XSIZE_fetch_and_add_full(volatile XCTYPE *addr, XCTYPE incr)
139   {
140     XCTYPE old;
141 
142     do
143       {
144         old = *(XCTYPE *)addr;
145       }
146     while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
147                                                            old + incr)));
148     return old;
149   }
150 # define AO_HAVE_XSIZE_fetch_and_add_full
151 #endif
152 
153 #if defined(AO_HAVE_XSIZE_compare_and_swap_acquire) \
154     && !defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
155   AO_ATTR_NO_SANITIZE_THREAD
156   AO_INLINE XCTYPE
157   AO_XSIZE_fetch_and_add_acquire(volatile XCTYPE *addr, XCTYPE incr)
158   {
159     XCTYPE old;
160 
161     do
162       {
163         old = *(XCTYPE *)addr;
164       }
165     while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_acquire(addr, old,
166                                                               old + incr)));
167     return old;
168   }
169 # define AO_HAVE_XSIZE_fetch_and_add_acquire
170 #endif
171 
172 #if defined(AO_HAVE_XSIZE_compare_and_swap_release) \
173     && !defined(AO_HAVE_XSIZE_fetch_and_add_release)
174   AO_ATTR_NO_SANITIZE_THREAD
175   AO_INLINE XCTYPE
176   AO_XSIZE_fetch_and_add_release(volatile XCTYPE *addr, XCTYPE incr)
177   {
178     XCTYPE old;
179 
180     do
181       {
182         old = *(XCTYPE *)addr;
183       }
184     while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_release(addr, old,
185                                                               old + incr)));
186     return old;
187   }
188 # define AO_HAVE_XSIZE_fetch_and_add_release
189 #endif
190 
191 #if defined(AO_HAVE_XSIZE_compare_and_swap) \
192     && !defined(AO_HAVE_XSIZE_fetch_and_add)
193   AO_ATTR_NO_SANITIZE_THREAD
194   AO_INLINE XCTYPE
195   AO_XSIZE_fetch_and_add(volatile XCTYPE *addr, XCTYPE incr)
196   {
197     XCTYPE old;
198 
199     do
200       {
201         old = *(XCTYPE *)addr;
202       }
203     while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap(addr, old,
204                                                       old + incr)));
205     return old;
206   }
207 # define AO_HAVE_XSIZE_fetch_and_add
208 #endif
209 
210 #if defined(AO_HAVE_XSIZE_fetch_and_add_full)
211 # if !defined(AO_HAVE_XSIZE_fetch_and_add_release)
212 #   define AO_XSIZE_fetch_and_add_release(addr, val) \
213                                 AO_XSIZE_fetch_and_add_full(addr, val)
214 #   define AO_HAVE_XSIZE_fetch_and_add_release
215 # endif
216 # if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
217 #   define AO_XSIZE_fetch_and_add_acquire(addr, val) \
218                                 AO_XSIZE_fetch_and_add_full(addr, val)
219 #   define AO_HAVE_XSIZE_fetch_and_add_acquire
220 # endif
221 # if !defined(AO_HAVE_XSIZE_fetch_and_add_write)
222 #   define AO_XSIZE_fetch_and_add_write(addr, val) \
223                                 AO_XSIZE_fetch_and_add_full(addr, val)
224 #   define AO_HAVE_XSIZE_fetch_and_add_write
225 # endif
226 # if !defined(AO_HAVE_XSIZE_fetch_and_add_read)
227 #   define AO_XSIZE_fetch_and_add_read(addr, val) \
228                                 AO_XSIZE_fetch_and_add_full(addr, val)
229 #   define AO_HAVE_XSIZE_fetch_and_add_read
230 # endif
231 #endif /* AO_HAVE_XSIZE_fetch_and_add_full */
232 
233 #if defined(AO_HAVE_XSIZE_fetch_and_add) && defined(AO_HAVE_nop_full) \
234     && !defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
235   AO_INLINE XCTYPE
236   AO_XSIZE_fetch_and_add_acquire(volatile XCTYPE *addr, XCTYPE incr)
237   {
238     XCTYPE result = AO_XSIZE_fetch_and_add(addr, incr);
239     AO_nop_full();
240     return result;
241   }
242 # define AO_HAVE_XSIZE_fetch_and_add_acquire
243 #endif
244 #if defined(AO_HAVE_XSIZE_fetch_and_add) && defined(AO_HAVE_nop_full) \
245     && !defined(AO_HAVE_XSIZE_fetch_and_add_release)
246 # define AO_XSIZE_fetch_and_add_release(addr, incr) \
247                 (AO_nop_full(), AO_XSIZE_fetch_and_add(addr, incr))
248 # define AO_HAVE_XSIZE_fetch_and_add_release
249 #endif
250 
251 #if !defined(AO_HAVE_XSIZE_fetch_and_add) \
252     && defined(AO_HAVE_XSIZE_fetch_and_add_release)
253 # define AO_XSIZE_fetch_and_add(addr, val) \
254                                 AO_XSIZE_fetch_and_add_release(addr, val)
255 # define AO_HAVE_XSIZE_fetch_and_add
256 #endif
257 #if !defined(AO_HAVE_XSIZE_fetch_and_add) \
258     && defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
259 # define AO_XSIZE_fetch_and_add(addr, val) \
260                                 AO_XSIZE_fetch_and_add_acquire(addr, val)
261 # define AO_HAVE_XSIZE_fetch_and_add
262 #endif
263 #if !defined(AO_HAVE_XSIZE_fetch_and_add) \
264     && defined(AO_HAVE_XSIZE_fetch_and_add_write)
265 # define AO_XSIZE_fetch_and_add(addr, val) \
266                                 AO_XSIZE_fetch_and_add_write(addr, val)
267 # define AO_HAVE_XSIZE_fetch_and_add
268 #endif
269 #if !defined(AO_HAVE_XSIZE_fetch_and_add) \
270     && defined(AO_HAVE_XSIZE_fetch_and_add_read)
271 # define AO_XSIZE_fetch_and_add(addr, val) \
272                                 AO_XSIZE_fetch_and_add_read(addr, val)
273 # define AO_HAVE_XSIZE_fetch_and_add
274 #endif
275 
276 #if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \
277     && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_XSIZE_fetch_and_add_full)
278 # define AO_XSIZE_fetch_and_add_full(addr, val) \
279                 (AO_nop_full(), AO_XSIZE_fetch_and_add_acquire(addr, val))
280 # define AO_HAVE_XSIZE_fetch_and_add_full
281 #endif
282 
283 #if !defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
284     && defined(AO_HAVE_XSIZE_fetch_and_add_write)
285 # define AO_XSIZE_fetch_and_add_release_write(addr, val) \
286                                 AO_XSIZE_fetch_and_add_write(addr, val)
287 # define AO_HAVE_XSIZE_fetch_and_add_release_write
288 #endif
289 #if !defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
290     && defined(AO_HAVE_XSIZE_fetch_and_add_release)
291 # define AO_XSIZE_fetch_and_add_release_write(addr, val) \
292                                 AO_XSIZE_fetch_and_add_release(addr, val)
293 # define AO_HAVE_XSIZE_fetch_and_add_release_write
294 #endif
295 
296 #if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
297     && defined(AO_HAVE_XSIZE_fetch_and_add_read)
298 # define AO_XSIZE_fetch_and_add_acquire_read(addr, val) \
299                                 AO_XSIZE_fetch_and_add_read(addr, val)
300 # define AO_HAVE_XSIZE_fetch_and_add_acquire_read
301 #endif
302 #if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
303     && defined(AO_HAVE_XSIZE_fetch_and_add_acquire)
304 # define AO_XSIZE_fetch_and_add_acquire_read(addr, val) \
305                                 AO_XSIZE_fetch_and_add_acquire(addr, val)
306 # define AO_HAVE_XSIZE_fetch_and_add_acquire_read
307 #endif
308 
309 #ifdef AO_NO_DD_ORDERING
310 # if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read)
311 #   define AO_XSIZE_fetch_and_add_dd_acquire_read(addr, val) \
312                                 AO_XSIZE_fetch_and_add_acquire_read(addr, val)
313 #   define AO_HAVE_XSIZE_fetch_and_add_dd_acquire_read
314 # endif
315 #else
316 # if defined(AO_HAVE_XSIZE_fetch_and_add)
317 #   define AO_XSIZE_fetch_and_add_dd_acquire_read(addr, val) \
318                                 AO_XSIZE_fetch_and_add(addr, val)
319 #   define AO_HAVE_XSIZE_fetch_and_add_dd_acquire_read
320 # endif
321 #endif /* !AO_NO_DD_ORDERING */
322 
323 /* XSIZE_fetch_and_add1 */
324 #if defined(AO_HAVE_XSIZE_fetch_and_add_full) \
325     && !defined(AO_HAVE_XSIZE_fetch_and_add1_full)
326 # define AO_XSIZE_fetch_and_add1_full(addr) \
327                                 AO_XSIZE_fetch_and_add_full(addr, 1)
328 # define AO_HAVE_XSIZE_fetch_and_add1_full
329 #endif
330 #if defined(AO_HAVE_XSIZE_fetch_and_add_release) \
331     && !defined(AO_HAVE_XSIZE_fetch_and_add1_release)
332 # define AO_XSIZE_fetch_and_add1_release(addr) \
333                                 AO_XSIZE_fetch_and_add_release(addr, 1)
334 # define AO_HAVE_XSIZE_fetch_and_add1_release
335 #endif
336 #if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \
337     && !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
338 # define AO_XSIZE_fetch_and_add1_acquire(addr) \
339                                 AO_XSIZE_fetch_and_add_acquire(addr, 1)
340 # define AO_HAVE_XSIZE_fetch_and_add1_acquire
341 #endif
342 #if defined(AO_HAVE_XSIZE_fetch_and_add_write) \
343     && !defined(AO_HAVE_XSIZE_fetch_and_add1_write)
344 # define AO_XSIZE_fetch_and_add1_write(addr) \
345                                 AO_XSIZE_fetch_and_add_write(addr, 1)
346 # define AO_HAVE_XSIZE_fetch_and_add1_write
347 #endif
348 #if defined(AO_HAVE_XSIZE_fetch_and_add_read) \
349     && !defined(AO_HAVE_XSIZE_fetch_and_add1_read)
350 # define AO_XSIZE_fetch_and_add1_read(addr) \
351                                 AO_XSIZE_fetch_and_add_read(addr, 1)
352 # define AO_HAVE_XSIZE_fetch_and_add1_read
353 #endif
354 #if defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
355     && !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write)
356 # define AO_XSIZE_fetch_and_add1_release_write(addr) \
357                                 AO_XSIZE_fetch_and_add_release_write(addr, 1)
358 # define AO_HAVE_XSIZE_fetch_and_add1_release_write
359 #endif
360 #if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
361     && !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read)
362 # define AO_XSIZE_fetch_and_add1_acquire_read(addr) \
363                                 AO_XSIZE_fetch_and_add_acquire_read(addr, 1)
364 # define AO_HAVE_XSIZE_fetch_and_add1_acquire_read
365 #endif
366 #if defined(AO_HAVE_XSIZE_fetch_and_add) \
367     && !defined(AO_HAVE_XSIZE_fetch_and_add1)
368 # define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add(addr, 1)
369 # define AO_HAVE_XSIZE_fetch_and_add1
370 #endif
371 
372 #if defined(AO_HAVE_XSIZE_fetch_and_add1_full)
373 # if !defined(AO_HAVE_XSIZE_fetch_and_add1_release)
374 #   define AO_XSIZE_fetch_and_add1_release(addr) \
375                                 AO_XSIZE_fetch_and_add1_full(addr)
376 #   define AO_HAVE_XSIZE_fetch_and_add1_release
377 # endif
378 # if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
379 #   define AO_XSIZE_fetch_and_add1_acquire(addr) \
380                                 AO_XSIZE_fetch_and_add1_full(addr)
381 #   define AO_HAVE_XSIZE_fetch_and_add1_acquire
382 # endif
383 # if !defined(AO_HAVE_XSIZE_fetch_and_add1_write)
384 #   define AO_XSIZE_fetch_and_add1_write(addr) \
385                                 AO_XSIZE_fetch_and_add1_full(addr)
386 #   define AO_HAVE_XSIZE_fetch_and_add1_write
387 # endif
388 # if !defined(AO_HAVE_XSIZE_fetch_and_add1_read)
389 #   define AO_XSIZE_fetch_and_add1_read(addr) \
390                                 AO_XSIZE_fetch_and_add1_full(addr)
391 #   define AO_HAVE_XSIZE_fetch_and_add1_read
392 # endif
393 #endif /* AO_HAVE_XSIZE_fetch_and_add1_full */
394 
395 #if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
396     && defined(AO_HAVE_XSIZE_fetch_and_add1_release)
397 # define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_release(addr)
398 # define AO_HAVE_XSIZE_fetch_and_add1
399 #endif
400 #if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
401     && defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
402 # define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_acquire(addr)
403 # define AO_HAVE_XSIZE_fetch_and_add1
404 #endif
405 #if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
406     && defined(AO_HAVE_XSIZE_fetch_and_add1_write)
407 # define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_write(addr)
408 # define AO_HAVE_XSIZE_fetch_and_add1
409 #endif
410 #if !defined(AO_HAVE_XSIZE_fetch_and_add1) \
411     && defined(AO_HAVE_XSIZE_fetch_and_add1_read)
412 # define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_read(addr)
413 # define AO_HAVE_XSIZE_fetch_and_add1
414 #endif
415 
416 #if defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) \
417     && defined(AO_HAVE_nop_full) \
418     && !defined(AO_HAVE_XSIZE_fetch_and_add1_full)
419 # define AO_XSIZE_fetch_and_add1_full(addr) \
420                         (AO_nop_full(), AO_XSIZE_fetch_and_add1_acquire(addr))
421 # define AO_HAVE_XSIZE_fetch_and_add1_full
422 #endif
423 
424 #if !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) \
425     && defined(AO_HAVE_XSIZE_fetch_and_add1_write)
426 # define AO_XSIZE_fetch_and_add1_release_write(addr) \
427                                 AO_XSIZE_fetch_and_add1_write(addr)
428 # define AO_HAVE_XSIZE_fetch_and_add1_release_write
429 #endif
430 #if !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) \
431     && defined(AO_HAVE_XSIZE_fetch_and_add1_release)
432 # define AO_XSIZE_fetch_and_add1_release_write(addr) \
433                                 AO_XSIZE_fetch_and_add1_release(addr)
434 # define AO_HAVE_XSIZE_fetch_and_add1_release_write
435 #endif
436 #if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) \
437     && defined(AO_HAVE_XSIZE_fetch_and_add1_read)
438 # define AO_XSIZE_fetch_and_add1_acquire_read(addr) \
439                                 AO_XSIZE_fetch_and_add1_read(addr)
440 # define AO_HAVE_XSIZE_fetch_and_add1_acquire_read
441 #endif
442 #if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) \
443     && defined(AO_HAVE_XSIZE_fetch_and_add1_acquire)
444 # define AO_XSIZE_fetch_and_add1_acquire_read(addr) \
445                                 AO_XSIZE_fetch_and_add1_acquire(addr)
446 # define AO_HAVE_XSIZE_fetch_and_add1_acquire_read
447 #endif
448 
449 #ifdef AO_NO_DD_ORDERING
450 # if defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read)
451 #   define AO_XSIZE_fetch_and_add1_dd_acquire_read(addr) \
452                                 AO_XSIZE_fetch_and_add1_acquire_read(addr)
453 #   define AO_HAVE_XSIZE_fetch_and_add1_dd_acquire_read
454 # endif
455 #else
456 # if defined(AO_HAVE_XSIZE_fetch_and_add1)
457 #   define AO_XSIZE_fetch_and_add1_dd_acquire_read(addr) \
458                                 AO_XSIZE_fetch_and_add1(addr)
459 #   define AO_HAVE_XSIZE_fetch_and_add1_dd_acquire_read
460 # endif
461 #endif /* !AO_NO_DD_ORDERING */
462 
463 /* XSIZE_fetch_and_sub1 */
464 #if defined(AO_HAVE_XSIZE_fetch_and_add_full) \
465     && !defined(AO_HAVE_XSIZE_fetch_and_sub1_full)
466 # define AO_XSIZE_fetch_and_sub1_full(addr) \
467                 AO_XSIZE_fetch_and_add_full(addr, (XCTYPE)(-1))
468 # define AO_HAVE_XSIZE_fetch_and_sub1_full
469 #endif
470 #if defined(AO_HAVE_XSIZE_fetch_and_add_release) \
471     && !defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
472 # define AO_XSIZE_fetch_and_sub1_release(addr) \
473                 AO_XSIZE_fetch_and_add_release(addr, (XCTYPE)(-1))
474 # define AO_HAVE_XSIZE_fetch_and_sub1_release
475 #endif
476 #if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \
477     && !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
478 # define AO_XSIZE_fetch_and_sub1_acquire(addr) \
479                 AO_XSIZE_fetch_and_add_acquire(addr, (XCTYPE)(-1))
480 # define AO_HAVE_XSIZE_fetch_and_sub1_acquire
481 #endif
482 #if defined(AO_HAVE_XSIZE_fetch_and_add_write) \
483     && !defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
484 # define AO_XSIZE_fetch_and_sub1_write(addr) \
485                 AO_XSIZE_fetch_and_add_write(addr, (XCTYPE)(-1))
486 # define AO_HAVE_XSIZE_fetch_and_sub1_write
487 #endif
488 #if defined(AO_HAVE_XSIZE_fetch_and_add_read) \
489     && !defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
490 # define AO_XSIZE_fetch_and_sub1_read(addr) \
491                 AO_XSIZE_fetch_and_add_read(addr, (XCTYPE)(-1))
492 # define AO_HAVE_XSIZE_fetch_and_sub1_read
493 #endif
494 #if defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \
495     && !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write)
496 # define AO_XSIZE_fetch_and_sub1_release_write(addr) \
497                 AO_XSIZE_fetch_and_add_release_write(addr, (XCTYPE)(-1))
498 # define AO_HAVE_XSIZE_fetch_and_sub1_release_write
499 #endif
500 #if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \
501     && !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read)
502 # define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \
503                 AO_XSIZE_fetch_and_add_acquire_read(addr, (XCTYPE)(-1))
504 # define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read
505 #endif
506 #if defined(AO_HAVE_XSIZE_fetch_and_add) \
507     && !defined(AO_HAVE_XSIZE_fetch_and_sub1)
508 # define AO_XSIZE_fetch_and_sub1(addr) \
509                 AO_XSIZE_fetch_and_add(addr, (XCTYPE)(-1))
510 # define AO_HAVE_XSIZE_fetch_and_sub1
511 #endif
512 
513 #if defined(AO_HAVE_XSIZE_fetch_and_sub1_full)
514 # if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
515 #   define AO_XSIZE_fetch_and_sub1_release(addr) \
516                                 AO_XSIZE_fetch_and_sub1_full(addr)
517 #   define AO_HAVE_XSIZE_fetch_and_sub1_release
518 # endif
519 # if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
520 #   define AO_XSIZE_fetch_and_sub1_acquire(addr) \
521                                 AO_XSIZE_fetch_and_sub1_full(addr)
522 #   define AO_HAVE_XSIZE_fetch_and_sub1_acquire
523 # endif
524 # if !defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
525 #   define AO_XSIZE_fetch_and_sub1_write(addr) \
526                                 AO_XSIZE_fetch_and_sub1_full(addr)
527 #   define AO_HAVE_XSIZE_fetch_and_sub1_write
528 # endif
529 # if !defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
530 #   define AO_XSIZE_fetch_and_sub1_read(addr) \
531                                 AO_XSIZE_fetch_and_sub1_full(addr)
532 #   define AO_HAVE_XSIZE_fetch_and_sub1_read
533 # endif
534 #endif /* AO_HAVE_XSIZE_fetch_and_sub1_full */
535 
536 #if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
537     && defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
538 # define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_release(addr)
539 # define AO_HAVE_XSIZE_fetch_and_sub1
540 #endif
541 #if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
542     && defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
543 # define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_acquire(addr)
544 # define AO_HAVE_XSIZE_fetch_and_sub1
545 #endif
546 #if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
547     && defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
548 # define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_write(addr)
549 # define AO_HAVE_XSIZE_fetch_and_sub1
550 #endif
551 #if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \
552     && defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
553 # define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_read(addr)
554 # define AO_HAVE_XSIZE_fetch_and_sub1
555 #endif
556 
557 #if defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) \
558     && defined(AO_HAVE_nop_full) \
559     && !defined(AO_HAVE_XSIZE_fetch_and_sub1_full)
560 # define AO_XSIZE_fetch_and_sub1_full(addr) \
561                         (AO_nop_full(), AO_XSIZE_fetch_and_sub1_acquire(addr))
562 # define AO_HAVE_XSIZE_fetch_and_sub1_full
563 #endif
564 
565 #if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) \
566     && defined(AO_HAVE_XSIZE_fetch_and_sub1_write)
567 # define AO_XSIZE_fetch_and_sub1_release_write(addr) \
568                                 AO_XSIZE_fetch_and_sub1_write(addr)
569 # define AO_HAVE_XSIZE_fetch_and_sub1_release_write
570 #endif
571 #if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) \
572     && defined(AO_HAVE_XSIZE_fetch_and_sub1_release)
573 # define AO_XSIZE_fetch_and_sub1_release_write(addr) \
574                                 AO_XSIZE_fetch_and_sub1_release(addr)
575 # define AO_HAVE_XSIZE_fetch_and_sub1_release_write
576 #endif
577 #if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) \
578     && defined(AO_HAVE_XSIZE_fetch_and_sub1_read)
579 # define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \
580                                 AO_XSIZE_fetch_and_sub1_read(addr)
581 # define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read
582 #endif
583 #if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) \
584     && defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire)
585 # define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \
586                                 AO_XSIZE_fetch_and_sub1_acquire(addr)
587 # define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read
588 #endif
589 
590 #ifdef AO_NO_DD_ORDERING
591 # if defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read)
592 #   define AO_XSIZE_fetch_and_sub1_dd_acquire_read(addr) \
593                                 AO_XSIZE_fetch_and_sub1_acquire_read(addr)
594 #   define AO_HAVE_XSIZE_fetch_and_sub1_dd_acquire_read
595 # endif
596 #else
597 # if defined(AO_HAVE_XSIZE_fetch_and_sub1)
598 #   define AO_XSIZE_fetch_and_sub1_dd_acquire_read(addr) \
599                                 AO_XSIZE_fetch_and_sub1(addr)
600 #   define AO_HAVE_XSIZE_fetch_and_sub1_dd_acquire_read
601 # endif
602 #endif /* !AO_NO_DD_ORDERING */
603 
604 /* XSIZE_and */
605 #if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
606     && !defined(AO_HAVE_XSIZE_and_full)
607   AO_ATTR_NO_SANITIZE_THREAD
608   AO_INLINE void
609   AO_XSIZE_and_full(volatile XCTYPE *addr, XCTYPE value)
610   {
611     XCTYPE old;
612 
613     do
614       {
615         old = *(XCTYPE *)addr;
616       }
617     while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
618                                                            old & value)));
619   }
620 # define AO_HAVE_XSIZE_and_full
621 #endif
622 
623 #if defined(AO_HAVE_XSIZE_and_full)
624 # if !defined(AO_HAVE_XSIZE_and_release)
625 #   define AO_XSIZE_and_release(addr, val) AO_XSIZE_and_full(addr, val)
626 #   define AO_HAVE_XSIZE_and_release
627 # endif
628 # if !defined(AO_HAVE_XSIZE_and_acquire)
629 #   define AO_XSIZE_and_acquire(addr, val) AO_XSIZE_and_full(addr, val)
630 #   define AO_HAVE_XSIZE_and_acquire
631 # endif
632 # if !defined(AO_HAVE_XSIZE_and_write)
633 #   define AO_XSIZE_and_write(addr, val) AO_XSIZE_and_full(addr, val)
634 #   define AO_HAVE_XSIZE_and_write
635 # endif
636 # if !defined(AO_HAVE_XSIZE_and_read)
637 #   define AO_XSIZE_and_read(addr, val) AO_XSIZE_and_full(addr, val)
638 #   define AO_HAVE_XSIZE_and_read
639 # endif
640 #endif /* AO_HAVE_XSIZE_and_full */
641 
642 #if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_release)
643 # define AO_XSIZE_and(addr, val) AO_XSIZE_and_release(addr, val)
644 # define AO_HAVE_XSIZE_and
645 #endif
646 #if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_acquire)
647 # define AO_XSIZE_and(addr, val) AO_XSIZE_and_acquire(addr, val)
648 # define AO_HAVE_XSIZE_and
649 #endif
650 #if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_write)
651 # define AO_XSIZE_and(addr, val) AO_XSIZE_and_write(addr, val)
652 # define AO_HAVE_XSIZE_and
653 #endif
654 #if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_read)
655 # define AO_XSIZE_and(addr, val) AO_XSIZE_and_read(addr, val)
656 # define AO_HAVE_XSIZE_and
657 #endif
658 
659 #if defined(AO_HAVE_XSIZE_and_acquire) && defined(AO_HAVE_nop_full) \
660     && !defined(AO_HAVE_XSIZE_and_full)
661 # define AO_XSIZE_and_full(addr, val) \
662                         (AO_nop_full(), AO_XSIZE_and_acquire(addr, val))
663 # define AO_HAVE_XSIZE_and_full
664 #endif
665 
666 #if !defined(AO_HAVE_XSIZE_and_release_write) \
667     && defined(AO_HAVE_XSIZE_and_write)
668 # define AO_XSIZE_and_release_write(addr, val) AO_XSIZE_and_write(addr, val)
669 # define AO_HAVE_XSIZE_and_release_write
670 #endif
671 #if !defined(AO_HAVE_XSIZE_and_release_write) \
672     && defined(AO_HAVE_XSIZE_and_release)
673 # define AO_XSIZE_and_release_write(addr, val) AO_XSIZE_and_release(addr, val)
674 # define AO_HAVE_XSIZE_and_release_write
675 #endif
676 #if !defined(AO_HAVE_XSIZE_and_acquire_read) \
677     && defined(AO_HAVE_XSIZE_and_read)
678 # define AO_XSIZE_and_acquire_read(addr, val) AO_XSIZE_and_read(addr, val)
679 # define AO_HAVE_XSIZE_and_acquire_read
680 #endif
681 #if !defined(AO_HAVE_XSIZE_and_acquire_read) \
682     && defined(AO_HAVE_XSIZE_and_acquire)
683 # define AO_XSIZE_and_acquire_read(addr, val) AO_XSIZE_and_acquire(addr, val)
684 # define AO_HAVE_XSIZE_and_acquire_read
685 #endif
686 
687 /* XSIZE_or */
688 #if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
689     && !defined(AO_HAVE_XSIZE_or_full)
690   AO_ATTR_NO_SANITIZE_THREAD
691   AO_INLINE void
692   AO_XSIZE_or_full(volatile XCTYPE *addr, XCTYPE value)
693   {
694     XCTYPE old;
695 
696     do
697       {
698         old = *(XCTYPE *)addr;
699       }
700     while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
701                                                            old | value)));
702   }
703 # define AO_HAVE_XSIZE_or_full
704 #endif
705 
706 #if defined(AO_HAVE_XSIZE_or_full)
707 # if !defined(AO_HAVE_XSIZE_or_release)
708 #   define AO_XSIZE_or_release(addr, val) AO_XSIZE_or_full(addr, val)
709 #   define AO_HAVE_XSIZE_or_release
710 # endif
711 # if !defined(AO_HAVE_XSIZE_or_acquire)
712 #   define AO_XSIZE_or_acquire(addr, val) AO_XSIZE_or_full(addr, val)
713 #   define AO_HAVE_XSIZE_or_acquire
714 # endif
715 # if !defined(AO_HAVE_XSIZE_or_write)
716 #   define AO_XSIZE_or_write(addr, val) AO_XSIZE_or_full(addr, val)
717 #   define AO_HAVE_XSIZE_or_write
718 # endif
719 # if !defined(AO_HAVE_XSIZE_or_read)
720 #   define AO_XSIZE_or_read(addr, val) AO_XSIZE_or_full(addr, val)
721 #   define AO_HAVE_XSIZE_or_read
722 # endif
723 #endif /* AO_HAVE_XSIZE_or_full */
724 
725 #if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_release)
726 # define AO_XSIZE_or(addr, val) AO_XSIZE_or_release(addr, val)
727 # define AO_HAVE_XSIZE_or
728 #endif
729 #if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_acquire)
730 # define AO_XSIZE_or(addr, val) AO_XSIZE_or_acquire(addr, val)
731 # define AO_HAVE_XSIZE_or
732 #endif
733 #if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_write)
734 # define AO_XSIZE_or(addr, val) AO_XSIZE_or_write(addr, val)
735 # define AO_HAVE_XSIZE_or
736 #endif
737 #if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_read)
738 # define AO_XSIZE_or(addr, val) AO_XSIZE_or_read(addr, val)
739 # define AO_HAVE_XSIZE_or
740 #endif
741 
742 #if defined(AO_HAVE_XSIZE_or_acquire) && defined(AO_HAVE_nop_full) \
743     && !defined(AO_HAVE_XSIZE_or_full)
744 # define AO_XSIZE_or_full(addr, val) \
745                         (AO_nop_full(), AO_XSIZE_or_acquire(addr, val))
746 # define AO_HAVE_XSIZE_or_full
747 #endif
748 
749 #if !defined(AO_HAVE_XSIZE_or_release_write) \
750     && defined(AO_HAVE_XSIZE_or_write)
751 # define AO_XSIZE_or_release_write(addr, val) AO_XSIZE_or_write(addr, val)
752 # define AO_HAVE_XSIZE_or_release_write
753 #endif
754 #if !defined(AO_HAVE_XSIZE_or_release_write) \
755     && defined(AO_HAVE_XSIZE_or_release)
756 # define AO_XSIZE_or_release_write(addr, val) AO_XSIZE_or_release(addr, val)
757 # define AO_HAVE_XSIZE_or_release_write
758 #endif
759 #if !defined(AO_HAVE_XSIZE_or_acquire_read) && defined(AO_HAVE_XSIZE_or_read)
760 # define AO_XSIZE_or_acquire_read(addr, val) AO_XSIZE_or_read(addr, val)
761 # define AO_HAVE_XSIZE_or_acquire_read
762 #endif
763 #if !defined(AO_HAVE_XSIZE_or_acquire_read) \
764     && defined(AO_HAVE_XSIZE_or_acquire)
765 # define AO_XSIZE_or_acquire_read(addr, val) AO_XSIZE_or_acquire(addr, val)
766 # define AO_HAVE_XSIZE_or_acquire_read
767 #endif
768 
769 /* XSIZE_xor */
770 #if defined(AO_HAVE_XSIZE_compare_and_swap_full) \
771     && !defined(AO_HAVE_XSIZE_xor_full)
772   AO_ATTR_NO_SANITIZE_THREAD
773   AO_INLINE void
774   AO_XSIZE_xor_full(volatile XCTYPE *addr, XCTYPE value)
775   {
776     XCTYPE old;
777 
778     do
779       {
780         old = *(XCTYPE *)addr;
781       }
782     while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old,
783                                                            old ^ value)));
784   }
785 # define AO_HAVE_XSIZE_xor_full
786 #endif
787 
788 #if defined(AO_HAVE_XSIZE_xor_full)
789 # if !defined(AO_HAVE_XSIZE_xor_release)
790 #   define AO_XSIZE_xor_release(addr, val) AO_XSIZE_xor_full(addr, val)
791 #   define AO_HAVE_XSIZE_xor_release
792 # endif
793 # if !defined(AO_HAVE_XSIZE_xor_acquire)
794 #   define AO_XSIZE_xor_acquire(addr, val) AO_XSIZE_xor_full(addr, val)
795 #   define AO_HAVE_XSIZE_xor_acquire
796 # endif
797 # if !defined(AO_HAVE_XSIZE_xor_write)
798 #   define AO_XSIZE_xor_write(addr, val) AO_XSIZE_xor_full(addr, val)
799 #   define AO_HAVE_XSIZE_xor_write
800 # endif
801 # if !defined(AO_HAVE_XSIZE_xor_read)
802 #   define AO_XSIZE_xor_read(addr, val) AO_XSIZE_xor_full(addr, val)
803 #   define AO_HAVE_XSIZE_xor_read
804 # endif
805 #endif /* AO_HAVE_XSIZE_xor_full */
806 
807 #if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_release)
808 # define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_release(addr, val)
809 # define AO_HAVE_XSIZE_xor
810 #endif
811 #if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_acquire)
812 # define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_acquire(addr, val)
813 # define AO_HAVE_XSIZE_xor
814 #endif
815 #if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_write)
816 # define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_write(addr, val)
817 # define AO_HAVE_XSIZE_xor
818 #endif
819 #if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_read)
820 # define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_read(addr, val)
821 # define AO_HAVE_XSIZE_xor
822 #endif
823 
824 #if defined(AO_HAVE_XSIZE_xor_acquire) && defined(AO_HAVE_nop_full) \
825     && !defined(AO_HAVE_XSIZE_xor_full)
826 # define AO_XSIZE_xor_full(addr, val) \
827                         (AO_nop_full(), AO_XSIZE_xor_acquire(addr, val))
828 # define AO_HAVE_XSIZE_xor_full
829 #endif
830 
831 #if !defined(AO_HAVE_XSIZE_xor_release_write) \
832     && defined(AO_HAVE_XSIZE_xor_write)
833 # define AO_XSIZE_xor_release_write(addr, val) AO_XSIZE_xor_write(addr, val)
834 # define AO_HAVE_XSIZE_xor_release_write
835 #endif
836 #if !defined(AO_HAVE_XSIZE_xor_release_write) \
837     && defined(AO_HAVE_XSIZE_xor_release)
838 # define AO_XSIZE_xor_release_write(addr, val) AO_XSIZE_xor_release(addr, val)
839 # define AO_HAVE_XSIZE_xor_release_write
840 #endif
841 #if !defined(AO_HAVE_XSIZE_xor_acquire_read) \
842     && defined(AO_HAVE_XSIZE_xor_read)
843 # define AO_XSIZE_xor_acquire_read(addr, val) AO_XSIZE_xor_read(addr, val)
844 # define AO_HAVE_XSIZE_xor_acquire_read
845 #endif
846 #if !defined(AO_HAVE_XSIZE_xor_acquire_read) \
847     && defined(AO_HAVE_XSIZE_xor_acquire)
848 # define AO_XSIZE_xor_acquire_read(addr, val) AO_XSIZE_xor_acquire(addr, val)
849 # define AO_HAVE_XSIZE_xor_acquire_read
850 #endif
851 
852 /* XSIZE_and/or/xor_dd_acquire_read are meaningless.    */
853