/*
 * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
 * Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
 * All rights reserved.
 * Copyright (c) 2009-2025 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * This file could be used for the following purposes:
 *   - get the complete collector as a single link object file (module);
 *   - enable more compiler optimizations.
 *
 * Tip: to get the highest level of compiler optimizations, the typical
 * compiler options to use (assuming gcc) are:
 * `-O3 -march=native -fprofile-generate`
 *
 * Warning: gcc for Linux (for C++ clients only): use `-fexceptions` both for
 * the collector library and the client as otherwise `GC_thread_exit_proc()`
 * is not guaranteed to be invoked (see the comments in `pthread_start.c`
 * file).
 */

/*
 * V note: the bundled libgc must be compiled with `-DALL_INTERIOR_POINTERS=1`
 * (set in `vlib/builtin/builtin_d_gcboehm.c.v`). Without it, the conservative
 * collector misses interior pointers into GC-allocated buffers — sumtype
 * payloads referenced via offset (e.g. `toml.ast_to_any`, `yaml.Any.to_json`)
 * get reclaimed mid-walk and `-gc boehm` builds crash with `signal 11`.
 * Keep this requirement in sync when re-amalgamating from bdwgc.
 */

#define GC_SINGLE_OBJ_BUILD

#ifndef __cplusplus
/* `static` is desirable here for more efficient linkage. */
/* TODO: Enable this in case of the compilation as C++ code. */
#  define GC_INNER STATIC
#  define GC_EXTERN GC_INNER
/* Note: `STATIC` macro is defined in `gcconfig.h` file. */
#endif

/* Small files go first... */
/*
 * Copyright (c) 2001 by Hewlett-Packard Company. All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1995 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1997 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999 by Hewlett-Packard Company.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * This is mostly an internal header file.  Typical clients should
 * not use it.  Clients that define their own object kinds with
 * debugging allocators will probably want to include this, however.
 * No attempt is made to keep the namespace clean.  This should not be
 * included from header files that are frequently included by clients.
 */

#ifndef GC_DBG_MLC_H
#define GC_DBG_MLC_H

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2008-2025 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

#ifndef GC_PRIVATE_H
#define GC_PRIVATE_H

#ifdef HAVE_CONFIG_H
/* include/config.h.  Generated from config.h.in by configure.  */
/* include/config.h.in.  Generated from configure.ac by autoheader.  */

/* Define to recognize all pointers to the interior of objects. */
#define ALL_INTERIOR_POINTERS 1

/* AO load, store and/or test-and-set primitives are implemented in
   `libatomic_ops` using locks. */
/* #undef BASE_ATOMIC_OPS_EMULATED */

/* Erroneously cleared dirty bits checking. Use only for debugging of the
   incremental collector. */
/* #undef CHECKSUMS */

/* Define to discover thread stack bounds on Darwin without trying to walk the
   frames on the stack. */
/* #undef DARWIN_DONT_PARSE_STACK */

/* Define to force debug headers on all objects. */
#define DBG_HDRS_ALL 1

/* Do not use user32.dll import library (Win32). */
/* #undef DONT_USE_USER32_DLL */

/* Define to support pointer mask/shift set at runtime. */
/* #undef DYNAMIC_POINTER_MASK */

/* Define to enable eCos target support. */
/* #undef ECOS */

/* Wine `getenv` may not return `NULL` for missing entry. */
/* #undef EMPTY_GETENV_RESULTS */

/* Define to enable alternative finalization interface. */
#define ENABLE_DISCLAIM 1

/* Define to enable internal debug assertions. */
/* #undef GC_ASSERTIONS */

/* Define to enable atomic uncollectible allocation. */
#define GC_ATOMIC_UNCOLLECTABLE 1

/* Use GCC atomic intrinsics instead of `libatomic_ops` primitives. */
#define GC_BUILTIN_ATOMIC 1

/* Define to build dynamic libraries with only API symbols exposed. */
/* #undef GC_DLL */

/* Skip the initial guess of data root sets. */
/* #undef GC_DONT_REGISTER_MAIN_STATIC_DATA */

/* Define to turn on `GC_suspend_thread` support (Linux only). */
#define GC_ENABLE_SUSPEND_THREAD 1

/* Define to include support for `gcj`. */
#define GC_GCJ_SUPPORT 1

/* Define if backtrace information is supported. */
/* #undef GC_HAVE_BUILTIN_BACKTRACE */

/* Define to use `pthread_sigmask` function if needed. */
/* #undef GC_HAVE_PTHREAD_SIGMASK */

/* Enable Win32 `DllMain`-based approach of threads registering. */
/* #undef GC_INSIDE_DLL */

/* Missing execinfo.h header. */
/* #undef GC_MISSING_EXECINFO_H */

/* Missing `sigsetjmp` function. */
/* #undef GC_NO_SIGSETJMP */

/* Disable threads discovery in the collector. */
/* #undef GC_NO_THREADS_DISCOVERY */

/* Read environment variables from the GC 'env' file. */
/* #undef GC_READ_ENV_FILE */

/* Define and export `GC_wcsdup` function. */
#define GC_REQUIRE_WCSDUP 1

/* Define to support platform-specific threads. */
#define GC_THREADS 1

/* Force the GC to use signals based on `SIGRTMIN+k`. */
/* #undef GC_USESIGRT_SIGNALS */

/* Define to cause the collector to redefine `malloc` and intercepted
   `pthreads` routines with their real names while using `dlsym` to refer to
   the original routines. */
/* #undef GC_USE_DLOPEN_WRAP */

/* The major version number of this GC release. */
#define GC_VERSION_MAJOR 8

/* The micro version number of this GC release. */
#define GC_VERSION_MICRO 0

/* The minor version number of this GC release. */
#define GC_VERSION_MINOR 3

/* Define to support pthreads-win32 or winpthreads. */
/* #undef GC_WIN32_PTHREADS */

/* Define to install `pthread_atfork` handlers by default. */
#define HANDLE_FORK 1

/* Define to use `dladdr` function. */
#define HAVE_DLADDR 1

/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1

/* Define to 1 if you have the 'dl_iterate_phdr' function. */
/* #undef HAVE_DL_ITERATE_PHDR */

/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1

/* `libatomic_ops` `AO_or` primitive implementation is lock-free. */
/* #undef HAVE_LOCKFREE_AO_OR */

/* Define to use `pthread_setname_np(const char *)` function. */
#define HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID 1

/* Define to use `pthread_setname_np(pthread_t, const char *)` function. */
/* #undef HAVE_PTHREAD_SETNAME_NP_WITH_TID */

/* Define to use `pthread_setname_np(pthread_t, const char *, void *)`
   function. */
/* #undef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG */

/* Define to use `pthread_set_name_np(pthread_t, const char *)` function. */
/* #undef HAVE_PTHREAD_SET_NAME_NP */

/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1

/* Define to 1 if you have the <stdio.h> header file. */
#define HAVE_STDIO_H 1

/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1

/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1

/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1

/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1

/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1

/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1

/* Do not define `DYNAMIC_LOADING` even if supported (i.e., build the
   collector with disabled tracing of dynamic library data roots). */
/* #undef IGNORE_DYNAMIC_LOADING */

/* Define to make it somewhat safer by default to finalize objects out of
   order by specifying a nonstandard finalization mark procedure. */
#define JAVA_FINALIZATION 1

/* Define to save back-pointers in debugging headers. */
#define KEEP_BACK_PTRS 1

/* Define to optimize for large heaps or root sets. */
/* #undef LARGE_CONFIG */

/* Define to the sub-directory where libtool stores uninstalled libraries. */
#define LT_OBJDIR ".libs/"

/* Define to build the collector with the support of the functionality to
   print max length of chain through unreachable objects ending in a reachable
   one. */
/* #undef MAKE_BACK_GRAPH */

/* Number of sequential garbage collections during those a candidate block for
   unmapping should be marked as free. */
#define MUNMAP_THRESHOLD 7

/* Define to not use system clock (cross compiling). */
/* #undef NO_CLOCK */

/* Disable debugging, like `GC_dump` and its callees. */
/* #undef NO_DEBUGGING */

/* Define to make the collector not allocate executable memory by default. */
#define NO_EXECUTE_PERMISSION 1

/* Missing `getcontext` function. */
/* #undef NO_GETCONTEXT */

/* Prohibit installation of `pthread_atfork` handlers. */
/* #undef NO_HANDLE_FORK */

/* Name of package */
#define PACKAGE "gc"

/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT "https://github.com/bdwgc/bdwgc/issues"

/* Define to the full name of this package. */
#define PACKAGE_NAME "gc"

/* Define to the full name and version of this package. */
#define PACKAGE_STRING "gc 8.3.0"

/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME "gc"

/* Define to the home page for this package. */
#define PACKAGE_URL ""

/* Define to the version of this package. */
#define PACKAGE_VERSION "8.3.0"

/* Define to enable parallel marking. */
#define PARALLEL_MARK 1

/* If defined, redirect `free` to this function. */
/* #undef REDIRECT_FREE */

/* If defined, redirect `malloc` to this function. */
/* #undef REDIRECT_MALLOC */

/* If defined, redirect `realloc` to this function. */
/* #undef REDIRECT_REALLOC */

/* The number of caller frames saved when allocating with the debugging API.
   */
/* #undef SAVE_CALL_COUNT */

/* Shorten the headers to minimize object size at the expense of checking for
   writes past the end. */
/* #undef SHORT_DBG_HDRS */

/* Define to tune the collector for small heap sizes. */
/* #undef SMALL_CONFIG */

/* Define to 1 if all of the C89 standard headers exist (not just the ones
   required in a freestanding environment). This macro is provided for
   backward compatibility; new code need not use it. */
#define STDC_HEADERS 1

/* Define to work around a Solaris 5.3 bug (see dyn_load.c). */
/* #undef SUNOS53_SHARED_LIB */

/* Define to enable thread-local allocation optimization. */
/* #undef THREAD_LOCAL_ALLOC */

/* Use Unicode (W) variant of Win32 API instead of ASCII (A) one. */
/* #undef UNICODE */

/* Define to use of compiler-support for thread-local variables. */
/* #undef USE_COMPILER_TLS */

/* Define to use `mmap` instead of `sbrk` to expand the heap. */
#define USE_MMAP 1

/* Define to return memory to OS with `munmap` calls. */
#define USE_MUNMAP 1

/* Use `rwlock` for the allocator lock instead of mutex. */
/* #undef USE_RWLOCK */

/* Define to use Win32 `VirtualAlloc` (instead of `sbrk` or `mmap`) to expand
   the heap. */
/* #undef USE_WINALLOC */

/* Define to support tracking `GC_malloc` and friends for heap profiling
   tools. */
/* #undef VALGRIND_TRACKING */

/* Version number of package */
#define VERSION "8.3.0"

/* The POSIX feature macro. */
/* #undef _POSIX_C_SOURCE */

/* Indicates the use of `pthreads` (NetBSD). */
/* #undef _PTHREADS */

/* Required define if using POSIX threads. */
#define _REENTRANT 1

/* Define to '__inline__' or '__inline' if that's what the C compiler
   calls it, or to nothing if 'inline' is not supported under any name.  */
#ifndef __cplusplus
/* #undef inline */
#endif

#endif

#if !defined(GC_BUILD) && !defined(NOT_GCBUILD)
#  define GC_BUILD
#endif

#if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__) \
     || defined(__CYGWIN__) || defined(HAVE_DLADDR)               \
     || (defined(__COSMOPOLITAN__) && defined(USE_MUNMAP))        \
     || defined(GC_HAVE_PTHREAD_SIGMASK)                          \
     || defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID)              \
     || defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG)         \
     || defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID))                \
    && !defined(_GNU_SOURCE)
/* Cannot test `LINUX`, since this must be defined before other includes. */
#  define _GNU_SOURCE 1
#endif

#if defined(__INTERIX) && !defined(_ALL_SOURCE)
#  define _ALL_SOURCE 1
#endif

#if (defined(DGUX) && defined(GC_THREADS) || defined(DGUX386_THREADS) \
     || defined(GC_DGUX386_THREADS))                                  \
    && !defined(_USING_POSIX4A_DRAFT10)
#  define _USING_POSIX4A_DRAFT10 1
#endif

#if defined(__MINGW32__) && !defined(__MINGW_EXCPT_DEFINE_PSDK) \
    && defined(__i386__)                                        \
    && defined(GC_EXTERN) /*< defined in `extra/gc.c` file */
/* See the description in `mark.c` file. */
#  define __MINGW_EXCPT_DEFINE_PSDK 1
#endif

#if defined(NO_DEBUGGING) && !defined(GC_ASSERTIONS) && !defined(NDEBUG)
/* To turn off assertion checking (in `atomic_ops.h` file). */
#  define NDEBUG 1
#endif

#ifndef GC_H
#  include "gc/gc.h"
#endif

#include <stdlib.h>
#if !defined(sony_news)
#  include <stddef.h>
#endif

#ifdef DGUX
#  include <sys/resource.h>
#  include <sys/time.h>
#endif

#ifdef BSD_TIME
#  include <sys/resource.h>
#  include <sys/time.h>
#endif

#ifdef PARALLEL_MARK
#  define AO_REQUIRE_CAS
#  if !defined(__GNUC__) && !defined(AO_ASSUME_WINDOWS98)
#    define AO_ASSUME_WINDOWS98
#  endif
#endif

#include "gc/gc_mark.h"
#include "gc/gc_tiny_fl.h"

typedef GC_word word;

#ifndef PTR_T_DEFINED
/*
 * A generic pointer to which we can add byte displacements and which
 * can be used for address comparisons.
 */
typedef char *ptr_t;
#  define PTR_T_DEFINED
#endif

#ifndef SIZE_MAX
#  include <limits.h>
#endif
#if defined(SIZE_MAX) && !defined(CPPCHECK)
/*
 * A constant representing maximum value for `size_t` type.  Note: an extra
 * cast is used to workaround some buggy `SIZE_MAX` definitions.
 */
#  define GC_SIZE_MAX ((size_t)SIZE_MAX)
#else
#  define GC_SIZE_MAX (~(size_t)0)
#endif

#if (GC_GNUC_PREREQ(3, 0) || defined(__clang__)) && !defined(LINT2)
/* Equivalent to `e`, but predict that usually `e` is true (false). */
#  define LIKELY(e) __builtin_expect(e, 1 /* `TRUE` */)
#  define UNLIKELY(e) __builtin_expect(e, 0 /* `FALSE` */)
#else
#  define LIKELY(e) (e)
#  define UNLIKELY(e) (e)
#endif /* __GNUC__ */

/*
 * Saturated addition of `size_t` values.  Used to avoid value wrap around
 * on overflow.  The arguments should have no side effects.
 */
#define SIZET_SAT_ADD(a, b) \
  (LIKELY((a) < GC_SIZE_MAX - (b)) ? (a) + (b) : GC_SIZE_MAX)

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 2000-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009-2025 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * This header is private to the collector.  It is almost always
 * included from `gc_priv.h` file.  However it is possible to include
 * it by itself if just the configuration macros are needed.  In that
 * case, a few declarations relying on types declared in `gc_priv.h`
 * file will be omitted.
 */

#ifndef GCCONFIG_H
#define GCCONFIG_H

#ifndef GC_H
#  ifdef HAVE_CONFIG_H
#  endif


#endif

#ifdef CPPCHECK
#  undef CLOCKS_PER_SEC
#  undef FIXUP_POINTER
#  undef POINTER_MASK
#  undef POINTER_SHIFT
#  undef REDIRECT_REALLOC
#  undef _MAX_PATH
#endif

#ifndef PTR_T_DEFINED
typedef char *ptr_t;
#  define PTR_T_DEFINED
#endif

#if !defined(sony_news)
#  include <stddef.h> /*< for `size_t`, etc. */
#endif

/*
 * Note: only wrap our own declarations, and not the included headers;
 * in this case, wrap our entire file, but temporarily unwrap/rewrap
 * around `#include` entities; macros do not need such wrapping.
 */
#ifdef __cplusplus
#  define EXTERN_C_BEGIN extern "C" {
#  define EXTERN_C_END } /* extern "C" */
#else
#  define EXTERN_C_BEGIN
#  define EXTERN_C_END
#endif

EXTERN_C_BEGIN

/* Convenient internal macro to test version of Clang. */
#if defined(__clang__) && defined(__clang_major__)
#  define GC_CLANG_PREREQ(major, minor) \
    ((__clang_major__ << 8) + __clang_minor__ >= ((major) << 8) + (minor))
#  define GC_CLANG_PREREQ_FULL(major, minor, patchlevel)          \
    (GC_CLANG_PREREQ(major, (minor) + 1)                          \
     || (__clang_major__ == (major) && __clang_minor__ == (minor) \
         && __clang_patchlevel__ >= (patchlevel)))
#else
#  define GC_CLANG_PREREQ(major, minor) 0 /*< `FALSE` */
#  define GC_CLANG_PREREQ_FULL(major, minor, patchlevel) 0
#endif

/*
 * Machine-dependent parameters.  Some tuning parameters can be found
 * near the top of `gc_priv.h` file.
 */

/* Machine-specific parts contributed by various people.  See `README` file. */

#if defined(__ANDROID__) && !defined(HOST_ANDROID)
/* A Linux-based OS.  `__ANDROID__` macro is defined by Android NDK gcc. */
#  define HOST_ANDROID 1
#endif

#if defined(TIZEN) && !defined(HOST_TIZEN)
/* A Linux-based OS. */
#  define HOST_TIZEN 1
#endif

#if defined(__SYMBIAN32__) && !defined(SYMBIAN)
#  define SYMBIAN
#  ifdef __WINS__
#    pragma data_seg(".data2")
#  endif
#endif

/* First a unified test for Linux. */
#if (defined(linux) || defined(__linux__) || defined(HOST_ANDROID) \
     || defined(HOST_TIZEN))                                       \
    && !defined(LINUX) && !defined(__native_client__)
#  define LINUX
#endif

/* And one for NetBSD. */
#if defined(__NetBSD__)
#  define NETBSD
#endif

/* And one for OpenBSD. */
#if defined(__OpenBSD__)
#  define OPENBSD
#endif

/* And one for FreeBSD. */
#if (defined(__FreeBSD__) || defined(__DragonFly__) \
     || defined(__FreeBSD_kernel__))                \
    && !defined(FREEBSD)                            \
    && !defined(GC_NO_FREEBSD) /*< Orbis compiler defines `__FreeBSD__` */
#  define FREEBSD
#endif

#if defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD)
#  define ANY_BSD
#endif

#if defined(__COSMOPOLITAN__)
#  define COSMO
#endif

#if defined(__EMBOX__)
#  define EMBOX
#endif

#if defined(__KOS__)
#  define KOS
#endif

#if defined(__QNX__) && !defined(QNX)
#  define QNX
#endif

#if defined(__serenity__)
#  define SERENITY
#endif

/* And one for Darwin. */
#if defined(macosx) || (defined(__APPLE__) && defined(__MACH__))
#  define DARWIN
EXTERN_C_END
#  include <TargetConditionals.h>
EXTERN_C_BEGIN
#endif

/* Determine the machine type. */
#if defined(__native_client__)
#  define NACL
#  if !defined(__portable_native_client__) && !defined(__arm__)
#    define I386
#    define mach_type_known
#  else
/* Here we will rely upon arch-specific defines. */
#  endif
#endif
#if defined(__aarch64__) && !defined(ANY_BSD) && !defined(COSMO) \
    && !defined(DARWIN) && !defined(LINUX) && !defined(KOS)      \
    && !defined(NN_BUILD_TARGET_PLATFORM_NX) && !defined(QNX)    \
    && !defined(SERENITY) && !defined(_WIN32)
#  define AARCH64
#  define NOSYS
#  define mach_type_known
#endif
#if defined(__arm) || defined(__arm__) || defined(__thumb__)
#  define ARM32
#  if defined(NACL) || defined(SYMBIAN)
#    define mach_type_known
#  elif !defined(ANY_BSD) && !defined(DARWIN) && !defined(LINUX)             \
      && !defined(QNX) && !defined(NN_PLATFORM_CTR)                          \
      && !defined(SN_TARGET_PSP2) && !defined(_WIN32) && !defined(__CEGCC__) \
      && !defined(GC_NO_NOSYS)
#    define NOSYS
#    define mach_type_known
#  endif
#endif
#if defined(__riscv) && !defined(ANY_BSD) && !defined(LINUX)
#  define RISCV
#  define NOSYS
#  define mach_type_known
#endif
#if defined(vax) || defined(__vax__)
#  define VAX
#  ifdef ultrix
#    define ULTRIX
#  else
#    define BSD
#  endif
#  define mach_type_known
#endif
#if defined(NETBSD) && defined(__vax__)
#  define VAX
#  define mach_type_known
#endif
#if (defined(mips) || defined(__mips) || defined(_mips)) \
    && !defined(__TANDEM) && !defined(ANY_BSD) && !defined(LINUX)
#  define MIPS
#  if defined(nec_ews) || defined(_nec_ews)
#    define EWS4800
#    define mach_type_known
#  elif defined(ultrix) || defined(__ultrix)
#    define ULTRIX
#    define mach_type_known
#  elif !defined(_WIN32_WCE) && !defined(__CEGCC__) && !defined(__MINGW32CE__)
#    define IRIX5 /*< or Irix 6.x */
#    define mach_type_known
#  endif /* !MSWINCE */
#endif
#if defined(DGUX) && (defined(i386) || defined(__i386__))
#  define I386
#  ifndef _USING_DGUX
#    define _USING_DGUX
#  endif
#  define mach_type_known
#endif
#if defined(sequent) && (defined(i386) || defined(__i386__))
#  define I386
#  define SEQUENT
#  define mach_type_known
#endif
#if (defined(sun) || defined(__sun)) && (defined(i386) || defined(__i386__))
#  define I386
#  define SOLARIS
#  define mach_type_known
#endif
#if (defined(sun) || defined(__sun)) && defined(__amd64)
#  define X86_64
#  define SOLARIS
#  define mach_type_known
#endif
#if (defined(__OS2__) || defined(__EMX__)) && defined(__32BIT__)
#  define I386
#  define OS2
#  define mach_type_known
#endif
#if (defined(sun) || defined(__sun)) && (defined(sparc) || defined(__sparc))
/* SunOS 5.x */
EXTERN_C_END
#  include <errno.h>
EXTERN_C_BEGIN
#  define SPARC
#  define SOLARIS
#  define mach_type_known
#elif defined(sparc) && defined(unix) && !defined(sun) && !defined(linux) \
    && !defined(ANY_BSD)
#  define SPARC
#  define DRSNX
#  define mach_type_known
#endif
#if defined(_IBMR2) /* `&& defined(_AIX)` */
#  define POWERPC
#  define AIX
#  define mach_type_known
#endif
#if defined(_M_XENIX) && defined(_M_SYSV) && defined(_M_I386)
/* TODO: The above test may need refinement. */
#  define I386
#  if defined(_SCO_ELF)
#    define SCO_ELF
#  else
#    define SCO
#  endif
#  define mach_type_known
#endif
#if defined(_PA_RISC1_0) || defined(_PA_RISC1_1) || defined(_PA_RISC2_0) \
    || defined(hppa) || defined(__hppa__)
#  define HP_PA
#  if !defined(LINUX) && !defined(HPUX) && !defined(OPENBSD)
#    define HPUX
#  endif
#  define mach_type_known
#endif
#if defined(__ia64) && (defined(_HPUX_SOURCE) || defined(__HP_aCC))
#  define IA64
#  ifndef HPUX
#    define HPUX
#  endif
#  define mach_type_known
#endif
#if (defined(__BEOS__) || defined(__HAIKU__)) && defined(_X86_)
#  define I386
#  define HAIKU
#  define mach_type_known
#endif
#if defined(__HAIKU__) && (defined(__amd64__) || defined(__x86_64__))
#  define X86_64
#  define HAIKU
#  define mach_type_known
#endif
#if defined(__alpha) || defined(__alpha__)
#  define ALPHA
#  if !defined(ANY_BSD) && !defined(LINUX)
#    define OSF1 /*< Compaq Tru64 UNIX (Digital UNIX) */
#  endif
#  define mach_type_known
#endif
#if defined(__rtems__) && (defined(i386) || defined(__i386__))
#  define I386
#  define RTEMS
#  define mach_type_known
#endif
#if defined(NeXT) && defined(mc68000)
#  define M68K
#  define NEXT
#  define mach_type_known
#endif
#if defined(NeXT) && (defined(i386) || defined(__i386__))
#  define I386
#  define NEXT
#  define mach_type_known
#endif
#if defined(bsdi) && (defined(i386) || defined(__i386__))
#  define I386
#  define BSDI
#  define mach_type_known
#endif
#if defined(__386BSD__) && !defined(mach_type_known)
#  define I386
#  define THREE86BSD
#  define mach_type_known
#endif
#if defined(_CX_UX) && defined(_M88K)
#  define M88K
#  define CX_UX
#  define mach_type_known
#endif
#if defined(DGUX) && defined(m88k)
#  define M88K
/* `DGUX` macro is already defined. */
#  define mach_type_known
#endif
#if defined(_WIN32_WCE) || defined(__CEGCC__) || defined(__MINGW32CE__)
/*
 * `SH3`, `SH4`, `MIPS` macros are already defined for the corresponding
 * architectures.
 */
#  if defined(SH3) || defined(SH4)
#    define SH
#  endif
#  if defined(x86) || defined(__i386__)
#    define I386
#  endif
#  if defined(_M_ARM) || defined(ARM) || defined(_ARM_)
#    define ARM32
#  endif
#  define MSWINCE
#  define mach_type_known
#else
#  if ((defined(_MSDOS) || defined(_MSC_VER)) && (_M_IX86 >= 300))          \
      || (defined(_WIN32) && !defined(__CYGWIN32__) && !defined(__CYGWIN__) \
          && !defined(__INTERIX) && !defined(SYMBIAN))                      \
      || defined(__MINGW32__)
#    if defined(__LP64__) || defined(_M_X64)
#      define X86_64
#    elif defined(_M_ARM)
#      define ARM32
#    elif defined(_M_ARM64)
#      define AARCH64
#    else /* _M_IX86 */
#      define I386
#    endif
#    ifdef _XBOX_ONE
#      define MSWIN_XBOX1
#    else
#      ifndef MSWIN32
#        define MSWIN32 /*< or Win64 */
#      endif
#      if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP)
#        define MSWINRT_FLAVOR
#      endif
#    endif
#    define mach_type_known
#  endif
#  if defined(_MSC_VER) && defined(_M_IA64)
#    define IA64
/*
 * Really Win64, but we do not treat 64-bit variants as a different
 * platform.
 */
#    define MSWIN32
#  endif
#endif /* !_WIN32_WCE && !__CEGCC__ && !__MINGW32CE__ */
#if defined(__DJGPP__)
#  define I386
#  ifndef DJGPP
/* MSDOS running the DJGPP port of gcc. */
#    define DJGPP
#  endif
#  define mach_type_known
#endif
#if defined(__CYGWIN32__) || defined(__CYGWIN__)
#  if defined(__LP64__)
#    define X86_64
#  else
#    define I386
#  endif
#  define CYGWIN32
#  define mach_type_known
#endif /* __CYGWIN__ */
#if defined(__INTERIX)
#  define I386
#  define INTERIX
#  define mach_type_known
#endif
#if defined(_UTS) && !defined(mach_type_known)
#  define S370
#  define UTS4
#  define mach_type_known
#endif
#if defined(__embedded__) && defined(PPC)
#  define POWERPC
#  define NOSYS
#  define mach_type_known
#endif
#if defined(__WATCOMC__) && defined(__386__)
#  define I386
#  if !defined(OS2) && !defined(MSWIN32) && !defined(DOS4GW)
#    if defined(__OS2__)
#      define OS2
#    elif defined(__WINDOWS_386__) || defined(__NT__)
#      define MSWIN32
#    else
#      define DOS4GW
#    endif
#  endif
#  define mach_type_known
#endif /* __WATCOMC__ && __386__ */
#if defined(__GNU__) && defined(__i386__)
/* The Debian Hurd running on generic PC. */
#  define HURD
#  define I386
#  define mach_type_known
#endif
#if defined(__GNU__) && defined(__x86_64__)
#  define HURD
#  define X86_64
#  define mach_type_known
#endif
#if defined(__TANDEM)
/* Nonstop S-series. */
/* FIXME: Should recognize Integrity series? */
#  define MIPS
#  define NONSTOP
#  define mach_type_known
#endif
#if defined(__tile__) && defined(LINUX)
#  ifdef __tilegx__
#    define TILEGX
#  else
#    define TILEPRO
#  endif
#  define mach_type_known
#endif
#if defined(NN_BUILD_TARGET_PLATFORM_NX)
#  define AARCH64
#  define NINTENDO_SWITCH
#  define mach_type_known
#endif
#if defined(__EMSCRIPTEN__) || defined(EMSCRIPTEN)
#  define WEBASSEMBLY
#  ifndef EMSCRIPTEN
#    define EMSCRIPTEN
#  endif
#  define mach_type_known
#endif
#if defined(__wasi__)
/* The WebAssembly System Interface (WASI). */
#  define WEBASSEMBLY
#  define WASI
#  define mach_type_known
#endif

#if defined(__aarch64__)                                      \
    && (defined(ANY_BSD) || defined(COSMO) || defined(DARWIN) \
        || defined(LINUX) || defined(KOS) || defined(QNX)     \
        || defined(SERENITY))
#  define AARCH64
#  define mach_type_known
#elif defined(__arc__) && defined(LINUX)
#  define ARC
#  define mach_type_known
#elif (defined(__arm) || defined(__arm__) || defined(__arm32__)               \
       || defined(__ARM__))                                                   \
    && (defined(ANY_BSD) || defined(DARWIN) || defined(LINUX) || defined(QNX) \
        || defined(NN_PLATFORM_CTR) || defined(SN_TARGET_PSP2))
#  define ARM32
#  define mach_type_known
#elif defined(__avr32__) && defined(LINUX)
#  define AVR32
#  define mach_type_known
#elif defined(__cris__) && defined(LINUX)
#  ifndef CRIS
#    define CRIS
#  endif
#  define mach_type_known
#elif defined(__e2k__) && defined(LINUX)
#  define E2K
#  define mach_type_known
#elif defined(__hexagon__) && defined(LINUX)
#  define HEXAGON
#  define mach_type_known
#elif (defined(__i386__) || defined(i386) || defined(__X86__)) \
    && (defined(ANY_BSD) || defined(DARWIN) || defined(EMBOX)  \
        || defined(LINUX) || defined(QNX) || defined(SERENITY))
#  define I386
#  define mach_type_known
#elif (defined(__ia64) || defined(__ia64__)) && defined(LINUX)
#  define IA64
#  define mach_type_known
#elif defined(__loongarch__) && defined(LINUX)
#  define LOONGARCH
#  define mach_type_known
#elif defined(__m32r__) && defined(LINUX)
#  define M32R
#  define mach_type_known
#elif ((defined(__m68k__) || defined(m68k))      \
       && (defined(NETBSD) || defined(OPENBSD))) \
    || (defined(__mc68000__) && defined(LINUX))
#  define M68K
#  define mach_type_known
#elif (defined(__mips) || defined(_mips) || defined(mips)) \
    && (defined(ANY_BSD) || defined(LINUX))
#  define MIPS
#  define mach_type_known
#elif (defined(__NIOS2__) || defined(__NIOS2) || defined(__nios2__)) \
    && defined(LINUX)
#  define NIOS2 /*< Altera NIOS2 */
#  define mach_type_known
#elif defined(__or1k__) && defined(LINUX)
#  define OR1K /*< OpenRISC (or1k) */
#  define mach_type_known
#elif (defined(__powerpc__) || defined(__powerpc64__) || defined(__ppc__) \
       || defined(__ppc64__) || defined(powerpc) || defined(powerpc64))   \
    && (defined(ANY_BSD) || defined(DARWIN) || defined(LINUX))
#  define POWERPC
#  define mach_type_known
#elif defined(__riscv) && (defined(ANY_BSD) || defined(LINUX))
#  define RISCV
#  define mach_type_known
#elif defined(__s390__) && defined(LINUX)
#  define S390
#  define mach_type_known
#elif defined(__sh__) \
    && (defined(LINUX) || defined(NETBSD) || defined(OPENBSD))
#  define SH
#  define mach_type_known
#elif (defined(__sparc) || defined(sparc)) \
    && (defined(ANY_BSD) || defined(LINUX))
#  define SPARC
#  define mach_type_known
#elif defined(__sw_64__) && defined(LINUX)
#  define SW_64
#  define mach_type_known
#elif (defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) \
       || defined(__X86_64__))                                        \
    && (defined(ANY_BSD) || defined(COSMO) || defined(DARWIN)         \
        || defined(LINUX) || defined(QNX) || defined(SERENITY))
#  define X86_64
#  define mach_type_known
#endif

/*
 * Feel free to add more clauses here.  Or manually define the machine
 * type here.  A machine type is characterized by the architecture.
 * Some machine types are further subdivided by OS.  Macros such as
 * `LINUX`, `FREEBSD`, etc. distinguish them.  The distinction in these
 * cases is usually the stack starting address.
 */

#if !defined(mach_type_known) && !defined(CPPCHECK)
#  error The collector has not been ported to this machine/OS combination
#endif

/*
 * The CPU architecture mapping is:
 *   - `AARCH64`: ARM AArch64 ILP32/64-bit (running `COSMO` environment,
 *     `DARWIN` (OS X or iOS), `KOS`, `LINUX`, `MSWIN32`, `NETBSD`,
 *     `NINTENDO_SWITCH`, `NOSYS` environment, `OPENBSD`, `QNX`, `SERENITY`);
 *   - `ALPHA`: DEC Alpha (running `FREEBSD`, `LINUX`, `NETBSD`, `OPENBSD`,
 *     `OSF1`);
 *   - `ARC`: Synopsys ARC (running `LINUX`);
 *   - `ARM32`: ARMv7 (running `DARWIN` (iOS), `FREEBSD`, `LINUX`, `MSWIN32`,
 *     `MSWINCE`, `NETBSD`, `NN_PLATFORM_CTR`, `NOSYS` environment,
 *     `OPENBSD`, `QNX`, `SN_TARGET_PSP2`, `SYMBIAN`);
 *   - `AVR32`: Atmel RISC (running `LINUX`);
 *   - `CRIS`: Axis Etrax (running `LINUX`);
 *   - `E2K`: Elbrus 2000 32/64-bit (running `LINUX`);
 *   - `HEXAGON`: Qualcomm Hexagon (running `LINUX`);
 *   - `HP_PA`: HP9000/700 and HP9000/800 32/64-bit (running `HPUX`, `LINUX`,
 *     `OPENBSD`);
 *   - `I386`: Intel 486/586/686 (running `BSDI`, `CYGWIN32` environment,
 *     `DARWIN` (macOS), `DGUX`, `DJGPP` environment, `DOS4GW` environment,
 *     `EMBOX`, `FREEBSD`, `HAIKU`, `HURD`, `INTERIX`, `LINUX`, `MSWIN32`,
 *     `MSWINCE`, `NACL` environment, `NETBSD`, `NEXT`, `OPENBSD`, `OS2`,
 *     `QNX`, `RTEMS`, `SCO`, `SCO_ELF`, `SEQUENT`, `SERENITY`, `SOLARIS`,
 *     `THREE86BSD`);
 *   - `IA64`: Intel IPF, e.g. Itanium 32/64-bit (running `HPUX`, `LINUX`,
 *     `MSWIN32`);
 *   - `LOONGARCH`: Loongson LoongArch 32/64-bit (running `LINUX`);
 *   - `M32R`: Renesas M32R (running `LINUX`);
 *   - `M68K`: Motorola 680x0 (running `LINUX`, `NETBSD`, `NEXT`, `OPENBSD`);
 *   - `M88K`: Motorola 88xx0 (running `CX_UX`, `DGUX`);
 *   - `MIPS`: R2000+ 32/64-bit (running `EWS4800`, `FREEBSD`, `IRIX5`,
 *     `LINUX`, `MSWINCE`, `NETBSD`, `NONSTOP`, `OPENBSD`, `ULTRIX`);
 *   - `NIOS2`: Altera NIOS2 (running `LINUX`);
 *   - `OR1K`: OpenRISC/or1k (running `LINUX`);
 *   - `POWERPC`: IBM/Apple PowerPC 32/64-bit (running `AIX`, `DARWIN`,
 *     `FREEBSD`, `LINUX`, `NETBSD`, `NOSYS` environment, `OPENBSD`,
 *     `SN_TARGET_PS3`);
 *   - `RISCV`: RISC-V 32/64-bit (running `FREEBSD`, `LINUX`, `NETBSD`,
 *     `NOSYS` environment, `OPENBSD`);
 *   - `S370`: A 370-like machine (running `UTS4`);
 *   - `S390`: A 390-like machine 32/64-bit (running `LINUX`);
 *   - `SH`: Hitachi SuperH (running `LINUX`, `MSWINCE`, `NETBSD`,
 *     `OPENBSD`);
 *   - `SH4`: Hitachi SH4 (running `MSWINCE`);
 *   - `SPARC`: SPARC v7/v8/v9 32/64-bit (running `DRSNX`, `FREEBSD`,
 *     `LINUX`, `NETBSD`, `OPENBSD`, `SOLARIS`);
 *   - `SW_64`: Sunway/Shenwei (running `LINUX`);
 *   - `TILEGX`: Tilera TILE-Gx 32/64-bit (running `LINUX`);
 *   - `TILEPRO`: Tilera TILEPro (running `LINUX`);
 *   - `VAX`: DEC VAX (running `BSD`, `ULTRIX`);
 *   - `WEBASSEMBLY`: WebAssembly/Wasm (running `EMSCRIPTEN` environment,
 *     `WASI` environment);
 *   - `X86_64`: AMD x86-64 ILP32/64-bit (running `COSMO` environment,
 *     `CYGWIN32` environment, `DARWIN` (macOS), `FREEBSD`, `HAIKU`,
 *     `HURD`, `LINUX`, `MSWIN32`, `MSWIN_XBOX1`, `NETBSD`, `OPENBSD`,
 *     `PLATFORM_GETMEM` environment, `QNX`, `SERENITY`, `SOLARIS`).
 */

/*
 * For each architecture and OS, the following need to be defined:
 *
 * `CPP_WORDSZ` is a simple integer constant representing the word size
 * in bits.  We assume byte addressability, where a byte has 8 bits.
 * We also assume `CPP_WORDSZ` is either 32 or 64.
 * (We care about the length of a pointer address, not hardware
 * bus widths.  Thus a 64-bit processor with a C compiler that uses
 * 32-bit pointers should use `CPP_WORDSZ` of 32, not 64.)
 *
 * `CPP_PTRSZ` is the pointer size in bits.  For most of the supported
 * targets, it is equal to `CPP_WORDSZ`.
 *
 * `MACH_TYPE` is a string representation of the machine type.
 * `OS_TYPE` is analogous for the OS.
 *
 * `ALIGNMENT` is the largest `n`, such that all pointer are guaranteed
 * to be aligned on `n`-byte boundary.  Defining it to be 1 will always
 * work, but will perform poorly.  Should not be larger than size of
 * a pointer.
 *
 * `DATASTART` is the beginning of the data segment.  On some platforms
 * `SEARCH_FOR_DATA_START` is defined.  The latter will cause
 * `GC_data_start` to be set to an address determined by accessing data
 * backwards from `_end` until an unmapped page is found.  `DATASTART` will
 * be defined to be `GC_data_start`.   On UNIX-like systems, the collector
 * will scan the area between `DATASTART` and `DATAEND` for root pointers.
 *
 * `DATAEND`, if not `end`, where `end` is defined as `extern int end[]`.
 * RTH suggests gaining access to linker script synth'd values with
 * this idiom instead of `&end`, where `end` is defined as `extern int end`.
 * Otherwise, "gcc will assume these are in .sdata/.sbss" and it will, e.g.,
 * cause failures on `alpha*-*-*` with `-msmall-data` or `-fpic` or
 * `mips-*-*` without any special options.
 *
 * `STACKBOTTOM` is the cold end of the stack, which is usually the
 * highest address in the stack.
 * Under OS/2, we have other ways of finding thread stacks.
 * For each machine, the following should:
 *   1. Define `STACK_GROWS_UP` if the stack grows toward higher addresses;
 *   2. Define exactly one of
 *      - `STACKBOTTOM` (should be defined to be an expression),
 *      - `HEURISTIC1`,
 *      - `SPECIFIC_MAIN_STACKBOTTOM`,
 *      - `HEURISTIC2`.
 *
 * If `STACKBOTTOM` is defined, then its value will be used directly (as
 * the stack bottom).  If `SPECIFIC_MAIN_STACKBOTTOM` is defined, then it
 * will be determined with a specific method appropriate for the operating
 * system.  Currently we look first for `__libc_stack_end` (currently only
 * if `USE_LIBC_PRIVATES` is defined), and if that fails, read it from
 * `/proc` pseudo-file.  (If `USE_LIBC_PRIVATES` is not defined and
 * `NO_PROC_STAT` is defined, we revert to `HEURISTIC2`.)
 * If either of the last two macros are defined, then `STACKBOTTOM` is
 * computed during collector startup using one of the following two
 * heuristics:
 *   - `HEURISTIC1`: Take an address inside `GC_init`'s frame, and round it
 *     up to the next multiple of `STACK_GRAN`;
 *   - `HEURISTIC2`: Take an address inside `GC_init`'s frame, increment it
 *     repeatedly in small steps (decrement if `STACK_GROWS_UP`), and read
 *     the value at each location, remember the value when the first
 *     Segmentation violation or Bus error is signaled, round that to the
 *     nearest plausible page boundary, and use that instead of
 *     `STACKBOTTOM`.
 *
 * Gustavo Rodriguez-Rivera points out that on most (all?) UNIX machines,
 * the value of `environ` is a pointer that can serve as `STACKBOTTOM`.
 * I expect that `HEURISTIC2` can be replaced by this approach, which
 * interferes far less with debugging.  However it has the disadvantage that
 * it is confused by a `putenv()` call before the collector is initialized.
 * This could be dealt with by intercepting `putenv()`...
 *
 * If no expression for `STACKBOTTOM` can be found, and neither of the above
 * heuristics are usable, the collector can still be used with all of the
 * above undefined, provided one of the following is done:
 *   1. `GC_mark_roots` can be changed to somehow mark from the correct
 *      stack(s) without reference to `STACKBOTTOM`.  This is appropriate for
 *      use in conjunction with thread packages, since there will be multiple
 *      stacks.  (Allocating thread stacks in the heap, and treating them as
 *      ordinary heap data objects is also possible as a last resort.
 *      However, this is likely to introduce significant amounts of excess
 *      storage retention unless the dead parts of the thread stacks are
 *      periodically cleared.)
 *   2. Client code may set `GC_stackbottom` before calling any `GC_`
 *      routines.  If the author of the client code owns the main program,
 *      this could be accomplished by introducing a new `main` function,
 *      calling `GC_call_with_gc_active()` which sets `GC_stackbottom` and
 *      then calls the original (real) `main` function.
 *
 * Each architecture may also define the style of virtual dirty bit
 * implementation to be used:
 *   - `GWW_VDB`: use Win32 `GetWriteWatch` primitive;
 *   - `MPROTECT_VDB`: write-protect the heap and catch faults;
 *   - `PROC_VDB`: use the SVR4 `/proc` primitives to read dirty bits;
 *   - `SOFT_VDB`: use the Linux `/proc` primitives to track dirty bits.
 *
 * The first and second one may be combined, in which case a runtime
 * selection will be made, based on `GetWriteWatch` availability.
 *
 * An architecture may define `DYNAMIC_LOADING` if `dyn_load.c` file
 * implements `GC_register_dynamic_libraries()` for the architecture.
 *
 * An architecture may define `PREFETCH(x)` to preload the cache with `*x`.
 * This defaults to gcc built-in operation (or a no-op for other compilers).
 *
 * `GC_PREFETCH_FOR_WRITE(x)` is used if `*x` is about to be written.
 *
 * An architecture may also define `CLEAR_DOUBLE(x)` to be a fast way to
 * clear 2 pointers at `GC_malloc`-aligned address `x`.  The default
 * implementation is just to store two `NULL` pointers.
 *
 * `HEAP_START` may be defined as the initial address hint for `mmap`-based
 * allocation.
 */

#ifdef LINUX
/* TODO: FreeBSD too? */
EXTERN_C_END
#  include <features.h> /*< for `__GLIBC__` and `__GLIBC_MINOR__`, at least */
EXTERN_C_BEGIN
#endif

/* Convenient internal macro to test `glibc` version (if compiled against). */
#if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
#  define GC_GLIBC_PREREQ(major, minor) \
    ((__GLIBC__ << 8) + __GLIBC_MINOR__ >= ((major) << 8) + (minor))
#else
#  define GC_GLIBC_PREREQ(major, minor) 0 /*< `FALSE` */
#endif

/*
 * Align a `ptr_t` pointer down/up to a given boundary.  The latter should
 * be a power of two.
 */
#if GC_CLANG_PREREQ(11, 0)
#  define PTR_ALIGN_DOWN(p, b) __builtin_align_down(p, b)
#  if defined(DARWIN) && defined(GC_SINGLE_OBJ_BUILD) && GC_CLANG_PREREQ(17, 0)
/* Workaround a crash in Apple clang-17. */
/* TODO: Disable for later clang versions when the bug is fixed. */
#    define PTR_ALIGN_UP(p, b) \
      ((ptr_t)__builtin_align_up((GC_uintptr_t)(p), b))
#  else
#    define PTR_ALIGN_UP(p, b) __builtin_align_up(p, b)
#  endif
#else
#  define PTR_ALIGN_DOWN(p, b) \
    ((ptr_t)((GC_uintptr_t)(p) & ~((GC_uintptr_t)(b) - (GC_uintptr_t)1)))
#  define PTR_ALIGN_UP(p, b)                                           \
    ((ptr_t)(((GC_uintptr_t)(p) + (GC_uintptr_t)(b) - (GC_uintptr_t)1) \
             & ~((GC_uintptr_t)(b) - (GC_uintptr_t)1)))
#endif

/*
 * If available, we can use `__builtin_unwind_init()` to push the relevant
 * registers onto the stack.
 */
#if GC_GNUC_PREREQ(2, 8)                                                   \
    && !GC_GNUC_PREREQ(11, 0) /*< broken at least in 11.2.0 on cygwin64 */ \
    && !defined(__INTEL_COMPILER) && !defined(__PATHCC__)                  \
    && !defined(__FUJITSU)                    /*< for FX10 system */       \
    && !(defined(POWERPC) && defined(DARWIN)) /*< for MacOS X 10.3.9 */    \
    && !defined(E2K) && !defined(RTEMS)                                    \
    && !defined(__ARMCC_VERSION) /*< does not exist in armcc gnu emu */    \
    && !(defined(__clang__)                                                \
         && defined(__ARM_ARCH_5TE__) /* clang-19 emits `vpush`/`vpop` */) \
    && (!defined(__clang__)                                                \
        || GC_CLANG_PREREQ(8, 0) /* was no-op in clang-3 at least */)
#  define HAVE_BUILTIN_UNWIND_INIT
#endif

#if (defined(__CC_ARM) || defined(CX_UX) || defined(DJGPP) || defined(EMBOX) \
     || defined(EWS4800) || defined(LINUX) || defined(OS2) || defined(RTEMS) \
     || defined(UTS4) || defined(MSWIN32) || defined(MSWINCE)                \
     || (defined(NOSYS) && defined(RISCV)))                                  \
    && !defined(NO_UNDERSCORE_SETJMP)
#  define NO_UNDERSCORE_SETJMP
#endif

/*
 * The common OS-specific definitions.  Should be applicable to all
 * (or most, at least) supported architectures.
 */

#ifdef CYGWIN32
#  define OS_TYPE "CYGWIN32"
#  define RETRY_GET_THREAD_CONTEXT
#  ifdef USE_WINALLOC
#    define GWW_VDB
#  elif defined(USE_MMAP)
#    define USE_MMAP_ANON
#  endif
#endif /* CYGWIN32 */

#ifdef COSMO
#  define OS_TYPE "COSMO"
#  ifndef USE_GET_STACKBASE_FOR_MAIN
#    define USE_GET_STACKBASE_FOR_MAIN
#  endif
extern int __data_start[] __attribute__((__weak__));
#  define DATASTART ((ptr_t)__data_start)
extern int _end[];
#  define DATAEND ((ptr_t)_end)
#  define USE_MMAP_ANON
#  ifndef HAVE_CLOCK_GETTIME
#    define HAVE_CLOCK_GETTIME 1
#  endif
#  ifndef HAVE_PTHREAD_SETNAME_NP_WITH_TID
/* Normally should be defined by `configure`, etc. */
#    define HAVE_PTHREAD_SETNAME_NP_WITH_TID 1
#  endif
#  if !defined(GC_THREADS) || defined(NO_HANDLE_FORK) \
      || defined(GC_NO_CAN_CALL_ATFORK)
#    define MPROTECT_VDB
/* FIXME: Otherwise `gctest` crashes in child process. */
#  endif
/* FIXME: A deadlock occurs in markers, thus disabled for now. */
#  undef PARALLEL_MARK
#endif /* COSMO */

#ifdef DARWIN
#  define OS_TYPE "DARWIN"
#  define DYNAMIC_LOADING
/*
 * TODO: See `get_end(3)`, `get_etext` and `get_end` should not be used.
 * These are not used when `dyld` support is enabled (it is the default).
 */
#  define DATASTART ((ptr_t)get_etext())
#  define DATAEND ((ptr_t)get_end())
#  define USE_MMAP_ANON
/* There seems to be some issues with try-lock hanging on Darwin. */
/* TODO: This should be looked into some more. */
#  define NO_PTHREAD_TRYLOCK
#  ifndef TARGET_OS_XR
#    define TARGET_OS_XR 0
#  endif
#  ifndef TARGET_OS_VISION
#    define TARGET_OS_VISION 0
#  endif
#endif /* DARWIN */

#ifdef EMBOX
#  define OS_TYPE "EMBOX"
extern int _modules_data_start[], _apps_bss_end[];
#  define DATASTART ((ptr_t)_modules_data_start)
#  define DATAEND ((ptr_t)_apps_bss_end)
/*
 * Note: the designated area might be quite large (several dozens of MBs)
 * as it includes `.data` and `.bss` of all apps and modules of the built
 * binary image.
 */
#endif /* EMBOX */

#ifdef FREEBSD
#  define OS_TYPE "FREEBSD"
#  define SPECIFIC_MAIN_STACKBOTTOM
#  ifdef __ELF__
#    define DYNAMIC_LOADING
#  endif
#  ifndef USE_MMAP
/* `sbrk()` is not available. */
#    define USE_MMAP 1
#  endif
#  if !defined(ALPHA) && !defined(SPARC)
extern char etext[];
#    define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)etext)
#    define DATASTART_USES_XGETDATASTART
#    ifndef REDIRECT_MALLOC
#      define MPROTECT_VDB
#    else
/* Similar as on Linux, `fread()` might use `malloc()`. */
#    endif
#  endif
#endif /* FREEBSD */

#ifdef HAIKU
#  define OS_TYPE "HAIKU"
#  define DYNAMIC_LOADING
/* Note: `DATASTART` is not used really, see `GC_register_main_static_data`. */
extern int etext[];
#  define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x1000)
#  ifndef USE_GET_STACKBASE_FOR_MAIN
#    define USE_GET_STACKBASE_FOR_MAIN
#  endif
#  define USE_MMAP_ANON
/*
 * On Haiku R1, at least, `pthreads` locks never spin but always call
 * into the kernel if the lock cannot be acquired with a simple atomic
 * operation.  (Up to 5x overall performance improvement of the
 * collector is observed by forcing use of spin locks.)
 */
#  ifndef USE_SPIN_LOCK
#    define USE_SPIN_LOCK
#  endif
/*
 * TODO: `MPROTECT_VDB` is not working correctly on anything other than
 * recent nightly Haiku OS builds (as of Nov 2024), and also it is
 * considerably slower than regular collecting, so do not enable it for now.
 */
EXTERN_C_END
#  include <OS.h>
EXTERN_C_BEGIN
#  define GETPAGESIZE() (unsigned)B_PAGE_SIZE
#  ifndef HAVE_CLOCK_GETTIME
#    define HAVE_CLOCK_GETTIME 1
#  endif
#endif /* HAIKU */

#ifdef HPUX
#  define OS_TYPE "HPUX"
extern int __data_start[];
#  define DATASTART ((ptr_t)__data_start)
#  ifdef USE_MMAP
#    define USE_MMAP_ANON
#  endif
#  define DYNAMIC_LOADING
#  define GETPAGESIZE() (unsigned)sysconf(_SC_PAGE_SIZE)
#endif /* HPUX */

#ifdef HURD
#  define OS_TYPE "HURD"
#  define HEURISTIC2
#  define SEARCH_FOR_DATA_START
extern int _end[];
#  define DATAEND ((ptr_t)_end)
/* TODO: `MPROTECT_VDB` is not quite working yet? */
#  define DYNAMIC_LOADING
#  define USE_MMAP_ANON
#endif /* HURD */

#ifdef LINUX
#  define OS_TYPE "LINUX"
#  if defined(FORCE_MPROTECT_BEFORE_MADVISE) || defined(PREFER_MMAP_PROT_NONE)
#    define COUNT_UNMAPPED_REGIONS
#  endif
#  define RETRY_TKILL_ON_EAGAIN
#  if !defined(MIPS) && !defined(POWERPC)
#    define SPECIFIC_MAIN_STACKBOTTOM
#  endif
#  if defined(__ELF__) && !defined(IA64)
#    define DYNAMIC_LOADING
#  endif
#  if defined(__ELF__) && !defined(ARC) && !defined(RISCV) && !defined(S390) \
      && !defined(TILEGX) && !defined(TILEPRO)
extern int _end[];
#    define DATAEND ((ptr_t)_end)
#  endif
#  if !defined(REDIRECT_MALLOC) && !defined(E2K)
/* Requires Linux 2.3.47 or later. */
#    define MPROTECT_VDB
#  else
/*
 * We seem to get random errors in the incremental mode, possibly because
 * the Linux threads implementation itself is a `malloc` client and cannot
 * deal with the signals.  `fread()` uses `malloc()` too.
 * In case of e2k, unless `-fsemi-spec-ld` (or `-O0`) option is passed
 * to gcc (both when compiling the collector library and the client),
 * a semi-speculative optimization may lead to `SIGILL` (with `ILL_ILLOPN`
 * `si_code`) instead of `SIGSEGV`.
 */
#  endif
#endif /* LINUX */

#ifdef KOS
#  define OS_TYPE "KOS"
#  ifndef USE_GET_STACKBASE_FOR_MAIN
/* Note: this requires `-lpthread` option. */
#    define USE_GET_STACKBASE_FOR_MAIN
#  endif
extern int __data_start[];
#  define DATASTART ((ptr_t)__data_start)
#endif /* KOS */

#ifdef MSWIN32
#  define OS_TYPE "MSWIN32"
/* `STACKBOTTOM` and `DATASTART` are handled specially in `os_dep.c` file. */
#  if !defined(CPPCHECK)
#    define DATAEND /*< not needed */
#  endif
#  if defined(USE_GLOBAL_ALLOC) && !defined(MSWINRT_FLAVOR)
/* Cannot pass `MEM_WRITE_WATCH` to `GlobalAlloc()`. */
#  else
#    define GWW_VDB
#  endif
#endif

#ifdef MSWINCE
#  define OS_TYPE "MSWINCE"
#  if !defined(CPPCHECK)
#    define DATAEND /*< not needed */
#  endif
#endif

#ifdef NACL
#  define OS_TYPE "NACL"
#  if defined(__GLIBC__)
#    define DYNAMIC_LOADING
#  endif
#  define DATASTART MAKE_CPTR(0x10020000)
extern int _end[];
#  define DATAEND ((ptr_t)_end)
#  define STACK_GRAN 0x10000
#  define HEURISTIC1
#  define NO_PTHREAD_GETATTR_NP
#  define USE_MMAP_ANON
/* FIXME: Not real page size. */
#  define GETPAGESIZE() 65536
#  define MAX_NACL_GC_THREADS 1024
#endif /* NACL */

#ifdef NETBSD
#  define OS_TYPE "NETBSD"
#  define HEURISTIC2
#  ifdef __ELF__
#    define SEARCH_FOR_DATA_START
#    define DYNAMIC_LOADING
#  elif !defined(MIPS)
/* TODO: Probably do not exclude it. */
extern char etext[];
#    define DATASTART ((ptr_t)etext)
#  endif
#  define MPROTECT_VDB
#endif /* NETBSD */

#ifdef NEXT
#  define OS_TYPE "NEXT"
#  define DATASTART ((ptr_t)get_etext())
#  define DATAEND /*< not needed */
#  undef USE_MUNMAP
#endif

#ifdef OPENBSD
#  define OS_TYPE "OPENBSD"
#  ifndef GC_THREADS
#    define HEURISTIC2
#  endif
#  ifdef __ELF__
extern int __data_start[], _end[];
#    define DATASTART ((ptr_t)__data_start)
#    define DATAEND ((ptr_t)_end)
#    define DYNAMIC_LOADING
#  else
extern char etext[];
#    define DATASTART ((ptr_t)etext)
#  endif
#  define MPROTECT_VDB
#endif /* OPENBSD */

#ifdef QNX
#  define OS_TYPE "QNX"
#  define SA_RESTART 0
#  ifndef SPECIFIC_MAIN_STACKBOTTOM
/* TODO: This is not used by default. */
#    define STACK_GRAN 0x1000000
#    define HEURISTIC1
#  endif
extern char etext[];
#  define DATASTART ((ptr_t)etext)
extern int _end[];
#  define DATAEND ((ptr_t)_end)
#endif /* QNX */

#ifdef SERENITY
#  define OS_TYPE "SERENITY"
extern int etext[], _end[];
#  define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x1000)
#  define DATAEND ((ptr_t)_end)
#  define DYNAMIC_LOADING
/* TODO: Enable `mprotect`-based VDB. */
#  define USE_MMAP_ANON
#endif /* SERENITY */

#ifdef SOLARIS
#  define OS_TYPE "SOLARIS"
extern int _end[];
#  define DATAEND ((ptr_t)_end)
#  if !defined(USE_MMAP) && defined(REDIRECT_MALLOC)
#    define USE_MMAP 1
/*
 * Otherwise we now use `calloc()`.  `mmap()` may result in the heap
 * interleaved with thread stacks, which can result in excessive
 * black-listing.  `sbrk()` is unusable since it does not interact
 * correctly with the system `malloc()`.
 */
#  endif
#  ifdef USE_MMAP
#    define HEAP_START ((word)0x40000000)
#  else
#    define HEAP_START ADDR(DATAEND)
#  endif
#  ifndef GC_THREADS
#    define MPROTECT_VDB
#  endif
#  define DYNAMIC_LOADING
/*
 * Define `STACKBOTTOM` as `(ptr_t)_start` worked through 2.7, but
 * reportedly breaks under 2.8.  It appears that the stack base is
 * a property of the executable, so this should not break old executables.
 * `HEURISTIC1` reportedly no longer works under Solaris 2.7.
 * `HEURISTIC2` probably works, but this appears to be preferable.
 * Apparently `USRSTACK` is defined to be `USERLIMIT`, but in some
 * installations that is undefined.  We work around this with a gross hack.
 */
EXTERN_C_END
#  include <sys/vmparam.h>
EXTERN_C_BEGIN
#  ifdef USERLIMIT
/* This should work everywhere, but does not. */
#    define STACKBOTTOM ((ptr_t)USRSTACK)
#  else
#    define HEURISTIC2
#  endif
#endif /* SOLARIS */

#ifdef SYMBIAN
#  define OS_TYPE "SYMBIAN"
#  define DATASTART ((ptr_t)ALIGNMENT) /*< cannot be `NULL` */
#  define DATAEND ((ptr_t)ALIGNMENT)
#  ifndef USE_MMAP
/* `sbrk()` is not available. */
#    define USE_MMAP 1
#  endif
#endif /* SYMBIAN */

/*
 * Below are the definitions specific to each supported architecture
 * and OS, grouped by the former.
 */

#ifdef M68K
#  define MACH_TYPE "M68K"
#  define CPP_WORDSZ 32
#  define ALIGNMENT 2 /*< not 4 */
#  ifdef OPENBSD
/* Nothing specific. */
#  endif
#  ifdef NETBSD
/* Nothing specific. */
#  endif
#  ifdef LINUX
#    ifdef __ELF__
#      if GC_GLIBC_PREREQ(2, 0)
#        define SEARCH_FOR_DATA_START
#      else
/*
 * Hideous kludge: `__environ` is the first word in platform `crt0.o` file,
 * and delimits the start of the data segment, no matter which `ld` options
 * were passed through.  We could use `_etext` instead, but that would
 * include `.rodata`, which may contain large read-only data tables that
 * we would rather not scan.
 */
extern char **__environ;
#        define DATASTART ((ptr_t)(&__environ))
#      endif
#    else
extern int etext[];
#      define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x1000)
#    endif
#  endif
#  ifdef NEXT
#    define STACKBOTTOM MAKE_CPTR(0x4000000)
#  endif
#endif

#ifdef POWERPC
#  define MACH_TYPE "POWERPC"
#  ifdef LINUX
#    if defined(__powerpc64__)
#      define CPP_WORDSZ 64
#      ifndef HBLKSIZE
#        define HBLKSIZE 4096
#      endif
#    else
#      define CPP_WORDSZ 32
#    endif
/*
 * `HEURISTIC1` has been reliably reported to fail for a 32-bit executable
 * on a 64-bit kernel.
 */
#    if defined(__bg__)
/*
 * The Linux Compute Node Kernel (used on BlueGene systems) does not
 * support the `SPECIFIC_MAIN_STACKBOTTOM` way.
 */
#      define HEURISTIC2
#      define NO_PTHREAD_GETATTR_NP
#    else
#      define SPECIFIC_MAIN_STACKBOTTOM
#    endif
#    define SEARCH_FOR_DATA_START
#    ifndef SOFT_VDB
#      define SOFT_VDB
#    endif
#  endif
#  ifdef DARWIN
#    if defined(__ppc64__)
#      define CPP_WORDSZ 64
#      define STACKBOTTOM MAKE_CPTR(0x7fff5fc00000)
#      define CACHE_LINE_SIZE 64
#      ifndef HBLKSIZE
#        define HBLKSIZE 4096
#      endif
#    else
#      define CPP_WORDSZ 32
#      define STACKBOTTOM MAKE_CPTR(0xc0000000)
#    endif
#    define MPROTECT_VDB
#    if defined(USE_PPC_PREFETCH) && defined(__GNUC__)
/* The performance impact of prefetches is untested. */
#      define PREFETCH(x) \
        __asm__ __volatile__("dcbt 0,%0" : : "r"((const void *)(x)))
#      define GC_PREFETCH_FOR_WRITE(x) \
        __asm__ __volatile__("dcbtst 0,%0" : : "r"((const void *)(x)))
#    endif
#  endif
#  ifdef OPENBSD
#    if defined(__powerpc64__)
#      define CPP_WORDSZ 64
#    else
#      define CPP_WORDSZ 32
#    endif
#  endif
#  ifdef FREEBSD
#    if defined(__powerpc64__)
#      define CPP_WORDSZ 64
#      ifndef HBLKSIZE
#        define HBLKSIZE 4096
#      endif
#    else
#      define CPP_WORDSZ 32
#    endif
#  endif
#  ifdef NETBSD
#    define CPP_WORDSZ 32
#  endif
#  ifdef SN_TARGET_PS3
#    define OS_TYPE "SN_TARGET_PS3"
#    define CPP_WORDSZ 32
#    define NO_GETENV
extern int _end[], __bss_start;
#    define DATASTART ((ptr_t)__bss_start)
#    define DATAEND ((ptr_t)_end)
#    define STACKBOTTOM ((ptr_t)ps3_get_stack_bottom())
void *ps3_get_mem(size_t lb);
#    define GET_MEM(lb) ps3_get_mem(lb)
/*
 * The current `LOCK()` implementation for PS3 explicitly uses
 * `pthread_mutex_lock()` for some reason.
 */
#    define NO_PTHREAD_TRYLOCK
#  endif
#  ifdef AIX
#    define OS_TYPE "AIX"
#    undef ALIGNMENT /*< in case it is defined */
#    undef IA64
/*
 * DOB: some AIX installs stupidly define `IA64` in platform
 * `sys/systemcfg.h` file.
 */
#    ifdef __64BIT__
#      define CPP_WORDSZ 64
#      define STACKBOTTOM MAKE_CPTR(0x1000000000000000)
#    else
#      define CPP_WORDSZ 32
extern int errno;
#      define STACKBOTTOM ((ptr_t)(&errno))
#    endif
#    define USE_MMAP_ANON
/*
 * From AIX linker man page:
 *   - `_text` specifies the first location of the program;
 *   - `_etext` specifies the first location after the program;
 *   - `_data` specifies the first location of the data;
 *   - `_edata` specifies the first location after the initialized data;
 *   - `_end` (or `end`) specifies the first location after all data.
 */
extern int _data[], _end[];
#    define DATASTART ((ptr_t)_data)
#    define DATAEND ((ptr_t)_end)
#    define MPROTECT_VDB
#    define DYNAMIC_LOADING
/*
 * Note: for really old versions of AIX, `DYNAMIC_LOADING` may have to
 * be removed.
 */
#  endif
#  ifdef NOSYS
#    define OS_TYPE "NOSYS"
#    define CPP_WORDSZ 32
extern void __end[], __dso_handle[];
#    define DATASTART ((ptr_t)__dso_handle) /*< OK, that is ugly */
#    define DATAEND ((ptr_t)__end)
/* Note: stack starts at 0xE0000000 for the simulator. */
#    define STACKBOTTOM PTR_ALIGN_UP(GC_approx_sp(), 0x10000000)
#  endif
#endif /* POWERPC */

#ifdef VAX
#  define MACH_TYPE "VAX"
#  define CPP_WORDSZ 32
/* Pointers are `longword`-aligned by C compiler v4.2. */
extern char etext[];
#  define DATASTART ((ptr_t)etext)
#  ifdef BSD
#    define OS_TYPE "BSD"
#    define STACK_GRAN 0x1000000
#    define HEURISTIC1
/* Note: `HEURISTIC2` may be OK, but it is hard to test. */
#  endif
#  ifdef ULTRIX
#    define OS_TYPE "ULTRIX"
#    define STACKBOTTOM MAKE_CPTR(0x7fffc800)
#  endif
#endif /* VAX */

#ifdef SPARC
#  define MACH_TYPE "SPARC"
#  if defined(__arch64__) || defined(__sparcv9)
#    define CPP_WORDSZ 64
#    define ELF_CLASS ELFCLASS64
#  else
#    define CPP_WORDSZ 32
#    define ALIGNMENT 4 /*< required by hardware */
#  endif
#  ifdef SOLARIS
extern int _etext[];
#    define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
#    define PROC_VDB
/*
 * `getpagesize()` appeared to be missing from at least one Solaris 5.4
 * installation.  Weird.
 */
#    define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE)
#  endif
#  ifdef DRSNX
#    define OS_TYPE "DRSNX"
extern int etext[];
#    define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)etext)
#    define MPROTECT_VDB
#    define STACKBOTTOM MAKE_CPTR(0xdfff0000)
#    define DYNAMIC_LOADING
#  endif
#  ifdef LINUX
extern int _etext[];
#    ifdef __arch64__
#      define DATASTART GC_SysVGetDataStart(0x100000, (ptr_t)_etext)
#    else
#      define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
#    endif
#  endif
#  ifdef OPENBSD
/* Nothing specific. */
#  endif
#  ifdef NETBSD
/* Nothing specific. */
#  endif
#  ifdef FREEBSD
extern char etext[];
#    define DATASTART ((ptr_t)(&etext))
#    define DATAEND ((ptr_t)GC_find_limit(DATASTART, TRUE))
#    define DATAEND_IS_FUNC
#    define GC_HAVE_DATAREGION2
extern char edata[], end[];
#    define DATASTART2 ((ptr_t)(&edata))
#    define DATAEND2 ((ptr_t)(&end))
#  endif
#endif /* SPARC */

#ifdef I386
#  define MACH_TYPE "I386"
#  if (defined(__LP64__) || defined(_WIN64)) && !defined(CPPCHECK)
#    error This should be handled as X86_64
#  endif
#  define CPP_WORDSZ 32
/*
 * The 4-byte alignment appears to hold for all 32-bit compilers
 * except Borland and Watcom.  If using the Borland (bcc32) or
 * Watcom (wcc386) compiler, `-a4` or `-zp4` option, respectively,
 * should be passed to the compiler, both for building the library
 * and client code.  (The alternate solution is to define
 * `FORCE_ALIGNMENT_ONE` macro but this would have significant
 * negative performance implications.)
 */
#  if defined(FORCE_ALIGNMENT_ONE) \
      && (defined(__BORLANDC__) || defined(__WATCOMC__))
#    define ALIGNMENT 1
#  endif
#  ifdef SEQUENT
#    define OS_TYPE "SEQUENT"
extern int etext[];
#    define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x1000)
#    define STACKBOTTOM MAKE_CPTR(0x3ffff000)
#  endif
#  ifdef HAIKU
/* Nothing specific. */
#  endif
#  ifdef HURD
/* Nothing specific. */
#  endif
#  ifdef EMBOX
/* Nothing specific. */
#  endif
#  ifdef NACL
/* Nothing specific. */
#  endif
#  ifdef QNX
/* Nothing specific. */
#  endif
#  ifdef SERENITY
/* Nothing specific. */
#  endif
#  ifdef SOLARIS
extern int _etext[];
#    define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)_etext)
#    define PROC_VDB
#  endif
#  ifdef SCO
#    define OS_TYPE "SCO"
extern int etext[];
#    define DATASTART \
      (PTR_ALIGN_UP((ptr_t)etext, 0x400000) + (ADDR(etext) & 0xfff))
#    define STACKBOTTOM MAKE_CPTR(0x7ffffffc)
#  endif
#  ifdef SCO_ELF
#    define OS_TYPE "SCO_ELF"
extern int etext[];
#    define DATASTART ((ptr_t)etext)
#    define STACKBOTTOM MAKE_CPTR(0x8048000)
#    define DYNAMIC_LOADING
#    define ELF_CLASS ELFCLASS32
#  endif
#  ifdef DGUX
#    define OS_TYPE "DGUX"
extern int _etext, _end;
#    define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)(&_etext))
#    define DATASTART_USES_XGETDATASTART
#    define DATAEND ((ptr_t)(&_end))
#    define HEURISTIC2
#    define DYNAMIC_LOADING
#    ifndef USE_MMAP
#      define USE_MMAP 1
#    endif
#    define MAP_FAILED ((void *)(~(GC_uintptr_t)0))
#    define HEAP_START ((word)0x40000000)
#  endif /* DGUX */
#  ifdef LINUX
/*
 * This encourages `mmap()` to give us low addresses, thus allowing the
 * heap to grow to ~3 GB.
 */
#    define HEAP_START ((word)0x1000)
#    ifdef __ELF__
#      if GC_GLIBC_PREREQ(2, 0) || defined(HOST_ANDROID)
#        define SEARCH_FOR_DATA_START
#      else
/* See the comment of the Linux/m68k case. */
extern char **__environ;
#        define DATASTART ((ptr_t)(&__environ))
#      endif
#      if !defined(GC_NO_SIGSETJMP)                                  \
          && (defined(HOST_TIZEN)                                    \
              || (defined(HOST_ANDROID)                              \
                  && !(GC_GNUC_PREREQ(4, 8) || GC_CLANG_PREREQ(3, 2) \
                       || __ANDROID_API__ >= 18)))
/*
 * Older Android NDK releases lack `sigsetjmp` in x86 `libc` (`setjmp` is
 * used instead to find `data_start`).  The bug is fixed in Android NDK r8e
 * (so, it is OK to use `sigsetjmp` if gcc-4.8+, clang-3.2+ or Android
 * API level 18+).
 */
#        define GC_NO_SIGSETJMP 1
#      endif
#    else
extern int etext[];
#      define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x1000)
#    endif
#    ifdef USE_I686_PREFETCH
/*
 * Empirically `prefetcht0` is much more effective at reducing cache miss
 * stalls for the targeted load instructions.  But it seems to interfere
 * enough with other cache traffic that the net result is worse than
 * `prefetchnta`.
 */
#      define PREFETCH(x) \
        __asm__ __volatile__("prefetchnta %0" : : "m"(*(char *)(x)))
#      ifdef FORCE_WRITE_PREFETCH
/*
 * Using prefetches for write seems to have a slight negative impact on
 * performance, at least for a PIII/500.
 */
#        define GC_PREFETCH_FOR_WRITE(x) \
          __asm__ __volatile__("prefetcht0 %0" : : "m"(*(char *)(x)))
#      else
#        define GC_NO_PREFETCH_FOR_WRITE
#      endif
#    elif defined(USE_3DNOW_PREFETCH)
#      define PREFETCH(x) \
        __asm__ __volatile__("prefetch %0" : : "m"(*(char *)(x)))
#      define GC_PREFETCH_FOR_WRITE(x) \
        __asm__ __volatile__("prefetchw %0" : : "m"(*(char *)(x)))
#    endif
#    if defined(__GLIBC__) && !defined(__UCLIBC__) \
        && !defined(GLIBC_TSX_BUG_FIXED)
/* Workaround lock elision implementation for some `glibc`. */
#      define GLIBC_2_19_TSX_BUG
EXTERN_C_END
#      include <gnu/libc-version.h> /*< for `gnu_get_libc_version()` */
EXTERN_C_BEGIN
#    endif
#    ifndef SOFT_VDB
#      define SOFT_VDB
#    endif
#  endif
#  ifdef CYGWIN32
#    define WOW64_THREAD_CONTEXT_WORKAROUND
#    define DATASTART ((ptr_t)GC_DATASTART) /*< defined in `gc.h` file */
#    define DATAEND ((ptr_t)GC_DATAEND)
#    ifndef USE_WINALLOC
/* `MPROTECT_VDB` does not work, it leads to a spurious exit. */
#    endif
#  endif
#  ifdef INTERIX
#    define OS_TYPE "INTERIX"
extern int _data_start__[], _bss_end__[];
#    define DATASTART ((ptr_t)_data_start__)
#    define DATAEND ((ptr_t)_bss_end__)
#    define STACKBOTTOM                                        \
      ({                                                       \
        ptr_t rv;                                              \
        __asm__ __volatile__("movl %%fs:4, %%eax" : "=a"(rv)); \
        rv;                                                    \
      })
#    define USE_MMAP_ANON
#  endif
#  ifdef OS2
#    define OS_TYPE "OS2"
/*
 * `STACKBOTTOM` and `DATASTART` are handled specially in `os_dep.c`
 * file.  OS/2 actually has the right system call!
 */
#    define DATAEND /*< not needed */
#    undef USE_MUNMAP
#    define GETPAGESIZE() os2_getpagesize()
#  endif
#  ifdef MSWIN32
#    define WOW64_THREAD_CONTEXT_WORKAROUND
#    define RETRY_GET_THREAD_CONTEXT
#    if defined(__BORLANDC__)
/*
 * TODO: VDB based on `VirtualProtect` and `SetUnhandledExceptionFilter`
 * does not work correctly.
 */
#    else
#      define MPROTECT_VDB
#    endif
#  endif
#  ifdef MSWINCE
/* Nothing specific. */
#  endif
#  ifdef DJGPP
#    define OS_TYPE "DJGPP"
EXTERN_C_END
#    include "stubinfo.h"
EXTERN_C_BEGIN
extern int etext[];
#    define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x200)
extern int __djgpp_stack_limit, _stklen;
#    define STACKBOTTOM (MAKE_CPTR(__djgpp_stack_limit) + _stklen)
#  endif
#  ifdef OPENBSD
/* Nothing specific. */
#  endif
#  ifdef FREEBSD
#    if defined(__GLIBC__)
extern int _end[];
#      define DATAEND ((ptr_t)_end)
#    endif
#  endif
#  ifdef NETBSD
/* Nothing specific. */
#  endif
#  ifdef THREE86BSD
#    define OS_TYPE "THREE86BSD"
#    define HEURISTIC2
extern char etext[];
#    define DATASTART ((ptr_t)etext)
#  endif
#  ifdef BSDI
#    define OS_TYPE "BSDI"
#    define HEURISTIC2
extern char etext[];
#    define DATASTART ((ptr_t)etext)
#  endif
#  ifdef NEXT
#    define STACKBOTTOM MAKE_CPTR(0xc0000000)
#  endif
#  ifdef RTEMS
#    define OS_TYPE "RTEMS"
EXTERN_C_END
#    include <sys/unistd.h>
EXTERN_C_BEGIN
extern int etext[];
#    define DATASTART ((ptr_t)etext)
void *rtems_get_stack_bottom(void);
#    define InitStackBottom rtems_get_stack_bottom()
#    define STACKBOTTOM ((ptr_t)InitStackBottom)
#    undef USE_MUNMAP
#  endif
#  ifdef DOS4GW
#    define OS_TYPE "DOS4GW"
extern long __nullarea;
extern char _end;
extern char *_STACKTOP;
/*
 * Depending on calling conventions Watcom C either precedes or does not
 * precede the names of the C variables with an underscore.
 * Make sure the startup code variables always have the same name.
 */
#    pragma aux __nullarea "*";
#    pragma aux _end "*";
#    define STACKBOTTOM ((ptr_t)_STACKTOP) /*< confused? me too */
#    define DATASTART ((ptr_t)(&__nullarea))
#    define DATAEND ((ptr_t)(&_end))
#    undef USE_MUNMAP
#    define GETPAGESIZE() 4096
#  endif
#  ifdef DARWIN
#    define DARWIN_DONT_PARSE_STACK 1
#    define STACKBOTTOM MAKE_CPTR(0xc0000000)
#    define MPROTECT_VDB
#  endif
#endif /* I386 */

#ifdef LOONGARCH
#  define MACH_TYPE "LOONGARCH"
#  define CPP_WORDSZ (__SIZEOF_SIZE_T__ * 8)
#  ifdef LINUX
#    pragma weak __data_start
extern int __data_start[];
#    define DATASTART ((ptr_t)__data_start)
#  endif
#endif /* LOONGARCH */

#ifdef SW_64
#  define MACH_TYPE "SW_64"
#  define CPP_WORDSZ 64
#  ifdef LINUX
/* Nothing specific. */
#  endif
#endif /* SW_64 */

#ifdef MIPS
#  define MACH_TYPE "MIPS"
#  ifdef LINUX
#    ifdef _MIPS_SZPTR
#      define CPP_WORDSZ _MIPS_SZPTR
#    else
#      define CPP_WORDSZ 32
#    endif
#    pragma weak __data_start
extern int __data_start[];
#    define DATASTART ((ptr_t)__data_start)
#    ifndef HBLKSIZE
#      define HBLKSIZE 4096
#    endif
#    if GC_GLIBC_PREREQ(2, 2)
#      define SPECIFIC_MAIN_STACKBOTTOM
#    else
#      define STACKBOTTOM MAKE_CPTR(0x7fff8000)
#    endif
#  endif
#  ifdef EWS4800
#    define OS_TYPE "EWS4800"
#    define HEURISTIC2
#    if defined(_MIPS_SZPTR) && (_MIPS_SZPTR == 64)
#      define CPP_WORDSZ _MIPS_SZPTR
extern int _fdata[], _end[];
#      define DATASTART ((ptr_t)_fdata)
#      define DATAEND ((ptr_t)_end)
#    else
#      define CPP_WORDSZ 32
extern int etext[], edata[];
#      define DATASTART \
        (PTR_ALIGN_UP((ptr_t)etext, 0x40000) + (ADDR(etext) & 0xffff))
#      define DATAEND ((ptr_t)edata)
#      define GC_HAVE_DATAREGION2
extern int _DYNAMIC_LINKING[], _gp[];
#      define DATASTART2                                               \
        (_DYNAMIC_LINKING ? PTR_ALIGN_UP((ptr_t)_gp + 0x8000, 0x40000) \
                          : (ptr_t)edata)
extern int end[];
#      define DATAEND2 ((ptr_t)end)
#    endif
#  endif
#  ifdef ULTRIX
#    define OS_TYPE "ULTRIX"
#    define CPP_WORDSZ 32
#    define HEURISTIC2
/*
 * Note: the actual beginning of the data segment could probably be
 * slightly higher since startup code allocates lots of stuff.
 */
#    define DATASTART MAKE_CPTR(0x10000000)
#  endif
#  ifdef IRIX5
#    define OS_TYPE "IRIX5"
#    ifdef _MIPS_SZPTR
#      define CPP_WORDSZ _MIPS_SZPTR
#    else
#      define CPP_WORDSZ 32
#    endif
#    define HEURISTIC2
extern int _fdata[];
#    define DATASTART ((ptr_t)_fdata)
/*
 * Lowest plausible heap address.  In the `USE_MMAP` case, we map there.
 * In either case it is used to identify heap sections so they are not
 * considered as roots.
 */
#    ifdef USE_MMAP
#      define HEAP_START ((word)0x30000000)
#    else
#      define HEAP_START ADDR(DATASTART)
#    endif
/* `MPROTECT_VDB` should work, but there is evidence of a breakage. */
#    define DYNAMIC_LOADING
#  endif
#  ifdef MSWINCE
#    define CPP_WORDSZ 32
#  endif
#  ifdef NETBSD
#    define CPP_WORDSZ 32
#    ifndef __ELF__
#      define DATASTART MAKE_CPTR(0x10000000)
#      define STACKBOTTOM MAKE_CPTR(0x7ffff000)
#    endif
#  endif
#  ifdef OPENBSD
#    define CPP_WORDSZ 64 /*< all OpenBSD/mips platforms are 64-bit */
#  endif
#  ifdef FREEBSD
#    define CPP_WORDSZ 32
#  endif
#  ifdef NONSTOP
#    define OS_TYPE "NONSTOP"
#    define CPP_WORDSZ 32
#    define DATASTART MAKE_CPTR(0x8000000)
extern char **environ;
#    define DATAEND ((ptr_t)(environ - 0x10))
#    define STACKBOTTOM MAKE_CPTR(0x4fffffff)
#    undef USE_MUNMAP
#  endif
#endif /* MIPS */

#ifdef NIOS2
#  define MACH_TYPE "NIOS2"
#  define CPP_WORDSZ 32
#  ifndef HBLKSIZE
#    define HBLKSIZE 4096
#  endif
#  ifdef LINUX
extern int __data_start[];
#    define DATASTART ((ptr_t)__data_start)
#  endif
#endif /* NIOS2 */

#ifdef OR1K
#  define MACH_TYPE "OR1K"
#  define CPP_WORDSZ 32
#  ifndef HBLKSIZE
#    define HBLKSIZE 4096
#  endif
#  ifdef LINUX
extern int __data_start[];
#    define DATASTART ((ptr_t)__data_start)
#  endif
#endif /* OR1K */

#ifdef HP_PA
#  define MACH_TYPE "HP_PA"
#  ifdef __LP64__
#    define CPP_WORDSZ 64
#  else
#    define CPP_WORDSZ 32
#  endif
#  define STACK_GROWS_UP
#  ifdef HPUX
#    ifndef GC_THREADS
#      define MPROTECT_VDB
#    endif
#    ifdef USE_HPUX_FIXED_STACKBOTTOM
/*
 * The following appears to work for 7xx systems running HP/UX 9.xx.
 * Furthermore, it might result in much faster collections than `HEURISTIC2`,
 * which may involve scanning segments that directly precede the stack.
 * It is not the default, since it may not work on older machine/OS
 * combinations. (Thanks to Raymond X.T. Nijssen for uncovering this.)
 * This technique also does not work with HP/UX 11.xx.  The stack size is
 * settable using the kernel `maxssiz` variable, and the size can be set
 * dynamically in HP/UX 11.23 and later.  It also does not handle
 * `SHMEM_MAGIC` binaries that have stack and data in the first quadrant.
 * This is from platform `/etc/conf/h/param.h` file.
 */
#      define STACKBOTTOM MAKE_CPTR(0x7b033000)
#    elif defined(USE_ENVIRON_POINTER)
/*
 * Gustavo Rodriguez-Rivera suggested changing `HEURISTIC2` to this.
 * Note that the collector must be initialized before the first `putenv()`
 * call.  Unfortunately, some clients do not obey.
 */
extern char **environ;
#      define STACKBOTTOM ((ptr_t)environ)
#    elif !defined(HEURISTIC2)
/* This uses `pst_vm_status` support. */
#      define SPECIFIC_MAIN_STACKBOTTOM
#    endif
#    ifndef __GNUC__
#      define PREFETCH(x)                   \
        do {                                \
          register long addr = (long)(x);   \
          (void)_asm("LDW", 0, 0, addr, 0); \
        } while (0)
#    endif
#  endif /* HPUX */
#  ifdef LINUX
#    define SEARCH_FOR_DATA_START
#  endif
#  ifdef OPENBSD
/* Nothing specific. */
#  endif
#endif /* HP_PA */

#ifdef ALPHA
#  define MACH_TYPE "ALPHA"
#  define CPP_WORDSZ 64
#  ifdef NETBSD
#    define ELFCLASS32 32
#    define ELFCLASS64 64
#    define ELF_CLASS ELFCLASS64
#  endif
#  ifdef OPENBSD
/* Nothing specific. */
#  endif
#  ifdef FREEBSD
extern char etext[];
#    define DATASTART ((ptr_t)(&etext))
#    define DATAEND ((ptr_t)GC_find_limit(DATASTART, TRUE))
#    define DATAEND_IS_FUNC
/*
 * Handle unmapped hole which `alpha*-*-freebsd[45]*` puts between
 * `etext` and `edata`.
 */
#    define GC_HAVE_DATAREGION2
extern char edata[], end[];
#    define DATASTART2 ((ptr_t)(&edata))
#    define DATAEND2 ((ptr_t)(&end))
/* `MPROTECT_VDB` is not yet supported at all on FreeBSD/alpha. */
#  endif
#  ifdef OSF1
#    define OS_TYPE "OSF1"
#    define DATASTART MAKE_CPTR(0x140000000)
extern int _end[];
#    define DATAEND ((ptr_t)(&_end))
extern char **environ;
/*
 * Round up from the value of `environ` to the nearest page boundary.
 * Probably this is broken if `putenv()` is called before the collector
 * initialization.
 */
#    define STACKBOTTOM PTR_ALIGN_UP((ptr_t)environ, getpagesize())
/*
 * Normally `HEURISTIC2` is too conservative, since the text segment
 * immediately follows the stack.  Hence we give an upper bound.
 * This is currently unused, since `HEURISTIC2` is not defined.
 */
extern int __start[];
#    define HEURISTIC2_LIMIT PTR_ALIGN_DOWN((ptr_t)__start, getpagesize())
#    ifndef GC_THREADS
/* FIXME: Unresolved signal issues with threads. */
#      define MPROTECT_VDB
#    endif
#    define DYNAMIC_LOADING
#  endif
#  ifdef LINUX
#    ifdef __ELF__
#      define SEARCH_FOR_DATA_START
#    else
#      define DATASTART MAKE_CPTR(0x140000000)
extern int _end[];
#      define DATAEND ((ptr_t)_end)
#    endif
#  endif
#endif /* ALPHA */

#ifdef IA64
#  define MACH_TYPE "IA64"
#  ifdef HPUX
#    ifdef _ILP32
#      define CPP_WORDSZ 32
/* Note: requires 8-byte alignment (granularity) for `malloc()`. */
#      define ALIGNMENT 4
#    else
#      if !defined(_LP64) && !defined(CPPCHECK)
#        error Unknown ABI
#      endif
#      define CPP_WORDSZ 64
/* Note: requires 16-byte alignment (granularity) for `malloc()`. */
#      define ALIGNMENT 8
#    endif
/*
 * Note that the collector must be initialized before the 1st `putenv`
 * call.
 */
extern char **environ;
#    define STACKBOTTOM ((ptr_t)environ)
/*
 * The following was empirically determined, and is probably not very
 * robust.  Note that the backing store base seems to be at a nice address
 * minus one page.
 */
#    define BACKING_STORE_DISPLACEMENT 0x1000000
#    define BACKING_STORE_ALIGNMENT 0x1000
/* Known to be wrong for recent HP/UX versions!!! */
#  endif
#  ifdef LINUX
#    define CPP_WORDSZ 64
/*
 * The following works on NUE and older kernels:
 * `define STACKBOTTOM MAKE_CPTR(0xa000000000000000l)`.
 */
/* TODO: `SPECIFIC_MAIN_STACKBOTTOM` does not work on NUE. */
/* We also need the base address of the register stack backing store. */
#    define SEARCH_FOR_DATA_START
#    ifdef __GNUC__
#      define DYNAMIC_LOADING
#    else
/*
 * In the Intel compiler environment, we seem to end up with statically
 * linked executables and an undefined reference to `_DYNAMIC`.
 */
#    endif
#    ifdef __GNUC__
#      ifndef __INTEL_COMPILER
#        define PREFETCH(x) __asm__("        lfetch  [%0]" : : "r"(x))
#        define GC_PREFETCH_FOR_WRITE(x) \
          __asm__("        lfetch.excl     [%0]" : : "r"(x))
#        define CLEAR_DOUBLE(x) \
          __asm__("        stf.spill       [%0]=f0" : : "r"((void *)(x)))
#      else
EXTERN_C_END
#        include <ia64intrin.h>
EXTERN_C_BEGIN
#        define PREFETCH(x) __lfetch(__lfhint_none, (x))
#        define GC_PREFETCH_FOR_WRITE(x) __lfetch(__lfhint_nta, (x))
#        define CLEAR_DOUBLE(x) __stf_spill((void *)(x), 0)
#      endif /* __INTEL_COMPILER */
#    endif
#  endif
#  ifdef MSWIN32
/* FIXME: This is a very partial guess.  There is no port, yet. */
#    if defined(_WIN64)
#      define CPP_WORDSZ 64
#    else
/* TODO: Is this possible? */
#      define CPP_WORDSZ 32
#    endif
#  endif
#endif /* IA64 */

#ifdef E2K
#  define MACH_TYPE "E2K"
#  ifdef __LP64__
#    define CPP_WORDSZ 64
#  else
#    define CPP_WORDSZ 32
#  endif
#  ifndef HBLKSIZE
#    define HBLKSIZE 4096
#  endif
#  ifdef LINUX
extern int __dso_handle[];
#    define DATASTART ((ptr_t)__dso_handle)
#    ifdef REDIRECT_MALLOC
#      define NO_PROC_FOR_LIBRARIES
#    endif
#  endif
#endif /* E2K */

#ifdef M88K
#  define MACH_TYPE "M88K"
#  define CPP_WORDSZ 32
#  define STACKBOTTOM MAKE_CPTR(0xf0000000) /*< determined empirically */
extern int etext[];
#  ifdef CX_UX
#    define OS_TYPE "CX_UX"
#    define DATASTART (PTR_ALIGN_UP((ptr_t)etext, 0x400000) + 0x10000)
#  endif
#  ifdef DGUX
#    define OS_TYPE "DGUX"
#    define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)etext)
#    define DATASTART_USES_XGETDATASTART
#  endif
#endif /* M88K */

#ifdef S370
/*
 * If this still works, and if anyone cares, this should probably be moved
 * to the `S390` category.
 */
#  define MACH_TYPE "S370"
#  define CPP_WORDSZ 32
#  define ALIGNMENT 4 /*< required by hardware */
#  ifdef UTS4
#    define OS_TYPE "UTS4"
extern int _etext[], _end[];
#    define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
#    define DATAEND ((ptr_t)_end)
#    define HEURISTIC2
#  endif
#endif /* S370 */

#ifdef S390
#  define MACH_TYPE "S390"
#  ifndef __s390x__
#    define CPP_WORDSZ 32
#  else
#    define CPP_WORDSZ 64
#    ifndef HBLKSIZE
#      define HBLKSIZE 4096
#    endif
#  endif
#  ifdef LINUX
extern int __data_start[] __attribute__((__weak__));
extern int _end[] __attribute__((__weak__));
#    define DATASTART ((ptr_t)__data_start)
#    define DATAEND ((ptr_t)_end)
#    define CACHE_LINE_SIZE 256
#    define GETPAGESIZE() 4096
#    ifndef SOFT_VDB
#      define SOFT_VDB
#    endif
#  endif
#endif /* S390 */

#ifdef AARCH64
#  define MACH_TYPE "AARCH64"
#  ifdef __ILP32__
#    define CPP_WORDSZ 32
#  else
#    define CPP_WORDSZ 64
#  endif
#  ifndef HBLKSIZE
#    define HBLKSIZE 4096
#  endif
#  ifdef LINUX
#    if defined(HOST_ANDROID)
#      define SEARCH_FOR_DATA_START
#    else
extern int __data_start[] __attribute__((__weak__));
#      define DATASTART ((ptr_t)__data_start)
#    endif
#  endif
#  ifdef COSMO
/* Empty. */
#  endif
#  ifdef DARWIN
/* OS X, iOS, visionOS */
#    define DARWIN_DONT_PARSE_STACK 1
#    define STACKBOTTOM MAKE_CPTR(0x16fdfffff)
#    if (TARGET_OS_IPHONE || TARGET_OS_XR || TARGET_OS_VISION)
/*
 * `MPROTECT_VDB` causes use of non-public API like `exc_server`, this
 * could be a reason for blocking the client application in the store.
 */
#    elif TARGET_OS_OSX
#      define MPROTECT_VDB
#    endif
#  endif
#  ifdef FREEBSD
/* Nothing specific. */
#  endif
#  ifdef NETBSD
#    define ELF_CLASS ELFCLASS64
#  endif
#  ifdef OPENBSD
/* Nothing specific. */
#  endif
#  ifdef NINTENDO_SWITCH
#    define OS_TYPE "NINTENDO_SWITCH"
#    define NO_HANDLE_FORK 1
extern int __bss_end[];
#    define DATASTART ((ptr_t)ALIGNMENT) /*< cannot be `NULL` */
#    define DATAEND ((ptr_t)(&__bss_end))
void *switch_get_stack_bottom(void);
#    define STACKBOTTOM ((ptr_t)switch_get_stack_bottom())
void *switch_get_mem(size_t lb);
#    define GET_MEM(lb) switch_get_mem(lb)
#    define GETPAGESIZE() 4096
#    undef USE_MMAP
#    undef USE_MUNMAP
#    ifndef HAVE_CLOCK_GETTIME
#      define HAVE_CLOCK_GETTIME 1
#    endif
#  endif
#  ifdef KOS
/* Nothing specific. */
#  endif
#  ifdef QNX
/* Nothing specific. */
#  endif
#  ifdef SERENITY
/* Nothing specific. */
#  endif
#  ifdef MSWIN32
/* UWP */
/* TODO: Enable `MPROTECT_VDB`. */
#  endif
#  ifdef NOSYS
#    define OS_TYPE "NOSYS"
/* `__data_start` is usually defined in the target linker script. */
extern int __data_start[];
#    define DATASTART ((ptr_t)__data_start)
extern void *__stack_base__;
#    define STACKBOTTOM ((ptr_t)__stack_base__)
#  endif
#endif /* AARCH64 */

#ifdef ARM32
#  define MACH_TYPE "ARM32"
#  define CPP_WORDSZ 32
#  ifdef LINUX
#    if GC_GLIBC_PREREQ(2, 0) || defined(HOST_ANDROID)
#      define SEARCH_FOR_DATA_START
#    else
/* See the comment of the Linux/m68k case. */
extern char **__environ;
#      define DATASTART ((ptr_t)(&__environ))
#    endif
#  endif
#  ifdef MSWINCE
/* Nothing specific. */
#  endif
#  ifdef FREEBSD
/* Nothing specific. */
#  endif
#  ifdef DARWIN
/* iOS */
#    define DARWIN_DONT_PARSE_STACK 1
#    define STACKBOTTOM MAKE_CPTR(0x30000000)
/* `MPROTECT_VDB` causes use of non-public API. */
#  endif
#  ifdef NETBSD
/* Nothing specific. */
#  endif
#  ifdef OPENBSD
/* Nothing specific. */
#  endif
#  ifdef QNX
/* Nothing specific. */
#  endif
#  ifdef SN_TARGET_PSP2
#    define OS_TYPE "SN_TARGET_PSP2"
#    define NO_HANDLE_FORK 1
#    ifndef HBLKSIZE
#      define HBLKSIZE 65536 /*< page size is 64 KB */
#    endif
#    define DATASTART ((ptr_t)ALIGNMENT)
#    define DATAEND ((ptr_t)ALIGNMENT)
void *psp2_get_stack_bottom(void);
#    define STACKBOTTOM ((ptr_t)psp2_get_stack_bottom())
void *psp2_get_mem(size_t lb);
#    define GET_MEM(lb) psp2_get_mem(lb)
#  endif
#  ifdef NN_PLATFORM_CTR
#    define OS_TYPE "NN_PLATFORM_CTR"
extern unsigned char Image$$ZI$$ZI$$Base[];
#    define DATASTART ((ptr_t)Image$$ZI$$ZI$$Base)
extern unsigned char Image$$ZI$$ZI$$Limit[];
#    define DATAEND ((ptr_t)Image$$ZI$$ZI$$Limit)
void *n3ds_get_stack_bottom(void);
#    define STACKBOTTOM ((ptr_t)n3ds_get_stack_bottom())
#  endif
#  ifdef MSWIN32
/* UWP */
/* TODO: Enable `MPROTECT_VDB`. */
#  endif
#  ifdef NOSYS
#    define OS_TYPE "NOSYS"
/* `__data_start` is usually defined in the target linker script. */
extern int __data_start[];
#    define DATASTART ((ptr_t)__data_start)
/* `__stack_base__` is set in platform `newlib/libc/sys/arm/crt0.S` file. */
extern void *__stack_base__;
#    define STACKBOTTOM ((ptr_t)__stack_base__)
#  endif
#  ifdef SYMBIAN
/* Nothing specific. */
#  endif
#endif /* ARM32 */

#ifdef CRIS
#  define MACH_TYPE "CRIS"
#  define CPP_WORDSZ 32
#  define ALIGNMENT 1
#  ifdef LINUX
#    define SEARCH_FOR_DATA_START
#  endif
#endif /* CRIS */

#if defined(SH) && !defined(SH4)
#  define MACH_TYPE "SH"
#  define CPP_WORDSZ 32
#  ifdef LINUX
#    define SEARCH_FOR_DATA_START
#  endif
#  ifdef NETBSD
/* Nothing specific. */
#  endif
#  ifdef OPENBSD
/* Nothing specific. */
#  endif
#  ifdef MSWINCE
/* Nothing specific. */
#  endif
#endif

#ifdef SH4
#  define MACH_TYPE "SH4"
#  define CPP_WORDSZ 32
#  ifdef MSWINCE
/* Nothing specific. */
#  endif
#endif /* SH4 */

#ifdef AVR32
#  define MACH_TYPE "AVR32"
#  define CPP_WORDSZ 32
#  ifdef LINUX
#    define SEARCH_FOR_DATA_START
#  endif
#endif /* AVR32 */

#ifdef M32R
#  define MACH_TYPE "M32R"
#  define CPP_WORDSZ 32
#  ifdef LINUX
#    define SEARCH_FOR_DATA_START
#  endif
#endif /* M32R */

#ifdef X86_64
#  define MACH_TYPE "X86_64"
#  ifdef __ILP32__
#    define CPP_WORDSZ 32
#  else
#    define CPP_WORDSZ 64
#  endif
#  ifndef HBLKSIZE
#    define HBLKSIZE 4096
#  endif
#  ifndef CACHE_LINE_SIZE
#    define CACHE_LINE_SIZE 64
#  endif
#  ifdef PLATFORM_GETMEM
#    define OS_TYPE "PLATFORM_GETMEM"
#    define DATASTART ((ptr_t)ALIGNMENT)
#    define DATAEND ((ptr_t)ALIGNMENT)
EXTERN_C_END
#    include <pthread.h>
EXTERN_C_BEGIN
void *platform_get_stack_bottom(void);
#    define STACKBOTTOM ((ptr_t)platform_get_stack_bottom())
void *platform_get_mem(size_t lb);
#    define GET_MEM(lb) platform_get_mem(lb)
#  endif
#  ifdef LINUX
#    define SEARCH_FOR_DATA_START
#    if defined(__GLIBC__) && !defined(__UCLIBC__)
/*
 * A workaround for GCF (Google Cloud Function) which does not support
 * `mmap()` for `/dev/zero` pseudo-file.  Should not cause any harm to
 * other targets.
 */
#      define USE_MMAP_ANON
#    endif
#    if defined(__GLIBC__) && !defined(__UCLIBC__) \
        && !defined(GETCONTEXT_FPU_BUG_FIXED)
/*
 * At present, there is a bug in `glibc` `getcontext()` on Linux/x86_64
 * (it clears FPU exception mask).  We define this macro to workaround it.
 */
/* TODO: This seems to be fixed in `glibc` 2.14. */
#      define GETCONTEXT_FPU_EXCMASK_BUG
#    endif
#    if defined(__GLIBC__) && !defined(__UCLIBC__) \
        && !defined(GLIBC_TSX_BUG_FIXED)
/* Workaround lock elision implementation for some `glibc`. */
#      define GLIBC_2_19_TSX_BUG
EXTERN_C_END
#      include <gnu/libc-version.h> /*< for `gnu_get_libc_version()` */
EXTERN_C_BEGIN
#    endif
#    ifndef SOFT_VDB
#      define SOFT_VDB
#    endif
#  endif
#  ifdef COSMO
/* Empty. */
#  endif
#  ifdef DARWIN
#    define DARWIN_DONT_PARSE_STACK 1
#    define STACKBOTTOM MAKE_CPTR(0x7fff5fc00000)
#    define MPROTECT_VDB
#  endif
#  ifdef FREEBSD
#    if defined(__GLIBC__)
extern int _end[];
#      define DATAEND ((ptr_t)_end)
#    endif
#    if defined(__DragonFly__)
/*
 * DragonFly BSD still has `vm.max_proc_mmap`, according to its
 * `mmap(2)` man page.
 */
#      define COUNT_UNMAPPED_REGIONS
#    endif
#  endif
#  ifdef NETBSD
/* Nothing specific. */
#  endif
#  ifdef OPENBSD
/* Nothing specific. */
#  endif
#  ifdef HAIKU
/* Nothing specific. */
#  endif
#  ifdef HURD
/* Nothing specific. */
#  endif
#  ifdef QNX
/* Nothing specific. */
#  endif
#  ifdef SERENITY
/* Nothing specific. */
#  endif
#  ifdef SOLARIS
#    define ELF_CLASS ELFCLASS64
extern int _etext[];
#    define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)_etext)
#    define PROC_VDB
#  endif
#  ifdef CYGWIN32
#    ifndef USE_WINALLOC
#      if defined(THREAD_LOCAL_ALLOC)
/*
 * TODO: For an unknown reason, thread-local allocations lead to spurious
 * process exit after the fault handler is once invoked.
 */
#      else
#        define MPROTECT_VDB
#      endif
#    endif
#  endif
#  ifdef MSWIN_XBOX1
#    define OS_TYPE "MSWIN_XBOX1"
#    define NO_GETENV
#    define DATASTART ((ptr_t)ALIGNMENT)
#    define DATAEND ((ptr_t)ALIGNMENT)
LONG64 durango_get_stack_bottom(void);
#    define STACKBOTTOM ((ptr_t)durango_get_stack_bottom())
#    define GETPAGESIZE() 4096
#    ifndef USE_MMAP
#      define USE_MMAP 1
#    endif
/* The following is from platform `sys/mman.h` file. */
#    define PROT_NONE 0
#    define PROT_READ 1
#    define PROT_WRITE 2
#    define PROT_EXEC 4
#    define MAP_PRIVATE 2
#    define MAP_FIXED 0x10
#    define MAP_FAILED ((void *)(~(GC_uintptr_t)0))
#  endif
#  ifdef MSWIN32
#    define RETRY_GET_THREAD_CONTEXT
#    if !defined(__GNUC__) || defined(__INTEL_COMPILER) \
        || (GC_GNUC_PREREQ(4, 7) && !defined(__MINGW64__))
/*
 * Older gcc and Mingw-w64 (both gcc and clang) do not support
 * `SetUnhandledExceptionFilter()` properly on x86_64.
 */
#      define MPROTECT_VDB
#    endif
#  endif
#endif /* X86_64 */

#ifdef ARC
#  define MACH_TYPE "ARC"
#  define CPP_WORDSZ 32
#  define CACHE_LINE_SIZE 64
#  ifdef LINUX
extern int __data_start[] __attribute__((__weak__));
#    define DATASTART ((ptr_t)__data_start)
#  endif
#endif /* ARC */

#ifdef HEXAGON
#  define MACH_TYPE "HEXAGON"
#  define CPP_WORDSZ 32
#  ifdef LINUX
#    if defined(__GLIBC__)
#      define SEARCH_FOR_DATA_START
#    elif !defined(CPPCHECK)
#      error Unknown Hexagon libc configuration
#    endif
#  endif
#endif /* HEXAGON */

#ifdef TILEPRO
#  define MACH_TYPE "TILEPRO"
#  define CPP_WORDSZ 32
#  define PREFETCH(x) __insn_prefetch(x)
#  define CACHE_LINE_SIZE 64
#  ifdef LINUX
extern int __data_start[];
#    define DATASTART ((ptr_t)__data_start)
#  endif
#endif /* TILEPRO */

#ifdef TILEGX
#  define MACH_TYPE "TILEGX"
#  define CPP_WORDSZ (__SIZEOF_PTRDIFF_T__ * 8)
#  if CPP_WORDSZ == 32
#    define CLEAR_DOUBLE(x) (void)(*(long long *)(x) = 0)
#  endif
#  define PREFETCH(x) __insn_prefetch_l1(x)
#  define CACHE_LINE_SIZE 64
#  ifdef LINUX
extern int __data_start[];
#    define DATASTART ((ptr_t)__data_start)
#  endif
#endif /* TILEGX */

#ifdef RISCV
#  define MACH_TYPE "RISCV"
#  define CPP_WORDSZ (__SIZEOF_SIZE_T__ * 8) /*< 32 or 64 */
#  ifdef FREEBSD
/* Nothing specific. */
#  endif
#  ifdef LINUX
extern int __data_start[] __attribute__((__weak__));
#    define DATASTART ((ptr_t)__data_start)
#  endif
#  ifdef NETBSD
/* Nothing specific. */
#  endif
#  ifdef OPENBSD
/* Nothing specific. */
#  endif
#  ifdef NOSYS
#    define OS_TYPE "NOSYS"
extern char etext[];
#    define DATASTART ((ptr_t)etext)
/* FIXME: `STACKBOTTOM` is wrong! */
extern char **environ;
#    define STACKBOTTOM ((ptr_t)environ)
/* TODO: Support 64K page size. */
#    define GETPAGESIZE() 4096
#  endif
#endif /* RISCV */

#ifdef WEBASSEMBLY
#  define MACH_TYPE "WEBASSEMBLY"
#  if defined(__wasm64__) && !defined(CPPCHECK)
#    error 64-bit WebAssembly is not yet supported
#  endif
#  define CPP_WORDSZ 32
/*
 * Emscripten does emulate `mmap` and `munmap`, but those should not be
 * used in the collector, since WebAssembly lacks the native support of
 * memory mapping.  Use `sbrk()` instead (by default).
 */
#  undef USE_MMAP
#  undef USE_MUNMAP
#  ifdef EMSCRIPTEN_TINY
void *emmalloc_memalign(size_t align, size_t lb);
#    define GET_MEM(lb) emmalloc_memalign(GC_page_size, lb)
#  endif
#  ifdef EMSCRIPTEN
#    define OS_TYPE "EMSCRIPTEN"
#    define DATASTART ((ptr_t)ALIGNMENT)
#    define DATAEND ((ptr_t)ALIGNMENT)
#    if defined(GC_THREADS) && !defined(CPPCHECK)
#      error No thread support yet
#    endif
#  endif
#  ifdef WASI
#    define OS_TYPE "WASI"
extern char __global_base, __heap_base;
#    define DATASTART ((ptr_t)(&__global_base))
#    define DATAEND ((ptr_t)(&__heap_base))
#    define STACKBOTTOM DATASTART
#    ifndef GC_NO_SIGSETJMP
#      define GC_NO_SIGSETJMP 1 /*< no support of signals */
#    endif
#    ifndef NO_CLOCK
#      define NO_CLOCK 1 /*< no support of `clock()` */
#    endif
#    if defined(GC_THREADS) && !defined(CPPCHECK)
#      error No thread support yet
#    endif
#  endif
#endif /* WEBASSEMBLY */

#if defined(CYGWIN32) || defined(MSWIN32) || defined(MSWINCE)
/* Note: it does not include Xbox One. */
#  define ANY_MSWIN
#endif

#if defined(GC_PTHREADS) || defined(GC_WIN32_THREADS)          \
    || ((defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH)  \
         || defined(SN_TARGET_PS3) || defined(SN_TARGET_PSP2)) \
        && defined(GC_THREADS))
#  define THREADS
#endif

/*
 * If the client app is known not to create threads (even indirectly in
 * the used libraries) and the collector is not multi-threaded, then the
 * collector could be built with `SINGLE_THREADED_PROCESS` macro defined.
 * But in practice the macro should never be defined.
 */
#if defined(THREADS) && defined(SINGLE_THREADED_PROCESS)
#  undef SINGLE_THREADED_PROCESS
#endif

#if defined(__CHERI_PURE_CAPABILITY__)
#  define CHERI_PURECAP
#endif

#if defined(__GLIBC__) && !defined(DONT_USE_LIBC_PRIVATES)
/* Use the stack-end marker of `glibc`. */
#  define USE_LIBC_PRIVATES
#endif

#ifdef NO_RETRY_GET_THREAD_CONTEXT
#  undef RETRY_GET_THREAD_CONTEXT
#endif

#if defined(LINUX) && defined(SPECIFIC_MAIN_STACKBOTTOM) \
    && defined(NO_PROC_STAT) && !defined(USE_LIBC_PRIVATES)
/*
 * This combination will fail, since we have no way to get the stack bottom.
 * Use `HEURISTIC2` instead.
 */
#  undef SPECIFIC_MAIN_STACKBOTTOM
#  define HEURISTIC2
/* This may still fail on some architectures like `IA64`.  We tried... */
#endif

#if defined(USE_MMAP_ANON) && !defined(USE_MMAP)
#  define USE_MMAP 1
#elif (defined(LINUX) || defined(OPENBSD)) && defined(USE_MMAP)
/*
 * The OS kernel may do a somewhat better job merging mappings with
 * anonymous mappings.
 */
#  define USE_MMAP_ANON
#endif

#if defined(CHERI_PURECAP) && defined(USE_MMAP)
/* TODO: Currently turned off to avoid downgrading permissions on CHERI. */
#  undef USE_MUNMAP
#endif

#if (defined(E2K) && defined(USE_PTR_HWTAG) || defined(CHERI_PURECAP)) \
    && !defined(NO_BLACK_LISTING)
/*
 * Misinterpreting of an integer is not possible on the platforms with
 * H/W-tagged pointers, thus the black-listing mechanism is redundant.
 */
#  define NO_BLACK_LISTING
#endif

#if defined(REDIRECT_MALLOC) && defined(THREADS) \
    && (defined(LINUX) || defined(NACL))
/* TODO: Unclear if NaCl really needs this. */
#  define REDIR_MALLOC_AND_LINUXTHREADS
#endif

#if defined(REDIR_MALLOC_AND_LINUXTHREADS) && !defined(NO_PROC_FOR_LIBRARIES) \
    && !defined(USE_PROC_FOR_LIBRARIES)
/*
 * NPTL allocates thread stacks with `mmap`, which is fine.  But it
 * keeps a cache of thread stacks.  Each thread stack contains a thread
 * control block (TCB).  The latter, in turn, contains a pointer to
 * (`sizeof(void*)` from the beginning of) the `dtv` for thread-local
 * storage, which is `calloc`-allocated.  If we do not scan the cached
 * thread stacks, we appear to lose the `dtv`.  This tends to result in
 * something that looks like a bogus `dtv` count, which tends to result
 * in a `memset()` call on a block that is way too large.  Sometimes
 * we are lucky and the process just dies...  There seems to be
 * a similar issue with some other memory allocated by the dynamic
 * loader.  This should be avoidable by either:
 *   - Defining `USE_PROC_FOR_LIBRARIES` here (that performs very poorly,
 *     precisely because we end up scanning cached stacks);
 *   - Have `calloc()` look at its callers (in spite of the fact that it
 *     is gross and disgusting).
 *
 * In fact, neither seems to suffice, probably in part because even
 * with `USE_PROC_FOR_LIBRARIES`, we do not scan parts of stack
 * segments that appear to be out of bounds.  Thus we actually do both,
 * which seems to yield the best results.
 */
#  define USE_PROC_FOR_LIBRARIES
#endif

#ifndef OS_TYPE
#  define OS_TYPE ""
#endif

#ifndef MACH_TYPE
#  define MACH_TYPE ""
#endif

#ifndef DATAEND
extern int end[];
#  define DATAEND ((ptr_t)end)
#endif

/*
 * Workaround for Android NDK clang-3.5+ (as of NDK r10e) which does
 * not provide correct `_end` symbol.  Unfortunately, alternate `__end__`
 * symbol is provided only by NDK `bfd` linker.
 */
#if defined(HOST_ANDROID) && defined(__clang__) && !defined(BROKEN_UUENDUU_SYM)
#  undef DATAEND
#  pragma weak __end__
extern int __end__[];
#  define DATAEND (__end__ != 0 ? (ptr_t)__end__ : (ptr_t)_end)
#endif

#if defined(SOLARIS) || defined(DRSNX) || defined(UTS4) \
    || (defined(LINUX) && defined(SPARC))
/* OS has SVR4 generic features.  Probably others also qualify. */
#  define SVR4
#  define DATASTART_USES_XGETDATASTART
#endif

#if defined(HAVE_SYS_TYPES_H)                                      \
    || !(defined(__CC_ARM) || defined(GC_NO_TYPES) || defined(OS2) \
         || defined(MSWINCE) || defined(SN_TARGET_PSP2))
EXTERN_C_END
#  if defined(COSMO) && defined(MPROTECT_VDB) && !defined(_GNU_SOURCE)
#    define _GNU_SOURCE 1
#  endif
#  include <sys/types.h>
EXTERN_C_BEGIN
#endif /* HAVE_SYS_TYPES_H */

#if defined(HAVE_UNISTD_H)                                                \
    || !(defined(GC_NO_TYPES) || defined(MSWIN32) || defined(MSWINCE)     \
         || defined(MSWIN_XBOX1) || defined(NINTENDO_SWITCH)              \
         || defined(NN_PLATFORM_CTR) || defined(OS2) || defined(SERENITY) \
         || defined(SN_TARGET_PSP2) || defined(__CC_ARM))
EXTERN_C_END
#  include <unistd.h>
EXTERN_C_BEGIN
#endif /* HAVE_UNISTD_H */

#if !defined(ANY_MSWIN) && !defined(GETPAGESIZE)
#  if defined(DGUX) || defined(HOST_ANDROID) || defined(HOST_TIZEN) \
      || defined(KOS) || defined(SERENITY)                          \
      || (defined(LINUX) && defined(SPARC))
#    define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE)
#  else
#    define GETPAGESIZE() (unsigned)getpagesize()
#  endif
#endif /* !ANY_MSWIN && !GETPAGESIZE */

#if defined(HOST_ANDROID) && !(__ANDROID_API__ >= 23)           \
    && ((defined(MIPS) && (CPP_WORDSZ == 32)) || defined(ARM32) \
        || defined(I386) /* but not x32 */)
/*
 * `tkill()` exists only on arm32/mips(32)/x86.
 * NDK r11+ deprecates `tkill()` but keeps it for Mono clients.
 */
#  define USE_TKILL_ON_ANDROID
#endif

#if defined(MPROTECT_VDB) && defined(__GLIBC__) && !GC_GLIBC_PREREQ(2, 2)
#  error glibc too old?
#endif

#if defined(SOLARIS) || defined(DRSNX)
/* OS has Solaris-style semi-undocumented interface to dynamic loader. */
#  define SOLARISDL
/* OS has Solaris-style signal handlers. */
#  define SUNOS5SIGS
#endif

#if (defined(FREEBSD)                                     \
     && (defined(__DragonFly__) || defined(__GLIBC__)     \
         || __FreeBSD_kernel__ >= 4 || __FreeBSD__ >= 4)) \
    || defined(HPUX)
#  define SUNOS5SIGS
#endif

#if defined(COSMO) || defined(HPUX) || defined(HURD) || defined(NETBSD) \
    || defined(SERENITY) || (defined(FREEBSD) && defined(SUNOS5SIGS))   \
    || (defined(IRIX5) && defined(_sigargs)) /*< Irix 5.x, not 6.x */
#  define USE_SEGV_SIGACT
/* We may also get `SIGBUS`. */
#  define USE_BUS_SIGACT
#elif defined(ANY_BSD) || defined(HAIKU) || defined(IRIX5) || defined(OSF1) \
    || defined(SUNOS5SIGS)
#  define USE_SEGV_SIGACT
#endif

#if !defined(GC_EXPLICIT_SIGNALS_UNBLOCK) && defined(SUNOS5SIGS) \
    && !defined(GC_NO_PTHREAD_SIGMASK)
#  define GC_EXPLICIT_SIGNALS_UNBLOCK
#endif

#if !defined(NO_SIGNALS_UNBLOCK_IN_MAIN) && defined(GC_NO_PTHREAD_SIGMASK)
#  define NO_SIGNALS_UNBLOCK_IN_MAIN
#endif

#ifndef PARALLEL_MARK
#  undef GC_PTHREADS_PARAMARK /*< just in case it is defined by client */
#elif defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK) \
    && !defined(__MINGW32__)
/*
 * Use `pthreads`-based parallel mark implementation.  Except for
 * MinGW 32/64 to workaround a deadlock in winpthreads-3.0b internals.
 */
#  define GC_PTHREADS_PARAMARK
#endif

#if !defined(NO_MARKER_SPECIAL_SIGMASK)                                 \
    && (defined(NACL) || defined(GC_WIN32_PTHREADS)                     \
        || (defined(GC_PTHREADS_PARAMARK) && defined(GC_WIN32_THREADS)) \
        || defined(GC_NO_PTHREAD_SIGMASK))
/*
 * Either there is no `pthread_sigmask()`, or the GC marker thread cannot
 * steal and drop user signal calls.
 */
#  define NO_MARKER_SPECIAL_SIGMASK
#endif

#if defined(NETBSD) && defined(THREADS)
#  define SIGRTMIN 33
#  define SIGRTMAX 63
/*
 * It seems to be necessary to wait until threads have restarted.
 * But it is unclear why that is the case.
 */
#  define GC_NETBSD_THREADS_WORKAROUND
#endif

#if defined(OPENBSD) && defined(THREADS)
EXTERN_C_END
#  include <sys/param.h>
EXTERN_C_BEGIN
#endif

#if defined(AIX) || defined(ANY_BSD) || defined(BSD) || defined(COSMO)     \
    || defined(DARWIN) || defined(DGUX) || defined(HAIKU) || defined(HPUX) \
    || defined(HURD) || defined(IRIX5) || defined(LINUX) || defined(OSF1)  \
    || defined(QNX) || defined(SERENITY) || defined(SVR4)
/* Basic UNIX-like system calls work. */
#  define UNIX_LIKE
#endif

#if defined(CPPCHECK)
#  undef CPP_WORDSZ
#  define CPP_WORDSZ (__SIZEOF_PTRDIFF_T__ * 8)
#elif CPP_WORDSZ != 32 && CPP_WORDSZ != 64
#  error Bad word size
#endif

#ifndef CPP_PTRSZ
#  ifdef CHERI_PURECAP
#    define CPP_PTRSZ (__SIZEOF_POINTER__ * 8)
#  else
#    define CPP_PTRSZ CPP_WORDSZ
#  endif
#endif

#ifndef CPPCHECK
#  if GC_SIZEOF_PTR * 8 != CPP_PTRSZ
#    error Bad pointer size
#  endif
#endif /* !CPPCHECK */

#ifndef ALIGNMENT
#  define ALIGNMENT (CPP_PTRSZ >> 3)
#endif

#if !defined(STACKBOTTOM) && (defined(ECOS) || defined(NOSYS)) \
    && !defined(CPPCHECK)
#  error Undefined STACKBOTTOM
#endif

#ifdef IGNORE_DYNAMIC_LOADING
#  undef DYNAMIC_LOADING
#endif

#if defined(SMALL_CONFIG) && !defined(GC_DISABLE_INCREMENTAL)
/* Presumably not worth the space it takes. */
#  define GC_DISABLE_INCREMENTAL
#endif

/* `USE_WINALLOC` is only an option for Cygwin. */
#ifndef CYGWIN32
#  undef USE_WINALLOC
#endif
#if defined(MSWIN32) || defined(MSWINCE)
#  define USE_WINALLOC 1
#endif

#ifdef USE_WINALLOC
#  undef USE_MMAP
#endif

#if defined(ANY_BSD) || defined(DARWIN) || defined(IRIX5) || defined(LINUX) \
    || defined(SERENITY) || defined(SOLARIS)                                \
    || ((defined(CYGWIN32) || defined(USE_MMAP) || defined(USE_MUNMAP))     \
        && !defined(USE_WINALLOC))
/* Try both `sbrk` and `mmap`, in that order. */
#  define MMAP_SUPPORTED
#endif

/*
 * Xbox One (DURANGO) may not need to be this aggressive, but the
 * default is likely too lax under heavy allocation pressure.
 * The platform does not have a virtual paging system, so it does not
 * have a large virtual address space that a standard x86_64 platform has.
 */
#if defined(USE_MUNMAP) && !defined(MUNMAP_THRESHOLD)     \
    && (defined(SN_TARGET_PS3) || defined(SN_TARGET_PSP2) \
        || defined(MSWIN_XBOX1))
#  define MUNMAP_THRESHOLD 3
#endif

#if defined(GC_DISABLE_INCREMENTAL) || defined(DEFAULT_VDB)
#  undef GWW_VDB
#  undef MPROTECT_VDB
#  undef PROC_VDB
#  undef SOFT_VDB
#endif

#ifdef NO_GWW_VDB
#  undef GWW_VDB
#endif

#ifdef NO_MPROTECT_VDB
#  undef MPROTECT_VDB
#endif

#ifdef NO_SOFT_VDB
#  undef SOFT_VDB
#endif

#if defined(SOFT_VDB) && defined(SOFT_VDB_LINUX_VER_STATIC_CHECK)
EXTERN_C_END
#  include <linux/version.h> /*< for `LINUX_VERSION`, `LINUX_VERSION_CODE` */
EXTERN_C_BEGIN
#  if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0)
/* Not reliable in Linux kernels prior to v3.18. */
#    undef SOFT_VDB
#  endif
#endif /* SOFT_VDB */

#ifdef GC_DISABLE_INCREMENTAL
#  undef CHECKSUMS
#endif

#if defined(BASE_ATOMIC_OPS_EMULATED)
/*
 * `GC_write_fault_handler()` cannot use lock-based atomic primitives
 * as this could lead to a deadlock.
 */
#  undef MPROTECT_VDB
#endif

#if defined(USE_PROC_FOR_LIBRARIES) && defined(LINUX) && defined(THREADS)
/* Incremental GC based on `mprotect` is incompatible with `/proc` roots. */
#  undef MPROTECT_VDB
#endif

#if defined(MPROTECT_VDB) && defined(GC_PREFER_MPROTECT_VDB)
/* Choose `MPROTECT_VDB` manually (if multiple strategies available). */
#  undef PROC_VDB
/* `GWW_VDB`, `SOFT_VDB` are handled in `os_dep.c` file. */
#endif

#ifdef PROC_VDB
/* Mutually exclusive VDB implementations (for now). */
#  undef MPROTECT_VDB
/* For a test purpose only. */
#  undef SOFT_VDB
#endif

#if defined(MPROTECT_VDB) && !defined(MSWIN32) && !defined(MSWINCE)
EXTERN_C_END
#  include <signal.h> /*< for `SA_SIGINFO`, `SIGBUS` */
EXTERN_C_BEGIN
#endif

#if defined(SIGBUS) && !defined(HAVE_SIGBUS) && !defined(CPPCHECK)
#  define HAVE_SIGBUS
#endif

#ifndef SA_SIGINFO
#  define NO_SA_SIGACTION
#endif

#if (defined(NO_SA_SIGACTION) || defined(GC_NO_SIGSETJMP))            \
    && defined(MPROTECT_VDB) && !defined(DARWIN) && !defined(MSWIN32) \
    && !defined(MSWINCE)
#  undef MPROTECT_VDB
#endif

#if !defined(DEFAULT_VDB) && !defined(GWW_VDB) && !defined(MPROTECT_VDB) \
    && !defined(PROC_VDB) && !defined(SOFT_VDB)                          \
    && !defined(GC_DISABLE_INCREMENTAL)
#  define DEFAULT_VDB
#endif

#if defined(CHECK_SOFT_VDB) && !defined(CPPCHECK)             \
    && (defined(GC_PREFER_MPROTECT_VDB) || !defined(SOFT_VDB) \
        || !defined(MPROTECT_VDB))
#  error Invalid config for CHECK_SOFT_VDB
#endif

#if (defined(GC_DISABLE_INCREMENTAL) || defined(BASE_ATOMIC_OPS_EMULATED) \
     || defined(REDIRECT_MALLOC) || defined(SMALL_CONFIG)                 \
     || defined(REDIRECT_MALLOC_IN_HEADER) || defined(CHECKSUMS))         \
    && !defined(NO_MANUAL_VDB)
/* TODO: Implement `CHECKSUMS` for manual VDB. */
#  define NO_MANUAL_VDB
#endif

#if !defined(PROC_VDB) && !defined(SOFT_VDB) \
    && !defined(NO_VDB_FOR_STATIC_ROOTS)
/* Cannot determine whether a static root page is dirty. */
#  define NO_VDB_FOR_STATIC_ROOTS
#endif

#if defined(MPROTECT_VDB) && !defined(DONT_COUNT_PROTECTED_REGIONS) \
    && !defined(COUNT_PROTECTED_REGIONS)                            \
    && (defined(LINUX) || defined(__DragonFly__))
#  define COUNT_PROTECTED_REGIONS
#endif

#if (defined(COUNT_PROTECTED_REGIONS) || defined(COUNT_UNMAPPED_REGIONS)) \
    && !defined(GC_UNMAPPED_REGIONS_SOFT_LIMIT)
/*
 * The default limit of `vm.max_map_count` on Linux is ~65530.
 * There is approximately one mapped region to every protected or
 * unmapped region.  Therefore if we aim to use up to half of
 * `vm.max_map_count` for the collector (leaving half for the rest
 * of the process), then the number of such regions should be one
 * quarter of `vm.max_map_count`.
 */
#  if defined(__DragonFly__)
#    define GC_UNMAPPED_REGIONS_SOFT_LIMIT (1000000 / 4)
#  else
#    define GC_UNMAPPED_REGIONS_SOFT_LIMIT 16384
#  endif
#endif

#if (((defined(ARM32) || defined(AVR32) || defined(MIPS) || defined(NIOS2) \
       || defined(OR1K))                                                   \
      && defined(UNIX_LIKE))                                               \
     || defined(DARWIN) || defined(HAIKU) || defined(HURD)                 \
     || defined(OPENBSD) || defined(QNX) || defined(RTEMS)                 \
     || defined(SERENITY) || defined(HOST_ANDROID)                         \
     || (defined(LINUX) && !defined(__gnu_linux__)))                       \
    && !defined(NO_GETCONTEXT)
#  define NO_GETCONTEXT 1
#endif

#if defined(MSWIN32) && !defined(CONSOLE_LOG) && defined(_MSC_VER) \
    && defined(_DEBUG) && !defined(NO_CRT)
/*
 * This should be included before the platform `intrin.h` file to
 * workaround some bug in Windows Kit (as of 10.0.17763) headers
 * causing redefinition of `_malloca` macro.
 */
EXTERN_C_END
#  include <crtdbg.h> /*< for `_CrtDbgReport` */
EXTERN_C_BEGIN
#endif

#ifndef PREFETCH
#  if (GC_GNUC_PREREQ(3, 0) || defined(__clang__)) && !defined(NO_PREFETCH)
#    define PREFETCH(x) __builtin_prefetch((x), 0, 0)
#  elif defined(_MSC_VER) && !defined(NO_PREFETCH)                      \
      && (defined(_M_IX86) || defined(_M_X64)) && !defined(_CHPE_ONLY_) \
      && (_MSC_VER >= 1900 /* VS 2015+ */)
EXTERN_C_END
#    include <intrin.h>
EXTERN_C_BEGIN
#    define PREFETCH(x) _mm_prefetch((const char *)(x), _MM_HINT_T0)
/* TODO: Support also `_M_ARM` and `_M_ARM64` (`__prefetch`). */
#  else
#    define PREFETCH(x) (void)0
#  endif
#endif /* !PREFETCH */

#ifndef GC_PREFETCH_FOR_WRITE
/*
 * The default `GC_PREFETCH_FOR_WRITE(x)` is defined in `gc_inline.h` file,
 * the latter one is included from `gc_priv.h` file.
 */
#endif

#ifndef CACHE_LINE_SIZE
#  define CACHE_LINE_SIZE 32 /*< wild guess */
#endif

#ifndef STATIC
#  ifdef GC_ASSERTIONS
#    define STATIC /*< ignore to aid debugging (or profiling) */
#  else
#    define STATIC static
#  endif
#endif

/*
 * Do we need the `GC_find_limit` machinery to find the end of
 * a data segment (or the backing store base)?
 */
#if defined(HEURISTIC2) || defined(SEARCH_FOR_DATA_START) || defined(IA64)    \
    || defined(DGUX) || defined(FREEBSD) || defined(OPENBSD) || defined(SVR4) \
    || (defined(HPUX) && defined(SPECIFIC_MAIN_STACKBOTTOM))                  \
    || (defined(CYGWIN32) && defined(I386) && defined(USE_MMAP)               \
        && !defined(USE_WINALLOC))                                            \
    || (defined(NETBSD) && defined(__ELF__))
#  define NEED_FIND_LIMIT
#endif

#if defined(LINUX)                                       \
    && (defined(USE_PROC_FOR_LIBRARIES) || defined(IA64) \
        || !defined(SMALL_CONFIG))
#  define NEED_PROC_MAPS
#endif

#if defined(LINUX) || defined(HURD) || defined(__GLIBC__)
#  define REGISTER_LIBRARIES_EARLY
/*
 * We sometimes use `dl_iterate_phdr`, which may acquire an internal lock.
 * This is not safe after the world has stopped.  So we must call
 * `GC_register_dynamic_libraries` before stopping the world.
 * For performance reasons, this may be beneficial on other platforms
 * as well, though it should be avoided on Windows.
 */
#endif /* LINUX */

#if defined(SEARCH_FOR_DATA_START)
extern ptr_t GC_data_start;
#  define DATASTART GC_data_start
#endif

#ifndef HEAP_START
#  define HEAP_START 0
#endif

#ifndef CLEAR_DOUBLE
#  define CLEAR_DOUBLE(x) \
    (void)(((ptr_t *)(x))[0] = NULL, ((ptr_t *)(x))[1] = NULL)
#endif

/*
 * Some `libc` implementations like `bionic`, `musl` and `glibc` 2.34+
 * do not have `libpthread.so` file because the `pthreads`-related code
 * is located in `libc.so` file, thus potential `calloc()` calls from
 * such code are forwarded to real (`libc`) `calloc()` without any
 * special handling on the collector side.  Checking `glibc` version at
 * compile time for the purpose seems to be fine.
 */
#if defined(REDIR_MALLOC_AND_LINUXTHREADS) && !defined(HAVE_LIBPTHREAD_SO) \
    && defined(__GLIBC__) && !GC_GLIBC_PREREQ(2, 34)
#  define HAVE_LIBPTHREAD_SO
#endif

#if defined(REDIR_MALLOC_AND_LINUXTHREADS) \
    && !defined(INCLUDE_LINUX_THREAD_DESCR)
/*
 * Will not work, since `libc` and the dynamic loader use thread locals,
 * sometimes as the only reference.
 */
#  define INCLUDE_LINUX_THREAD_DESCR
#endif

#ifndef CPPCHECK
#  if defined(GC_AIX_THREADS) && !defined(AIX)                            \
      || (defined(GC_DARWIN_THREADS) && !defined(DARWIN))                 \
      || (defined(GC_DGUX386_THREADS) && !defined(DGUX))                  \
      || (defined(GC_FREEBSD_THREADS) && !defined(FREEBSD))               \
      || (defined(GC_HAIKU_THREADS) && !defined(HAIKU))                   \
      || (defined(GC_HPUX_THREADS) && !defined(HPUX))                     \
      || (defined(GC_IRIX_THREADS) && !defined(IRIX5))                    \
      || (defined(GC_LINUX_THREADS) && !defined(LINUX) && !defined(NACL)) \
      || (defined(GC_NETBSD_THREADS) && !defined(NETBSD))                 \
      || (defined(GC_OPENBSD_THREADS) && !defined(OPENBSD))               \
      || (defined(GC_OSF1_THREADS) && !defined(OSF1))                     \
      || (defined(GC_RTEMS_PTHREADS) && !defined(RTEMS))                  \
      || (defined(GC_SOLARIS_THREADS) && !defined(SOLARIS))               \
      || (defined(GC_WIN32_THREADS) && !defined(ANY_MSWIN)                \
          && !defined(MSWIN_XBOX1))
#    error Inconsistent configuration
#  elif defined(GC_WIN32_PTHREADS) && defined(CYGWIN32)
#    error Inconsistent configuration (GC_PTHREADS)
#  endif
#  if defined(PARALLEL_MARK) && !defined(THREADS)
#    error Invalid config: PARALLEL_MARK requires GC_THREADS
#  endif
#  if defined(GWW_VDB) && !defined(USE_WINALLOC)
#    error Invalid config: GWW_VDB requires USE_WINALLOC
#  endif
#  if (defined(GC_FINDLEAK_DELAY_FREE) && defined(SHORT_DBG_HDRS)) \
      || ((defined(FIND_LEAK) || defined(GC_FINDLEAK_DELAY_FREE))  \
          && defined(NO_FIND_LEAK))
#    error Invalid config: FIND_LEAK and NO_FIND_LEAK are mutually exclusive
#  endif
#endif /* !CPPCHECK */

#if defined(NO_FIND_LEAK) && !defined(DONT_USE_ATEXIT)
#  define DONT_USE_ATEXIT
#endif

/* Whether `GC_page_size` is to be set to a value other than page size. */
#if defined(CYGWIN32) && (defined(MPROTECT_VDB) || defined(USE_MUNMAP)) \
    || (!defined(ANY_MSWIN) && !defined(WASI) && !defined(USE_MMAP)     \
        && (defined(GC_DISABLE_INCREMENTAL) || defined(DEFAULT_VDB)))
/*
 * Cygwin: use the allocation granularity instead.
 * Other than WASI and Windows: use `HBLKSIZE` instead (unless `mmap()`
 * is used).
 */
#  define ALT_PAGESIZE_USED
#  ifndef GC_NO_VALLOC
/* Nonetheless, we need the real page size is some extra functions. */
#    define REAL_PAGESIZE_NEEDED
#  endif
#endif

#if defined(GC_PTHREADS) && !defined(DARWIN) && !defined(GC_WIN32_THREADS) \
    && !defined(PLATFORM_STOP_WORLD) && !defined(SN_TARGET_PSP2)
#  define PTHREAD_STOP_WORLD_IMPL
#endif

#if defined(PTHREAD_STOP_WORLD_IMPL) && !defined(NACL)
#  define SIGNAL_BASED_STOP_WORLD
#endif

#if (defined(E2K) || defined(HP_PA) || defined(IA64) || defined(M68K) \
     || defined(NO_SA_SIGACTION))                                     \
    && defined(SIGNAL_BASED_STOP_WORLD)
#  define SUSPEND_HANDLER_NO_CONTEXT
#endif

#if (defined(MSWIN32) || defined(MSWINCE)                      \
     || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))) \
    && !defined(NO_CRT) && !defined(NO_WRAP_MARK_SOME)
/*
 * Under rare conditions, we may end up marking from nonexistent memory.
 * Hence we need to be prepared to recover by running `GC_mark_some` with
 * a suitable handler in place.
 */
/* TODO: Should we also define it for Cygwin? */
#  define WRAP_MARK_SOME
#endif

#if !defined(MSWIN32) && !defined(MSWINCE) || defined(__GNUC__) \
    || defined(NO_CRT)
#  define NO_SEH_AVAILABLE
#endif

#ifdef GC_WIN32_THREADS
/* The number of copied registers in `copy_ptr_regs()`. */
#  if defined(I386)
#    ifdef WOW64_THREAD_CONTEXT_WORKAROUND
#      define PUSHED_REGS_COUNT 9
#    else
#      define PUSHED_REGS_COUNT 7
#    endif
#  elif defined(X86_64)
#    ifdef XMM_CANT_STORE_PTRS
/* If pointers cannot be located in Xmm registers. */
#      define PUSHED_REGS_COUNT 15
#    else
/*
 * gcc-13 may store pointers into SIMD registers when certain compiler
 * optimizations are enabled.
 */
#      define PUSHED_REGS_COUNT (15 + 32)
#    endif
#  elif defined(SHx)
#    define PUSHED_REGS_COUNT 15
#  elif defined(ARM32)
#    define PUSHED_REGS_COUNT 13
#  elif defined(AARCH64)
#    define PUSHED_REGS_COUNT 30
#  elif defined(MIPS) || defined(ALPHA)
#    define PUSHED_REGS_COUNT 28
#  elif defined(PPC)
#    define PUSHED_REGS_COUNT 29
#  endif
#endif /* GC_WIN32_THREADS */

#if !defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK)
#  undef HAVE_PTHREAD_SETNAME_NP_WITH_TID
#  undef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG
#  undef HAVE_PTHREAD_SET_NAME_NP
#endif

#if !(defined(GC_PTHREADS) || defined(GC_PTHREADS_PARAMARK) \
      || (defined(MPROTECT_VDB) && defined(DARWIN)))
#  undef HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID
#endif

#if defined(USE_RWLOCK) || defined(GC_DISABLE_SUSPEND_THREAD)
/*
 * At least in the Linux threads implementation, `rwlock` primitives are
 * not atomic in respect to signals, and suspending externally a thread
 * which is running inside `pthread_rwlock_rdlock()` may lead to a deadlock.
 */
/* TODO: As a workaround `GC_suspend_thread()` API is disabled. */
#  undef GC_ENABLE_SUSPEND_THREAD
#endif

#ifndef GC_NO_THREADS_DISCOVERY
#  if defined(DARWIN) && defined(THREADS)
/* Task-based thread registration requires stack-frame-walking code. */
#    if defined(DARWIN_DONT_PARSE_STACK)
#      define GC_NO_THREADS_DISCOVERY
#    endif
#  elif defined(GC_WIN32_THREADS)
/*
 * `DllMain`-based thread registration is currently incompatible with
 * thread-local allocation, `pthreads` and WinCE.
 */
#    if (!defined(GC_DLL) && !defined(GC_INSIDE_DLL)) || defined(GC_PTHREADS) \
        || defined(MSWINCE) || defined(NO_CRT) || defined(THREAD_LOCAL_ALLOC)
#      define GC_NO_THREADS_DISCOVERY
#    endif
#  else
#    define GC_NO_THREADS_DISCOVERY
#  endif
#endif /* !GC_NO_THREADS_DISCOVERY */

#if defined(GC_DISCOVER_TASK_THREADS) && defined(GC_NO_THREADS_DISCOVERY) \
    && !defined(CPPCHECK)
#  error Defined both GC_DISCOVER_TASK_THREADS and GC_NO_THREADS_DISCOVERY
#endif

#if defined(PARALLEL_MARK) && !defined(DEFAULT_STACK_MAYBE_SMALL) \
    && (defined(DGUX) || defined(HPUX)                            \
        || defined(NO_GETCONTEXT) /* e.g. musl */)
/* TODO: Test default stack size in configure. */
#  define DEFAULT_STACK_MAYBE_SMALL
#endif

#ifdef PARALLEL_MARK
/* The minimum stack size for a marker thread. */
#  define MIN_STACK_SIZE (8 * HBLKSIZE * sizeof(ptr_t))
#endif

#if defined(HOST_ANDROID) && !defined(THREADS) \
    && !defined(USE_GET_STACKBASE_FOR_MAIN)
/*
 * Always use `pthread_attr_getstack` on Android (`-lpthread` option
 * is not needed to be specified manually) since Linux-specific
 * `os_main_stackbottom()` causes application crash if invoked inside
 * Dalvik VM.
 */
#  define USE_GET_STACKBASE_FOR_MAIN
#endif

/*
 * Outline `pthreads` primitives to use in `GC_get_stack_base()` and
 * `GC_get_main_stack_base()`.
 */
#if ((defined(FREEBSD) && defined(__GLIBC__) /* kFreeBSD */)               \
     || defined(COSMO) || defined(HAIKU) || defined(LINUX) || defined(KOS) \
     || defined(NETBSD))                                                   \
    && !defined(NO_PTHREAD_GETATTR_NP)
#  define HAVE_PTHREAD_GETATTR_NP 1
#elif defined(FREEBSD) && !defined(__GLIBC__) \
    && !defined(NO_PTHREAD_ATTR_GET_NP)
#  define HAVE_PTHREAD_NP_H 1 /*< requires include `pthread_np.h` file */
#  define HAVE_PTHREAD_ATTR_GET_NP 1
#endif

#if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
    && defined(USE_GET_STACKBASE_FOR_MAIN) && !defined(STACKBOTTOM)         \
    && !defined(HEURISTIC1) && !defined(HEURISTIC2) && !defined(STACK_GRAN) \
    && !defined(SPECIFIC_MAIN_STACKBOTTOM)
/* Dummy definitions; rely on `pthread_attr_getstack` actually. */
#  define HEURISTIC1
#  define STACK_GRAN 0x1000000
#endif

#if !defined(HAVE_CLOCK_GETTIME) && defined(_POSIX_TIMERS) \
    && (defined(CYGWIN32) || (defined(LINUX) && defined(__USE_POSIX199309)))
#  define HAVE_CLOCK_GETTIME 1
#endif

#if defined(GC_PTHREADS) && !defined(E2K) && !defined(IA64)   \
    && (!defined(DARWIN) || defined(DARWIN_DONT_PARSE_STACK)) \
    && !defined(SN_TARGET_PSP2) && !defined(REDIRECT_MALLOC)
/*
 * Note: unimplemented in case of redirection of `malloc()` because
 * the client-provided function might call some `pthreads` primitive
 * which, in turn, may use `malloc()` internally.
 */
#  define STACKPTR_CORRECTOR_AVAILABLE
#endif

#if defined(UNIX_LIKE) && defined(THREADS) && !defined(NO_CANCEL_SAFE) \
    && !defined(HOST_ANDROID)
/*
 * Make the code cancellation-safe.  This basically means that we ensure
 * that cancellation requests are ignored while we are in the collector.
 * This applies only to POSIX deferred cancellation; we do not handle POSIX
 * asynchronous cancellation.  Note that this only works if
 * `pthread_setcancelstate` is async-signal-safe, at least in the absence of
 * asynchronous cancellation.  This appears to be true for `glibc`, though it
 * is not documented.  Without that assumption, there seems to be no way to
 * safely wait in a signal handler, which we need to do for thread suspension.
 * Also note that little other code appears to be cancellation-safe.
 * Hence it may make sense to turn this off for performance.
 */
#  define CANCEL_SAFE
#endif

#ifdef CANCEL_SAFE
#  define IF_CANCEL(x) x
#else
#  define IF_CANCEL(x) /*< empty */
#endif

#if defined(DARWIN) && defined(MPROTECT_VDB)   \
    && !defined(NO_DESC_CATCH_EXCEPTION_RAISE) \
    && !defined(FORCE_DESC_CATCH_EXCEPTION_RAISE) && GC_CLANG_PREREQ(17, 0)
/*
 * Workaround "REFERENCED_DYNAMICALLY flag on _catch_exception_raise"
 * linker deprecation warnings on macOS 15.4.
 */
#  define NO_DESC_CATCH_EXCEPTION_RAISE
#endif

#if !defined(CAN_HANDLE_FORK) && !defined(NO_HANDLE_FORK)                 \
    && !defined(HAVE_NO_FORK)                                             \
    && ((defined(GC_PTHREADS) && !defined(NACL)                           \
         && !defined(GC_WIN32_PTHREADS) && !defined(USE_WINALLOC))        \
        || (defined(DARWIN) && defined(MPROTECT_VDB) /* `&& !THREADS` */) \
        || (defined(HANDLE_FORK) && defined(GC_PTHREADS)))
/*
 * Attempts (where supported and requested) to make `GC_malloc` work in
 * a child process forked from a multi-threaded parent process.
 */
#  define CAN_HANDLE_FORK
#endif

/*
 * Workaround "failed to create new win32 semaphore" Cygwin fatal error
 * during semaphores fixup-after-fork.
 */
#if defined(CYGWIN32) && defined(THREADS) && defined(CAN_HANDLE_FORK) \
    && !defined(CYGWIN_SEM_FIXUP_AFTER_FORK_BUG_FIXED)                \
    && !defined(EMULATE_PTHREAD_SEMAPHORE)
#  define EMULATE_PTHREAD_SEMAPHORE
#endif

#if defined(CAN_HANDLE_FORK) && !defined(CAN_CALL_ATFORK)      \
    && !defined(GC_NO_CAN_CALL_ATFORK) && !defined(HOST_TIZEN) \
    && !defined(HURD) && (!defined(HOST_ANDROID) || __ANDROID_API__ >= 21)
/* Have working `pthread_atfork()`. */
#  define CAN_CALL_ATFORK
#endif

#if !defined(CAN_HANDLE_FORK) && !defined(HAVE_NO_FORK) \
    && !(defined(CYGWIN32) || defined(SOLARIS) || defined(UNIX_LIKE))
#  define HAVE_NO_FORK
#endif

#if !defined(USE_MARK_BITS) && !defined(USE_MARK_BYTES) \
    && defined(PARALLEL_MARK)
/* Minimize compare-and-swap usage. */
#  define USE_MARK_BYTES
#endif

#if (defined(MSWINCE) && !defined(__CEGCC__) || defined(MSWINRT_FLAVOR)) \
    && !defined(NO_GETENV)
#  define NO_GETENV
#endif

#if (defined(NO_GETENV) || defined(MSWINCE)) && !defined(NO_GETENV_WIN32)
#  define NO_GETENV_WIN32
#endif

#if !defined(MSGBOX_ON_ERROR) && !defined(NO_MSGBOX_ON_ERROR)                \
    && defined(MSWIN32) && !defined(MSWINRT_FLAVOR) && !defined(MSWIN_XBOX1) \
    && !defined(SMALL_CONFIG)
/*
 * Show a Windows message box with "OK" button on a GC fatal error.
 * Client application is terminated once the user clicks the button.
 */
#  define MSGBOX_ON_ERROR
#endif

#ifndef STRTOULL
#  if defined(_WIN64) && !defined(__GNUC__)
#    define STRTOULL _strtoui64
#  elif defined(_LLP64) || defined(__LLP64__) || defined(_WIN64)
#    define STRTOULL strtoull
#  else
/* `strtoul()` fits since `sizeof(long)` is not less than `sizeof(word)`. */
#    define STRTOULL strtoul
#  endif
#endif /* !STRTOULL */

#ifndef GC_WORD_C
#  if defined(_WIN64) && !defined(__GNUC__)
#    define GC_WORD_C(val) val##ui64
#  elif defined(_LLP64) || defined(__LLP64__) || defined(_WIN64)
#    define GC_WORD_C(val) val##ULL
#  else
#    define GC_WORD_C(val) ((word)val##UL)
#  endif
#endif /* !GC_WORD_C */

#if defined(__has_feature)
/* `__has_feature()` is supported. */
#  if __has_feature(address_sanitizer)
#    define ADDRESS_SANITIZER
#  endif
#  if __has_feature(memory_sanitizer)
#    define MEMORY_SANITIZER
#  endif
#  if __has_feature(thread_sanitizer) && defined(THREADS)
#    define THREAD_SANITIZER
#  endif
#else
#  ifdef __SANITIZE_ADDRESS__
/* gcc v4.8+ */
#    define ADDRESS_SANITIZER
#  endif
#  if defined(__SANITIZE_THREAD__) && defined(THREADS)
/* gcc v7.1+ */
#    define THREAD_SANITIZER
#  endif
#endif /* !__has_feature */

#if defined(SPARC)
/* Stack clearing is crucial, and we include assembly code to do it well. */
#  define ASM_CLEAR_CODE
#endif

/*
 * Can we save call chain in objects for debugging?  Set `NFRAMES`
 * (number of saved frames) and `NARGS` (number of arguments for each
 * frame) to reasonable values for the platform.
 * Define `SAVE_CALL_CHAIN` if we can.  `SAVE_CALL_COUNT` can be
 * specified at build time, though we feel free to adjust it slightly.
 * Define `NEED_CALLINFO` if we either save the call stack or
 * `GC_ADD_CALLER` is defined.  Note: `GC_CAN_SAVE_CALL_STACKS` is
 * defined (for certain platforms) in `gc_config_macros.h` file.
 */
#if defined(SPARC)                         \
    || ((defined(I386) || defined(X86_64)) \
        && (defined(LINUX) || defined(__GLIBC__)))
/*
 * Linux/x86: `SAVE_CALL_CHAIN` is supported if the code is compiled to save
 * frame pointers by default, i.e. no `-fomit-frame-pointer` flag is given.
 */
#  define CAN_SAVE_CALL_ARGS
#endif

#if defined(SAVE_CALL_COUNT) && !defined(GC_ADD_CALLER) \
    && defined(GC_CAN_SAVE_CALL_STACKS)
#  define SAVE_CALL_CHAIN
#endif

#ifdef SAVE_CALL_CHAIN
/* Number of arguments to save for each call. */
#  if defined(SAVE_CALL_NARGS) && defined(CAN_SAVE_CALL_ARGS)
#    define NARGS SAVE_CALL_NARGS
#  else
#    define NARGS 0
#  endif
/* Number of frames to save.  Even for alignment reasons. */
#  if !defined(SAVE_CALL_COUNT) || defined(CPPCHECK)
#    define NFRAMES 6
#  else
#    define NFRAMES ((SAVE_CALL_COUNT + 1) & ~1)
#  endif
#  define NEED_CALLINFO
#elif defined(GC_ADD_CALLER)
#  define NFRAMES 1
#  define NARGS 0
#  define NEED_CALLINFO
#endif

#if (defined(FREEBSD) || (defined(DARWIN) && !defined(_POSIX_C_SOURCE)) \
     || (defined(SOLARIS)                                               \
         && (!defined(_XOPEN_SOURCE) || defined(__EXTENSIONS__)))       \
     || defined(LINUX))                                                 \
    && !defined(HAVE_DLADDR)
#  define HAVE_DLADDR 1
#endif

#if defined(MAKE_BACK_GRAPH) && !defined(DBG_HDRS_ALL)
#  define DBG_HDRS_ALL 1
#endif

#if defined(POINTER_MASK) && !defined(POINTER_SHIFT)
#  define POINTER_SHIFT 0
#elif !defined(POINTER_MASK) && defined(POINTER_SHIFT)
#  define POINTER_MASK GC_WORD_MAX
#endif

#if defined(FIXUP_POINTER)
/* Custom `FIXUP_POINTER(p)`. */
#  define NEED_FIXUP_POINTER
#elif defined(DYNAMIC_POINTER_MASK)
#  define FIXUP_POINTER(p) \
    (p = (ptr_t)((((word)(p)) & GC_pointer_mask) << GC_pointer_shift))
#  undef POINTER_MASK
#  undef POINTER_SHIFT
#  define NEED_FIXUP_POINTER
#elif defined(POINTER_MASK)
/*
 * Note: extra parentheses around custom-defined `POINTER_MASK` and
 *`POINTER_SHIFT` are intentional.
 */
#  define FIXUP_POINTER(p) \
    (p = (ptr_t)(((word)(p) & (POINTER_MASK)) << (POINTER_SHIFT)))
#  define NEED_FIXUP_POINTER
#else
#  define FIXUP_POINTER(p) (void)(p)
#endif

#ifdef LINT2
/*
 * A macro (based on a tricky expression) to prevent false warnings
 * like "Array compared to 0", "Comparison of identical expressions",
 * "Untrusted loop bound" output by some static code analysis tools.
 * The argument should not be a literal value.  The result is
 * converted to `word` type.  (Actually, `GC_word` is used instead of
 * `word` type as the latter might be undefined at the place of use.)
 */
#  define COVERT_DATAFLOW(w) (~(GC_word)(w) ^ (~(GC_word)0))
#else
#  define COVERT_DATAFLOW(w) ((GC_word)(w))
#endif

#if CPP_PTRSZ > CPP_WORDSZ
/* TODO: Cannot use tricky operations on a pointer. */
#  define COVERT_DATAFLOW_P(p) ((ptr_t)(p))
#else
#  define COVERT_DATAFLOW_P(p) ((ptr_t)COVERT_DATAFLOW(p))
#endif

#if defined(REDIRECT_MALLOC) && defined(THREADS) && !defined(LINUX) \
    && !defined(REDIRECT_MALLOC_IN_HEADER)
/*
 * May work on other platforms (e.g. Darwin) provided the client
 * ensures all the client threads are registered with the collector,
 * e.g. by using the preprocessor-based interception of the thread
 * primitives (i.e., define `GC_THREADS` and include `gc.h` file from
 * all the client files those are using `pthread_create` and friends).
 */
#endif

EXTERN_C_END

#endif /* GCCONFIG_H */


#ifdef __cplusplus
typedef bool GC_bool;
#elif defined(__BORLANDC__) || defined(__WATCOMC__)
typedef int GC_bool;
#else
typedef char GC_bool;
#endif

#if defined(__cplusplus) && !defined(ANY_MSWIN)
/* Avoid macro redefinition on a Windows platform. */
#  define TRUE true
#  define FALSE false
#else
#  define TRUE 1
#  define FALSE 0
#endif

#if !defined(GC_ATOMIC_UNCOLLECTABLE) && defined(ATOMIC_UNCOLLECTABLE)
/* For compatibility with old-style naming. */
#  define GC_ATOMIC_UNCOLLECTABLE
#endif

#ifndef GC_INNER
/*
 * This tagging macro must be used at the start of every variable definition
 * which is declared with `GC_EXTERN`.  Should be also used for the GC-scope
 * function definitions and prototypes.  Must not be used in `gcconfig.h`
 * file.  Should not be used for the debugging-only functions.
 */
#  if defined(GC_DLL) && defined(__GNUC__) && !defined(ANY_MSWIN)
#    if GC_GNUC_PREREQ(4, 0) && !defined(GC_NO_VISIBILITY)
/* See the corresponding `GC_API` definition. */
#      define GC_INNER __attribute__((__visibility__("hidden")))
#    else
/* The attribute is unsupported. */
#      define GC_INNER /*< empty */
#    endif
#  else
#    define GC_INNER /*< empty */
#  endif

#  define GC_EXTERN extern GC_INNER
/*
 * Used only for the GC-scope variables (prefixed with `GC_`) declared
 * in the private header files.  Must not be used for thread-local
 * variables.  Must not be used in `gcconfig.h` file.
 * The corresponding variable definition must start with `GC_INNER`.
 * Should not be used for the debugging- or profiling-only variables.
 * As of now, there are some other exceptions, e.g. for the variables
 * that are known to be used by some popular clients.
 */
#endif /* !GC_INNER */

#ifdef __cplusplus
/* `register` storage specifier is deprecated in C++11. */
#  define REGISTER /*< empty */
#else
/*
 * Used only for several local variables in the performance-critical
 * functions.  Should not be used for new code.
 */
#  define REGISTER register
#endif

#if defined(CPPCHECK)
#  define MACRO_BLKSTMT_BEGIN {
#  define MACRO_BLKSTMT_END }
#  define LOCAL_VAR_INIT_OK = 0 /*< to avoid "uninit var" false positive */
#else
#  define MACRO_BLKSTMT_BEGIN do {
#  define MACRO_BLKSTMT_END \
    }                       \
    while (0)
#  define LOCAL_VAR_INIT_OK /*< empty */
#endif

#if defined(M68K) && defined(__GNUC__)
/*
 * By default, `__alignof__(void *)` is 2 on m68k architecture.
 * Use this attribute to have the machine-word alignment (i.e. 4-byte one
 * on the given 32-bit architecture).
 */
#  define GC_ATTR_PTRT_ALIGNED __attribute__((__aligned__(sizeof(ptr_t))))
#else
#  define GC_ATTR_PTRT_ALIGNED /*< empty */
#endif

#ifdef CHERI_PURECAP
#  include <cheriintrin.h>
#endif

EXTERN_C_BEGIN

typedef GC_uintptr_t GC_funcptr_uint;
#define FUNCPTR_IS_DATAPTR

typedef unsigned int unsigned32;

#define hblk GC_hblk_s
struct hblk;

typedef struct hblkhdr hdr;

EXTERN_C_END

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

#ifndef GC_HEADERS_H
#define GC_HEADERS_H

#if !defined(GC_PRIVATE_H) && !defined(CPPCHECK)
#  error gc_hdrs.h should be included from gc_priv.h
#endif

#if CPP_WORDSZ != 32 && CPP_WORDSZ < 36 && !defined(CPPCHECK)
#  error Get a real machine
#endif

EXTERN_C_BEGIN

/*
 * The 2-level tree data structure that is used to find block headers.
 * If there are more than 32 bits in a pointer, the top level is a hash
 * table.
 *
 * This defines `HDR()`, `GET_HDR()`, and `SET_HDR()`, the main macros
 * used to retrieve and set object headers.
 *
 * We take advantage of a header lookup cache.  This is a locally declared
 * direct mapped cache, used inside the marker.  The `HC_GET_HDR()` macro
 * uses and maintains this cache.  Assuming we get reasonable hit rates,
 * this saves a few memory references from each pointer validation.
 */

#if CPP_WORDSZ > 32
#  define HASH_TL
#endif

/* Define appropriate out-degrees for each of the two tree levels. */
#if defined(LARGE_CONFIG) || !defined(SMALL_CONFIG)
#  define LOG_BOTTOM_SZ 10
#else
/* Keep top index size reasonable with smaller blocks. */
#  define LOG_BOTTOM_SZ 11
#endif
#define BOTTOM_SZ (1 << LOG_BOTTOM_SZ)

#ifndef HASH_TL
#  define LOG_TOP_SZ (CPP_WORDSZ - LOG_BOTTOM_SZ - LOG_HBLKSIZE)
#else
#  define LOG_TOP_SZ 11
#endif
#define TOP_SZ (1 << LOG_TOP_SZ)

#ifdef COUNT_HDR_CACHE_HITS
extern word GC_hdr_cache_hits; /*< used for debugging/profiling */
extern word GC_hdr_cache_misses;
#  define HC_HIT() (void)(++GC_hdr_cache_hits)
#  define HC_MISS() (void)(++GC_hdr_cache_misses)
#else
#  define HC_HIT() (void)0
#  define HC_MISS() (void)0
#endif

typedef struct hce {
  word block_addr; /*< right-shifted by `LOG_HBLKSIZE` */
  hdr *hce_hdr;
} hdr_cache_entry;

#define HDR_CACHE_SIZE 8 /*< a power of two */

#define DECLARE_HDR_CACHE hdr_cache_entry hdr_cache[HDR_CACHE_SIZE]

#define INIT_HDR_CACHE BZERO(hdr_cache, sizeof(hdr_cache))

#define HCE(h) (hdr_cache + ((ADDR(h) >> LOG_HBLKSIZE) & (HDR_CACHE_SIZE - 1)))

#define HCE_VALID_FOR(hce, h) ((hce)->block_addr == (ADDR(h) >> LOG_HBLKSIZE))

#define HCE_HDR(h) ((hce)->hce_hdr)

#ifdef PRINT_BLACK_LIST
/*
 * Handle a header cache miss.  Returns a pointer to the header
 * corresponding to `p`, if the latter can possibly be a valid object
 * pointer, and `NULL` otherwise.  Guaranteed to return `NULL` for
 * a pointer past the first page of an object unless both
 * `GC_all_interior_pointers` is set and `p` is in fact a valid object
 * pointer.  Never returns a pointer to a free `hblk`.
 */
GC_INNER hdr *GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce,
                                   ptr_t source);

#  define HEADER_CACHE_MISS(p, hce, source) \
    GC_header_cache_miss(p, hce, source)
#else
GC_INNER hdr *GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce);
#  define HEADER_CACHE_MISS(p, hce, source) GC_header_cache_miss(p, hce)
#endif

/*
 * Set `hhdr` to the header for `p`.  Analogous to `GET_HDR()` below,
 * except that in the case of large objects, it gets the header for the
 * object beginning if `GC_all_interior_pointers` is true.  Sets `hhdr`
 * to `NULL` if `p` points to somewhere other than the first page of an
 * object, and it is not a valid pointer to the object.
 */
#define HC_GET_HDR(p, hhdr, source)                \
  { /*< cannot use `do ... while (0)` here */      \
    hdr_cache_entry *hce = HCE(p);                 \
    if (LIKELY(HCE_VALID_FOR(hce, p))) {           \
      HC_HIT();                                    \
      hhdr = hce->hce_hdr;                         \
    } else {                                       \
      hhdr = HEADER_CACHE_MISS(p, hce, source);    \
      if (NULL == hhdr)                            \
        break; /*< go to the enclosing loop end */ \
    }                                              \
  }

typedef struct bi {
  /*
   * The bottom-level index contains one of three kinds of values:
   *   - 0 means we are not responsible for this block, or this is
   *     a block other than the first one in a free block;
   *   - 1 < (long)`x` <= `MAX_JUMP` means the block starts at least
   *     `x * HBLKSIZE` bytes before the current address;
   *   - a valid pointer points to a `hdr` structure (the above cannot
   *     be valid pointers due to the `GET_MEM()` return convention).
   */
  hdr *index[BOTTOM_SZ];

  /*
   * All indices are linked in the ascending and descending orders,
   * respectively.
   */
  struct bi *asc_link;
  struct bi *desc_link;

  word key; /*< high-order address bits */
#ifdef HASH_TL
  struct bi *hash_link; /*< hash chain link */
#endif
} bottom_index;

#define MAX_JUMP (HBLKSIZE - 1)

#define HDR_FROM_BI(bi, p) \
  (bi)->index[(ADDR(p) >> LOG_HBLKSIZE) & (BOTTOM_SZ - 1)]
#ifndef HASH_TL
#  define BI(p) GC_top_index[ADDR(p) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE)]
#  define HDR_INNER(p) HDR_FROM_BI(BI(p), p)
#  ifdef SMALL_CONFIG
#    define HDR(p) GC_find_header(p)
#  else
#    define HDR(p) HDR_INNER(p)
#  endif
#  define GET_BI(p, bottom_indx) (void)((bottom_indx) = BI(p))
#  define GET_HDR(p, hhdr) (void)((hhdr) = HDR(p))
#  define SET_HDR(p, hhdr) (void)(HDR_INNER(p) = (hhdr))
#  define GET_HDR_ADDR(p, ha) (void)((ha) = &HDR_INNER(p))
#else
/* A hash function for the tree top level. */
#  define TL_HASH(hi) ((hi) & (TOP_SZ - 1))
/* Set `bottom_indx` to point to the bottom index for address `p`. */
#  define GET_BI(p, bottom_indx)                                    \
    do {                                                            \
      REGISTER word hi = ADDR(p) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE); \
      REGISTER bottom_index *_bi = GC_top_index[TL_HASH(hi)];       \
      while (_bi->key != hi && _bi != GC_all_nils)                  \
        _bi = _bi->hash_link;                                       \
      (bottom_indx) = _bi;                                          \
    } while (0)
#  define GET_HDR_ADDR(p, ha)     \
    do {                          \
      REGISTER bottom_index *bi;  \
      GET_BI(p, bi);              \
      (ha) = &HDR_FROM_BI(bi, p); \
    } while (0)
#  define GET_HDR(p, hhdr)  \
    do {                    \
      REGISTER hdr **_ha;   \
      GET_HDR_ADDR(p, _ha); \
      (hhdr) = *_ha;        \
    } while (0)
#  define SET_HDR(p, hhdr)          \
    do {                            \
      REGISTER bottom_index *bi;    \
      GET_BI(p, bi);                \
      GC_ASSERT(bi != GC_all_nils); \
      HDR_FROM_BI(bi, p) = (hhdr);  \
    } while (0)
#  define HDR(p) GC_find_header(p)
#endif

/*
 * Is the result a forwarding address to someplace closer to the
 * beginning of the block or `NULL`?
 */
#define IS_FORWARDING_ADDR_OR_NIL(hhdr) ((size_t)ADDR(hhdr) <= MAX_JUMP)

/*
 * Get an `HBLKSIZE`-aligned address closer to the beginning of the
 * block `h`.  Assumes that `hhdr` is equal to `HDR(h)`,
 * `IS_FORWARDING_ADDR(hhdr)` is true and `hhdr` is not `NULL`.
 * `HDR(result)` is expected to be non-`NULL`.
 */
#define FORWARDED_ADDR(h, hhdr) \
  ((struct hblk *)(h) - (size_t)(GC_uintptr_t)(hhdr))

EXTERN_C_END

#endif /* GC_HEADERS_H */


#ifndef GC_ATTR_NO_SANITIZE_ADDR
#  ifndef ADDRESS_SANITIZER
#    define GC_ATTR_NO_SANITIZE_ADDR /*< empty */
#  elif GC_CLANG_PREREQ(3, 8)
#    define GC_ATTR_NO_SANITIZE_ADDR \
      __attribute__((__no_sanitize__("address")))
#  else
#    define GC_ATTR_NO_SANITIZE_ADDR __attribute__((__no_sanitize_address__))
#  endif
#endif /* !GC_ATTR_NO_SANITIZE_ADDR */

#ifndef GC_ATTR_NO_SANITIZE_MEMORY
#  ifndef MEMORY_SANITIZER
#    define GC_ATTR_NO_SANITIZE_MEMORY /*< empty */
#  elif GC_CLANG_PREREQ(3, 8)
#    define GC_ATTR_NO_SANITIZE_MEMORY \
      __attribute__((__no_sanitize__("memory")))
#  else
#    define GC_ATTR_NO_SANITIZE_MEMORY __attribute__((__no_sanitize_memory__))
#  endif
#endif /* !GC_ATTR_NO_SANITIZE_MEMORY */

#ifndef GC_ATTR_NO_SANITIZE_THREAD
#  ifndef THREAD_SANITIZER
#    define GC_ATTR_NO_SANITIZE_THREAD /*< empty */
#  elif GC_CLANG_PREREQ(3, 8)
#    define GC_ATTR_NO_SANITIZE_THREAD \
      __attribute__((__no_sanitize__("thread")))
#  else
/*
 * It seems that `no_sanitize_thread` attribute has no effect if the
 * function is inlined (as of gcc-11.1.0, at least).
 */
#    define GC_ATTR_NO_SANITIZE_THREAD \
      GC_ATTR_NOINLINE __attribute__((__no_sanitize_thread__))
#  endif
#endif /* !GC_ATTR_NO_SANITIZE_THREAD */

#define GC_ATTR_NO_SANITIZE_ADDR_MEM_THREAD           \
  GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY \
      GC_ATTR_NO_SANITIZE_THREAD

#ifndef UNUSED_ARG
#  define UNUSED_ARG(arg) ((void)(arg))
#endif

#ifdef HAVE_CONFIG_H
/* The `inline` keyword is determined by `AC_C_INLINE` of `autoconf`. */
#  define GC_INLINE static inline
#elif defined(_MSC_VER) || defined(__INTEL_COMPILER) || defined(__DMC__) \
    || (GC_GNUC_PREREQ(3, 0) && defined(__STRICT_ANSI__))                \
    || defined(__BORLANDC__) || defined(__WATCOMC__)
#  define GC_INLINE static __inline
#elif GC_GNUC_PREREQ(3, 0) || defined(__sun)
#  define GC_INLINE static inline
#else
#  define GC_INLINE static
#endif

#ifndef GC_ATTR_NOINLINE
#  if GC_GNUC_PREREQ(4, 0)
#    define GC_ATTR_NOINLINE __attribute__((__noinline__))
#  elif _MSC_VER >= 1400
#    define GC_ATTR_NOINLINE __declspec(noinline)
#  else
#    define GC_ATTR_NOINLINE /*< empty */
#  endif
#endif

#ifndef GC_API_OSCALL
/* This is used to identify GC routines called by name from OS. */
#  if defined(__GNUC__)
#    if GC_GNUC_PREREQ(4, 0) && !defined(GC_NO_VISIBILITY)
/* Same as `GC_API` macro if `GC_DLL` one is defined. */
#      define GC_API_OSCALL extern __attribute__((__visibility__("default")))
#    else
/* The attribute is unsupported. */
#      define GC_API_OSCALL extern
#    endif
#  else
#    define GC_API_OSCALL GC_API
#  endif
#endif

#ifndef GC_API_PRIV
#  define GC_API_PRIV GC_API
#endif

#ifndef GC_API_PATCHABLE
#  define GC_API_PATCHABLE GC_ATTR_NOINLINE GC_API
#endif

#if defined(THREADS) && !defined(NN_PLATFORM_CTR)
/*
 * Copyright (c) 2017 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * This is a private collector header which provides an implementation of
 * `libatomic_ops` subset primitives sufficient for the collector assuming
 * that gcc atomic intrinsics are available (and have the correct
 * implementation).  This is enabled by defining `GC_BUILTIN_ATOMIC` macro.
 * Otherwise, `libatomic_ops` library is used to define the primitives.
 */

#ifndef GC_ATOMIC_OPS_H
#define GC_ATOMIC_OPS_H

#ifdef GC_BUILTIN_ATOMIC




#  ifdef __cplusplus
extern "C" {
#  endif

typedef size_t AO_t;

#  ifdef GC_PRIVATE_H /*< i.e. have `GC_INLINE` */
#    define AO_INLINE GC_INLINE
#  else
#    define AO_INLINE static __inline
#  endif

#  if !defined(THREAD_SANITIZER) && !defined(GC_PRIVATE_H)
/* Similar to that in `gcconfig.h` file. */
#    if defined(__has_feature)
#      if __has_feature(thread_sanitizer)
#        define THREAD_SANITIZER
#      endif
#    elif defined(__SANITIZE_THREAD__)
#      define THREAD_SANITIZER
#    endif
#  endif /* !THREAD_SANITIZER && !GC_PRIVATE_H */

typedef unsigned char AO_TS_t;
#  define AO_TS_CLEAR 0
#  define AO_TS_INITIALIZER ((AO_TS_t)AO_TS_CLEAR)
#  if defined(__GCC_ATOMIC_TEST_AND_SET_TRUEVAL) && !defined(CPPCHECK)
#    define AO_TS_SET __GCC_ATOMIC_TEST_AND_SET_TRUEVAL
#  else
#    define AO_TS_SET (AO_TS_t)1 /*< true */
#  endif
#  define AO_CLEAR(p) __atomic_clear(p, __ATOMIC_RELEASE)
#  define AO_test_and_set_acquire(p) \
    (__atomic_test_and_set(p, __ATOMIC_ACQUIRE) ? AO_TS_SET : AO_TS_CLEAR)
#  define AO_HAVE_test_and_set_acquire

#  define AO_compiler_barrier() __atomic_signal_fence(__ATOMIC_SEQ_CST)

#  if defined(THREAD_SANITIZER) && !defined(AO_USE_ATOMIC_THREAD_FENCE)
/*
 * Workaround a compiler warning (reported by gcc-11, at least) that
 * `__atomic_thread_fence` is unsupported with thread sanitizer.
 */
AO_INLINE void
AO_nop_full(void)
{
  volatile AO_TS_t dummy = AO_TS_INITIALIZER;
  (void)__atomic_test_and_set(&dummy, __ATOMIC_SEQ_CST);
}
#  else
#    define AO_nop_full() __atomic_thread_fence(__ATOMIC_SEQ_CST)
#  endif
#  define AO_HAVE_nop_full

#  define AO_fetch_and_add(p, v) __atomic_fetch_add(p, v, __ATOMIC_RELAXED)
#  define AO_HAVE_fetch_and_add
#  define AO_fetch_and_add1(p) AO_fetch_and_add(p, 1)
#  define AO_HAVE_fetch_and_add1
#  define AO_fetch_and_sub1(p) AO_fetch_and_add(p, ~(AO_t)0 /* -1 */)
#  define AO_HAVE_fetch_and_sub1

#  define AO_or(p, v) (void)__atomic_or_fetch(p, v, __ATOMIC_RELAXED)
#  define AO_HAVE_or

#  define AO_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
#  define AO_HAVE_load
#  define AO_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
#  define AO_HAVE_load_acquire
/*
 * `AO_load_acquire_read(p)` is not defined as it is unused, but we need
 * its `AO_HAVE_` macro defined.
 */
#  define AO_HAVE_load_acquire_read

#  define AO_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
#  define AO_HAVE_store
#  define AO_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
#  define AO_HAVE_store_release
#  define AO_store_release_write(p, v) AO_store_release(p, v)
#  define AO_HAVE_store_release_write

#  define AO_char_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
#  define AO_HAVE_char_load
#  define AO_char_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
#  define AO_HAVE_char_store
#  define AO_char_fetch_and_add1(p) __atomic_fetch_add(p, 1, __ATOMIC_RELAXED)
#  define AO_HAVE_char_fetch_and_add1

#  ifdef AO_REQUIRE_CAS
AO_INLINE int
AO_compare_and_swap_release(volatile AO_t *p, AO_t ov, AO_t nv)
{
  return (int)__atomic_compare_exchange_n(p, &ov, nv, 0, __ATOMIC_RELEASE,
                                          __ATOMIC_RELAXED /* on fail */);
}
#    define AO_HAVE_compare_and_swap_release
#  endif

#  ifdef __cplusplus
} /* extern "C" */
#  endif

#  ifndef NO_LOCKFREE_AO_OR
/* `__atomic_or_fetch()` is assumed to be lock-free. */
#    define HAVE_LOCKFREE_AO_OR 1
#  endif

#else
/* Fall back to `libatomic_ops`. */
#  include "atomic_ops.h"

/*
 * `AO_compiler_barrier`, `AO_load` and `AO_store` should be defined
 * for all targets; the rest of the primitives are guaranteed to exist
 * only if `AO_REQUIRE_CAS` is defined (or if the corresponding
 * `AO_HAVE_` macro is defined).  i686 and x86_64 targets have
 * `AO_nop_full`, `AO_load_acquire`, `AO_store_release`, at least.
 */
#  if (!defined(AO_HAVE_load) || !defined(AO_HAVE_store)) && !defined(CPPCHECK)
#    error AO_load or AO_store is missing; probably old version of atomic_ops
#  endif

#endif /* !GC_BUILTIN_ATOMIC */

#if defined(GC_BUILTIN_ATOMIC) || defined(__CHERI_PURE_CAPABILITY__)
/*
 * Assume that gcc atomic intrinsics are available (and have correct
 * implementation).  `p` should be of a pointer to `ptr_t` (`char *`) value.
 */
#  define GC_cptr_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
#  define GC_cptr_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
#  define GC_cptr_load_acquire_read(p) GC_cptr_load_acquire(p)
#  define GC_cptr_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
#  define GC_cptr_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
#  define GC_cptr_store_release_write(p, v) GC_cptr_store_release(p, v)
#  ifdef AO_REQUIRE_CAS
AO_INLINE int
GC_cptr_compare_and_swap(char *volatile *p, char *ov, char *nv)
{
  return (int)__atomic_compare_exchange_n(p, &ov, nv, 0, __ATOMIC_RELAXED,
                                          __ATOMIC_RELAXED);
}
#  endif
#else
/*
 * Redirect to the `AO_` primitives.  Assume the size of `AO_t` matches
 * that of a pointer.
 */
#  define GC_cptr_load(p) (char *)AO_load((volatile AO_t *)(p))
#  define GC_cptr_load_acquire(p) (char *)AO_load_acquire((volatile AO_t *)(p))
#  define GC_cptr_load_acquire_read(p) \
    (char *)AO_load_acquire_read((volatile AO_t *)(p))
#  define GC_cptr_store(p, v) AO_store((volatile AO_t *)(p), (AO_t)(v))
#  define GC_cptr_store_release(p, v) \
    AO_store_release((volatile AO_t *)(p), (AO_t)(v))
#  define GC_cptr_store_release_write(p, v) \
    AO_store_release_write((volatile AO_t *)(p), (AO_t)(v))
#  ifdef AO_REQUIRE_CAS
#    define GC_cptr_compare_and_swap(p, ov, nv) \
      AO_compare_and_swap((volatile AO_t *)(p), (AO_t)(ov), (AO_t)(nv))
#  endif
#endif /* !GC_BUILTIN_ATOMIC */

#endif /* GC_ATOMIC_OPS_H */

#  ifndef AO_HAVE_compiler_barrier
#    define AO_HAVE_compiler_barrier 1
#  endif
#endif

#ifdef ANY_MSWIN
#  ifndef WIN32_LEAN_AND_MEAN
#    define WIN32_LEAN_AND_MEAN 1
#  endif
#  define NOSERVICE
#  include <windows.h>

/* This is included strictly after the platform `windows.h` file. */
#  include <winbase.h>
#endif /* ANY_MSWIN */

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
 * Copyright (c) 2008-2025 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

#ifndef GC_LOCKS_H
#define GC_LOCKS_H

#if !defined(GC_PRIVATE_H) && !defined(CPPCHECK)
#  error gc_locks.h should be included from gc_priv.h
#endif

/*
 * Mutual exclusion between allocator/collector routines.  Needed if
 * there is more than one allocator thread.  Note that `I_HOLD_LOCK`,
 * `I_DONT_HOLD_LOCK` and `I_HOLD_READER_LOCK` macros are used only
 * positively in assertions, and may return `TRUE` in the "do not know"
 * case.
 */

#ifdef THREADS

EXTERN_C_BEGIN

#  if defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH)
extern void GC_lock(void);
extern void GC_unlock(void);
#    define UNCOND_LOCK() GC_lock()
#    define UNCOND_UNLOCK() GC_unlock()
#    ifdef GC_ASSERTIONS
#      define SET_LOCK_HOLDER() (void)0
#    endif
#  endif

#  if (!defined(AO_HAVE_test_and_set_acquire) || defined(GC_WIN32_THREADS) \
       || defined(LINT2) || defined(RTEMS) || defined(SN_TARGET_PS3)       \
       || defined(BASE_ATOMIC_OPS_EMULATED) || defined(USE_RWLOCK))        \
      && defined(GC_PTHREADS)
#    define USE_PTHREAD_LOCKS
#    undef USE_SPIN_LOCK
#    if (defined(GC_WIN32_THREADS) || defined(LINT2) || defined(USE_RWLOCK)) \
        && !defined(NO_PTHREAD_TRYLOCK)
/*
 * `pthread_mutex_trylock` may not win in `GC_lock` on Win32, due to
 * built-in support for spinning first?
 */
#      define NO_PTHREAD_TRYLOCK
#    endif
#  endif

#  if defined(GC_WIN32_THREADS) && !defined(USE_PTHREAD_LOCKS) \
      || defined(GC_PTHREADS)
/* A value which is not equal to `NUMERIC_THREAD_ID(id)` for any thread. */
#    define NO_THREAD ((unsigned long)(-1L))
#    ifdef GC_ASSERTIONS
GC_EXTERN unsigned long GC_lock_holder;
#      define UNSET_LOCK_HOLDER() (void)(GC_lock_holder = NO_THREAD)
#    endif
#  endif /* GC_WIN32_THREADS || GC_PTHREADS */

#  if defined(GC_WIN32_THREADS) && !defined(USE_PTHREAD_LOCKS)
#    ifdef USE_RWLOCK
GC_EXTERN SRWLOCK GC_allocate_ml;
#    else
GC_EXTERN CRITICAL_SECTION GC_allocate_ml;
#    endif
#    ifdef GC_ASSERTIONS
#      define SET_LOCK_HOLDER() (void)(GC_lock_holder = GetCurrentThreadId())
#      define I_HOLD_LOCK() \
        (!GC_need_to_lock || GC_lock_holder == GetCurrentThreadId())
#      ifdef THREAD_SANITIZER
#        define I_DONT_HOLD_LOCK() TRUE /*< conservatively say yes */
#      else
#        define I_DONT_HOLD_LOCK() \
          (!GC_need_to_lock || GC_lock_holder != GetCurrentThreadId())
#      endif
#      ifdef USE_RWLOCK
#        define UNCOND_READER_LOCK()               \
          {                                        \
            GC_ASSERT(I_DONT_HOLD_LOCK());         \
            AcquireSRWLockShared(&GC_allocate_ml); \
          }
#        define UNCOND_READER_UNLOCK()             \
          {                                        \
            GC_ASSERT(I_DONT_HOLD_LOCK());         \
            ReleaseSRWLockShared(&GC_allocate_ml); \
          }
#        define UNCOND_LOCK()                         \
          {                                           \
            GC_ASSERT(I_DONT_HOLD_LOCK());            \
            AcquireSRWLockExclusive(&GC_allocate_ml); \
            SET_LOCK_HOLDER();                        \
          }
#        define UNCOND_UNLOCK()                       \
          {                                           \
            GC_ASSERT(I_HOLD_LOCK());                 \
            UNSET_LOCK_HOLDER();                      \
            ReleaseSRWLockExclusive(&GC_allocate_ml); \
          }
#      else
#        define UNCOND_LOCK()                      \
          {                                        \
            GC_ASSERT(I_DONT_HOLD_LOCK());         \
            EnterCriticalSection(&GC_allocate_ml); \
            SET_LOCK_HOLDER();                     \
          }
#        define UNCOND_UNLOCK()                    \
          {                                        \
            GC_ASSERT(I_HOLD_LOCK());              \
            UNSET_LOCK_HOLDER();                   \
            LeaveCriticalSection(&GC_allocate_ml); \
          }
#      endif
#    else
#      ifdef USE_RWLOCK
#        define UNCOND_READER_LOCK() AcquireSRWLockShared(&GC_allocate_ml)
#        define UNCOND_READER_UNLOCK() ReleaseSRWLockShared(&GC_allocate_ml)
#        define UNCOND_LOCK() AcquireSRWLockExclusive(&GC_allocate_ml)
#        define UNCOND_UNLOCK() ReleaseSRWLockExclusive(&GC_allocate_ml)
#      else
#        define UNCOND_LOCK() EnterCriticalSection(&GC_allocate_ml)
#        define UNCOND_UNLOCK() LeaveCriticalSection(&GC_allocate_ml)
#      endif
#    endif /* !GC_ASSERTIONS */
#  elif defined(GC_PTHREADS)
EXTERN_C_END
#    include <pthread.h>
EXTERN_C_BEGIN
/*
 * POSIX allows `pthread_t` to be a structure type, though it rarely is.
 * Unfortunately, we need to use a `pthread_t` to index a data structure.
 * It also helps if comparisons do not involve a function call.
 * Hence we introduce platform-dependent macros to compare `pthread_t` ids
 * and to map them to integers (of `unsigned long` type).  This mapping
 * does not need to result in different values for each thread, though
 * that should be true as much as possible.
 */
#    if !defined(GC_WIN32_PTHREADS)
#      define NUMERIC_THREAD_ID(id) ((unsigned long)(GC_uintptr_t)(id))
#      define THREAD_EQUAL(id1, id2) ((id1) == (id2))
#      define NUMERIC_THREAD_ID_UNIQUE
#    elif defined(__WINPTHREADS_VERSION_MAJOR) /*< winpthreads */
#      define NUMERIC_THREAD_ID(id) ((unsigned long)(id))
#      define THREAD_EQUAL(id1, id2) ((id1) == (id2))
/* `NUMERIC_THREAD_ID()` is 32-bit and, thus, not unique on Win64. */
#      ifndef _WIN64
#        define NUMERIC_THREAD_ID_UNIQUE
#      endif
#    else /* pthreads-win32 */
#      define NUMERIC_THREAD_ID(id) ((unsigned long)(word)(id.p))
/*
 * The platform on which `pthread_t` is a structure.
 * Using documented internal details of pthreads-win32 library.
 * Faster than `pthread_equal()`.  Should not change with the
 * future versions of pthreads-win32 library.
 */
#      define THREAD_EQUAL(id1, id2) (id1.p == id2.p && id1.x == id2.x)
/*
 * Generic definitions based on `pthread_equal()` always work but will
 * result in poor performance (as `NUMERIC_THREAD_ID()` might give
 * a constant value) and weak assertion checking.
 */
#      undef NUMERIC_THREAD_ID_UNIQUE
#    endif

#    ifdef SN_TARGET_PSP2
EXTERN_C_END
#      include "psp2-support.h"
EXTERN_C_BEGIN
GC_EXTERN WapiMutex GC_allocate_ml_PSP2;
#      define UNCOND_LOCK()                           \
        {                                             \
          int res;                                    \
          GC_ASSERT(I_DONT_HOLD_LOCK());              \
          res = PSP2_MutexLock(&GC_allocate_ml_PSP2); \
          GC_ASSERT(0 == res);                        \
          (void)res;                                  \
          SET_LOCK_HOLDER();                          \
        }
#      define UNCOND_UNLOCK()                           \
        {                                               \
          int res;                                      \
          GC_ASSERT(I_HOLD_LOCK());                     \
          UNSET_LOCK_HOLDER();                          \
          res = PSP2_MutexUnlock(&GC_allocate_ml_PSP2); \
          GC_ASSERT(0 == res);                          \
          (void)res;                                    \
        }

#    elif (!defined(THREAD_LOCAL_ALLOC) || defined(USE_SPIN_LOCK))   \
        && !defined(USE_PTHREAD_LOCKS) && !defined(THREAD_SANITIZER) \
        && !defined(USE_RWLOCK)
/*
 * In the `THREAD_LOCAL_ALLOC` case, the allocator lock tends to
 * be held for long periods, if it is held at all.
 * Thus spinning and sleeping for fixed periods are likely to result
 * in significant wasted time.  We thus rely mostly on queued locks.
 */
#      undef USE_SPIN_LOCK
#      define USE_SPIN_LOCK
GC_INNER void GC_lock(void);
#      ifdef GC_ASSERTIONS
#        define UNCOND_LOCK()                                            \
          {                                                              \
            GC_ASSERT(I_DONT_HOLD_LOCK());                               \
            if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_SET) \
              GC_lock();                                                 \
            SET_LOCK_HOLDER();                                           \
          }
#        define UNCOND_UNLOCK()          \
          {                              \
            GC_ASSERT(I_HOLD_LOCK());    \
            UNSET_LOCK_HOLDER();         \
            AO_CLEAR(&GC_allocate_lock); \
          }
#      else
#        define UNCOND_LOCK()                                            \
          {                                                              \
            if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_SET) \
              GC_lock();                                                 \
          }
#        define UNCOND_UNLOCK() AO_CLEAR(&GC_allocate_lock)
#      endif /* !GC_ASSERTIONS */
#    else
#      ifndef USE_PTHREAD_LOCKS
#        define USE_PTHREAD_LOCKS
#      endif
#    endif /* THREAD_LOCAL_ALLOC || USE_PTHREAD_LOCKS */
#    ifdef USE_PTHREAD_LOCKS
EXTERN_C_END
#      include <pthread.h>
EXTERN_C_BEGIN
#      ifdef GC_ASSERTIONS
GC_INNER void GC_lock(void);
#        define UNCOND_LOCK()              \
          {                                \
            GC_ASSERT(I_DONT_HOLD_LOCK()); \
            GC_lock();                     \
            SET_LOCK_HOLDER();             \
          }
#      endif
#      ifdef USE_RWLOCK
GC_EXTERN pthread_rwlock_t GC_allocate_ml;
#        ifdef GC_ASSERTIONS
#          define UNCOND_READER_LOCK()                      \
            {                                               \
              GC_ASSERT(I_DONT_HOLD_LOCK());                \
              (void)pthread_rwlock_rdlock(&GC_allocate_ml); \
            }
#          define UNCOND_READER_UNLOCK()                    \
            {                                               \
              GC_ASSERT(I_DONT_HOLD_LOCK());                \
              (void)pthread_rwlock_unlock(&GC_allocate_ml); \
            }
#          define UNCOND_UNLOCK()                           \
            {                                               \
              GC_ASSERT(I_HOLD_LOCK());                     \
              UNSET_LOCK_HOLDER();                          \
              (void)pthread_rwlock_unlock(&GC_allocate_ml); \
            }
#        else
#          define UNCOND_READER_LOCK() \
            (void)pthread_rwlock_rdlock(&GC_allocate_ml)
#          define UNCOND_READER_UNLOCK() UNCOND_UNLOCK()
#          define UNCOND_LOCK() (void)pthread_rwlock_wrlock(&GC_allocate_ml)
#          define UNCOND_UNLOCK() (void)pthread_rwlock_unlock(&GC_allocate_ml)
#        endif /* !GC_ASSERTIONS */
#      else
GC_EXTERN pthread_mutex_t GC_allocate_ml;
#        ifdef GC_ASSERTIONS
#          define UNCOND_UNLOCK()                    \
            {                                        \
              GC_ASSERT(I_HOLD_LOCK());              \
              UNSET_LOCK_HOLDER();                   \
              pthread_mutex_unlock(&GC_allocate_ml); \
            }
#        else
#          if defined(NO_PTHREAD_TRYLOCK)
#            define UNCOND_LOCK() pthread_mutex_lock(&GC_allocate_ml)
#          else
GC_INNER void GC_lock(void);
#            define UNCOND_LOCK()                                \
              {                                                  \
                if (pthread_mutex_trylock(&GC_allocate_ml) != 0) \
                  GC_lock();                                     \
              }
#          endif
#          define UNCOND_UNLOCK() pthread_mutex_unlock(&GC_allocate_ml)
#        endif /* !GC_ASSERTIONS */
#      endif
#    endif /* USE_PTHREAD_LOCKS */
#    ifdef GC_ASSERTIONS
/* The allocator lock holder. */
#      define SET_LOCK_HOLDER() \
        (void)(GC_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
#      define I_HOLD_LOCK() \
        (!GC_need_to_lock   \
         || GC_lock_holder == NUMERIC_THREAD_ID(pthread_self()))
#      if !defined(NUMERIC_THREAD_ID_UNIQUE) || defined(THREAD_SANITIZER)
#        define I_DONT_HOLD_LOCK() TRUE /*< conservatively say yes */
#      else
#        define I_DONT_HOLD_LOCK() \
          (!GC_need_to_lock        \
           || GC_lock_holder != NUMERIC_THREAD_ID(pthread_self()))
#      endif
#    endif /* GC_ASSERTIONS */
#    if !defined(GC_WIN32_THREADS)
/*
 * A hint that we are in the collector and holding the allocator lock
 * for an extended period.
 */
GC_EXTERN volatile unsigned char GC_collecting;

#      ifdef AO_HAVE_char_store
#        if defined(GC_ASSERTIONS) && defined(AO_HAVE_char_fetch_and_add1)
/* Ensure `ENTER_GC()` is not used recursively. */
#          define ENTER_GC() GC_ASSERT(!AO_char_fetch_and_add1(&GC_collecting))
#        else
#          define ENTER_GC() AO_char_store(&GC_collecting, TRUE)
#        endif
#        define EXIT_GC() AO_char_store(&GC_collecting, FALSE)
#      else
#        define ENTER_GC() (void)(GC_collecting = TRUE)
#        define EXIT_GC() (void)(GC_collecting = FALSE)
#      endif
#    endif
#  endif /* GC_PTHREADS */
#  if defined(GC_ALWAYS_MULTITHREADED) \
      && (defined(USE_PTHREAD_LOCKS) || defined(USE_SPIN_LOCK))
#    define GC_need_to_lock TRUE
#    define set_need_to_lock() (void)0
#  else
#    if defined(GC_ALWAYS_MULTITHREADED) && !defined(CPPCHECK)
#      error Runtime initialization of the allocator lock is needed!
#    endif
#    undef GC_ALWAYS_MULTITHREADED
#    ifdef THREAD_SANITIZER
/*
 * To workaround TSan false positive (e.g., when `GC_pthread_create()` is
 * called from multiple threads in parallel), do not set `GC_need_to_lock`
 * if it is already set.
 */
#      define set_need_to_lock()                     \
        (void)(*(GC_bool volatile *)&GC_need_to_lock \
                   ? FALSE                           \
                   : (GC_need_to_lock = TRUE))
#    else
#      define set_need_to_lock() (void)(GC_need_to_lock = TRUE)
/* We are multi-threaded now. */
#    endif
#  endif

EXTERN_C_END

#else /* !THREADS */
#  define LOCK() (void)0
#  define UNLOCK() (void)0
#  ifdef GC_ASSERTIONS
/*
 * `I_HOLD_LOCK()` and `I_DONT_HOLD_LOCK()` are used only in positive
 * assertions or to test whether we still need to acquire the allocator
 * lock; `TRUE` works in either case.
 */
#    define I_HOLD_LOCK() TRUE
#    define I_DONT_HOLD_LOCK() TRUE
#  endif
#endif /* !THREADS */

#if defined(UNCOND_LOCK) && !defined(LOCK)
#  if (defined(LINT2) && defined(USE_PTHREAD_LOCKS)) \
      || defined(GC_ALWAYS_MULTITHREADED)
/*
 * Instruct code analysis tools not to care about `GC_need_to_lock`
 * influence to `LOCK`/`UNLOCK` semantic.
 */
#    define LOCK() UNCOND_LOCK()
#    define UNLOCK() UNCOND_UNLOCK()
#    ifdef UNCOND_READER_LOCK
#      define READER_LOCK() UNCOND_READER_LOCK()
#      define READER_UNLOCK() UNCOND_READER_UNLOCK()
#    endif
#  else
/* At least two thread running; need to lock. */
#    define LOCK()           \
      do {                   \
        if (GC_need_to_lock) \
          UNCOND_LOCK();     \
      } while (0)
#    define UNLOCK()         \
      do {                   \
        if (GC_need_to_lock) \
          UNCOND_UNLOCK();   \
      } while (0)
#    ifdef UNCOND_READER_LOCK
#      define READER_LOCK()       \
        do {                      \
          if (GC_need_to_lock)    \
            UNCOND_READER_LOCK(); \
        } while (0)
#      define READER_UNLOCK()       \
        do {                        \
          if (GC_need_to_lock)      \
            UNCOND_READER_UNLOCK(); \
        } while (0)
#    endif
#  endif
#endif /* UNCOND_LOCK && !LOCK */

#ifdef READER_LOCK
#  define HAS_REAL_READER_LOCK
/* TODO: Implement I_HOLD_READER_LOCK, conservatively say yes for now. */
#  define I_HOLD_READER_LOCK() TRUE
#else
#  define READER_LOCK() LOCK()
#  define READER_UNLOCK() UNLOCK()
#  ifdef GC_ASSERTIONS
/*
 * A macro to check that the allocator lock is held at least in the
 * reader mode.
 */
#    define I_HOLD_READER_LOCK() I_HOLD_LOCK()
#  endif
#endif /* !READER_LOCK */

/*
 * A variant of `READER_UNLOCK()` which ensures that data written
 * before the unlock will be visible to the thread which acquires the
 * allocator lock in the exclusive mode.  But according to some `rwlock`
 * documentation: writers synchronize with prior writers and readers.
 */
#define READER_UNLOCK_RELEASE() READER_UNLOCK()

#ifndef ENTER_GC
#  define ENTER_GC()
#  define EXIT_GC()
#endif

#endif /* GC_LOCKS_H */


#ifdef GC_ASSERTIONS
#  define GC_ASSERT(e)                                                   \
    do {                                                                 \
      if (UNLIKELY(!(e))) {                                              \
        GC_err_printf("Assertion failure: %s:%d\n", __FILE__, __LINE__); \
        ABORT("assertion failure");                                      \
      }                                                                  \
    } while (0)
#else
#  define GC_ASSERT(e)
#endif

#include "gc/gc_inline.h"

/*
 * Prevent certain compiler warnings by making a pointer-related cast
 * through a "pointer-sized" numeric type.
 */
#define CAST_THRU_UINTPTR(t, x) ((t)(GC_uintptr_t)(x))

#define CAST_AWAY_VOLATILE_PVOID(p) \
  CAST_THRU_UINTPTR(/* no volatile */ void *, p)

/*
 * Convert an `unsigned` value to a `void` pointer.  Typically used to
 * print a numeric value using "%p" format specifier.  The pointer is not
 * supposed to be dereferenced.
 */
#define NUMERIC_TO_VPTR(v) ((void *)(GC_uintptr_t)(v))

/* Create a `ptr_t` pointer from a number (of `word` type). */
#define MAKE_CPTR(w) ((ptr_t)(GC_uintptr_t)(word)(w))

#define GC_WORD_MAX (~(word)0)

/* Convert given pointer to its address.  Result is of `word` type. */
#ifdef CHERI_PURECAP
#  define ADDR(p) cheri_address_get(p)
#else
#  define ADDR(p) ((word)(GC_uintptr_t)(p))
#endif

#define ADDR_LT(p, q) GC_ADDR_LT(p, q)
#define ADDR_GE(p, q) (!ADDR_LT(p, q))

/*
 * Check whether pointer `p` is in range [`s`, `e_p1`).
 * `p` should not have side effects.
 */
#define ADDR_INSIDE(p, s, e_p1) (ADDR_GE(p, s) && ADDR_LT(p, e_p1))

/* Handy definitions to compare and adjust pointers in a stack. */
#ifdef STACK_GROWS_UP
#  define HOTTER_THAN(p, q) ADDR_LT(q, p) /*< inverse */
#  define MAKE_COOLER(p, d) \
    (void)((p) -= ADDR(p) > (word)((d) * sizeof(*(p))) ? (d) : 0)
#  define MAKE_HOTTER(p, d) (void)((p) += (d))
#else
#  define HOTTER_THAN(p, q) ADDR_LT(p, q)
#  define MAKE_COOLER(p, d) \
    (void)((p)              \
           += ADDR(p) <= (word)(GC_WORD_MAX - (d) * sizeof(*(p))) ? (d) : 0)
#  define MAKE_HOTTER(p, d) (void)((p) -= (d))
#endif /* !STACK_GROWS_UP */

/* Clear/set flags (given by a mask) in a pointer. */
#define CPTR_CLEAR_FLAGS(p, mask) \
  (ptr_t)((GC_uintptr_t)(p) & ~(GC_uintptr_t)(word)(mask))
#define CPTR_SET_FLAGS(p, mask) (ptr_t)((GC_uintptr_t)(p) | (word)(mask))

/* Easily changeable parameters are below. */

#ifdef ALL_INTERIOR_POINTERS
/*
 * Forces all pointers into the interior of an object to be considered valid.
 * Also causes the sizes of all objects to be inflated by at least one byte.
 * This should suffice to guarantee that in the presence of a compiler that
 * does not perform garbage-collector-unsafe optimizations, all portable,
 * strictly ANSI conforming C programs should be safely usable with `malloc`
 * replaced by `GC_malloc` and `free` calls removed.  There are several
 * disadvantages:
 *   1. There are probably no interesting, portable, strictly ANSI-conforming
 *      C programs;
 *   2. This option makes it hard for the collector to allocate space that is
 *      not "pointed to" by integers, etc.  (Under SunOS 4.x with a statically
 *      linked `libc`, we empirically observed that it would be difficult to
 *      allocate individual objects larger than 100 KB; even if only smaller
 *      objects are allocated, more swap space is likely to be needed;
 *      fortunately, much of this will never be touched.)
 *
 * If you can easily avoid using this option, do.  If not, try to keep
 * individual objects small.  This is really controlled at startup, through
 * `GC_all_interior_pointers` variable.
 */
#endif

EXTERN_C_BEGIN

#ifndef GC_NO_FINALIZATION
/*
 * If `GC_finalize_on_demand` is not set, invoke eligible finalizers.
 * Otherwise: call `(*GC_finalizer_notifier)()` if there are finalizers to
 * be run, and we have not called this procedure yet this collection cycle.
 */
GC_INNER void GC_notify_or_invoke_finalizers(void);

/*
 * Perform all indicated finalization actions on unmarked objects.
 * Unreachable finalizable objects are enqueued for processing by
 * `GC_invoke_finalizers()`.  Cause disappearing links to disappear
 * and unreachable objects to be enqueued for finalization.
 * Invoked with the allocator lock held but the world is running.
 */
GC_INNER void GC_finalize(void);

#  ifndef GC_TOGGLE_REFS_NOT_NEEDED
/* Process the "toggle-refs" before GC starts. */
GC_INNER void GC_process_togglerefs(void);
#  endif
#  ifndef SMALL_CONFIG
GC_INNER void GC_print_finalization_stats(void);
#  endif
#else
#  define GC_notify_or_invoke_finalizers() (void)0
#endif /* GC_NO_FINALIZATION */

#if !defined(DONT_ADD_BYTE_AT_END)
#  ifdef LINT2
/*
 * Explicitly instruct the code analysis tool that `GC_all_interior_pointers`
 * is assumed to have only value of 0 or 1.
 */
#    define EXTRA_BYTES ((size_t)(GC_all_interior_pointers ? 1 : 0))
#  else
#    define EXTRA_BYTES ((size_t)GC_all_interior_pointers)
#  endif
#  define MAX_EXTRA_BYTES 1
#else
#  define EXTRA_BYTES 0
#  define MAX_EXTRA_BYTES 0
#endif

#ifdef LARGE_CONFIG
#  define MINHINCR 64
#  define MAXHINCR 4096
#else
/*
 * Minimum heap increment, in blocks of `HBLKSIZE`.  Note: must be multiple
 * of largest page size.
 */
#  define MINHINCR 16

/* Maximum heap increment, in blocks. */
#  define MAXHINCR 2048
#endif /* !LARGE_CONFIG */

/* Stack saving for debugging. */

#ifdef NEED_CALLINFO
struct callinfo {
  GC_return_addr_t ci_pc; /*< `pc` of caller, not callee */
#  if NARGS > 0
  GC_hidden_pointer ci_arg[NARGS]; /*< hide to avoid retention */
#  endif
#  if (NFRAMES * (NARGS + 1)) % 2 == 1
  /* Likely alignment problem. */
  ptr_t ci_dummy;
#  endif
};

#  ifdef SAVE_CALL_CHAIN
/*
 * Fill in the `pc` and argument information for up to `NFRAMES` of
 * my callers.  Ignore my frame and my callers frame.
 */
GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]);
#  endif

/* Print `info` to `stderr`.  We do not hold the allocator lock. */
GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]);
#endif /* NEED_CALLINFO */

EXTERN_C_END

/*
 * Macros to ensure same formatting of C array/struct/union initializer
 * across multiple versions of clang-format.
 */
#define C_INITIALIZER_BEGIN {
#define C_INITIALIZER_END }

/* OS interface routines. */

#ifndef NO_CLOCK
#  ifdef BSD_TIME
#    undef CLOCK_TYPE
#    undef GET_TIME
#    undef MS_TIME_DIFF
#    define CLOCK_TYPE struct timeval
#    define CLOCK_TYPE_INITIALIZER C_INITIALIZER_BEGIN 0, 0 C_INITIALIZER_END
#    define GET_TIME(x)                  \
      do {                               \
        struct rusage rusage;            \
        getrusage(RUSAGE_SELF, &rusage); \
        x = rusage.ru_utime;             \
      } while (0)

/*
 * Compute time difference.  `a` time is expected to be not earlier
 * than `b` one; the result has `unsigned long` type.
 */
#    define MS_TIME_DIFF(a, b)                                            \
      ((unsigned long)((long)(a.tv_sec - b.tv_sec) * 1000                 \
                       + (long)(a.tv_usec - b.tv_usec) / 1000             \
                       - (a.tv_usec < b.tv_usec                           \
                                  && (long)(a.tv_usec - b.tv_usec) % 1000 \
                                         != 0                             \
                              ? 1                                         \
                              : 0)))

/*
 * The nanosecond part of the time difference.  The total time difference
 * could be computed as:
 * `MS_TIME_DIFF(a, b) * 1000000 + NS_FRAC_TIME_DIFF(a, b)`.
 */
#    define NS_FRAC_TIME_DIFF(a, b)                                          \
      ((unsigned long)((a.tv_usec < b.tv_usec                                \
                                && (long)(a.tv_usec - b.tv_usec) % 1000 != 0 \
                            ? 1000L                                          \
                            : 0)                                             \
                       + (long)(a.tv_usec - b.tv_usec) % 1000)               \
       * 1000)

#  elif defined(MSWIN32) || defined(MSWINCE) || defined(WINXP_USE_PERF_COUNTER)
#    if defined(MSWINRT_FLAVOR) || defined(WINXP_USE_PERF_COUNTER)
#      define CLOCK_TYPE ULONGLONG
/*
 * Note: two standalone `if` statements below are used to avoid MS VC
 * false warning (FP) about potentially uninitialized `tc` variable.
 */
#      define GET_TIME(x)                                              \
        do {                                                           \
          LARGE_INTEGER freq, tc;                                      \
          if (!QueryPerformanceFrequency(&freq))                       \
            ABORT("QueryPerformanceFrequency requires WinXP+");        \
          if (!QueryPerformanceCounter(&tc))                           \
            ABORT("QueryPerformanceCounter failed");                   \
          x = (CLOCK_TYPE)((double)tc.QuadPart / freq.QuadPart * 1e9); \
          /* TODO: Call QueryPerformanceFrequency once at GC init. */  \
        } while (0)
#      define MS_TIME_DIFF(a, b) ((unsigned long)(((a) - (b)) / 1000000UL))
#      define NS_FRAC_TIME_DIFF(a, b) \
        ((unsigned long)(((a) - (b)) % 1000000UL))
#    else
#      define CLOCK_TYPE DWORD
#      define GET_TIME(x) (void)(x = GetTickCount())
#      define MS_TIME_DIFF(a, b) ((unsigned long)((a) - (b)))
#      define NS_FRAC_TIME_DIFF(a, b) 0UL
#    endif /* !WINXP_USE_PERF_COUNTER */

#  elif defined(NN_PLATFORM_CTR)
#    define CLOCK_TYPE long long
EXTERN_C_BEGIN
CLOCK_TYPE n3ds_get_system_tick(void);
CLOCK_TYPE n3ds_convert_tick_to_ms(CLOCK_TYPE tick);
EXTERN_C_END
#    define GET_TIME(x) (void)(x = n3ds_get_system_tick())
#    define MS_TIME_DIFF(a, b) \
      ((unsigned long)n3ds_convert_tick_to_ms((a) - (b)))
/* TODO: Implement NS_FRAC_TIME_DIFF(). */
#    define NS_FRAC_TIME_DIFF(a, b) 0UL

#  elif defined(HAVE_CLOCK_GETTIME)
#    include <time.h>
#    define CLOCK_TYPE struct timespec
#    define CLOCK_TYPE_INITIALIZER C_INITIALIZER_BEGIN 0, 0 C_INITIALIZER_END
#    if defined(_POSIX_MONOTONIC_CLOCK) && !defined(NINTENDO_SWITCH)
#      define GET_TIME(x)                               \
        do {                                            \
          if (clock_gettime(CLOCK_MONOTONIC, &x) == -1) \
            ABORT("clock_gettime failed");              \
        } while (0)
#    else
#      define GET_TIME(x)                              \
        do {                                           \
          if (clock_gettime(CLOCK_REALTIME, &x) == -1) \
            ABORT("clock_gettime failed");             \
        } while (0)
#    endif
#    define MS_TIME_DIFF(a, b)                                            \
      /* `a.tv_nsec - b.tv_nsec` is in range -1e9 to 1e9, exclusively. */ \
      ((unsigned long)((a).tv_nsec + (1000000L * 1000 - (b).tv_nsec))     \
           / 1000000UL                                                    \
       + ((unsigned long)((a).tv_sec - (b).tv_sec) * 1000UL) - 1000UL)
#    define NS_FRAC_TIME_DIFF(a, b)                                   \
      ((unsigned long)((a).tv_nsec + (1000000L * 1000 - (b).tv_nsec)) \
       % 1000000UL)

#  else /* !BSD_TIME && !LINUX && !NN_PLATFORM_CTR && !MSWIN32 */
#    include <time.h>
#    if defined(FREEBSD) && !defined(CLOCKS_PER_SEC)
#      include <machine/limits.h>
#      define CLOCKS_PER_SEC CLK_TCK
#    endif
#    if !defined(CLOCKS_PER_SEC)
/*
 * This is technically a bug in the implementation.
 * ANSI requires that `CLOCKS_PER_SEC` be defined.  But at least under
 * SunOS 4.1.1, it is not.  Also note that the combination of ANSI C
 * and POSIX is incredibly gross here.  The type `clock_t` is used by
 * both `clock()` and `times()`.  But on some machines these use
 * different notions of a clock tick, `CLOCKS_PER_SEC` seems to apply
 * only to `clock()`.  Hence we use it here.  On many machines,
 * including SunOS, `clock()` actually uses units of microseconds (that
 * are not really clock ticks).
 */
#      define CLOCKS_PER_SEC 1000000
#    endif
#    define CLOCK_TYPE clock_t
#    define GET_TIME(x) (void)(x = clock())
#    define MS_TIME_DIFF(a, b)                            \
      (CLOCKS_PER_SEC % 1000 == 0                         \
           ? (unsigned long)((a) - (b))                   \
                 / (unsigned long)(CLOCKS_PER_SEC / 1000) \
           : ((unsigned long)((a) - (b)) * 1000)          \
                 / (unsigned long)CLOCKS_PER_SEC)
/*
 * Avoid using `double` type since some targets (like ARM) might
 * require `-lm` option for `double`-to-`long` conversion.
 */
#    define NS_FRAC_TIME_DIFF(a, b)                                         \
      (CLOCKS_PER_SEC <= 1000                                               \
           ? 0UL                                                            \
           : (unsigned long)(CLOCKS_PER_SEC <= (clock_t)1000000UL           \
                                 ? (((a) - (b))                             \
                                    * ((clock_t)1000000UL / CLOCKS_PER_SEC) \
                                    % 1000)                                 \
                                       * 1000                               \
                                 : (CLOCKS_PER_SEC                          \
                                            <= (clock_t)1000000UL * 1000    \
                                        ? ((a) - (b))                       \
                                              * ((clock_t)1000000UL * 1000  \
                                                 / CLOCKS_PER_SEC)          \
                                        : (((a) - (b)) * (clock_t)1000000UL \
                                           * 1000)                          \
                                              / CLOCKS_PER_SEC)             \
                                       % (clock_t)1000000UL))
#  endif /* !BSD_TIME && !MSWIN32 */
#  ifndef CLOCK_TYPE_INITIALIZER
/*
 * This is used to initialize `CLOCK_TYPE` variables (to some value)
 * to avoid "variable might be uninitialized" compiler warnings.
 */
#    define CLOCK_TYPE_INITIALIZER 0
#  endif
#endif /* !NO_CLOCK */

/* We use `bzero()` and `bcopy()` internally.  They may not be available. */
#if defined(M68K) && defined(NEXT) || defined(VAX)
#  define BCOPY_EXISTS
#elif defined(DARWIN)
#  include <string.h>
#  define BCOPY_EXISTS
#endif

#if !defined(BCOPY_EXISTS) || defined(CPPCHECK)
#  include <string.h>
#  define BCOPY(x, y, n) memcpy(y, x, (size_t)(n))
#  define BZERO(x, n) memset(x, 0, (size_t)(n))
#else
#  define BCOPY(x, y, n) bcopy((void *)(x), (void *)(y), (size_t)(n))
#  define BZERO(x, n) bzero((void *)(x), (size_t)(n))
#endif

EXTERN_C_BEGIN

#if defined(CPPCHECK) && defined(ANY_MSWIN)
#  undef TEXT
#  ifdef UNICODE
#    define TEXT(s) L##s
#  else
#    define TEXT(s) s
#  endif
#endif /* CPPCHECK && ANY_MSWIN */

/* Stop and restart mutator threads. */
#if defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) \
    || defined(GC_WIN32_THREADS) || defined(GC_PTHREADS)
GC_INNER void GC_stop_world(void);
GC_INNER void GC_start_world(void);
#  define STOP_WORLD() GC_stop_world()
#  define START_WORLD() GC_start_world()
#else
/* Just do a sanity check: we are not inside `GC_do_blocking()`. */
#  define STOP_WORLD() GC_ASSERT(GC_blocked_sp == NULL)
#  define START_WORLD()
#endif

/* Abandon ship. */
#ifdef SMALL_CONFIG
#  define GC_on_abort(msg) (void)0 /*< be silent on abort */
#else
GC_API_PRIV GC_abort_func GC_on_abort;
#endif
#if defined(CPPCHECK)
#  define ABORT(msg)    \
    {                   \
      GC_on_abort(msg); \
      abort();          \
    }
#else
#  if defined(MSWIN_XBOX1) && !defined(DebugBreak)
#    define DebugBreak() __debugbreak()
#  elif defined(MSWINCE) && !defined(DebugBreak) \
      && (!defined(UNDER_CE) || (defined(__MINGW32CE__) && !defined(ARM32)))
/*
 * This simplifies linking for WinCE (and, probably, does not
 * hurt debugging much); use `-D DebugBreak=DebugBreak` to override
 * this behavior if really needed.  This is also a workaround for
 * x86mingw32ce toolchain (if it is still declaring `DebugBreak()`
 * instead of defining it as a macro).
 */
#    define DebugBreak() _exit(-1) /*< there is no `abort()` in WinCE */
#  endif
#  if defined(MSWIN32) && (defined(NO_DEBUGGING) || defined(LINT2))
/*
 * A more user-friendly abort after showing fatal message.
 * Exit on error without running "at-exit" callbacks.
 */
#    define ABORT(msg) (GC_on_abort(msg), _exit(-1))
#  elif defined(MSWINCE) && defined(NO_DEBUGGING)
#    define ABORT(msg) (GC_on_abort(msg), ExitProcess(-1))
#  elif defined(MSWIN32) || defined(MSWINCE)
#    if defined(_CrtDbgBreak) && defined(_DEBUG) && defined(_MSC_VER)
#      define ABORT(msg)                          \
        {                                         \
          GC_on_abort(msg);                       \
          _CrtDbgBreak() /*< `__debugbreak()` */; \
        }
#    else
#      define ABORT(msg)    \
        {                   \
          GC_on_abort(msg); \
          DebugBreak();     \
        }
/*
 * Note: on a WinCE box, this could be silently ignored (i.e., the program
 * is not aborted); `DebugBreak()` is a statement in some toolchains.
 */
#    endif
#  else /* !MSWIN32 */
#    define ABORT(msg) (GC_on_abort(msg), abort())
#  endif
#endif /* !CPPCHECK */

/*
 * For the abort message with 1 .. 3 arguments.  `C_msg` and `C_fmt`
 * should be literals.  `C_msg` should not contain format specifiers.
 * Arguments should match their format specifiers.
 */
#define ABORT_ARG1(C_msg, C_fmt, arg1)               \
  MACRO_BLKSTMT_BEGIN                                \
  GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", arg1); \
  ABORT(C_msg);                                      \
  MACRO_BLKSTMT_END
#define ABORT_ARG2(C_msg, C_fmt, arg1, arg2)               \
  MACRO_BLKSTMT_BEGIN                                      \
  GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", arg1, arg2); \
  ABORT(C_msg);                                            \
  MACRO_BLKSTMT_END
#define ABORT_ARG3(C_msg, C_fmt, arg1, arg2, arg3)               \
  MACRO_BLKSTMT_BEGIN                                            \
  GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", arg1, arg2, arg3); \
  ABORT(C_msg);                                                  \
  MACRO_BLKSTMT_END

/*
 * Same as `ABORT` but does not have a `noreturn` attribute.
 * I.e. `ABORT` on a dummy condition (which is always true).
 */
#define ABORT_RET(msg)                                                \
  if ((GC_funcptr_uint)GC_current_warn_proc == ~(GC_funcptr_uint)0) { \
  } else                                                              \
    ABORT(msg)

/* Exit process abnormally, but without making a mess (e.g. out of memory). */
#define EXIT() (GC_on_abort(NULL), exit(1 /* `EXIT_FAILURE` */))

/*
 * Print warning message, e.g. almost out of memory.  The argument (if any)
 * format specifier should be: "%s", "%p", "%"`WARN_PRIdPTR` or
 * "%"`WARN_PRIuPTR`.
 */
#define WARN(msg, arg) \
  GC_current_warn_proc("GC Warning: " msg, (GC_uintptr_t)(arg))
GC_EXTERN GC_warn_proc GC_current_warn_proc;

/*
 * Print format type macro for decimal `GC_signed_word` value passed to
 * `WARN()`.  This could be redefined for Win64 or LLP64, but typically
 * should not be done as the `WARN` format string is, possibly,
 * processed on the client side, so non-standard print type modifiers
 * (like MS "I64d") should be avoided here if possible.
 * TODO: Assuming `sizeof(void *)` is equal to `sizeof(long)` or this
 * is a little-endian machine.
 */
#ifndef WARN_PRIdPTR
#  define WARN_PRIdPTR "ld"
#  define WARN_PRIuPTR "lu"
#endif

/*
 * A tagging macro (for a code static analyzer) to indicate that the
 * string obtained from an untrusted source (e.g., `argv[]`, `getenv`)
 * is safe to use in a vulnerable operation (e.g., `open`, `exec`).
 */
#define TRUSTED_STRING(s) COVERT_DATAFLOW_P(s)

#ifdef GC_READ_ENV_FILE
/*
 * This routine scans `GC_envfile_content` for the specified environment
 * variable (and returns its value if found).
 */
GC_INNER char *GC_envfile_getenv(const char *name);

/* Get the process environment entry. */
#  define GETENV(name) GC_envfile_getenv(name)
#elif defined(NO_GETENV) && !defined(CPPCHECK)
#  define GETENV(name) NULL
#elif defined(EMPTY_GETENV_RESULTS) && !defined(CPPCHECK)
/* Workaround for a reputed Wine bug. */
GC_INLINE char *
fixed_getenv(const char *name)
{
  char *value = getenv(name);
  return value != NULL && *value != '\0' ? value : NULL;
}
#  define GETENV(name) fixed_getenv(name)
#else
#  define GETENV(name) getenv(name)
#endif

EXTERN_C_END

#if defined(DARWIN)
#  include <mach/thread_status.h>
#  ifndef MAC_OS_X_VERSION_MAX_ALLOWED
/* Include this header just to import the above macro. */
#    include <AvailabilityMacros.h>
#  endif
#  if defined(POWERPC)
#    if CPP_WORDSZ == 32
#      define GC_THREAD_STATE_T ppc_thread_state_t
#    else
#      define GC_THREAD_STATE_T ppc_thread_state64_t
#      define GC_MACH_THREAD_STATE PPC_THREAD_STATE64
#      define GC_MACH_THREAD_STATE_COUNT PPC_THREAD_STATE64_COUNT
#    endif
#  elif defined(I386) || defined(X86_64)
#    if CPP_WORDSZ == 32
#      if defined(i386_THREAD_STATE_COUNT) \
          && !defined(x86_THREAD_STATE32_COUNT)
/* Use old naming convention for i686. */
#        define GC_THREAD_STATE_T i386_thread_state_t
#        define GC_MACH_THREAD_STATE i386_THREAD_STATE
#        define GC_MACH_THREAD_STATE_COUNT i386_THREAD_STATE_COUNT
#      else
#        define GC_THREAD_STATE_T x86_thread_state32_t
#        define GC_MACH_THREAD_STATE x86_THREAD_STATE32
#        define GC_MACH_THREAD_STATE_COUNT x86_THREAD_STATE32_COUNT
#      endif
#    else
#      define GC_THREAD_STATE_T x86_thread_state64_t
#      define GC_MACH_THREAD_STATE x86_THREAD_STATE64
#      define GC_MACH_THREAD_STATE_COUNT x86_THREAD_STATE64_COUNT
#    endif
#  elif defined(ARM32) && defined(ARM_UNIFIED_THREAD_STATE) \
      && !defined(CPPCHECK)
#    define GC_THREAD_STATE_T arm_unified_thread_state_t
#    define GC_MACH_THREAD_STATE ARM_UNIFIED_THREAD_STATE
#    define GC_MACH_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
#  elif defined(ARM32)
#    define GC_THREAD_STATE_T arm_thread_state_t
#    ifdef ARM_MACHINE_THREAD_STATE_COUNT
#      define GC_MACH_THREAD_STATE ARM_MACHINE_THREAD_STATE
#      define GC_MACH_THREAD_STATE_COUNT ARM_MACHINE_THREAD_STATE_COUNT
#    endif
#  elif defined(AARCH64)
#    define GC_THREAD_STATE_T arm_thread_state64_t
#    define GC_MACH_THREAD_STATE ARM_THREAD_STATE64
#    define GC_MACH_THREAD_STATE_COUNT ARM_THREAD_STATE64_COUNT
#  elif !defined(CPPCHECK)
#    error define GC_THREAD_STATE_T
#  endif
#  ifndef GC_MACH_THREAD_STATE
#    define GC_MACH_THREAD_STATE MACHINE_THREAD_STATE
#    define GC_MACH_THREAD_STATE_COUNT MACHINE_THREAD_STATE_COUNT
#  endif

/*
 * Try to work out the right way to access thread state structure members.
 * The structure has different definition in different Darwin versions.
 * This now defaults to the (older) names without `__`, thus hopefully
 * not breaking any existing `Makefile.direct` builds.
 */
#  if __DARWIN_UNIX03
#    define THREAD_FLD_NAME(x) __##x
#  else
#    define THREAD_FLD_NAME(x) x
#  endif
#  if defined(ARM32) && defined(ARM_UNIFIED_THREAD_STATE)
#    define THREAD_FLD(x) ts_32.THREAD_FLD_NAME(x)
#  else
#    define THREAD_FLD(x) THREAD_FLD_NAME(x)
#  endif
#endif /* DARWIN */

#ifndef WASI
#  include <setjmp.h>
#endif

#include <stdio.h>

#if defined(CAN_HANDLE_FORK) && defined(GC_PTHREADS)
#  include <pthread.h> /*< for `pthread_t` */
#endif

#if __STDC_VERSION__ >= 201112L
#  include <assert.h> /*< for `static_assert` */
#endif

EXTERN_C_BEGIN

/* Definitions depending on `word` size. */

#define modWORDSZ(n) ((n) & (CPP_WORDSZ - 1)) /*< `n mod size_of_word` */
#define divWORDSZ(n) ((n) / CPP_WORDSZ)

#define SIGNB ((word)1 << (CPP_WORDSZ - 1))
#define SIZET_SIGNB (GC_SIZE_MAX ^ (GC_SIZE_MAX >> 1))

#if CPP_PTRSZ / 8 != ALIGNMENT
#  define UNALIGNED_PTRS
#endif

#define BYTES_TO_GRANULES(lb) ((lb) / GC_GRANULE_BYTES)
#define GRANULES_TO_BYTES(lg) (GC_GRANULE_BYTES * (lg))
#define BYTES_TO_PTRS(lb) ((lb) / sizeof(ptr_t))
#define PTRS_TO_BYTES(lpw) ((lpw) * sizeof(ptr_t))
#define GRANULES_TO_PTRS(lg) (GC_GRANULE_PTRS * (lg))

/*
 * Convert size in bytes to that in pointers rounding up (but not adding
 * extra byte at end).
 */
#define BYTES_TO_PTRS_ROUNDUP(lb) BYTES_TO_PTRS((lb) + sizeof(ptr_t) - 1)

/* Size parameters. */

/*
 * Heap block size, in bytes.  Should be a power of two.
 * Incremental collection with `MPROTECT_VDB` currently requires the
 * page size to be a multiple of `HBLKSIZE`.  Since most modern
 * architectures support variable page sizes down to 4 KB, and i686 and
 * x86_64 are generally 4 KB, we now default to 4 KB, except for:
 *   - Alpha: seems to be used with 8 KB pages;
 *   - `SMALL_CONFIG`: want less block-level fragmentation.
 */
#ifndef HBLKSIZE
#  if defined(SMALL_CONFIG) && !defined(LARGE_CONFIG)
#    define CPP_LOG_HBLKSIZE 10
#  elif defined(ALPHA)
#    define CPP_LOG_HBLKSIZE 13
#  else
#    define CPP_LOG_HBLKSIZE 12
#  endif
#else
#  if HBLKSIZE == 512
#    define CPP_LOG_HBLKSIZE 9
#  elif HBLKSIZE == 1024
#    define CPP_LOG_HBLKSIZE 10
#  elif HBLKSIZE == 2048
#    define CPP_LOG_HBLKSIZE 11
#  elif HBLKSIZE == 4096
#    define CPP_LOG_HBLKSIZE 12
#  elif HBLKSIZE == 8192
#    define CPP_LOG_HBLKSIZE 13
#  elif HBLKSIZE == 16384
#    define CPP_LOG_HBLKSIZE 14
#  elif HBLKSIZE == 32768
#    define CPP_LOG_HBLKSIZE 15
#  elif HBLKSIZE == 65536
#    define CPP_LOG_HBLKSIZE 16
#  elif !defined(CPPCHECK)
#    error Bad HBLKSIZE value
#  endif
#  undef HBLKSIZE
#endif

#define LOG_HBLKSIZE ((size_t)CPP_LOG_HBLKSIZE)
#define HBLKSIZE ((size_t)1 << CPP_LOG_HBLKSIZE)

#define GC_SQRT_SIZE_MAX ((((size_t)1) << (sizeof(size_t) * 8 / 2)) - 1)

/*
 * Maximum size of objects supported by free list (larger objects are
 * allocated directly with `GC_alloc_large()`, by rounding to the next
 * multiple of `HBLKSIZE`).
 */
#define MAXOBJBYTES (HBLKSIZE >> 1)
#define MAXOBJGRANULES BYTES_TO_GRANULES(MAXOBJBYTES)

#define divHBLKSZ(n) ((n) >> LOG_HBLKSIZE)

/*
 * Equivalent to subtracting one `hblk` pointer from another.  We do it
 * this way because a compiler should find it hard to use an integer
 * division instead of a shift.  The bundled SunOS 4.1 otherwise sometimes
 * pessimizes the subtraction to involve a call to `.div`.
 */
#define HBLK_PTR_DIFF(p, q) divHBLKSZ((ptr_t)p - (ptr_t)q)

#define modHBLKSZ(n) ((n) & (HBLKSIZE - 1))

#define HBLKPTR(objptr) \
  ((struct hblk *)PTR_ALIGN_DOWN((ptr_t)(objptr), HBLKSIZE))
#define HBLKDISPL(objptr) modHBLKSZ((size_t)ADDR(objptr))

/* Same as `HBLKPTR` but points to the first block in the page. */
#define HBLK_PAGE_ALIGNED(objptr) \
  ((struct hblk *)PTR_ALIGN_DOWN((ptr_t)(objptr), GC_page_size))

/* Round up allocation size (in bytes) to a multiple of a granule. */
#define ROUNDUP_GRANULE_SIZE(lb) /*< `lb` should have no side-effect */ \
  (SIZET_SAT_ADD(lb, GC_GRANULE_BYTES - 1) & ~(size_t)(GC_GRANULE_BYTES - 1))

/*
 * Round up byte allocation request (after adding `EXTRA_BYTES`) to
 * a multiple of a granule, then convert it to granules.
 */
#define ALLOC_REQUEST_GRANS(lb) /*< `lb` should have no side-effect */ \
  BYTES_TO_GRANULES(SIZET_SAT_ADD(lb, GC_GRANULE_BYTES - 1 + EXTRA_BYTES))

#if MAX_EXTRA_BYTES == 0
#  define ADD_EXTRA_BYTES(lb) (lb)
#  define SMALL_OBJ(lb) LIKELY((lb) <= MAXOBJBYTES)
#else
#  define ADD_EXTRA_BYTES(lb) /*< `lb` should have no side-effect */ \
    SIZET_SAT_ADD(lb, EXTRA_BYTES)

/*
 * This really just tests that `lb` is not greater than
 * `MAXOBJBYTES - EXTRA_BYTES`, but we try to avoid looking up `EXTRA_BYTES`.
 */
#  define SMALL_OBJ(lb) /*< `lb` should have no side-effect */ \
    (LIKELY((lb) <= MAXOBJBYTES - MAX_EXTRA_BYTES)             \
     || (lb) <= MAXOBJBYTES - EXTRA_BYTES)
#endif

/*
 * Hash table representation of sets of pages.  Implements a map from
 * `HBLKSIZE`-aligned chunks of the address space to one bit each.
 * This assumes it is OK to spuriously set bits, e.g. because multiple
 * addresses are represented by a single location.  Used by
 * black-listing code, and perhaps by dirty bit maintenance code.
 */
#ifndef LOG_PHT_ENTRIES
#  ifdef LARGE_CONFIG
#    if CPP_WORDSZ == 32
/*
 * Collisions are impossible (because of a 4 GB space limit).
 * Each table takes 128 KB, some of which may never be touched.
 */
#      define LOG_PHT_ENTRIES 20
#    else
/*
 * Collisions likely at 2M blocks, which is greater than 8 GB.
 * Each table takes 256 KB, some of which may never be touched.
 */
#      define LOG_PHT_ENTRIES 21
#    endif
#  elif !defined(SMALL_CONFIG)
/*
 * Collisions are likely if heap grows to more than 256K blocks,
 * which is greater than 1 GB.  Each hash table occupies 32 KB.
 * Even for somewhat smaller heaps, say half of that, collisions may
 * be an issue because we blacklist addresses outside the heap.
 */
#    define LOG_PHT_ENTRIES 18
#  else
/*
 * Collisions are likely if heap grows to more than 32K blocks,
 * which is 128 MB.  Each hash table occupies 4 KB.
 */
#    define LOG_PHT_ENTRIES 15
#  endif
#endif /* !LOG_PHT_ENTRIES */

#define PHT_ENTRIES (1 << LOG_PHT_ENTRIES)
#define PHT_SIZE (PHT_ENTRIES > CPP_WORDSZ ? PHT_ENTRIES / CPP_WORDSZ : 1)
typedef word page_hash_table[PHT_SIZE];

#define PHT_HASH(p) ((size_t)((ADDR(p) >> LOG_HBLKSIZE) & (PHT_ENTRIES - 1)))

#define get_pht_entry_from_index(bl, index) \
  (((bl)[divWORDSZ(index)] >> modWORDSZ(index)) & 1)
#define set_pht_entry_from_index(bl, index) \
  (void)((bl)[divWORDSZ(index)] |= (word)1 << modWORDSZ(index))

#if defined(THREADS) && defined(AO_HAVE_or)
/*
 * And, one more variant for `GC_add_to_black_list_normal` and
 *`GC_add_to_black_list_stack` (invoked indirectly by `GC_do_local_mark()`)
 * and `async_set_pht_entry_from_index()` (invoked by `GC_dirty()` or the
 * write fault handler).
 */
#  define set_pht_entry_from_index_concurrent(bl, index) \
    AO_or((volatile AO_t *)&(bl)[divWORDSZ(index)],      \
          (AO_t)1 << modWORDSZ(index))
#  ifdef MPROTECT_VDB
#    define set_pht_entry_from_index_concurrent_volatile(bl, index) \
      set_pht_entry_from_index_concurrent(bl, index)
#  endif
#else
#  define set_pht_entry_from_index_concurrent(bl, index) \
    set_pht_entry_from_index(bl, index)
#  ifdef MPROTECT_VDB
/*
 * Same as `set_pht_entry_from_index` but avoiding the compound assignment
 * for a `volatile` array.
 */
#    define set_pht_entry_from_index_concurrent_volatile(bl, index) \
      (void)((bl)[divWORDSZ(index)]                                 \
             = (bl)[divWORDSZ(index)] | ((word)1 << modWORDSZ(index)))
#  endif
#endif

/* Heap blocks. */

/*
 * The upper bound.  We allocate 1 bit per allocation granule.
 * If `MARK_BIT_PER_OBJ` is not defined, we use every `n`-th bit, where
 * `n` is the number of allocation granules per object.  Otherwise, we only
 * use the initial group of mark bits, and it is safe to allocate smaller
 * header for large objects.
 */
#define MARK_BITS_PER_HBLK (HBLKSIZE / GC_GRANULE_BYTES)

#ifndef MARK_BIT_PER_OBJ
/*
 * We maintain layout maps for heap blocks containing objects of
 * a given size.  Each entry in this map describes a byte offset
 * (displacement) and has the following type.
 */
#  if (1 << (CPP_LOG_HBLKSIZE - 1)) / GC_GRANULE_BYTES <= 0x100
typedef unsigned char hb_map_entry_t;
#  else
typedef unsigned short hb_map_entry_t;
#  endif
#endif /* !MARK_BIT_PER_OBJ */

struct hblkhdr {
  /*
   * Link field for `hblk` free list and for lists of chunks waiting to
   * be reclaimed.
   */
  struct hblk *hb_next;

  struct hblk *hb_prev; /*< backwards link for free list */

  struct hblk *hb_block; /*< the corresponding block */

  /*
   * Kind of objects in the block.  Each kind identifies a mark
   * procedure and a set of list headers.  Sometimes called regions.
   */
  unsigned char hb_obj_kind;

  unsigned char hb_flags;

  /* Ignore pointers that do not point to the first `hblk` of this object. */
#define IGNORE_OFF_PAGE 1

  /*
   * This is a free block, which has been unmapped from the address space.
   * `GC_remap()` must be invoked on it before it can be reallocated.
   * Set only if `USE_MUNMAP` macro is defined.
   */
#define WAS_UNMAPPED 2

  /* Block is free, i.e. not in use. */
#define FREE_BLK 4

#ifdef ENABLE_DISCLAIM
  /* This kind has a callback on reclaim. */
#  define HAS_DISCLAIM 8

  /*
   * Mark from all objects, marked or not.  Used to mark objects needed
   * by reclaim notifier.
   */
#  define MARK_UNCONDITIONALLY 0x10
#endif

#ifndef MARK_BIT_PER_OBJ
#  define LARGE_BLOCK 0x20
#endif

  /*
   * Value of `GC_gc_no` when block was last allocated or swept.
   * May wrap.  For a free block, this is maintained only for `USE_MUNMAP`,
   * and indicates when the header was allocated, or when the size of the
   * block last changed.
   */
  unsigned short hb_last_reclaimed;

#ifdef MARK_BIT_PER_OBJ
#  define LARGE_INV_SZ ((unsigned32)1 << 16)

  /*
   * A good upper bound for `2**32 / hb_sz`.
   * For large objects, we use `LARGE_INV_SZ`.
   */
  unsigned32 hb_inv_sz;
#endif

  /*
   * If in use, size in bytes, of objects in the block.
   * Otherwise, the size of the whole free block.  We assume that this is
   * convertible to `GC_signed_word` without generating a negative result.
   * We avoid generating free blocks larger than that.
   */
  size_t hb_sz;

  /* Object descriptor for marking.  See `gc_mark.h` file. */
  word hb_descr;

#ifndef MARK_BIT_PER_OBJ
  /*
   * A table of remainders `mod BYTES_TO_GRANULES(hb_sz)` essentially,
   * except for large blocks.  See `GC_obj_map`.
   */
  hb_map_entry_t *hb_map;
#endif

#ifdef PARALLEL_MARK
  /*
   * Number of set mark bits, excluding the one always set at the end.
   * Currently it is updated concurrently and hence only approximate.
   * But a zero value does guarantee that the block contains
   * no marked objects.  Ensuring this property means that we never
   * decrement it to zero during a collection, and hence the count may
   * be one too high.  Due to concurrent updates, an arbitrary number
   * of increments, but not all of them (!) may be lost, hence it may,
   * in theory, be much too low.  The count may also be too high
   * if multiple mark threads mark the same object due to a race.
   */
  volatile AO_t hb_n_marks;
#else
  /* Without parallel marking, the count is accurate. */
  size_t hb_n_marks;
#endif

#ifdef USE_MARK_BYTES
  /*
   * Unlike the other case, this is in units of bytes.  Since we force
   * certain alignment, we need at most one mark bit per a granule.
   * But we do allocate and set one extra mark bit to avoid
   * an explicit check for the partial object at the end of each block.
   */
#  define HB_MARKS_SZ (MARK_BITS_PER_HBLK + 1)
  union {
    /*
     * The `i`-th byte is 1 if the object starting at granule `i`
     * or object `i` is marked, 0 otherwise.  The mark bit for the
     * "one past the end" object is always set to avoid a special case
     * test in the marker.
     */
    char _hb_marks[HB_MARKS_SZ];
    word dummy; /*< force word alignment of mark bytes */
  } _mark_byte_union;
#  define hb_marks _mark_byte_union._hb_marks
#else
#  define HB_MARKS_SZ (MARK_BITS_PER_HBLK / CPP_WORDSZ + 1)

#  if defined(PARALLEL_MARK) || (defined(THREAD_SANITIZER) && defined(THREADS))
  volatile AO_t hb_marks[HB_MARKS_SZ];
#  else
  word hb_marks[HB_MARKS_SZ];
#  endif
#endif /* !USE_MARK_BYTES */
};

/* A "random" mark bit index for assertions. */
#define ANY_INDEX 23

/* Heap block body. */

#define HBLK_WORDS (HBLKSIZE / sizeof(word))
#define HBLK_GRANULES (HBLKSIZE / GC_GRANULE_BYTES)

/*
 * The number of objects in a block dedicated to a certain size.
 * May erroneously yield zero (instead of one) for large objects.
 */
#define HBLK_OBJS(sz_in_bytes) (HBLKSIZE / (sz_in_bytes))

struct hblk {
  char hb_body[HBLKSIZE];
};

#define HBLK_IS_FREE(hhdr) (((hhdr)->hb_flags & FREE_BLK) != 0)

#define OBJ_SZ_TO_BLOCKS(lb) divHBLKSZ((lb) + HBLKSIZE - 1)

/*
 * Size of block (in units of `HBLKSIZE`) needed to hold objects of
 * given `lb` (in bytes).  The checked variant prevents wrap around.
 */
#define OBJ_SZ_TO_BLOCKS_CHECKED(lb) /*< `lb` should have no side-effect */ \
  divHBLKSZ(SIZET_SAT_ADD(lb, HBLKSIZE - 1))

/* The object free-list link. */
#define obj_link(p) (*(void **)(p))

/*
 * Root sets.  Logically private to `mark_rts.c` file.  But we do not
 * want the tables scanned, so we put them here.
 */

/* The maximum number of ranges that can be registered as static roots. */
#ifdef LARGE_CONFIG
#  define MAX_ROOT_SETS 8192
#elif !defined(SMALL_CONFIG)
#  define MAX_ROOT_SETS 2048
#else
#  define MAX_ROOT_SETS 512
#endif

/* Maximum number of segments that can be excluded from root sets. */
#define MAX_EXCLUSIONS (MAX_ROOT_SETS / 4)

/* A data structure for excluded static roots. */
struct exclusion {
  ptr_t e_start;
  ptr_t e_end;
};

/*
 * A data structure for list of root sets.  We keep a hash table, so that
 * we can filter out duplicate additions.  Under Win32, we need to do
 * a better job of filtering overlaps, so we resort to sequential search,
 * and pay the price.
 */
struct roots {
  ptr_t r_start; /*< multiple of pointer size */
  ptr_t r_end;   /*< multiple of pointer size and greater than `r_start` */
#ifndef ANY_MSWIN
  struct roots *r_next;
#endif
  /* Delete before registering new dynamic libraries if set. */
  GC_bool r_tmp;
};

#ifndef ANY_MSWIN
/* Size of hash table index to roots. */
#  define LOG_RT_SIZE 6

/* `RT_SIZE` should be a power of 2, may be not equal to `MAX_ROOT_SETS`. */
#  define RT_SIZE (1 << LOG_RT_SIZE)
#endif

#if (!defined(MAX_HEAP_SECTS) || defined(CPPCHECK)) \
    && (defined(ANY_MSWIN) || defined(USE_PROC_FOR_LIBRARIES))
#  ifdef LARGE_CONFIG
#    if CPP_WORDSZ > 32
#      define MAX_HEAP_SECTS 81920
#    else
#      define MAX_HEAP_SECTS 7680
#    endif
#  elif defined(SMALL_CONFIG) && !defined(USE_PROC_FOR_LIBRARIES)
#    if defined(PARALLEL_MARK) && (defined(MSWIN32) || defined(CYGWIN32))
#      define MAX_HEAP_SECTS 384
#    else
#      define MAX_HEAP_SECTS 128 /*< roughly 256 MB (`128 * 2048 * 1024`) */
#    endif
#  elif CPP_WORDSZ > 32
#    define MAX_HEAP_SECTS 1024 /*< roughly 8 GB */
#  else
#    define MAX_HEAP_SECTS 512 /*< roughly 4 GB */
#  endif
#endif /* !MAX_HEAP_SECTS */

typedef struct GC_ms_entry {
  ptr_t mse_start; /*< beginning of object, pointer-aligned one */
#ifdef PARALLEL_MARK
  volatile AO_t mse_descr;
#else
  /*
   * The descriptor; the low-order two bits are tags, as described
   * in `gc_mark.h` file.
   */
  word mse_descr;
#endif
} mse;

/*
 * Current state of marking.  Used to remember where we are during the
 * concurrent marking.
 */
typedef int mark_state_t;

struct disappearing_link;
struct finalizable_object;

struct dl_hashtbl_s {
  struct disappearing_link **head;
  size_t entries;
  unsigned log_size;
};

struct fnlz_roots_s {
  struct finalizable_object **fo_head;
  /* List of objects that should be finalized now. */
  struct finalizable_object *finalize_now;
};

union toggle_ref_u {
  /* The least significant bit is used to distinguish between choices. */
  void *strong_ref;
  GC_hidden_pointer weak_ref;
};

/*
 * Extended descriptors.  `GC_typed_mark_proc` understands these.
 * These are used for simple objects that are larger than what can
 * be described by a `BITMAP_BITS`-sized bitmap.
 */
typedef struct {
  /*
   * The least significant bit corresponds to the first "pointer-sized"
   * word.
   */
  word ed_bitmap;
  GC_bool ed_continued; /*< next entry is continuation */
} typed_ext_descr_t;

struct HeapSect {
  ptr_t hs_start;
  size_t hs_bytes;
};

#ifdef MAKE_BACK_GRAPH
/* The maximum in-degree we handle directly. */
#  define BACKGRAPH_MAX_IN 10

struct back_edges_s {
  /* Number of edges, including those in continuation structures. */
  word n_edges;

  unsigned short flags;

  /*
   * If `height` is greater than zero, then keeps the `GC_gc_no` value
   * when it was computed.  If it was computed this cycle, then it is
   * current.  If it was computed during the last cycle, then it belongs
   * to the old height, which is only saved for live objects referenced by
   * dead ones.  This may grow due to references from newly dead objects.
   */
  unsigned short height_gc_no;

  /*
   * Longest path through unreachable nodes to this node that we found
   * using depth first search.
   */
  GC_signed_word height;

  ptr_t edges[BACKGRAPH_MAX_IN];

  /*
   * Pointer to continuation structure; we use only the edges field in
   * the continuation.  Also used as a free-list link.
   */
  struct back_edges_s *cont;
};
#endif

/*
 * Lists of all heap blocks and free lists as well as other random data
 * structures that should not be scanned by the collector.  These are
 * grouped together in a structure so that they can be easily skipped by
 * `GC_push_conditional_with_exclusions()`.  The ordering is weird to
 * make `GC_malloc` faster by keeping the important fields sufficiently
 * close together that a single load of a base register will do.
 * Scalars that could easily appear to be pointers are also put here.
 * The main fields should precede any conditionally included fields, where
 * possible.
 */
struct _GC_arrays {
#define GC_heapsize GC_arrays._heapsize
  word _heapsize; /*< heap size in bytes (value never goes down) */

#define GC_requested_heapsize GC_arrays._requested_heapsize
  word _requested_heapsize; /*< heap size due to explicit expansion */

#define GC_heapsize_on_gc_disable GC_arrays._heapsize_on_gc_disable
  word _heapsize_on_gc_disable;

#define GC_heapsize_at_forced_unmap GC_arrays._heapsize_at_forced_unmap
  word _heapsize_at_forced_unmap; /*< accessed with the allocator lock held */

#define GC_last_heap_addr GC_arrays._last_heap_addr
  word _last_heap_addr;

  /*
   * Total bytes contained in blocks on the free list of large objects.
   * (A large object is the one that occupies a block of at least
   * two `HBLKSIZE`.)
   */
#define GC_large_free_bytes GC_arrays._large_free_bytes
  word _large_free_bytes;

  /* Total number of bytes in allocated large objects blocks. */
#define GC_large_allocd_bytes GC_arrays._large_allocd_bytes
  word _large_allocd_bytes;

  /*
   * Maximum number of bytes that were ever allocated in large object blocks.
   * This is used to help decide when it is safe to split up a large block.
   */
#define GC_max_large_allocd_bytes GC_arrays._max_large_allocd_bytes
  word _max_large_allocd_bytes;

  /* Number of bytes allocated before this collection cycle. */
#define GC_bytes_allocd_before_gc GC_arrays._bytes_allocd_before_gc
  word _bytes_allocd_before_gc;

#define GC_our_mem_bytes GC_arrays._our_mem_bytes
  word _our_mem_bytes;

  /* Number of bytes allocated during this collection cycle. */
#define GC_bytes_allocd GC_arrays._bytes_allocd
  word _bytes_allocd;

  /*
   * Number of black-listed bytes dropped during GC cycle as a result
   * of repeated scanning during allocation attempts.  These are treated
   * largely as allocated, even though they are not useful to the client.
   */
#define GC_bytes_dropped GC_arrays._bytes_dropped
  word _bytes_dropped;

  /*
   * Approximate number of bytes in objects (and headers) that became
   * ready for finalization in the last collection.
   */
#define GC_bytes_finalized GC_arrays._bytes_finalized
  word _bytes_finalized;

  /*
   * Number of explicitly deallocated bytes of memory since last
   * collection.
   */
#define GC_bytes_freed GC_arrays._bytes_freed
  word _bytes_freed;

  /*
   * Bytes of memory explicitly deallocated while finalizers were running.
   * Used to approximate size of memory explicitly deallocated by finalizers.
   */
#define GC_finalizer_bytes_freed GC_arrays._finalizer_bytes_freed
  word _finalizer_bytes_freed;

  /*
   * Pointer to the first (lowest address) `bottom_index` entity;
   * assumes the allocator lock is held.
   */
#define GC_all_bottom_indices GC_arrays._all_bottom_indices
  bottom_index *_all_bottom_indices;

  /*
   * Pointer to the last (highest address) `bottom_index` entity;
   * assumes the allocator lock is held.
   */
#define GC_all_bottom_indices_end GC_arrays._all_bottom_indices_end
  bottom_index *_all_bottom_indices_end;

#define GC_scratch_free_ptr GC_arrays._scratch_free_ptr
  ptr_t _scratch_free_ptr;

#define GC_hdr_free_list GC_arrays._hdr_free_list
  hdr *_hdr_free_list;

#define GC_scratch_end_addr GC_arrays._scratch_end_addr
  word _scratch_end_addr; /*< the end point of the current scratch area */

#if defined(IRIX5) || (defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX))
#  define USE_SCRATCH_LAST_END_PTR
  /*
   * The address of the end point of the last obtained scratch area.
   * Used by `GC_register_dynamic_libraries()`.
   */
#  define GC_scratch_last_end_addr GC_arrays._scratch_last_end_addr
  word _scratch_last_end_addr;
#endif

#if defined(GC_ASSERTIONS) || defined(MAKE_BACK_GRAPH) \
    || defined(INCLUDE_LINUX_THREAD_DESCR)             \
    || (defined(KEEP_BACK_PTRS) && ALIGNMENT == 1)
#  define SET_REAL_HEAP_BOUNDS

  /*
   * Similar to `GC_least_plausible_heap_addr` and
   * `GC_greatest_plausible_heap_addr` but do not include future
   * (potential) heap expansion.  Both variables are zero initially.
   */
#  define GC_least_real_heap_addr GC_arrays._least_real_heap_addr
#  define GC_greatest_real_heap_addr GC_arrays._greatest_real_heap_addr
  word _least_real_heap_addr;
  word _greatest_real_heap_addr;
#endif

#define GC_used_heap_size_after_full GC_arrays._used_heap_size_after_full
  word _used_heap_size_after_full;

  /* Number of explicitly managed bytes of storage at last collection. */
#define GC_non_gc_bytes_at_gc GC_arrays._non_gc_bytes_at_gc
  word _non_gc_bytes_at_gc;

  /* The number of extra calls to `GC_mark_some` that we have made. */
#define GC_mark_deficit GC_arrays._mark_deficit
  size_t _mark_deficit;

#ifndef NO_BLACK_LISTING
  /*
   * Counter of the cases when found block by `GC_allochblk_nth` is
   * black-listed completely.
   */
#  define GC_drop_blacklisted_count GC_arrays._drop_blacklisted_count
  unsigned _drop_blacklisted_count;

  /* Number of warnings suppressed so far. */
#  define GC_large_alloc_warn_suppressed GC_arrays._large_alloc_warn_suppressed
  long _large_alloc_warn_suppressed;

  /* Non-stack false references seen at last full collection. */
#  define GC_old_normal_bl GC_arrays._old_normal_bl
  word *_old_normal_bl;
  /* Non-stack false references seen since last full collection. */
#  define GC_incomplete_normal_bl GC_arrays._incomplete_normal_bl
  word *_incomplete_normal_bl;

#  define GC_old_stack_bl GC_arrays._old_stack_bl
#  define GC_incomplete_stack_bl GC_arrays._incomplete_stack_bl
  word *_old_stack_bl;
  word *_incomplete_stack_bl;
#endif

  /* The limits of stack for `GC_mark_some()` and friends. */
#define GC_mark_stack GC_arrays._mark_stack
#define GC_mark_stack_limit GC_arrays._mark_stack_limit
  mse *_mark_stack;
  mse *_mark_stack_limit;

  /*
   * All ranges between `GC_mark_stack` (incl.) and `GC_mark_stack_top`
   * (incl.) still need to be marked from.
   */
#define GC_mark_stack_top GC_arrays._mark_stack_top
#ifdef PARALLEL_MARK
  /* Updated only with the mark lock held, but read asynchronously. */
  mse *volatile _mark_stack_top;

#  define GC_mark_no GC_arrays._mark_no
  word _mark_no; /*< protected by the mark lock */

  /*
   * Number of bytes of memory allocated since we released the allocator lock.
   * Instead of reacquiring the allocator lock just to add this in, we add it
   * in the next time we reacquire the allocator lock.  (Atomically adding it
   * does not work, since we would have to atomically update it in
   * `GC_malloc`, which is too expensive.)
   */
#  define GC_bytes_allocd_tmp GC_arrays._bytes_allocd_tmp
  volatile AO_t _bytes_allocd_tmp;
#else
  mse *_mark_stack_top;
#endif

#ifdef DYNAMIC_POINTER_MASK
  /*
   * Both mask and shift are zeros by default; if mask is zero, then
   * correct it to ~0 at the collector initialization.
   */
#  define GC_pointer_mask GC_arrays._pointer_mask
#  define GC_pointer_shift GC_arrays._pointer_shift
  word _pointer_mask;
  unsigned char _pointer_shift;
#endif

#ifdef THREADS
#  ifdef USE_SPIN_LOCK
#    define GC_allocate_lock GC_arrays._allocate_lock
  volatile AO_TS_t _allocate_lock;
#  endif
#  if !defined(HAVE_LOCKFREE_AO_OR) && defined(AO_HAVE_test_and_set_acquire) \
      && (!defined(NO_MANUAL_VDB) || defined(MPROTECT_VDB))
#    define NEED_FAULT_HANDLER_LOCK
#    define GC_fault_handler_lock GC_arrays._fault_handler_lock
  volatile AO_TS_t _fault_handler_lock;
#  endif
#  if !(defined(GC_ALWAYS_MULTITHREADED) \
        && (defined(USE_PTHREAD_LOCKS) || defined(USE_SPIN_LOCK)))
#    define GC_need_to_lock GC_arrays._need_to_lock
  GC_bool _need_to_lock;
#  endif

#  ifdef GC_ASSERTIONS
#    define GC_thr_initialized GC_arrays._thr_initialized
  GC_bool _thr_initialized;
#  endif
#  if defined(GC_USE_DLOPEN_WRAP) && !defined(GC_WIN32_THREADS) \
      && !defined(PLATFORM_THREADS) && !defined(SN_TARGET_PSP2)
#    define GC_syms_wrap_initialized GC_arrays._syms_wrap_initialized
  GC_bool _syms_wrap_initialized;
#  endif

#  define GC_roots_were_cleared GC_arrays._roots_were_cleared
  GC_bool _roots_were_cleared;
#else
#  ifndef GC_NO_FINALIZATION
/*
 * The variables to minimize the level of recursion when a client
 * finalizer allocates memory.
 */
#    define GC_finalizer_nested GC_arrays._finalizer_nested
#    define GC_finalizer_skipped GC_arrays._finalizer_skipped
  unsigned char _finalizer_nested;
  unsigned short _finalizer_skipped;
#  endif
#endif

  /* Has `GC_init()` been run? */
#define GC_is_initialized GC_arrays._is_initialized
  GC_bool _is_initialized;

  /*
   * Do we need a larger mark stack?  May be set by client-supplied
   * mark routines.
   */
#define GC_mark_stack_too_small GC_arrays._mark_stack_too_small
  GC_bool _mark_stack_too_small;

  /* Are there collectible marked objects in the heap? */
#define GC_objects_are_marked GC_arrays._objects_are_marked
  GC_bool _objects_are_marked;

#ifndef GC_DISABLE_INCREMENTAL
#  define GC_should_start_incremental_collection \
    GC_arrays._should_start_incremental_collection
  GC_bool _should_start_incremental_collection;
#endif

#ifndef GC_NO_FINALIZATION
  /* Avoid the work if unreachable finalizable objects are not used. */
  /* TODO: Turn this variable into a counter. */
#  define GC_need_unreachable_finalization \
    GC_arrays._need_unreachable_finalization
  GC_bool _need_unreachable_finalization;
#endif

#ifndef DONT_USE_ATEXIT
#  ifdef SMALL_CONFIG
#    define GC_skip_collect_atexit FALSE
#  else
  /*
   * A dedicated variable to avoid a garbage collection on abort.
   * `GC_find_leak` cannot be used for this purpose as otherwise
   * TSan finds a data race (between `GC_default_on_abort` and, e.g.,
   * `GC_finish_collection`).
   */
#    define GC_skip_collect_atexit GC_arrays._skip_collect_atexit
  GC_bool _skip_collect_atexit;
#  endif
#endif

#if defined(NO_FIND_LEAK) && defined(SHORT_DBG_HDRS)
#  define GC_debugging_initialized GC_arrays._debugging_initialized
  GC_bool _debugging_initialized;
#  define GC_debugging_started FALSE
#else
#  define GC_debugging_initialized GC_debugging_started
  /* `GC_debug_malloc()` has been called, once at least. */
#  define GC_debugging_started GC_arrays._debugging_started
  GC_bool _debugging_started;

#  define GC_have_errors GC_arrays._have_errors
#  ifdef AO_HAVE_store
  volatile AO_t _have_errors;
#  else
  GC_bool _have_errors;
#  endif
#endif

#define GC_explicit_typing_initialized GC_arrays._explicit_typing_initialized
#ifdef AO_HAVE_load_acquire
  volatile AO_t _explicit_typing_initialized;
#else
  GC_bool _explicit_typing_initialized;
#endif

  /* Indicate whether a full collection due to heap growth is needed. */
#define GC_need_full_gc GC_arrays._need_full_gc
  GC_bool _need_full_gc;

#ifndef NO_CLOCK
  /*
   * Do performance measurements if set to true (e.g., accumulation of the
   * total time of full collections).
   */
#  define GC_measure_performance GC_arrays._measure_performance
  GC_bool _measure_performance;

  /*
   * Variables for world-stop average delay time statistic computation.
   * `GC_world_stopped_total_divisor` is incremented every world stop and
   * halved when reached its maximum (or upon `GC_world_stopped_total_time`
   * overflow).  In milliseconds.
   */
  /* TODO: Store the nanosecond part. */
#  define GC_world_stopped_total_time GC_arrays._world_stopped_total_time
#  define GC_world_stopped_total_divisor GC_arrays._world_stopped_total_divisor
  unsigned _world_stopped_total_time;
  unsigned _world_stopped_total_divisor;
#endif

#ifndef NO_FIND_LEAK
#  define GC_n_leaked GC_arrays._n_leaked
  unsigned _n_leaked;
#endif
#ifndef SHORT_DBG_HDRS
#  define GC_n_smashed GC_arrays._n_smashed
  unsigned _n_smashed;
#endif

  /* Number of bytes in the accessible composite objects. */
#define GC_composite_in_use GC_arrays._composite_in_use
  word _composite_in_use;

  /* Number of bytes in the accessible atomic objects. */
#define GC_atomic_in_use GC_arrays._atomic_in_use
  word _atomic_in_use;

  /* GC number of latest successful `GC_expand_hp_inner()` call. */
#define GC_last_heap_growth_gc_no GC_arrays._last_heap_growth_gc_no
  word _last_heap_growth_gc_no;

  /*
   * Number of bytes of memory reclaimed minus the number of bytes originally
   * on free lists that we had to drop.  Protected by the allocator lock.
   */
#define GC_bytes_found GC_arrays._bytes_found
  GC_signed_word _bytes_found;

#ifndef GC_GET_HEAP_USAGE_NOT_NEEDED
  /*
   * Number of bytes reclaimed before this collection cycle; used for
   * statistics only.
   */
#  define GC_reclaimed_bytes_before_gc GC_arrays._reclaimed_bytes_before_gc
  word _reclaimed_bytes_before_gc;
#endif

#ifdef USE_MUNMAP
#  define GC_unmapped_bytes GC_arrays._unmapped_bytes
  word _unmapped_bytes;
#else
#  define GC_unmapped_bytes 0
#endif

#if defined(COUNT_UNMAPPED_REGIONS) && defined(USE_MUNMAP)
#  define GC_num_unmapped_regions GC_arrays._num_unmapped_regions
  GC_signed_word _num_unmapped_regions;
#else
#  define GC_num_unmapped_regions 0
#endif

#define GC_all_nils GC_arrays._all_nils
  bottom_index *_all_nils;

#define GC_scan_ptr GC_arrays._scan_ptr
  struct hblk *_scan_ptr;

#ifdef PARALLEL_MARK
#  define GC_main_local_mark_stack GC_arrays._main_local_mark_stack
  mse *_main_local_mark_stack;

  /*
   * The lowest entry on mark stack that may not be empty.
   * Updated only by the initiating thread.
   */
#  define GC_first_nonempty GC_arrays._first_nonempty
  volatile ptr_t _first_nonempty;
#endif

#ifdef ENABLE_TRACE
#  define GC_trace_ptr GC_arrays._trace_ptr
  ptr_t _trace_ptr;
#endif

#if !defined(ALWAYS_SMALL_CLEAR_STACK) && !defined(STACK_NOT_SCANNED) \
    && !defined(THREADS)
  /*
   * Coolest stack pointer value from which we have already cleared
   * the stack.
   */
#  define GC_min_sp GC_arrays._min_sp
  ptr_t _min_sp;

  /*
   * The "hottest" stack pointer value we have seen recently.
   * Degrades over time.
   */
#  define GC_high_water GC_arrays._high_water
  ptr_t _high_water;
#endif

#if CPP_PTRSZ > CPP_WORDSZ
#  define GC_noop_sink_ptr GC_arrays._noop_sink_ptr
  volatile ptr_t _noop_sink_ptr;
#endif

#define GC_noop_sink GC_arrays._noop_sink
#if defined(AO_HAVE_store) && defined(THREAD_SANITIZER)
  volatile AO_t _noop_sink;
#else
  volatile word _noop_sink;
#endif

#if !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION)
  /* Saved number of disappearing links for stats printing. */
#  define GC_old_dl_entries GC_arrays._old_dl_entries
  size_t _old_dl_entries;
#  ifndef GC_LONG_REFS_NOT_NEEDED
#    define GC_old_ll_entries GC_arrays._old_ll_entries
  size_t _old_ll_entries;
#  endif
#endif

#define GC_mark_stack_size GC_arrays._mark_stack_size
  size_t _mark_stack_size;

#define GC_mark_state GC_arrays._mark_state
  mark_state_t _mark_state; /*< initialized to `MS_NONE` (0) */

#define GC_capacity_heap_sects GC_arrays._capacity_heap_sects
  size_t _capacity_heap_sects;

#define GC_n_heap_sects GC_arrays._n_heap_sects
  size_t _n_heap_sects; /*< number of separately added heap sections */

#ifdef ANY_MSWIN
#  define GC_n_heap_bases GC_arrays._n_heap_bases
  size_t _n_heap_bases; /*< see `GC_heap_bases[]` */
#endif

#ifdef USE_PROC_FOR_LIBRARIES
  /* Number of `GET_MEM`-allocated memory sections. */
#  define GC_n_memory GC_arrays._n_memory
  word _n_memory;
#endif

#ifdef GC_GCJ_SUPPORT
#  define GC_last_finalized_no GC_arrays._last_finalized_no
#  define GC_gcjobjfreelist GC_arrays._gcjobjfreelist
  word _last_finalized_no;
  ptr_t *_gcjobjfreelist;
#endif

#define GC_fo_entries GC_arrays._fo_entries
  size_t _fo_entries;

#ifndef GC_NO_FINALIZATION
#  define GC_dl_hashtbl GC_arrays._dl_hashtbl
#  define GC_fnlz_roots GC_arrays._fnlz_roots
#  define GC_log_fo_table_size GC_arrays._log_fo_table_size
#  ifndef GC_LONG_REFS_NOT_NEEDED
#    define GC_ll_hashtbl GC_arrays._ll_hashtbl
  struct dl_hashtbl_s _ll_hashtbl;
#  endif
  struct dl_hashtbl_s _dl_hashtbl;
  struct fnlz_roots_s _fnlz_roots;
  unsigned _log_fo_table_size;

#  ifndef GC_TOGGLE_REFS_NOT_NEEDED
#    define GC_toggleref_arr GC_arrays._toggleref_arr
#    define GC_toggleref_array_size GC_arrays._toggleref_array_size
#    define GC_toggleref_array_capacity GC_arrays._toggleref_array_capacity
  union toggle_ref_u *_toggleref_arr;
  size_t _toggleref_array_size;
  size_t _toggleref_array_capacity;
#  endif
#endif

#ifdef TRACE_BUF
#  define GC_trace_buf_pos GC_arrays._trace_buf_pos
  size_t _trace_buf_pos; /*< an index in the circular buffer */
#endif

  /*
   * How many consecutive collection/expansion failures?
   * Reset by `GC_allochblk()`.
   */
#define GC_alloc_fail_count GC_arrays._alloc_fail_count
  unsigned _alloc_fail_count;

#ifdef ENABLE_DISCLAIM
#  define GC_finalized_kind GC_arrays._finalized_kind
  unsigned _finalized_kind;
#endif

#ifndef NO_CLOCK
#  define GC_full_gc_total_time GC_arrays._full_gc_total_time
#  define GC_stopped_mark_total_time GC_arrays._stopped_mark_total_time
#  define GC_full_gc_total_ns_frac GC_arrays._full_gc_total_ns_frac
#  define GC_stopped_mark_total_ns_frac GC_arrays._stopped_mark_total_ns_frac
  unsigned long _full_gc_total_time; /*< in ms, may wrap */
  unsigned long _stopped_mark_total_time;
  unsigned32 _full_gc_total_ns_frac; /*< fraction of 1 ms */
  unsigned32 _stopped_mark_total_ns_frac;
#endif

#ifdef GC_WIN32_THREADS
  /* Largest index in `dll_thread_table` that was ever used. */
#  define GC_max_thread_index GC_arrays._max_thread_index
  volatile LONG _max_thread_index;
#endif

  /* Total size of registered root sections. */
#define GC_root_size GC_arrays._root_size
  word _root_size;

  /* `GC_static_roots[0..n_root_sets-1]` contains the valid root sets. */
#define n_root_sets GC_arrays._n_root_sets
  size_t _n_root_sets;

#define GC_excl_table_entries GC_arrays._excl_table_entries
  size_t _excl_table_entries; /*< number of entries in use */

#define GC_ed_size GC_arrays._ed_size
  size_t _ed_size; /*< current size of above arrays */

#define GC_avail_descr GC_arrays._avail_descr
  size_t _avail_descr; /*< next available slot */

#if defined(CAN_HANDLE_FORK) && defined(GC_PTHREADS)
  /* Value of `pthread_self()` of the thread which called `fork()`. */
#  define GC_parent_pthread_self GC_arrays._parent_pthread_self
  pthread_t _parent_pthread_self;
#endif

  /* Points to array of extended descriptors. */
#define GC_ext_descriptors GC_arrays._ext_descriptors
  typed_ext_descr_t *_ext_descriptors;

  /*
   * Table of user-defined mark procedures.  There is a small number
   * of these, which can be referenced by `DS_PROC` mark descriptors.
   * See `gc_mark.h` file.
   */
#define GC_mark_procs GC_arrays._mark_procs
  GC_mark_proc _mark_procs[GC_MAX_MARK_PROCS];

  /*
   * `GC_valid_offsets[i]` implies
   * `GC_modws_valid_offsets[i % sizeof(ptr_t)]`.
   */
#define GC_modws_valid_offsets GC_arrays._modws_valid_offsets
  char _modws_valid_offsets[sizeof(ptr_t)];

#ifdef MAKE_BACK_GRAPH
  /* Points to never-used `back_edges` space. */
#  define GC_n_back_edge_structs GC_arrays._n_back_edge_structs
  int _n_back_edge_structs;

#  define GC_back_edge_space GC_arrays._back_edge_space
  struct back_edges_s *_back_edge_space;

  /* Pointer to free list of deallocated `back_edges` structures. */
#  define GC_avail_back_edges GC_arrays._avail_back_edges
  struct back_edges_s *_avail_back_edges;

  /*
   * Table of objects that are currently on the depth-first search stack.
   * Only objects with in-degree one are in this table.  Other objects are
   * identified using `HEIGHT_IN_PROGRESS`.
   */
  /* FIXME: This data structure needs improvement. */
#  define GC_backgraph_in_progress_space GC_arrays._backgraph_in_progress_space
#  define GC_backgraph_in_progress_size GC_arrays._backgraph_in_progress_size
#  define GC_backgraph_n_in_progress GC_arrays._backgraph_n_in_progress
  ptr_t *_backgraph_in_progress_space;
  size_t _backgraph_in_progress_size;
  size_t _backgraph_n_in_progress;

#  define GC_backgraph_max_deepest_h GC_arrays._backgraph_max_deepest_h
#  define GC_backgraph_deepest_height GC_arrays._backgraph_deepest_height
#  define GC_backgraph_deepest_obj GC_arrays._backgraph_deepest_obj
  word _backgraph_max_deepest_h;
  word _backgraph_deepest_height;
  ptr_t _backgraph_deepest_obj;
#endif

#ifdef GC_READ_ENV_FILE
  /*
   * The content of the `.gc.env` file with CR and LF replaced to '\0'.
   * `NULL` if the file is missing or empty.  Otherwise, always ends
   * with '\0' (designating the end of the file).
   */
#  define GC_envfile_content GC_arrays._envfile_content
  char *_envfile_content;

  /* Length of `GC_envfile_content` (if non-`NULL`). */
#  define GC_envfile_length GC_arrays._envfile_length
  size_t _envfile_length;
#endif

#ifndef ANY_MSWIN
  /*
   * The hash table header.  Used only to check whether a range
   * is already present.
   */
#  define GC_root_index GC_arrays._root_index
  struct roots *_root_index[RT_SIZE];
#endif

#if defined(SAVE_CALL_CHAIN) && !defined(DONT_SAVE_TO_LAST_STACK) \
    && (!defined(REDIRECT_MALLOC) || !defined(GC_HAVE_BUILTIN_BACKTRACE))
  /*
   * Stack at last garbage collection.  Useful for debugging mysterious
   * object disappearances.  In the multi-threaded case, we currently only
   * save the calling stack.  Not supported in case of `malloc` redirection
   * because `backtrace()` may call `malloc()`.
   */
  struct callinfo _last_stack[NFRAMES];
#  define SAVE_CALLERS_TO_LAST_STACK() GC_save_callers(GC_arrays._last_stack)
#else
#  define SAVE_CALLERS_TO_LAST_STACK() (void)0
#endif

  /* Free list for `NORMAL` objects. */
#define GC_objfreelist GC_arrays._objfreelist
  void *_objfreelist[MAXOBJGRANULES + 1];

  /* Free list for atomic objects. */
#define GC_aobjfreelist GC_arrays._aobjfreelist
  void *_aobjfreelist[MAXOBJGRANULES + 1];

  /*
   * Uncollectible but traced objects.  Objects on this and `_auobjfreelist`
   * are always marked, except during garbage collections.
   */
#define GC_uobjfreelist GC_arrays._uobjfreelist
  void *_uobjfreelist[MAXOBJGRANULES + 1];

#ifdef GC_ATOMIC_UNCOLLECTABLE
  /* Atomic uncollectible but traced objects. */
#  define GC_auobjfreelist GC_arrays._auobjfreelist
  void *_auobjfreelist[MAXOBJGRANULES + 1];
#endif

  /*
   * Number of granules to allocate when asked for a certain number of bytes
   * (plus `EXTRA_BYTES`).  Should be accessed with the allocator lock held.
   */
#define GC_size_map GC_arrays._size_map
  size_t _size_map[MAXOBJBYTES + 1];

#ifndef MARK_BIT_PER_OBJ
  /*
   * If the element is not `NULL`, then it points to a map of valid object
   * addresses.  `GC_obj_map[lg][i]` is `i % lg`.  This is now used purely
   * to replace a division in the marker by a table lookup.
   * `GC_obj_map[0]` is used for large objects and contains all nonzero
   * entries.  This gets us out of the marker fast path without an extra test.
   */
#  define GC_obj_map GC_arrays._obj_map
  hb_map_entry_t *_obj_map[MAXOBJGRANULES + 1];

#  define OBJ_MAP_LEN BYTES_TO_GRANULES(HBLKSIZE)
#endif

#define VALID_OFFSET_SZ HBLKSIZE
  /*
   * A nonzero `GC_valid_offsets[i]` means `i` is registered as
   * a displacement.
   */
#define GC_valid_offsets GC_arrays._valid_offsets
  char _valid_offsets[VALID_OFFSET_SZ];

#ifndef GC_DISABLE_INCREMENTAL
  /* Pages that were dirty at last `GC_read_dirty()` call. */
#  define GC_grungy_pages GC_arrays._grungy_pages
  page_hash_table _grungy_pages;

  /* Pages dirtied since last `GC_read_dirty()` call. */
#  define GC_dirty_pages GC_arrays._dirty_pages
#  ifdef MPROTECT_VDB
  volatile
#  endif
      page_hash_table _dirty_pages;
#endif

#if (defined(CHECKSUMS) && (defined(GWW_VDB) || defined(SOFT_VDB))) \
    || defined(PROC_VDB)
  /* A table to indicate the pages ever dirtied. */
#  define GC_written_pages GC_arrays._written_pages
  page_hash_table _written_pages;
#endif

  /* Heap segments potentially containing client objects. */
#define GC_heap_sects GC_arrays._heap_sects
  struct HeapSect *_heap_sects;

#if defined(USE_PROC_FOR_LIBRARIES)
  /* All `GET_MEM`-allocated memory.  Includes block headers and the like. */
#  define GC_our_memory GC_arrays._our_memory
  struct HeapSect _our_memory[MAX_HEAP_SECTS];
#endif

#ifdef ANY_MSWIN
  /* Start address of memory regions obtained from OS. */
#  define GC_heap_bases GC_arrays._heap_bases
  ptr_t _heap_bases[MAX_HEAP_SECTS];
#endif

#ifdef MSWINCE
  /* Committed lengths of memory regions obtained from OS. */
#  define GC_heap_lengths GC_arrays._heap_lengths
  word _heap_lengths[MAX_HEAP_SECTS];
#endif

#define GC_static_roots GC_arrays._static_roots
  struct roots _static_roots[MAX_ROOT_SETS];

  /* Array of exclusions, ascending address order. */
#define GC_excl_table GC_arrays._excl_table
  struct exclusion _excl_table[MAX_EXCLUSIONS];

  /*
   * The block header index.  Each entry points to a `bottom_index` entity.
   * On a 32-bit machine, it points to the index for a set of the high-order
   * bits equal to the index.  For longer addresses, we hash the high-order
   * bits to compute the index in `GC_top_index`, and each entry points to
   * a hash chain.  The last entry in each chain is `GC_all_nils`.
   */
#define GC_top_index GC_arrays._top_index
  bottom_index *_top_index[TOP_SZ];

#ifdef ECOS
#  ifndef ECOS_GC_MEMORY_SIZE
#    define ECOS_GC_MEMORY_SIZE (448 * 1024)
#  endif
#  define GC_ecos_brk_idx GC_arrays._ecos_brk_idx
#  define GC_ecos_memory GC_arrays._ecos_memory
  size_t _ecos_brk_idx;
  char _ecos_memory[ECOS_GC_MEMORY_SIZE];
#endif
};

GC_API_PRIV struct _GC_arrays GC_arrays;
#define beginGC_arrays ((ptr_t)(&GC_arrays))
#define endGC_arrays (beginGC_arrays + sizeof(GC_arrays))

/* Object kinds. */
#ifndef MAXOBJKINDS
#  ifdef SMALL_CONFIG
#    define MAXOBJKINDS 16
#  else
#    define MAXOBJKINDS 24
#  endif
#endif
GC_EXTERN struct obj_kind {
  /*
   * Array of free-list headers for this kind of object.  Point either
   * to `GC_arrays` or to storage allocated with `GC_scratch_alloc()`.
   */
  void **ok_freelist;

  /*
   * List headers for lists of blocks waiting to be swept.
   * Indexed by object size in granules.
   */
  struct hblk **ok_reclaim_list;

  /* Descriptor template for objects in this block. */
  word ok_descriptor;

  /*
   * Add object size in bytes to descriptor template to obtain descriptor.
   * Otherwise the template is used as is.
   */
  GC_bool ok_relocate_descr;

  /* Clear objects before putting them on the free list. */
  GC_bool ok_init;

#ifdef ENABLE_DISCLAIM
  /*
   * Mark from all, including unmarked, objects in block.
   * Used to protect objects reachable from reclaim notifiers.
   */
  GC_bool ok_mark_unconditionally;

  /*
   * The disclaim procedure is called before `obj` is reclaimed, but
   * must also tolerate being called with object from free list.
   * A nonzero exit prevents object from being reclaimed.
   */
  int(GC_CALLBACK *ok_disclaim_proc)(void * /* `obj` */);

#  define OK_DISCLAIM_INITZ /* comma */ , FALSE, 0
#else
#  define OK_DISCLAIM_INITZ /*< empty */
#endif
} GC_obj_kinds[MAXOBJKINDS];

#define beginGC_obj_kinds ((ptr_t)(&GC_obj_kinds[0]))
#define endGC_obj_kinds (beginGC_obj_kinds + sizeof(GC_obj_kinds))

/* The predefined kinds. */
#define PTRFREE GC_I_PTRFREE
#define NORMAL GC_I_NORMAL
#define UNCOLLECTABLE 2
#ifdef GC_ATOMIC_UNCOLLECTABLE
#  define AUNCOLLECTABLE 3
#  define IS_UNCOLLECTABLE(k) (((k) & ~1) == UNCOLLECTABLE)
#  define GC_N_KINDS_INITIAL_VALUE 4
#else
#  define IS_UNCOLLECTABLE(k) ((k) == UNCOLLECTABLE)
#  define GC_N_KINDS_INITIAL_VALUE 3
#endif

GC_EXTERN unsigned GC_n_kinds;

/* May mean the allocation granularity size, not page size. */
GC_EXTERN size_t GC_page_size;

#ifdef REAL_PAGESIZE_NEEDED
GC_EXTERN size_t GC_real_page_size;
#else
#  define GC_real_page_size GC_page_size
#endif

/*
 * Get heap memory from the OS.
 * Note that `sbrk`-like allocation is preferred, since it usually
 * makes it possible to merge consecutively allocated chunks.
 * It also avoids unintended recursion with `REDIRECT_MALLOC` macro
 * defined.  `GET_MEM()` argument should be of `size_t` type and
 * have no side-effect.  `GET_MEM()` returns `HBLKSIZE`-aligned chunk
 * (`NULL` means a failure).  In case of `MMAP_SUPPORTED`, the argument
 * must also be a multiple of a physical page size.
 * `GET_MEM` is currently not assumed to retrieve zero-filled space.
 */
/* TODO: Take advantage of `GET_MEM()` returning a zero-filled space. */
#if defined(ANY_MSWIN) || defined(MSWIN_XBOX1) || defined(OS2)
GC_INNER void *GC_get_mem(size_t lb);
#  define GET_MEM(lb) GC_get_mem(lb)
#  if defined(CYGWIN32) && !defined(USE_WINALLOC)
#    define NEED_UNIX_GET_MEM
#  endif
#elif defined(DOS4GW) || defined(EMBOX) || defined(KOS) || defined(NEXT) \
    || defined(NONSTOP) || defined(RTEMS) || defined(__CC_ARM)           \
    || (defined(SOLARIS) && !defined(USE_MMAP))
/* TODO: Use `page_alloc()` directly on Embox. */
#  if defined(REDIRECT_MALLOC) && !defined(CPPCHECK)
#    error Malloc redirection is unsupported
#  endif
#  define GET_MEM(lb)                                                  \
    ((void *)HBLKPTR((ptr_t)calloc(1, SIZET_SAT_ADD(lb, GC_page_size)) \
                     + GC_page_size - 1))
#elif !defined(GET_MEM)
GC_INNER void *GC_unix_get_mem(size_t lb);
#  define GET_MEM(lb) GC_unix_get_mem(lb)
#  define NEED_UNIX_GET_MEM
#endif

/*
 * Round up allocation size to a multiple of a page size.
 * `GC_setpagesize()` is assumed to be already invoked.
 */
#define ROUNDUP_PAGESIZE(lb) /*< `lb` should have no side-effect */ \
  (SIZET_SAT_ADD(lb, GC_page_size - 1) & ~(GC_page_size - 1))

/*
 * Same as `ROUNDUP_PAGESIZE` but is used to make `GET_MEM()` argument
 * safe.
 */
#ifdef MMAP_SUPPORTED
#  define ROUNDUP_PAGESIZE_IF_MMAP(lb) ROUNDUP_PAGESIZE(lb)
#else
#  define ROUNDUP_PAGESIZE_IF_MMAP(lb) (lb)
#endif

#ifdef ANY_MSWIN
GC_EXTERN SYSTEM_INFO GC_sysinfo;

/*
 * Is `p` the start of either the `malloc` heap, or of one of the collector
 * heap sections?
 */
GC_INNER GC_bool GC_is_heap_base(const void *p);
#endif

#ifdef GC_GCJ_SUPPORT
/* Note: `GC_hblkfreelist` and `GC_free_bytes` remain visible to GNU `gcj`. */
extern struct hblk *GC_hblkfreelist[];
extern word GC_free_bytes[];
#endif

/* This is used by `GC_do_blocking()`. */
struct blocking_data {
  GC_fn_type fn;
  void *client_data; /*< and result */
};

/* This is used by `GC_call_with_gc_active`, `GC_push_all_stack_sections`. */
struct GC_traced_stack_sect_s {
  ptr_t saved_stack_ptr;
#ifdef IA64
  ptr_t saved_backing_store_ptr;
  ptr_t backing_store_end;
#endif
  struct GC_traced_stack_sect_s *prev;
};

#ifdef THREADS
/*
 * Process all "traced stack sections" - scan entire stack except for
 * frames belonging to the user functions invoked by `GC_do_blocking`.
 */
GC_INNER void
GC_push_all_stack_sections(ptr_t lo, ptr_t hi,
                           struct GC_traced_stack_sect_s *traced_stack_sect);

/*
 * The total size, in bytes, of all stacks.
 * Updated on every `GC_push_all_stacks()` call.
 */
GC_EXTERN word GC_total_stacksize;

#else
/* Note: `NULL` value means we are not inside `GC_do_blocking()` call. */
GC_EXTERN ptr_t GC_blocked_sp;

/*
 * Points to the "frame" data held in stack by the innermost
 * `GC_call_with_gc_active()`.  `NULL` if no such "frame" active.
 */
GC_EXTERN struct GC_traced_stack_sect_s *GC_traced_stack_sect;
#endif /* !THREADS */

#if defined(E2K) && defined(THREADS) || defined(IA64)
/*
 * The bottom of the register stack of the primordial thread.
 * E2K: holds the offset (`ps_ofs`) instead of a pointer.
 */
GC_EXTERN ptr_t GC_register_stackbottom;
#endif

#ifdef IA64
/* Similar to `GC_push_all_stack_sections` but for IA-64 registers store. */
GC_INNER void GC_push_all_register_sections(
    ptr_t bs_lo, ptr_t bs_hi, GC_bool eager,
    struct GC_traced_stack_sect_s *traced_stack_sect);
#endif

/*
 * Mark bit operations.
 *
 * The marks are in a reserved area of each heap block.
 * Each object or granule has one mark bit associated with it.
 * Only those corresponding to the beginning of an object are used.
 */

/*
 * Retrieve, set, clear the `n`-th mark bit in a given heap block.
 * (Recall that bit `n` corresponds to `n`-th object or allocation granule
 * relative to the beginning of the block, including unused space.)
 */

#ifdef USE_MARK_BYTES
#  define mark_bit_from_hdr(hhdr, n) ((hhdr)->hb_marks[n])
#  define set_mark_bit_from_hdr(hhdr, n) (void)((hhdr)->hb_marks[n] = 1)
#  define clear_mark_bit_from_hdr(hhdr, n) (void)((hhdr)->hb_marks[n] = 0)
#else
/* Set mark bit correctly, even if mark bits may be concurrently accessed. */
#  if defined(PARALLEL_MARK) || (defined(THREAD_SANITIZER) && defined(THREADS))
/*
 * Workaround TSan false positive: there is no race between
 * `mark_bit_from_hdr` and `set_mark_bit_from_hdr` when `n` is different
 * (alternatively, `USE_MARK_BYTES` could be used).  If TSan is off, then
 * `AO_or()` is used only if we define `USE_MARK_BITS` macro explicitly.
 */
#    define OR_WORD(addr, bits) AO_or(addr, bits)
#  else
#    define OR_WORD(addr, bits) (void)(*(addr) |= (bits))
#  endif
#  define mark_bit_from_hdr(hhdr, n) \
    (((hhdr)->hb_marks[divWORDSZ(n)] >> modWORDSZ(n)) & (word)1)
#  define set_mark_bit_from_hdr(hhdr, n) \
    OR_WORD((hhdr)->hb_marks + divWORDSZ(n), (word)1 << modWORDSZ(n))
#  define clear_mark_bit_from_hdr(hhdr, n)                                    \
    (void)(((word *)CAST_AWAY_VOLATILE_PVOID((hhdr)->hb_marks))[divWORDSZ(n)] \
           &= ~((word)1 << modWORDSZ(n)))
#endif /* !USE_MARK_BYTES */

#ifdef MARK_BIT_PER_OBJ
/*
 * Get the mark bit index corresponding to the given byte offset and
 * size (in bytes).
 */
#  define MARK_BIT_NO(offset, sz) ((offset) / (sz))

/* Spacing between useful mark bits. */
#  define MARK_BIT_OFFSET(sz) 1

/* Position of final, always set, mark bit. */
#  define FINAL_MARK_BIT(sz) ((sz) > MAXOBJBYTES ? 1 : HBLK_OBJS(sz))
#else
#  define MARK_BIT_NO(offset, sz) BYTES_TO_GRANULES(offset)
#  define MARK_BIT_OFFSET(sz) BYTES_TO_GRANULES(sz)
#  define FINAL_MARK_BIT(sz)                 \
    ((sz) > MAXOBJBYTES ? MARK_BITS_PER_HBLK \
                        : BYTES_TO_GRANULES(HBLK_OBJS(sz) * (sz)))
#endif /* !MARK_BIT_PER_OBJ */

/* Important internal collector routines. */

/* Return the current stack pointer, approximately. */
GC_INNER ptr_t GC_approx_sp(void);

/*
 * Same as `GC_approx_sp` but a macro.  `sp` should be a local variable
 * of `volatile` `ptr_t` type.
 */
#if (defined(E2K) && defined(__clang__)         \
     || (defined(S390) && __clang_major__ < 8)) \
    && !defined(CPPCHECK)
/*
 * Workaround some bugs in clang:
 *   - "undefined reference to llvm.frameaddress" error (clang-9/e2k);
 *   - a crash in SystemZTargetLowering of libLLVM-3.8 (s390).
 */
#  define STORE_APPROX_SP_TO(sp) (void)(sp = (ptr_t)(&sp))
#elif defined(CPPCHECK)                              \
    || ((__GNUC__ >= 4 /* `GC_GNUC_PREREQ(4, 0)` */) \
        && !defined(STACK_NOT_SCANNED))
/* TODO: Use `GC_GNUC_PREREQ` after fixing a bug in cppcheck. */
/* Note: l-value is passed instead of pointer to `sp` (because of cppcheck). */
#  define STORE_APPROX_SP_TO(sp) (void)(sp = (ptr_t)__builtin_frame_address(0))
#else
#  define STORE_APPROX_SP_TO(sp) (void)(sp = (ptr_t)(&sp))
#endif

/* Have we allocated enough to amortize a collection? */
GC_INNER GC_bool GC_should_collect(void);

/*
 * Get the next block whose address is at least `h`.  Returned block
 * is managed by the collector.  The block must be in use unless
 * `allow_free` is TRUE.  Return `NULL` if there is no such block.
 */
GC_INNER struct hblk *GC_next_block(struct hblk *h, GC_bool allow_free);

/*
 * Get the last (highest address) block whose address is at most `h`.
 * Returned block is managed by the collector, but may or may not be in use.
 * Return `NULL` if there is no such block.
 */
GC_INNER struct hblk *GC_prev_block(struct hblk *h);

GC_INNER void GC_mark_init(void);

/*
 * Clear mark bits in all allocated heap blocks (i.e. for all heap objects).
 * This invalidates the marker invariant, and sets `GC_mark_state` to
 * reflect this.  (This implicitly starts marking to reestablish the
 * invariant.)
 */
GC_INNER void GC_clear_marks(void);

/*
 * Tell the marker that marked objects may point to unmarked ones, and
 * roots may point to unmarked objects.  Reset mark stack.
 */
GC_INNER void GC_invalidate_mark_state(void);

/*
 * Perform a small amount of marking.  We try to touch roughly a page
 * of memory.  Returns quickly if no collection is in progress.
 * Returns `TRUE` if we just finished a mark phase.
 * `cold_gc_frame` argument is an address inside a frame of the
 * collector that remains valid until all marking is complete;
 * `NULL` value indicates that it is OK to miss some register values.
 * In the case of an incremental collection, the world may be running.
 */
GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame);

/*
 * Initiate a garbage collection.  Initiates a full collection if the
 * mark state is invalid; otherwise it is a partial one.
 */
GC_INNER void GC_initiate_gc(void);

/*
 * Is a collection in progress?  Note that this can return `TRUE` in
 * the non-incremental case, if a collection has been abandoned and
 * the mark state is now `MS_INVALID`.
 */
GC_INNER GC_bool GC_collection_in_progress(void);

/*
 * Push contents of the symbol residing in the static roots area excluded
 * from scanning by the collector for a reason.  Note: it should be used only
 * for symbols of relatively small size (containing one or several pointers).
 */
#define GC_PUSH_ALL_SYM(sym) GC_push_all_eager(&(sym), &(sym) + 1)

/* Same as `GC_push_all` but consider interior pointers as valid. */
GC_INNER void GC_push_all_stack(ptr_t b, ptr_t t);

#ifdef NO_VDB_FOR_STATIC_ROOTS
#  define GC_push_conditional_static(b, t, all) \
    ((void)(all), GC_push_all(b, t))
#else
/*
 * Same as `GC_push_conditional` (does either of `GC_push_all` or
 * `GC_push_selected` depending on the third argument) but the caller
 * guarantees the region belongs to the registered static roots.
 */
GC_INNER void GC_push_conditional_static(void *b, void *t, GC_bool all);
#endif

#if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK)
/*
 * Similar to `GC_push_conditional` but scans the whole region immediately.
 * `GC_mark_local` does not handle memory protection faults yet.
 * So, the static data regions are scanned immediately by `GC_push_roots`.
 */
GC_INNER void GC_push_conditional_eager(void *bottom, void *top, GC_bool all);
#endif

/*
 * In the multi-threaded case, we push part of the current thread stack
 * with `GC_push_all_eager` when we push the registers.  This gets the
 * callee-save registers that may disappear.  The remainder of the stacks
 * are scheduled for scanning in `(*GC_push_other_roots)()`, which is
 * thread-package-specific.
 */

/*
 * Push all or dirty roots.  Call the mark routine (`GC_push_one` for
 * a single pointer, `GC_push_conditional` on groups of pointers) on every
 * top level accessible pointer.  If not `all`, then arrange to push only
 * possibly altered values.  `cold_gc_frame` is an address inside
 * a collector frame that remains valid until all marking is complete;
 * a `NULL` pointer indicates that it is OK to miss some register values.
 */
GC_INNER void GC_push_roots(GC_bool all, ptr_t cold_gc_frame);

/*
 * Push system or application specific roots onto the mark stack.
 * In some environments (e.g. a multi-threaded one) this is predefined
 * to be nonzero.  A client-supplied replacement should also call the
 * original function.  Remains externally visible as used by some
 * well-known 3rd-party software (e.g., ECL) currently.
 */
GC_API_PRIV GC_push_other_roots_proc GC_push_other_roots;

#ifdef THREADS
GC_INNER void GC_push_thread_structures(void);
#endif

/*
 * A pointer set to `GC_push_typed_structures_proc` lazily so that we can
 * avoid linking in the typed allocation support if the latter is unused.
 */
GC_EXTERN void (*GC_push_typed_structures)(void);

typedef void (*GC_with_callee_saves_func)(ptr_t arg, void *context);

/*
 * Ensure that either registers are pushed, or callee-save registers are
 * somewhere on the stack, and then call `fn(arg, ctxt)`.  `ctxt` is either
 * a pointer to a `ucontext_t` entity we generated, or `NULL`.  Could be
 * called with or w/o the allocator lock held; could be called from a signal
 * handler as well.
 */
GC_INNER void GC_with_callee_saves_pushed(GC_with_callee_saves_func fn,
                                          ptr_t arg);

#if defined(IA64) || defined(SPARC)
/*
 * Cause all stacked registers to be saved in memory.  Return a pointer to
 * the top of the corresponding memory stack.
 */
ptr_t GC_save_regs_in_stack(void);
#endif

#ifdef E2K
#  include <asm/e2k_syswork.h>
#  include <errno.h>
#  include <sys/syscall.h>

#  if defined(CPPCHECK)
/*
 * Workaround "Uninitialized bs_lo" and "obsolete alloca() called"
 * false positive (FP) warnings.
 */
#    define PS_ALLOCA_BUF(pbuf, sz) \
      (void)(GC_noop1_ptr(pbuf), *(pbuf) = (ptr_t)__builtin_alloca(sz))
#  else
#    define PS_ALLOCA_BUF(pbuf, sz) (void)(*(pbuf) = (ptr_t)alloca(sz))
#  endif

/*
 * Approximate size (in bytes) of the obtained procedure stack part
 * belonging to `syscall()` itself.
 */
#  define PS_SYSCALL_TAIL_BYTES 0x100

/*
 * Determine the current size of the whole procedure stack.  The size
 * is valid only within the current function.
 */
#  define GET_PROCEDURE_STACK_SIZE_INNER(psz_ull)                            \
    do {                                                                     \
      *(psz_ull) = 0; /*< might be redundant */                              \
      if (syscall(__NR_access_hw_stacks, E2K_GET_PROCEDURE_STACK_SIZE, NULL, \
                  NULL, 0, psz_ull)                                          \
          == -1)                                                             \
        ABORT_ARG1("Cannot get size of procedure stack", ": errno= %d",      \
                   errno);                                                   \
      GC_ASSERT(*(psz_ull) > 0 && *(psz_ull) % sizeof(ptr_t) == 0);          \
    } while (0)

#  ifdef THREADS
#    define PS_COMPUTE_ADJUSTED_OFS(padj_ps_ofs, ps_ofs, ofs_sz_ull)      \
      do {                                                                \
        if ((ofs_sz_ull) <= (ps_ofs) /* `&& ofs_sz_ull > 0` */)           \
          ABORT_ARG2("Incorrect size of procedure stack",                 \
                     ": ofs= %lu, size= %lu", (unsigned long)(ps_ofs),    \
                     (unsigned long)(ofs_sz_ull));                        \
        *(padj_ps_ofs) = (ps_ofs) > (unsigned)PS_SYSCALL_TAIL_BYTES       \
                             ? (ps_ofs) - (unsigned)PS_SYSCALL_TAIL_BYTES \
                             : 0;                                         \
      } while (0)
#  else
/* A simplified variant of the above assuming `ps_ofs` is a zero const. */
#    define PS_COMPUTE_ADJUSTED_OFS(padj_ps_ofs, ps_ofs, ofs_sz_ull) \
      do {                                                           \
        GC_STATIC_ASSERT((ps_ofs) == 0);                             \
        (void)(ofs_sz_ull);                                          \
        *(padj_ps_ofs) = 0;                                          \
      } while (0)
#  endif /* !THREADS */

/*
 * Copy procedure (register) stack to a stack-allocated buffer.
 * Usable from a signal handler.  The buffer (`*pbuf`) is valid only
 * within the current function.  `ps_ofs` designates the offset in the
 * procedure stack to copy the contents from.  Note: this macro cannot
 * be changed to a function because `alloca()` and both `syscall()`
 * should be called in the context of the caller.
 */
#  define GET_PROCEDURE_STACK_LOCAL(ps_ofs, pbuf, psz)                      \
    do {                                                                    \
      unsigned long long ofs_sz_ull;                                        \
      size_t adj_ps_ofs;                                                    \
                                                                            \
      GET_PROCEDURE_STACK_SIZE_INNER(&ofs_sz_ull);                          \
      PS_COMPUTE_ADJUSTED_OFS(&adj_ps_ofs, ps_ofs, ofs_sz_ull);             \
      *(psz) = (size_t)ofs_sz_ull - adj_ps_ofs;                             \
      /* Allocate buffer on the stack; cannot return `NULL`. */             \
      PS_ALLOCA_BUF(pbuf, *(psz));                                          \
      /* Copy the procedure stack at the given offset to the buffer. */     \
      for (;;) {                                                            \
        ofs_sz_ull = adj_ps_ofs;                                            \
        if (syscall(__NR_access_hw_stacks, E2K_READ_PROCEDURE_STACK_EX,     \
                    &ofs_sz_ull, *(pbuf), *(psz), NULL)                     \
            != -1)                                                          \
          break;                                                            \
        if (errno != EAGAIN)                                                \
          ABORT_ARG2("Cannot read procedure stack", ": sz= %lu, errno= %d", \
                     (unsigned long)(*(psz)), errno);                       \
      }                                                                     \
    } while (0)
#endif /* E2K */

#if defined(E2K) && defined(USE_PTR_HWTAG)
/* Load value and get tag of the target memory. */
#  if defined(__ptr64__)
#    define LOAD_TAGGED_VALUE(v, tag, p)                        \
      do {                                                      \
        ptr_t val;                                              \
        __asm__ __volatile__("ldd, sm %[adr], 0x0, %[val]\n\t"  \
                             "gettagd %[val], %[tag]\n"         \
                             : [val] "=r"(val), [tag] "=r"(tag) \
                             : [adr] "r"(p));                   \
        v = val;                                                \
      } while (0)
#  elif !defined(CPPCHECK)
#    error Unsupported -march for e2k target
#  endif

#  define LOAD_PTR_OR_CONTINUE(v, p) \
    {                                \
      int tag LOCAL_VAR_INIT_OK;     \
      LOAD_TAGGED_VALUE(v, tag, p);  \
      if (tag != 0)                  \
        continue;                    \
    }
#elif defined(CHERI_PURECAP)
#  define HAS_TAG_AND_PERM_LOAD(cap) \
    (cheri_tag_get(cap) != 0 && (cheri_perms_get(cap) & CHERI_PERM_LOAD) != 0)

#  define LOAD_PTR_OR_CONTINUE(v, p)                                         \
    {                                                                        \
      word base_addr;                                                        \
      v = *(ptr_t *)(p);                                                     \
      if (!HAS_TAG_AND_PERM_LOAD(v))                                         \
        continue;                                                            \
      base_addr = cheri_base_get(v);                                         \
      if (ADDR(v) < base_addr || ADDR(v) >= base_addr + cheri_length_get(v)) \
        continue;                                                            \
    }

#  define CAPABILITY_COVERS_RANGE(cap, b_addr, e_addr) \
    (cheri_base_get(cap) <= (b_addr)                   \
     && cheri_base_get(cap) + cheri_length_get(cap) >= (e_addr))
#  define SPANNING_CAPABILITY(cap, b_addr, e_addr)                       \
    (cheri_tag_get(cap) && CAPABILITY_COVERS_RANGE(cap, b_addr, e_addr)  \
     && (cheri_perms_get(cap) & (CHERI_PERM_LOAD | CHERI_PERM_LOAD_CAP)) \
            != 0)
#else
#  define LOAD_PTR_OR_CONTINUE(v, p) (void)(v = *(ptr_t *)(p))
#endif /* !CHERI_PURECAP */

#if defined(DARWIN) && defined(THREADS)
/*
 * If `p` points to an object, mark it and push contents on the mark stack.
 * Pointer recognition test always accepts interior pointers, i.e. this is
 * appropriate for pointers found on the thread stack.
 */
void GC_push_one(word p);
#endif

/*
 * Mark and push (i.e. gray) a single object `p` onto the main mark stack.
 * Consider `p` to be valid if it is an interior pointer.  The object `p`
 * has passed a preliminary pointer validity test, but we do not definitely
 * know whether it is valid.  Mark bits are not atomically updated; thus
 * this must be the only thread setting them.
 */
#if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
GC_INNER void GC_mark_and_push_stack(ptr_t p, ptr_t source);
#else
GC_INNER void GC_mark_and_push_stack(ptr_t p);
#endif

/* Is the block with the given header containing no pointers? */
#define IS_PTRFREE(hhdr) (0 == (hhdr)->hb_descr)

/* Clear all mark bits in the header. */
GC_INNER void GC_clear_hdr_marks(hdr *hhdr);

/* Set all mark bits in the header.  Used for uncollectible blocks. */
GC_INNER void GC_set_hdr_marks(hdr *hhdr);

/* Set all mark bits associated with a free list. */
GC_INNER void GC_set_fl_marks(ptr_t);

#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
/*
 * Check that all mark bits associated with a free list are set.
 * Abort if not.
 */
void GC_check_fl_marks(void **);
#endif

/*
 * Add [`b`,`e`) to the root set.  Adding the same interval a second
 * time is a moderately fast no-op, and hence benign.  We do not handle
 * different but overlapping intervals efficiently.  (But we do handle
 * them correctly.)  `tmp` specifies that the interval may be deleted
 * before re-registering dynamic libraries.
 */
GC_INNER void GC_add_roots_inner(ptr_t b, ptr_t e, GC_bool tmp);

#ifdef USE_PROC_FOR_LIBRARIES
/*
 * Remove given range from every static root which intersects with the range.
 * `GC_remove_tmp_roots` is assumed to be called before this function is
 * called (repeatedly) by `GC_register_map_entries`.
 */
GC_INNER void GC_remove_roots_subregion(ptr_t b, ptr_t e);
#endif

/*
 * Inform the collector that a certain section of statically allocated
 * memory contains no pointers to garbage-collected memory.
 * The range boundaries should be properly aligned and valid.
 */
GC_INNER void GC_exclude_static_roots_inner(ptr_t start, ptr_t finish);

#if defined(ANY_MSWIN) || defined(DYNAMIC_LOADING)
/* Add dynamic library data sections to the root set. */
GC_INNER void GC_register_dynamic_libraries(void);
#endif

/*
 * Remove and re-register dynamic libraries if we are configured to do
 * that at each collection.
 */
GC_INNER void GC_cond_register_dynamic_libraries(void);

/* Machine-dependent startup routines. */

/*
 * Get the cold end of the stack of the primordial thread.  This is always
 * called from the main (primordial) thread.
 */
GC_INNER ptr_t GC_get_main_stack_base(void);

#ifdef IA64
/* Get the cold end of register stack. */
GC_INNER ptr_t GC_get_register_stack_base(void);
#endif

GC_INNER void GC_register_data_segments(void);

#ifdef THREADS
/* Both are invoked from `GC_init()` only. */
GC_INNER void GC_thr_init(void);

/*
 * Perform all initializations, including those that may require allocation,
 * e.g. initialize thread-local free lists if used.  Called by `GC_init()`.
 */
GC_INNER void GC_init_parallel(void);

#  ifndef DONT_USE_ATEXIT
GC_INNER GC_bool GC_is_main_thread(void);
#  endif
#else
#  ifdef TRACE_BUF
void GC_add_trace_entry(const char *caller_fn_name, ptr_t arg1, ptr_t arg2);
#  endif
#endif /* !THREADS */

#ifdef NO_BLACK_LISTING
#  define GC_bl_init() (void)0
/* Do not define `GC_bl_init_no_interiors()`. */
#  define GC_ADD_TO_BLACK_LIST_NORMAL(p, source) ((void)(p))
#  define GC_ADD_TO_BLACK_LIST_STACK(p, source) ((void)(p))
#  define GC_promote_black_lists() (void)0
#  define GC_unpromote_black_lists() (void)0
#else

/*
 * If we need a block of `n` bytes, and we have a block of `n + BL_LIMIT`
 * bytes available, and `n` is greater than `BL_LIMIT`, but all possible
 * positions in it are black-listed, we just use it anyway (and print
 * a warning, if warnings are enabled).  This risks subsequently leaking
 * the block due to a false reference.  But not using the block risks
 * unreasonable immediate heap growth.
 */
#  define BL_LIMIT GC_black_list_spacing

/*
 * Average number of bytes between black-listed blocks.  Approximate.
 * Counts only blocks that are "stack black-listed", i.e. that are
 * problematic in the interior of an object.
 */
GC_EXTERN word GC_black_list_spacing;

/*
 * The interval between unsuppressed warnings about repeated allocation
 * of a very large block.
 */
GC_EXTERN long GC_large_alloc_warn_interval;

/* Initialize the black listing mechanism. */
GC_INNER void GC_bl_init(void);
GC_INNER void GC_bl_init_no_interiors(void);

#  ifdef PRINT_BLACK_LIST
/*
 * Register bits as a possible future false reference from the heap
 * or static data.  The argument `p` is not a valid pointer reference,
 * but it falls inside the plausible heap bounds.
 */
GC_INNER void GC_add_to_black_list_normal(ptr_t p, ptr_t source);
#    define GC_ADD_TO_BLACK_LIST_NORMAL(p, source) \
      if (GC_all_interior_pointers) {              \
        GC_add_to_black_list_stack(p, source);     \
      } else                                       \
        GC_add_to_black_list_normal(p, source)
GC_INNER void GC_add_to_black_list_stack(ptr_t p, ptr_t source);
#    define GC_ADD_TO_BLACK_LIST_STACK(p, source) \
      GC_add_to_black_list_stack(p, source)
#  else
GC_INNER void GC_add_to_black_list_normal(ptr_t p);
#    define GC_ADD_TO_BLACK_LIST_NORMAL(p, source) \
      if (GC_all_interior_pointers) {              \
        GC_add_to_black_list_stack(p);             \
      } else                                       \
        GC_add_to_black_list_normal(p)
GC_INNER void GC_add_to_black_list_stack(ptr_t p);
#    define GC_ADD_TO_BLACK_LIST_STACK(p, source) GC_add_to_black_list_stack(p)
#  endif /* PRINT_BLACK_LIST */

/*
 * Declare an end to a black listing phase.  (I.e. signal the completion of
 * a collection.)  Turn the incomplete black lists into new black lists, etc.
 */
GC_INNER void GC_promote_black_lists(void);

/*
 * Approximately undo the effect of `GC_promote_black_lists()`.
 * This actually loses some information, but only in a reasonably safe way.
 */
GC_INNER void GC_unpromote_black_lists(void);
#endif

/*
 * The collector internal memory allocation for small objects.
 * Deallocation is not possible.  May return `NULL`.
 */
GC_INNER ptr_t GC_scratch_alloc(size_t bytes);

#ifdef GWW_VDB
/* `GC_scratch_recycle_no_gww()` is not used. */
#else
#  define GC_scratch_recycle_no_gww GC_scratch_recycle_inner
#endif
/* Reuse the memory region by the heap. */
GC_INNER void GC_scratch_recycle_inner(void *ptr, size_t sz);

#ifndef MARK_BIT_PER_OBJ
/*
 * Add a heap block map for objects of a size in granules to `GC_obj_map`.
 * A size of zero is used for large objects.  Returns `FALSE` on failure.
 */
GC_INNER GC_bool GC_add_map_entry(size_t lg);
#endif

/*
 * Same as `GC_register_displacement` but assuming the allocator lock
 * is already held.
 */
GC_INNER void GC_register_displacement_inner(size_t offset);

/*
 * Allocate a new heap block for small objects of size `lg` (in granules)
 * and `kind`.  Add all of the block's objects to the free list for objects
 * of that size.  Set all mark bits if objects are uncollectible.
 * Will fail to do anything if out of memory.
 */
GC_INNER void GC_new_hblk(size_t lg, int kind);

/*
 * Build a free list for objects of size `lg` (in granules) inside heap
 * block `h`.  Clear objects inside `h` if `clear` argument is set.
 * Add `list` to the end of the free list we build.  Return the new
 * free list.  Normally called by `GC_new_hblk()`, but this could also
 * be called without the allocator lock, if we ensure that there is no
 * concurrent collection which might reclaim objects that we have not
 * yet allocated.
 */
GC_INNER ptr_t GC_build_fl(struct hblk *h, ptr_t list, size_t lg,
                           GC_bool clear);

/*
 * Allocate (and return pointer to) a heap block for objects of the
 * given size and alignment (in bytes), searching over the appropriate
 * free block lists; inform the marker that the found block is valid
 * for objects of the indicated size.  Assumes (as implied by the argument
 * name) that `EXTRA_BYTES` value is already added to the size, if needed.
 * The client is responsible for clearing the block, if needed.
 * Note: we set `GC_obj_map` field in the header correctly; the caller
 * is responsible for building an object's free list in the block.
 */
GC_INNER struct hblk *GC_allochblk(size_t lb_adjusted, int kind,
                                   unsigned flags, size_t align_m1);

/*
 * Deallocate (free) a heap block and mark it as invalid.  Coalesce it
 * with its neighbors if possible.  All mark words are assumed to be cleared.
 */
GC_INNER void GC_freehblk(struct hblk *p);

/*  Miscellaneous GC routines. */

/*
 * This explicitly increases the size of the heap.  It is used internally,
 * but may also be invoked from `GC_expand_hp` by client.  The argument is
 * in units of `HBLKSIZE`.  (An argument of zero is treated as 1.)
 * Returns `FALSE` on failure.
 */
GC_INNER GC_bool GC_expand_hp_inner(word n);

/*
 * Restore unmarked objects to free lists, or (if `abort_if_found` is `TRUE`)
 * report them.  (I.e. perform `GC_reclaim_block()` on the entire heap,
 * after first clearing small-object free lists if we are not just looking
 * for leaks.)  Sweeping of small object pages is largely deferred.
 */
GC_INNER void GC_start_reclaim(GC_bool abort_if_found);

/*
 * Sweep blocks of the indicated object size (in granules) and kind
 * until either the appropriate nonempty free list is found, or there
 * are no more blocks to sweep.
 */
GC_INNER void GC_continue_reclaim(size_t lg, int kind);

/*
 * Reclaim all small blocks waiting to be reclaimed.  Abort and return
 * `FALSE` when/if `(*stop_func)()` returns `TRUE`.  If this returns `TRUE`,
 * then it is safe to restart the world with incorrectly cleared mark bits.
 * If `ignore_old`, then reclaim only blocks that have been reclaimed
 * recently, and discard the rest.  `stop_func` may be 0.
 */
GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old);

/*
 * Generic procedure to rebuild a free list in `hbp` with header `hhdr`,
 * with objects of size `sz` bytes.  Add `list` to the end of the free list.
 * Add the number of reclaimed bytes to `*pcount`.  Note: it could be called
 * directly from `GC_malloc_many`.
 */
GC_INNER ptr_t GC_reclaim_generic(struct hblk *hbp, hdr *hhdr, size_t sz,
                                  GC_bool init, ptr_t list, word *pcount);

/*
 * Is given heap block completely unmarked (i.e. contains no marked objects)?
 * This does not require the block to be in physical memory.
 */
GC_INNER GC_bool GC_block_empty(const hdr *hhdr);

/* Always returns 0 (`FALSE`). */
GC_INNER int GC_CALLBACK GC_never_stop_func(void);

/*
 * Stop-the-world garbage collection.  The caller must have acquired
 * the allocator lock.  If `stop_func` is not `GC_never_stop_func`, then
 * abort if `stop_func` returns `TRUE`.  Return `TRUE` if we successfully
 * completed the collection (otherwise the collection is aborted).
 */
GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func stop_func);

#define GC_gcollect_inner() (void)GC_try_to_collect_inner(GC_never_stop_func)

#ifdef THREADS
/*
 * We may currently be in thread creation or destruction.  Only set to `TRUE`
 * while the allocator lock is held.  When set, it is OK to run the garbage
 * collection from an unknown thread.  Protected by the allocator lock.
 */
GC_EXTERN GC_bool GC_in_thread_creation;
#endif

/*
 * Do `n_blocks` units of a garbage collection work, if appropriate.
 * A unit is an amount appropriate for `HBLKSIZE` bytes of allocation.
 */
GC_INNER void GC_collect_a_little_inner(size_t n_blocks);

GC_INNER void *GC_malloc_kind_aligned_global(size_t lb, int kind,
                                             size_t align_m1);

GC_INNER void *GC_generic_malloc_aligned(size_t lb, int kind, unsigned flags,
                                         size_t align_m1);

/*
 * Allocate an object of the given `kind` but assuming the allocator
 * lock is already held.  Should not be used to directly allocate
 * objects requiring special handling on allocation.  `flags` argument
 * should be 0 or `IGNORE_OFF_PAGE`; in the latter case the client
 * guarantees there will always be a pointer to the beginning (i.e.
 * within the first `hblk`) of the object while it is live.
 */
GC_INNER void *GC_generic_malloc_inner(size_t lb, int kind, unsigned flags);

/*
 * Collect or expand heap in an attempt make the indicated number of
 * free blocks available.  Should be called until the blocks are
 * available (setting `retry` value to `TRUE` unless this is the first
 * call in a loop) or until it fails by returning `FALSE`.  The `flags`
 * argument should be `IGNORE_OFF_PAGE` or 0.
 */
GC_INNER GC_bool GC_collect_or_expand(word needed_blocks, unsigned flags,
                                      GC_bool retry);

/*
 * Make the indicated object free list nonempty, and return its head (the
 * first object on the free list).  The object must be removed from the free
 * list by the caller.  The size is in granules.
 */
GC_INNER ptr_t GC_allocobj(size_t lg, int kind);

#ifdef GC_ADD_CALLER
/*
 * `GC_DBG_EXTRAS` is used by the collector debug API functions (unlike
 * `GC_EXTRAS` used by the debug API macros) thus `GC_RETURN_ADDR_PARENT`
 * (pointing to client caller) should be used if possible.
 */
#  ifdef GC_HAVE_RETURN_ADDR_PARENT
#    define GC_DBG_EXTRAS GC_RETURN_ADDR_PARENT, NULL, 0
#  else
#    define GC_DBG_EXTRAS GC_RETURN_ADDR, NULL, 0
#  endif
#else
#  define GC_DBG_EXTRAS "unknown", 0
#endif /* !GC_ADD_CALLER */

#ifdef GC_COLLECT_AT_MALLOC
/*
 * Parameter to force collection at every `malloc` of size greater or
 * equal to the given value.  This might be handy during debugging.
 * Note: this variable is visible outside for debugging purpose.
 */
extern size_t GC_dbg_collect_at_malloc_min_lb;

#  define GC_DBG_COLLECT_AT_MALLOC(lb) \
    (void)((lb) >= GC_dbg_collect_at_malloc_min_lb ? (GC_gcollect(), 0) : 0)
#else
#  define GC_DBG_COLLECT_AT_MALLOC(lb) (void)0
#endif /* !GC_COLLECT_AT_MALLOC */

/* Allocation routines that bypass the thread-local cache. */

#if defined(THREAD_LOCAL_ALLOC) && defined(GC_GCJ_SUPPORT)
/*
 * Allocate an object, clear it, and store the pointer to the type
 * structure ("vtable" in `gcj`).  This adds a byte at the end of the
 * object if `GC_malloc` would.
 */
GC_INNER void *GC_core_gcj_malloc(size_t lb, const void *vtable_ptr,
                                  unsigned flags);
#endif

GC_INNER void GC_init_headers(void);

/*
 * Install a header for block `h`.  Return `NULL` on failure, or the
 * uninitialized header otherwise.
 */
GC_INNER hdr *GC_install_header(struct hblk *h);

/*
 * Set up forwarding counts for block `h` of size `sz`.  Return `FALSE`
 * on failure.
 */
GC_INNER GC_bool GC_install_counts(struct hblk *h, size_t sz);

/* Remove the header for block `h`. */
GC_INNER void GC_remove_header(struct hblk *h);

/* Remove forwarding counts for `h`. */
GC_INNER void GC_remove_counts(struct hblk *h, size_t sz);

/* A non-macro variant of the header location routine. */
GC_INNER hdr *GC_find_header(const void *h);

/*
 * Get `HBLKSIZE`-aligned heap memory chunk from the OS and add the
 * chunk to `GC_our_memory`.  Return `NULL` if out of memory.
 */
GC_INNER ptr_t GC_os_get_mem(size_t bytes);

#if defined(NO_FIND_LEAK) && defined(SHORT_DBG_HDRS)
#  define GC_print_all_errors() (void)0
#  define GC_check_heap() (void)0
#  define GC_print_all_smashed() (void)0
#else
/*
 * Print smashed and leaked objects, if any.  Clear the lists of such
 * objects.  Called without the allocator lock held.
 */
GC_INNER void GC_print_all_errors(void);

/*
 * Check that all objects in the heap with debugging info are intact.
 * Add any that are not to `GC_smashed` list.
 */
GC_EXTERN void (*GC_check_heap)(void);

/* Print `GC_smashed` list if it is not empty.  Then clear the list. */
GC_EXTERN void (*GC_print_all_smashed)(void);
#endif

/*
 * If possible, print (using `GC_err_printf()`) a more detailed
 * description (terminated with "\n") of the object referred to by `p`.
 */
GC_EXTERN void (*GC_print_heap_obj)(ptr_t p);

GC_INNER void GC_default_print_heap_obj_proc(ptr_t p);

#if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
/*
 * Print an address map of the process.  The caller should hold the
 * allocator lock.
 */
void GC_print_address_map(void);
#endif

#ifdef NO_FIND_LEAK
#  define GC_find_leak_inner FALSE
#else
#  define GC_find_leak_inner GC_find_leak
#  ifndef SHORT_DBG_HDRS
/*
 * Do not immediately deallocate object on `free()` in the find-leak mode,
 * just mark it as freed (and deallocate it after collection).
 */
GC_EXTERN GC_bool GC_findleak_delay_free;
#  endif
#endif /* !NO_FIND_LEAK */

#if defined(NO_FIND_LEAK) && defined(SHORT_DBG_HDRS)
#  define get_have_errors() FALSE
#elif defined(AO_HAVE_store)
#  define GC_SET_HAVE_ERRORS() AO_store(&GC_have_errors, (AO_t)TRUE)
#  define get_have_errors() \
    ((GC_bool)AO_load(&GC_have_errors)) /*< no barrier */
#else
#  define GC_SET_HAVE_ERRORS() (void)(GC_have_errors = TRUE)
/*
 * We saw a smashed or leaked object.  Call error printing routine
 * occasionally.  It is OK to read it not acquiring the allocator lock.
 * Once set to `TRUE`, it is never cleared.
 */
#  define get_have_errors() GC_have_errors
#endif /* !AO_HAVE_store */

#define VERBOSE 2
#if !defined(NO_CLOCK) || !defined(SMALL_CONFIG)
/*
 * Value of 1 generates basic collector log; `VERBOSE` generates additional
 * messages.
 */
GC_EXTERN int GC_print_stats;
#else /* SMALL_CONFIG */
/*
 * Defined as a macro to aid the compiler to remove the relevant message
 * character strings from the executable (with a particular level of
 * optimizations).
 */
#  define GC_print_stats 0
#endif

#ifdef KEEP_BACK_PTRS
/* Number of random backtraces to generate for each collection. */
GC_EXTERN long GC_backtraces;
#endif

/*
 * A trivial (linear congruential) pseudo-random numbers generator,
 * safe for the concurrent usage.
 */
#define GC_RAND_MAX ((int)(~0U >> 1))
#if defined(AO_HAVE_store) && defined(THREAD_SANITIZER)
#  define GC_RAND_STATE_T volatile AO_t
#  define GC_RAND_NEXT(pseed) GC_rand_next(pseed)
GC_INLINE int
GC_rand_next(GC_RAND_STATE_T *pseed)
{
  AO_t next = (AO_t)((AO_load(pseed) * (unsigned32)1103515245UL + 12345)
                     & (unsigned32)((unsigned)GC_RAND_MAX));
  AO_store(pseed, next);
  return (int)next;
}
#else
#  define GC_RAND_STATE_T unsigned32
#  define GC_RAND_NEXT(pseed) /*< overflow and race are OK */      \
    (int)(*(pseed) = (*(pseed) * (unsigned32)1103515245UL + 12345) \
                     & (unsigned32)((unsigned)GC_RAND_MAX))
#endif

#ifdef MAKE_BACK_GRAPH
GC_EXTERN GC_bool GC_print_back_height;
void GC_print_back_graph_stats(void);
#endif

#ifdef THREADS
/*
 * Explicitly deallocate the object when we already hold the allocator lock.
 * Only used for internally allocated objects.
 */
GC_INNER void GC_free_inner(void *p);
#endif

#ifdef VALGRIND_TRACKING
#  define FREE_PROFILER_HOOK(p) GC_free_profiler_hook(p)
#else
#  define FREE_PROFILER_HOOK(p) (void)(p)
#endif

/*
 * Macros used for collector internal allocation.  These assume the
 * allocator lock is held.
 */
#ifdef DBG_HDRS_ALL

/*
 * An allocation function for internal use.  Normally internally allocated
 * objects do not have debug information.  But in this case, we need to make
 * sure that all objects have debug headers.
 */
GC_INNER void *GC_debug_generic_malloc_inner(size_t lb, int kind,
                                             unsigned flags);

#  define GC_INTERNAL_MALLOC(lb, k) GC_debug_generic_malloc_inner(lb, k, 0)
#  define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(lb, k) \
    GC_debug_generic_malloc_inner(lb, k, IGNORE_OFF_PAGE)
#  ifdef THREADS
/* Used internally; we assume it is called correctly. */
GC_INNER void GC_debug_free_inner(void *p);

#    define GC_INTERNAL_FREE GC_debug_free_inner
#  else
#    define GC_INTERNAL_FREE GC_debug_free
#  endif
#else
#  define GC_INTERNAL_MALLOC(lb, k) GC_generic_malloc_inner(lb, k, 0)
#  define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(lb, k) \
    GC_generic_malloc_inner(lb, k, IGNORE_OFF_PAGE)
#  ifdef THREADS
#    define GC_INTERNAL_FREE GC_free_inner
#  else
#    define GC_INTERNAL_FREE GC_free
#  endif
#endif /* !DBG_HDRS_ALL */

/* Memory unmapping routines. */
#ifdef USE_MUNMAP

/*
 * Unmap blocks that have not been recently touched.  This is the only
 * way blocks are ever unmapped.
 */
GC_INNER void GC_unmap_old(unsigned threshold);

/*
 * Merge all unmapped blocks that are adjacent to other free blocks.
 * This may involve remapping, since all blocks are either fully mapped
 * or fully unmapped.  Returns `TRUE` if at least one block was merged.
 */
GC_INNER GC_bool GC_merge_unmapped(void);

GC_INNER void GC_unmap(ptr_t start, size_t bytes);
GC_INNER void GC_remap(ptr_t start, size_t bytes);

/*
 * Two adjacent blocks have already been unmapped and are about to be merged.
 * Unmap the whole block.  This typically requires that we unmap a small
 * section in the middle that was not previously unmapped due to alignment
 * constraints.
 */
GC_INNER void GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2,
                           size_t bytes2);
#endif

#ifdef CAN_HANDLE_FORK
/*
 * Fork-handling mode:
 *   - 0 means no `fork` handling is requested (but client could anyway
 *     call `fork()` provided it is surrounded with `GC_atfork_prepare`,
 *     `GC_atfork_parent`, `GC_atfork_child` calls);
 *   - (-1) means the collector tries to use `pthread_at_fork()` if it is
 *     available (if it succeeds, then `GC_handle_fork` value is changed to
 *     one), a portable client should nonetheless surround `fork()` with
 *     `GC_atfork_prepare()` and the accompanying routines (for the case
 *     of `pthread_at_fork()` failure or absence);
 *   - 1 (or other values) means client fully relies on `pthread_at_fork`
 *     (so if it is missing or failed, then `abort` occurs in `GC_init()`),
 *     `GC_atfork_prepare` and the accompanying routines are no-op in such
 *     a case.
 *
 * Note: the value is examined by `GC_thr_init`.
 */
GC_EXTERN int GC_handle_fork;

#  ifdef THREADS
#    if defined(SOLARIS) && !defined(_STRICT_STDC)
/* Update `pthreads` id in the child process right after `fork`. */
GC_INNER void GC_stackbase_info_update_after_fork(void);
#    else
#      define GC_stackbase_info_update_after_fork() (void)0
#    endif
#  endif
#endif /* CAN_HANDLE_FORK */

#ifdef NO_MANUAL_VDB
#  define GC_manual_vdb FALSE
#  define GC_auto_incremental GC_incremental
#  define GC_dirty(p) (void)(p)
#  define REACHABLE_AFTER_DIRTY(p) (void)(p)
#else
/*
 * The incremental collection is in the manual VDB mode.
 * Assumes `GC_incremental` is `TRUE`.  Should not be modified once
 * `GC_incremental` is set to `TRUE`.
 */
GC_EXTERN GC_bool GC_manual_vdb;

#  define GC_auto_incremental (GC_incremental && !GC_manual_vdb)

/*
 * Manually mark the page containing `p` as dirty.  Logically, this
 * dirties the entire object.  Does not require locking.
 * Exported and marked as `noinline` for the purpose of some clients that
 * need to patch the symbol when using write barrier validation.
 */
GC_API_PATCHABLE void GC_dirty_inner(const void *p);

#  define GC_dirty(p) (GC_manual_vdb ? GC_dirty_inner(p) : (void)0)
#  define REACHABLE_AFTER_DIRTY(p) GC_reachable_here(p)
#endif /* !NO_MANUAL_VDB */

#ifdef GC_DISABLE_INCREMENTAL
#  define GC_incremental FALSE
#else
/*
 * Using incremental/generational collection.  Assumes dirty bits are
 * being maintained.
 */
GC_EXTERN GC_bool GC_incremental;

/* Virtual dirty bit (VDB) implementations; each one exports the following. */

/*
 * Initialize the virtual dirty bit implementation.  Returns `TRUE` if
 * virtual dirty bits are maintained (otherwise it is OK to be called again
 * if the client calls `GC_enable_incremental()` once more).
 */
GC_INNER GC_bool GC_dirty_init(void);

/*
 * Retrieve system dirty bits for the heap to a local buffer (unless
 * `output_unneeded`).  The caller should set `output_unneeded` to indicate
 * that reading of the retrieved dirty bits is not planned till the next
 * retrieval.  Restore the systems notion of which pages are dirty.
 * We assume that either the world is stopped or it is OK to lose dirty bits
 * while it is happening (`GC_enable_incremental()` is the caller and
 * `output_unneeded` is `TRUE` at least if the multi-threading support is on).
 */
GC_INNER void GC_read_dirty(GC_bool output_unneeded);

/*
 * Is the `HBLKSIZE`-sized page at `h` marked dirty in the local buffer?
 * If the actual page size is different, this returns `TRUE` if any of
 * the pages overlapping `h` are dirty.  This routine may err on the side
 * of labeling pages as dirty (and this implementation does).
 */
GC_INNER GC_bool GC_page_was_dirty(struct hblk *h);

/*
 * Block `h` is about to be written or allocated shortly.  Ensure that
 * all pages containing any part of the `nblocks` `hblk` entities starting
 * at `h` are no longer write-protected (by the virtual dirty bit
 * implementation).  I.e., this is a call that:
 *   - hints that [`h`, `h + nblocks`) is about to be written;
 *   - guarantees that protection is removed;
 *   - may speed up some virtual dirty bit implementations;
 *   - may be essential if we need to ensure that pointer-free system
 *     call buffers in the heap are not protected.
 */
GC_INNER void GC_remove_protection(struct hblk *h, size_t nblocks,
                                   GC_bool is_ptrfree);

#  if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB)
/* Is VDB working for static roots? */
GC_INNER GC_bool GC_is_vdb_for_static_roots(void);
#  endif

#  ifdef CAN_HANDLE_FORK
#    if defined(PROC_VDB) || defined(SOFT_VDB) \
        || (defined(MPROTECT_VDB) && defined(DARWIN) && defined(THREADS))
/*
 * Update pid-specific resources (like `/proc` file descriptors) needed
 * by the dirty bits implementation after `fork` in the child process.
 */
GC_INNER void GC_dirty_update_child(void);
#    else
#      define GC_dirty_update_child() (void)0
#    endif
#  endif /* CAN_HANDLE_FORK */

#  if defined(MPROTECT_VDB) && defined(DARWIN)
EXTERN_C_END
#    include <pthread.h>
EXTERN_C_BEGIN
#    ifdef THREADS
GC_INNER int GC_inner_pthread_create(pthread_t *t,
                                     GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
                                     void *(*fn)(void *), void *arg);
#    else
#      define GC_inner_pthread_create pthread_create
#    endif
#  endif
#endif /* !GC_DISABLE_INCREMENTAL */

#if defined(COUNT_PROTECTED_REGIONS) && defined(MPROTECT_VDB)
/*
 * Do actions on heap growth, if needed, to prevent hitting the OS kernel
 * limit on the VM map regions.
 */
GC_INNER void GC_handle_protected_regions_limit(void);
#else
#  define GC_handle_protected_regions_limit() (void)0
#endif

/* Same as `GC_base` but accepts and returns a pointer to `const` object. */
#define GC_base_C(p) ((const void *)GC_base(GC_CAST_AWAY_CONST_PVOID(p)))

/* Some debugging print routines. */
void GC_print_block_list(void);
void GC_print_hblkfreelist(void);
void GC_print_heap_sects(void);
void GC_print_static_roots(void);

#ifdef KEEP_BACK_PTRS
/*
 * Store back pointer to `source` in `dest`, if that appears to be
 * possible.  This is not completely safe, since we may mistakenly
 * conclude that `dest` has a debugging wrapper.  But the error
 * probability is very small, and this should not be used in
 * production code.  We assume that `dest` is the real base pointer.
 * `source` should usually be a pointer to the interior of an object.
 */
GC_INNER void GC_store_back_pointer(ptr_t source, ptr_t dest);

GC_INNER void GC_marked_for_finalization(ptr_t dest);
#  define GC_STORE_BACK_PTR(source, dest) GC_store_back_pointer(source, dest)
#  define GC_MARKED_FOR_FINALIZATION(dest) GC_marked_for_finalization(dest)
#else
#  define GC_STORE_BACK_PTR(source, dest) (void)(source)
#  define GC_MARKED_FOR_FINALIZATION(dest)
#endif /* !KEEP_BACK_PTRS */

/* Make arguments appear live to compiler. */
void GC_noop6(word, word, word, word, word, word);

#ifndef GC_ATTR_FORMAT_PRINTF
#  if GC_GNUC_PREREQ(3, 0)
#    define GC_ATTR_FORMAT_PRINTF(spec_argnum, first_checked) \
      __attribute__((__format__(__printf__, spec_argnum, first_checked)))
#  else
#    define GC_ATTR_FORMAT_PRINTF(spec_argnum, first_checked)
#  endif
#endif

/* Logging and diagnostic output. */

/*
 * `GC_printf` is used typically on client explicit print requests.
 * A variant of `printf` that does not allocate, 1 KB total output length.
 * (It uses `sprintf()` internally; hopefully the latter does not allocate
 * memory for `long` arguments.)  For all `GC_*_printf` routines,
 * it is recommended to put "\n" at the end of `format` string (for the
 * output atomicity).
 */
GC_API_PRIV void GC_printf(const char *format, ...)
    GC_ATTR_FORMAT_PRINTF(1, 2);
GC_API_PRIV void GC_err_printf(const char *format, ...)
    GC_ATTR_FORMAT_PRINTF(1, 2);

/*
 * The basic logging routine.  Typically, it is called directly only inside
 * various `DEBUG_*` blocks.
 */
GC_API_PRIV void GC_log_printf(const char *format, ...)
    GC_ATTR_FORMAT_PRINTF(1, 2);

#ifndef GC_ANDROID_LOG
#  define GC_PRINT_STATS_FLAG (GC_print_stats != 0)
#  define GC_INFOLOG_PRINTF GC_COND_LOG_PRINTF
/*
 * The "verbose" logging routine which is called only if `GC_print_stats`
 * is `VERBOSE`.
 */
#  define GC_verbose_log_printf GC_log_printf
#else
extern GC_bool GC_quiet;
#  define GC_PRINT_STATS_FLAG (!GC_quiet)
/* `INFO`/`DBG` loggers are enabled even if `GC_print_stats` is off. */
#  ifndef GC_INFOLOG_PRINTF
#    define GC_INFOLOG_PRINTF \
      if (GC_quiet) {         \
      } else                  \
        GC_info_log_printf
#  endif
GC_INNER void GC_info_log_printf(const char *format, ...)
    GC_ATTR_FORMAT_PRINTF(1, 2);
GC_INNER void GC_verbose_log_printf(const char *format, ...)
    GC_ATTR_FORMAT_PRINTF(1, 2);
#endif /* GC_ANDROID_LOG */

#if defined(SMALL_CONFIG) || defined(GC_ANDROID_LOG)
#  define GC_ERRINFO_PRINTF GC_INFOLOG_PRINTF
#else
#  define GC_ERRINFO_PRINTF GC_log_printf
#endif

/*
 * Convenient wrapper macros over `GC_log_printf()` and
 * `GC_verbose_log_printf()`.
 */
#define GC_COND_LOG_PRINTF       \
  if (LIKELY(!GC_print_stats)) { \
  } else                         \
    GC_log_printf
#define GC_VERBOSE_LOG_PRINTF              \
  if (LIKELY(GC_print_stats != VERBOSE)) { \
  } else                                   \
    GC_verbose_log_printf
#ifndef GC_DBGLOG_PRINTF
#  define GC_DBGLOG_PRINTF      \
    if (!GC_PRINT_STATS_FLAG) { \
    } else                      \
      GC_log_printf
#endif

/* Write `s` to `stderr`, but do not buffer, do not add newlines, do not... */
void GC_err_puts(const char *s);

/*
 * A handy macro for logging size values (of `word` type) in KiB, rounding
 * to nearest value.
 */
#define TO_KiB_UL(v) ((unsigned long)(((v) + ((1 << 9) - 1)) >> 10))

#ifdef USE_MUNMAP
GC_EXTERN unsigned GC_unmap_threshold; /*< defined in `alloc.c` file */

/*
 * Force memory unmapping on every collection.  Has no effect on
 * implicitly-initiated collections.
 */
GC_EXTERN GC_bool GC_force_unmap_on_gcollect;
#endif

#ifdef MSWIN32
GC_EXTERN GC_bool GC_no_win32_dlls; /*< defined in `os_dep.c` file */

/* Is this a Windows NT derivative (i.e. NT, Win2K, XP or later)? */
GC_EXTERN GC_bool GC_wnt;
#endif

#ifdef THREADS
#  if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
GC_EXTERN CRITICAL_SECTION GC_write_cs;
#    ifdef GC_ASSERTIONS
/*
 * Set to `TRUE` only if `GC_stop_world()` has acquired `GC_write_cs`.
 * Protected by `GC_write_cs`.
 */
GC_EXTERN GC_bool GC_write_disabled;
#    endif
#  endif /* MSWIN32 || MSWINCE */
#  ifdef NEED_FAULT_HANDLER_LOCK
/*
 * Acquire the spin lock we use to update dirty bits.  Threads should
 * not get stopped holding it.  But we may acquire and release it during
 * `GC_remove_protection()` call.
 */
#    define GC_acquire_dirty_lock() \
      do { /* Empty. */             \
      } while (AO_test_and_set_acquire(&GC_fault_handler_lock) == AO_TS_SET)
#    define GC_release_dirty_lock() AO_CLEAR(&GC_fault_handler_lock)
#  else
#    define GC_acquire_dirty_lock() (void)0
#    define GC_release_dirty_lock() (void)0
#  endif
#  ifdef MSWINCE
GC_EXTERN GC_bool GC_dont_query_stack_min;
#  endif
#elif defined(IA64)
/* Value returned from register flushing routine (`ar.bsp`). */
GC_EXTERN ptr_t GC_save_regs_ret_val;
#endif /* !THREADS */

#ifdef THREAD_LOCAL_ALLOC
GC_EXTERN GC_bool GC_world_stopped; /*< defined in `alloc.c` file */

/*
 * We must explicitly mark `ptrfree` and `gcj` free lists, since the
 * free list links would not otherwise be found.  We also set them in
 * the normal free lists, since that involves touching less memory than
 * if we scanned them normally.
 */
GC_INNER void GC_mark_thread_local_free_lists(void);
#endif

#if defined(GLIBC_2_19_TSX_BUG) && defined(GC_PTHREADS_PARAMARK)
/* Parse string like `<major>[.<minor>[<tail>]]` and return `major` value. */
GC_INNER int GC_parse_version(int *pminor, const char *pverstr);
#endif

#if defined(MPROTECT_VDB) && defined(GWW_VDB)
/*
 * Returns `TRUE` if `GetWriteWatch()` is available.  May be called
 * repeatedly.  May be called with or without the allocator lock held.
 */
GC_INNER GC_bool GC_gww_dirty_init(void);
#endif

#if defined(CHECKSUMS) || defined(PROC_VDB)
/* Could any valid GC heap pointer ever have been written to this page? */
GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk *h);
#endif

#ifndef GC_NO_DEINIT
GC_INNER void GC_reset_freelist(void);
GC_INNER void GC_reset_obj_kinds(void);
#  ifdef CHECKSUMS
void GC_reset_check_page(void);
#  endif
#  ifdef THREADS
GC_INNER void GC_reset_threads(void);
#  endif
#  ifdef THREAD_LOCAL_ALLOC
GC_INNER void GC_reset_thread_local_initialization(void);
#  endif
#endif

#ifdef CHECKSUMS
#  ifdef MPROTECT_VDB
void GC_record_fault(struct hblk *h);
#  endif
void GC_check_dirty(void);
#endif

GC_INNER void GC_setpagesize(void);

GC_INNER void GC_initialize_offsets(void);

#if defined(REDIR_MALLOC_AND_LINUXTHREADS) \
    && !defined(REDIRECT_MALLOC_IN_HEADER)
GC_INNER void GC_init_lib_bounds(void);
#else
#  define GC_init_lib_bounds() (void)0
#endif

#ifdef REDIR_MALLOC_AND_LINUXTHREADS
/*
 * Find the text (code) mapping for the library whose name, after
 * stripping the directory part, starts with `nm`.
 */
GC_INNER GC_bool GC_text_mapping(const char *nm, ptr_t *startp, ptr_t *endp);
#endif

#if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
GC_INNER void GC_add_current_malloc_heap(void);
#endif

#ifdef MAKE_BACK_GRAPH
/*
 * Rebuild the representation of the backward reachability graph.
 * Does not examine mark bits.  Could be called before collection.
 */
GC_INNER void GC_build_back_graph(void);

GC_INNER void GC_traverse_back_graph(void);
#endif

#ifdef MSWIN32
GC_INNER void GC_init_win32(void);