c/zlib
1
0
Fork 0
mirror of https://github.com/madler/zlib synced 2025-03-28 21:13:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge f6c9911bf8ca3926333e1cbb4e287cef31c1db70 into 5a82f71ed1dfc0bec044d9702463dbdf84ea3b71

Browse Source
This commit is contained in:
Eddy S. 2025-03-21 10:29:45 +00:00 committed by GitHub
commit 48c8433a5a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 494 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@ -76,6 +76,10 @@ if(NOT ZLIB_CONF_WRITTEN)
mark_as_advanced(ZLIB_CONF_WRITTEN)
endif(NOT ZLIB_CONF_WRITTEN)
#
# Add contrib code
#
add_subdirectory(contrib/s390x)
#
# Check to see if we have large file support
#
@ -201,6 +205,7 @@ if(ZLIB_BUILD_SHARED)
UNIX
AND NOT APPLE
AND NOT (CMAKE_SYSTEM_NAME STREQUAL AIX))
target_link_libraries(zlib PRIVATE $<TARGET_NAME_IF_EXISTS:zlib_crc32_vx>)
endif(ZLIB_BUILD_SHARED)
if(ZLIB_BUILD_STATIC)
@ -223,6 +228,7 @@ if(ZLIB_BUILD_STATIC)
set_target_properties(
zlibstatic PROPERTIES EXPORT_NAME ZLIBSTATIC OUTPUT_NAME
z${zlib_static_suffix})
target_link_libraries(zlibstatic PRIVATE $<TARGET_NAME_IF_EXISTS:zlib_crc32_vx>)
endif(ZLIB_BUILD_STATIC)
if(ZLIB_INSTALL)

17
Makefile.in
View File

@ -27,6 +27,7 @@ LDFLAGS=
TEST_LIBS=-L. libz.a
LDSHARED=$(CC)
CPP=$(CC) -E
VGFMAFLAG=
STATICLIB=libz.a
SHAREDLIB=libz.so
@ -164,6 +165,12 @@ adler32.o: $(SRCDIR)adler32.c
crc32.o: $(SRCDIR)crc32.c
$(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)crc32.c
crc32-vx.o: $(SRCDIR)contrib/s390x/crc32-vx.c
$(CC) $(CFLAGS) $(VGFMAFLAG) $(ZINC) -c -o $@ $(SRCDIR)contrib/s390x/crc32-vx.c
s390x-functable.o: $(SRCDIR)contrib/s390x/s390x-functable.c
$(CC) $(CFLAGS) $(VGFMAFLAG) $(ZINC) -c -o $@ $(SRCDIR)contrib/s390x/s390x-functable.c
deflate.o: $(SRCDIR)deflate.c
$(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)deflate.c
@ -214,6 +221,16 @@ crc32.lo: $(SRCDIR)crc32.c
$(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/crc32.o $(SRCDIR)crc32.c
-@mv objs/crc32.o $@
crc32-vx.lo: $(SRCDIR)contrib/s390x/crc32-vx.c
-@mkdir objs 2>/dev/null || test -d objs
$(CC) $(SFLAGS) $(VGFMAFLAG) $(ZINC) -DPIC -c -o objs/crc32-vx.o $(SRCDIR)contrib/s390x/crc32-vx.c
-@mv objs/crc32-vx.o $@
s390x-functable.lo: $(SRCDIR)contrib/s390x/s390x-functable.c
-@mkdir objs 2>/dev/null || test -d objs
$(CC) $(SFLAGS) $(VGFMAFLAG) $(ZINC) -DPIC -c -o objs/s390x-functable.o $(SRCDIR)contrib/s390x/s390x-functable.c
-@mv objs/s390x-functable.o $@
deflate.lo: $(SRCDIR)deflate.c
-@mkdir objs 2>/dev/null || test -d objs
$(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/deflate.o $(SRCDIR)deflate.c

52
configure vendored
View File

@ -870,6 +870,55 @@ EOF
fi
fi
# check for ibm s390x build
HAVE_S390X=0
# preset the compiler specific flags
if test $clang -eq 1; then
VGFMAFLAG=-fzvector
else
VGFMAFLAG=-mzarch
fi
cat > $test.c <<EOF
#ifndef __s390x__
#error
#endif
#include <vecintrin.h>
int main(void) {
unsigned long long a __attribute__((vector_size(16))) = { 0 };
unsigned long long b __attribute__((vector_size(16))) = { 0 };
unsigned char c __attribute__((vector_size(16))) = { 0 };
c = vec_gfmsum_accum_128(a, b, c);
return c[0];
}
EOF
# cflags already contains a valid march
if try $CC -c $CFLAGS $VGFMAFLAG $test.c; then
echo "Checking for s390x build ... Yes." | tee -a configure.log
HAVE_S390X=1
# or set march for our compile units
elif try $CC -c $CFLAGS $VGFMAFLAG -march=z13 $test.c; then
echo "Checking for s390x build (march=z13) ... Yes." | tee -a configure.log
HAVE_S390X=1
VGFMAFLAG="$VGFMAFLAG -march=z13"
# else we are not on s390x
else
echo "Checking for s390x build ... No." | tee -a configure.log
fi
# prepare compiling for s390x
if test $HAVE_S390X -eq 1; then
CFLAGS="$CFLAGS -DHAVE_S390X_VX"
SFLAGS="$SFLAGS -DHAVE_S390X_VX"
OBJC="$OBJC crc32-vx.o s390x-functable.o"
PIC_OBJC="$PIC_OBJC crc32-vx.lo s390x-functable.lo"
else
# this is not a s390x build
VGFMAFLAG=""
fi
# show the results in the log
echo >> configure.log
echo ALL = $ALL >> configure.log
@ -901,6 +950,8 @@ echo mandir = $mandir >> configure.log
echo prefix = $prefix >> configure.log
echo sharedlibdir = $sharedlibdir >> configure.log
echo uname = $uname >> configure.log
echo HAVE_S390X = $HAVE_S390X >> configure.log
echo VGFMAFLAG = $VGFMAFLAG >> configure.log
# update Makefile with the configure results
sed < ${SRCDIR}Makefile.in "
@ -912,6 +963,7 @@ sed < ${SRCDIR}Makefile.in "
/^LDFLAGS *=/s#=.*#=$LDFLAGS#
/^LDSHARED *=/s#=.*#=$LDSHARED#
/^CPP *=/s#=.*#=$CPP#
/^VGFMAFLAG *=/s#=.*#=$VGFMAFLAG#
/^STATICLIB *=/s#=.*#=$STATICLIB#
/^SHAREDLIB *=/s#=.*#=$SHAREDLIB#
/^SHAREDLIBV *=/s#=.*#=$SHAREDLIBV#

19
contrib/functable/functable.h Normal file
View File

@ -0,0 +1,19 @@
#ifndef Z_FUNCTABLE_H__
#define Z_FUNCTABLE_H__
#include "../../zutil.h"
#include "../../zonce.h"
struct zfunctable_s {
unsigned long (*crc32_z)(unsigned long crc, const unsigned char FAR *buf,
z_size_t len);
/* int (*deflate)(z_streamp strm, int flush); */
/* int (*inflate)(z_streamp strm, int flush); */
};
extern struct zfunctable_s ZLIB_INTERNAL arch_functable;
extern once_t ZLIB_INTERNAL arch_functable_init_done;
/* to be implemented by architecture specific code */
void ZLIB_INTERNAL arch_functable_init(void);
#endif

54
contrib/s390x/CMakeLists.txt Normal file
View File

@ -0,0 +1,54 @@
option(ZLIB_CRC32VX "Enable building S390-CRC32VX implementation" ON)
#
# Check for IBM S390X extensions
#
if(ZLIB_CRC32VX)
# preset the compiler specific flags
if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
set(VGFMAFLAG "-fzvector")
else()
set(VGFMAFLAG "-mzarch")
endif(CMAKE_C_COMPILER_ID STREQUAL "Clang")
set(S390X_VX_TEST
"#ifndef __s390x__ \n\
#error \n\
#endif \n\
#include <vecintrin.h> \n\
int main(void) { \
unsigned long long a __attribute__((vector_size(16))) = { 0 }; \
unsigned long long b __attribute__((vector_size(16))) = { 0 }; \
unsigned char c __attribute__((vector_size(16))) = { 0 }; \
c = vec_gfmsum_accum_128(a, b, c); \
return c[0]; \
}")
# cflags already contains a valid march
set(CMAKE_REQUIRED_FLAGS "${VGFMAFLAG}")
check_c_source_compiles("${S390X_VX_TEST}" HAS_S390X_VX_SUPPORT)
unset(CMAKE_REQUIRED_FLAGS)
# or set march for our compile units
if(NOT HAS_S390X_VX_SUPPORT)
set(CMAKE_REQUIRED_FLAGS "${VGFMAFLAG} -march=z13")
check_c_source_compiles("${S390X_VX_TEST}" HAS_Z13_S390X_VX_SUPPORT)
unset(CMAKE_REQUIRED_FLAGS )
list(APPEND VGFMAFLAG "-march=z13")
endif(NOT HAS_S390X_VX_SUPPORT)
# prepare compiling for s390x
if(HAS_S390X_VX_SUPPORT OR HAS_Z13_S390X_VX_SUPPORT)
add_library(
zlib_crc32_vx OBJECT
s390x-functable.c
crc32-vx.c
../functable/functable.h)
set_source_files_properties(
crc32-vx.c
PROPERTIES COMPILE_OPTIONS "${VGFMAFLAG}")
target_compile_definitions(
zlib_crc32_vx PUBLIC HAVE_S390X_VX=1)
endif(HAS_S390X_VX_SUPPORT OR HAS_Z13_S390X_VX_SUPPORT)
endif(ZLIB_CRC32VX)

235
contrib/s390x/crc32-vx.c Normal file
View File

@ -0,0 +1,235 @@
/*
* Hardware-accelerated CRC-32 variants for Linux on z Systems
*
* Use the z/Architecture Vector Extension Facility to accelerate the
* computing of bitreflected CRC-32 checksums.
*
* This CRC-32 implementation algorithm is bitreflected and processes
* the least-significant bit first (Little-Endian).
*
* This code was originally written by Hendrik Brueckner
* <brueckner@linux.vnet.ibm.com> for use in the Linux kernel and has been
* relicensed under the zlib license.
*/
#include "../../zutil.h"
#include "../../zonce.h"
#include <stdint.h>
#include <stdio.h>
#include <vecintrin.h>
#ifdef __clang__
# if ((__clang_major__ == 18) || (__clang_major__ == 19 && (__clang_minor__ < 1 || (__clang_minor__ == 1 && __clang_patchlevel__ < 2))))
# error CRC32-VX optimizations are broken due to compiler bug in Clang versions: 18.0.0 <= clang_version < 19.1.2. \
Either disable the zlib CRC32-VX optimization, or switch to another compiler/compiler version.
# endif
#endif
#define VX_MIN_LEN 64
#define VX_ALIGNMENT 16L
#define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
typedef unsigned char uv16qi __attribute__((vector_size(16)));
typedef unsigned int uv4si __attribute__((vector_size(16)));
typedef unsigned long long uv2di __attribute__((vector_size(16)));
uint32_t crc32_le_vgfm_16(uint32_t crc, const unsigned char *buf, size_t len) {
/*
* The CRC-32 constant block contains reduction constants to fold and
* process particular chunks of the input data stream in parallel.
*
* For the CRC-32 variants, the constants are precomputed according to
* these definitions:
*
* R1 = [(x4*128+32 mod P'(x) << 32)]' << 1
* R2 = [(x4*128-32 mod P'(x) << 32)]' << 1
* R3 = [(x128+32 mod P'(x) << 32)]' << 1
* R4 = [(x128-32 mod P'(x) << 32)]' << 1
* R5 = [(x64 mod P'(x) << 32)]' << 1
* R6 = [(x32 mod P'(x) << 32)]' << 1
*
* The bitreflected Barret reduction constant, u', is defined as
* the bit reversal of floor(x**64 / P(x)).
*
* where P(x) is the polynomial in the normal domain and the P'(x) is the
* polynomial in the reversed (bitreflected) domain.
*
* CRC-32 (IEEE 802.3 Ethernet, ...) polynomials:
*
* P(x) = 0x04C11DB7
* P'(x) = 0xEDB88320
*/
const uv16qi perm_le2be = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; /* BE->LE mask */
const uv2di r2r1 = {0x1C6E41596, 0x154442BD4}; /* R2, R1 */
const uv2di r4r3 = {0x0CCAA009E, 0x1751997D0}; /* R4, R3 */
const uv2di r5 = {0, 0x163CD6124}; /* R5 */
const uv2di ru_poly = {0, 0x1F7011641}; /* u' */
const uv2di crc_poly = {0, 0x1DB710641}; /* P'(x) << 1 */
/*
* Load the initial CRC value.
*
* The CRC value is loaded into the rightmost word of the
* vector register and is later XORed with the LSB portion
* of the loaded input data.
*/
uv2di v0 = {0, 0};
v0 = (uv2di)vec_insert(crc, (uv4si)v0, 3);
/* Load a 64-byte data chunk and XOR with CRC */
uv2di v1 = vec_perm(((uv2di *)buf)[0], ((uv2di *)buf)[0], perm_le2be);
uv2di v2 = vec_perm(((uv2di *)buf)[1], ((uv2di *)buf)[1], perm_le2be);
uv2di v3 = vec_perm(((uv2di *)buf)[2], ((uv2di *)buf)[2], perm_le2be);
uv2di v4 = vec_perm(((uv2di *)buf)[3], ((uv2di *)buf)[3], perm_le2be);
v1 ^= v0;
buf += 64;
len -= 64;
while (len >= 64) {
/* Load the next 64-byte data chunk */
uv16qi part1 = vec_perm(((uv16qi *)buf)[0], ((uv16qi *)buf)[0], perm_le2be);
uv16qi part2 = vec_perm(((uv16qi *)buf)[1], ((uv16qi *)buf)[1], perm_le2be);
uv16qi part3 = vec_perm(((uv16qi *)buf)[2], ((uv16qi *)buf)[2], perm_le2be);
uv16qi part4 = vec_perm(((uv16qi *)buf)[3], ((uv16qi *)buf)[3], perm_le2be);
/*
* Perform a GF(2) multiplication of the doublewords in V1 with
* the R1 and R2 reduction constants in V0. The intermediate result
* is then folded (accumulated) with the next data chunk in PART1 and
* stored in V1. Repeat this step for the register contents
* in V2, V3, and V4 respectively.
*/
v1 = (uv2di)vec_gfmsum_accum_128(r2r1, v1, part1);
v2 = (uv2di)vec_gfmsum_accum_128(r2r1, v2, part2);
v3 = (uv2di)vec_gfmsum_accum_128(r2r1, v3, part3);
v4 = (uv2di)vec_gfmsum_accum_128(r2r1, v4, part4);
buf += 64;
len -= 64;
}
/*
* Fold V1 to V4 into a single 128-bit value in V1. Multiply V1 with R3
* and R4 and accumulating the next 128-bit chunk until a single 128-bit
* value remains.
*/
v1 = (uv2di)vec_gfmsum_accum_128(r4r3, v1, (uv16qi)v2);
v1 = (uv2di)vec_gfmsum_accum_128(r4r3, v1, (uv16qi)v3);
v1 = (uv2di)vec_gfmsum_accum_128(r4r3, v1, (uv16qi)v4);
while (len >= 16) {
/* Load next data chunk */
v2 = vec_perm(*(uv2di *)buf, *(uv2di *)buf, perm_le2be);
/* Fold next data chunk */
v1 = (uv2di)vec_gfmsum_accum_128(r4r3, v1, (uv16qi)v2);
buf += 16;
len -= 16;
}
/*
* Set up a vector register for byte shifts. The shift value must
* be loaded in bits 1-4 in byte element 7 of a vector register.
* Shift by 8 bytes: 0x40
* Shift by 4 bytes: 0x20
*/
uv16qi v9 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
v9 = vec_insert((unsigned char)0x40, v9, 7);
/*
* Prepare V0 for the next GF(2) multiplication: shift V0 by 8 bytes
* to move R4 into the rightmost doubleword and set the leftmost
* doubleword to 0x1.
*/
v0 = vec_srb(r4r3, (uv2di)v9);
v0[0] = 1;
/*
* Compute GF(2) product of V1 and V0. The rightmost doubleword
* of V1 is multiplied with R4. The leftmost doubleword of V1 is
* multiplied by 0x1 and is then XORed with rightmost product.
* Implicitly, the intermediate leftmost product becomes padded
*/
v1 = (uv2di)vec_gfmsum_128(v0, v1);
/*
* Now do the final 32-bit fold by multiplying the rightmost word
* in V1 with R5 and XOR the result with the remaining bits in V1.
*
* To achieve this by a single VGFMAG, right shift V1 by a word
* and store the result in V2 which is then accumulated. Use the
* vector unpack instruction to load the rightmost half of the
* doubleword into the rightmost doubleword element of V1; the other
* half is loaded in the leftmost doubleword.
* The vector register with CONST_R5 contains the R5 constant in the
* rightmost doubleword and the leftmost doubleword is zero to ignore
* the leftmost product of V1.
*/
v9 = vec_insert((unsigned char)0x20, v9, 7);
v2 = vec_srb(v1, (uv2di)v9);
v1 = vec_unpackl((uv4si)v1); /* Split rightmost doubleword */
v1 = (uv2di)vec_gfmsum_accum_128(r5, v1, (uv16qi)v2);
/*
* Apply a Barret reduction to compute the final 32-bit CRC value.
*
* The input values to the Barret reduction are the degree-63 polynomial
* in V1 (R(x)), degree-32 generator polynomial, and the reduction
* constant u. The Barret reduction result is the CRC value of R(x) mod
* P(x).
*
* The Barret reduction algorithm is defined as:
*
* 1. T1(x) = floor( R(x) / x^32 ) GF2MUL u
* 2. T2(x) = floor( T1(x) / x^32 ) GF2MUL P(x)
* 3. C(x) = R(x) XOR T2(x) mod x^32
*
* Note: The leftmost doubleword of vector register containing
* CONST_RU_POLY is zero and, thus, the intermediate GF(2) product
* is zero and does not contribute to the final result.
*/
/* T1(x) = floor( R(x) / x^32 ) GF2MUL u */
v2 = vec_unpackl((uv4si)v1);
v2 = (uv2di)vec_gfmsum_128(ru_poly, v2);
/*
* Compute the GF(2) product of the CRC polynomial with T1(x) in
* V2 and XOR the intermediate result, T2(x), with the value in V1.
* The final result is stored in word element 2 of V2.
*/
v2 = vec_unpackl((uv4si)v2);
v2 = (uv2di)vec_gfmsum_accum_128(crc_poly, v2, (uv16qi)v1);
return ((uv4si)v2)[2];
}
unsigned long s390_crc32_vx(unsigned long crc, const unsigned char FAR *buf, z_size_t len)
{
uintptr_t prealign, aligned, remaining;
if (buf == Z_NULL) return 0UL;
if (len < VX_MIN_LEN + VX_ALIGN_MASK)
return crc32_z(crc, buf, len);
if ((uintptr_t)buf & VX_ALIGN_MASK) {
prealign = VX_ALIGNMENT - ((uintptr_t)buf & VX_ALIGN_MASK);
len -= prealign;
crc = crc32_z(crc, buf, prealign);
buf += prealign;
}
aligned = len & ~VX_ALIGN_MASK;
remaining = len & VX_ALIGN_MASK;
crc = crc32_le_vgfm_16(crc ^ 0xffffffff, buf, (size_t)aligned) ^ 0xffffffff;
if (remaining)
crc = crc32_z(crc, buf + aligned, remaining);
return crc;
}

24
contrib/s390x/s390x-functable.c Normal file
View File

@ -0,0 +1,24 @@
#include "../../contrib/functable/functable.h"
#include "../../zutil.h"
#include <stdio.h>
#define USE_GETAUXVAL 1
#include <sys/auxv.h>
unsigned long ZLIB_INTERNAL s390_crc32_vx(unsigned long crc, const unsigned char FAR *buf,
z_size_t len);
once_t ZLIB_INTERNAL arch_functable_init_done = ONCE_INIT;
struct zfunctable_s ZLIB_INTERNAL arch_functable = {
.crc32_z = NULL
};
void ZLIB_INTERNAL arch_functable_init(void){
unsigned long hwcap = getauxval(AT_HWCAP);
if (hwcap & HWCAP_S390_VX)
arch_functable.crc32_z = s390_crc32_vx;
else
arch_functable.crc32_z = crc32_z;
}

82
crc32.c
View File

@ -19,7 +19,9 @@
MAKECRCH can be #defined to write out crc32.h. A main() routine is also
produced, so that this one source file can be compiled to an executable.
*/
#include "zonce.h"
#include "contrib/functable/functable.h"
#ifdef MAKECRCH
# include <stdio.h>
# ifndef DYNAMIC_CRC_TABLE
@ -204,81 +206,6 @@ local z_crc_t FAR crc_table[256];
local void write_table64(FILE *, const z_word_t FAR *, int);
#endif /* MAKECRCH */
/*
Define a once() function depending on the availability of atomics. If this is
compiled with DYNAMIC_CRC_TABLE defined, and if CRCs will be computed in
multiple threads, and if atomics are not available, then get_crc_table() must
be called to initialize the tables and must return before any threads are
allowed to compute or combine CRCs.
*/
/* Definition of once functionality. */
typedef struct once_s once_t;
/* Check for the availability of atomics. */
#if defined(__STDC__) && __STDC_VERSION__ >= 201112L && \
!defined(__STDC_NO_ATOMICS__)
#include <stdatomic.h>
/* Structure for once(), which must be initialized with ONCE_INIT. */
struct once_s {
atomic_flag begun;
atomic_int done;
};
#define ONCE_INIT {ATOMIC_FLAG_INIT, 0}
/*
Run the provided init() function exactly once, even if multiple threads
invoke once() at the same time. The state must be a once_t initialized with
ONCE_INIT.
*/
local void once(once_t *state, void (*init)(void)) {
if (!atomic_load(&state->done)) {
if (atomic_flag_test_and_set(&state->begun))
while (!atomic_load(&state->done))
;
else {
init();
atomic_store(&state->done, 1);
}
}
}
#else /* no atomics */
/* Structure for once(), which must be initialized with ONCE_INIT. */
struct once_s {
volatile int begun;
volatile int done;
};
#define ONCE_INIT {0, 0}
/* Test and set. Alas, not atomic, but tries to minimize the period of
vulnerability. */
local int test_and_set(int volatile *flag) {
int was;
was = *flag;
*flag = 1;
return was;
}
/* Run the provided init() function once. This is not thread-safe. */
local void once(once_t *state, void (*init)(void)) {
if (!state->done) {
if (test_and_set(&state->begun))
while (!state->done)
;
else {
init();
state->done = 1;
}
}
}
#endif
/* State for once(). */
local once_t made = ONCE_INIT;
@ -1014,7 +941,12 @@ unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf,
/* ========================================================================= */
unsigned long ZEXPORT crc32(unsigned long crc, const unsigned char FAR *buf,
uInt len) {
#ifdef HAVE_S390X_VX
once(&arch_functable_init_done, arch_functable_init);
return arch_functable.crc32_z(crc, buf, len);
#else
return crc32_z(crc, buf, len);
#endif
}
/* ========================================================================= */

80
zonce.h Normal file
View File

@ -0,0 +1,80 @@
#ifndef _ZONCE_H__
#define _ZONCE_H__
#include "zutil.h"
/*
Define a once() function depending on the availability of atomics. If this is
compiled with DYNAMIC_CRC_TABLE defined, and if CRCs will be computed in
multiple threads, and if atomics are not available, then get_crc_table() must
be called to initialize the tables and must return before any threads are
allowed to compute or combine CRCs.
*/
/* Definition of once functionality. */
typedef struct once_s once_t;
/* Check for the availability of atomics. */
#if defined(__STDC__) && __STDC_VERSION__ >= 201112L && \
!defined(__STDC_NO_ATOMICS__)
#include <stdatomic.h>
/* Structure for once(), which must be initialized with ONCE_INIT. */
struct once_s {
atomic_flag begun;
atomic_int done;
};
#define ONCE_INIT {ATOMIC_FLAG_INIT, 0}
/*
Run the provided init() function exactly once, even if multiple threads
invoke once() at the same time. The state must be a once_t initialized with
ONCE_INIT.
*/
local inline void once(once_t *state, void (*init)(void)) {
if (!atomic_load(&state->done)) {
if (atomic_flag_test_and_set(&state->begun))
while (!atomic_load(&state->done))
;
else {
init();
atomic_store(&state->done, 1);
}
}
}
#else /* no atomics */
/* Structure for once(), which must be initialized with ONCE_INIT. */
struct once_s {
volatile int begun;
volatile int done;
};
#define ONCE_INIT {0, 0}
/* Test and set. Alas, not atomic, but tries to minimize the period of
vulnerability. */
local inline int test_and_set(int volatile *flag) {
int was;
was = *flag;
*flag = 1;
return was;
}
/* Run the provided init() function once. This is not thread-safe. */
local inline void once(once_t *state, void (*init)(void)) {
if (!state->done) {
if (test_and_set(&state->begun))
while (!state->done)
;
else {
init();
state->done = 1;
}
}
}
#endif
#endif