From: Stefan Liebler Date: Tue, 28 Oct 2025 14:21:18 +0000 (+0100) Subject: Remove support for lock elision. X-Git-Url: https://www.sourceware.org/git/?a=commitdiff_plain;h=b9579342c6;p=glibc.git Remove support for lock elision. The support for lock elision was already deprecated with glibc 2.42: commit 77438db8cfa6ee66b3906230156bdae11c49a195 "Mark support for lock elision as deprecated." See also discussions: https://sourceware.org/pipermail/libc-alpha/2025-July/168492.html This patch removes the architecture specific support for lock elision for x86, powerpc and s390 by removing the elision-conf.h, elision-conf.c, elision-lock.c, elision-timed.c, elision-unlock.c, elide.h, htm.h/hle.h files. Those generic files are also removed. The architecture specific structures are adjusted and the elision fields are marked as unused. See struct_mutex.h files. Furthermore in struct_rwlock.h, the leftover __rwelision was also removed. Those were originally removed with commit 0377a7fde6dfcc078dda29a1225d7720a0931357 "nptl: Remove rwlock elision definitions" and by chance reintroduced with commit 7df8af43ad1cd8ce527444de50bee6f35eebe071 "nptl: Add struct_rwlock.h" The common code (e.g. the pthread_mutex-files) are changed back to the time before lock elision was introduced with the x86-support: - commit 1cdbe579482c07e9f4bb3baa4864da2d3e7eb837 "Add the low level infrastructure for pthreads lock elision with TSX" - commit b023e4ca99f5e81f90d87d23cd267ef2abd2388c "Add new internal mutex type flags for elision." - commit 68cc29355f3334c7ad18f648ff9a6383a0916d23 "Add minimal test suite changes for elision enabled kernels" - commit e8c659d74e011346785355eeef03b7fb6f533c61 "Add elision to pthread_mutex_{try,timed,un}lock" - commit 49186d21ef2d87986bccaf0a7c45c48c91b265f3 "Disable elision for any pthread_mutexattr_settype call" - commit 1717da59aed9612becd56aaa1249aac695af4c8a "Add a configure option to enable lock elision and disable by default" Elision is removed also from the tunables, the initialization part, the pretty-printers and the manual. Some extra handling in the testsuite is removed as well as the full tst-mutex10 testcase, which tested a race while enabling lock elision. I've also searched the code for "elision", "elide", "transaction" and e.g. cleaned some comments. I've run the testsuite on x86_64 and s390x and run the build-many-glibcs.py script. Thanks to Sachin Monga, this patch is also tested on powerpc. A NEWS entry also mentions the removal. Reviewed-by: Wilco Dijkstra --- diff --git a/NEWS b/NEWS index af2f84f88b..a2618545d5 100644 --- a/NEWS +++ b/NEWS @@ -42,6 +42,9 @@ Deprecated and removed features, and other changes affecting compatibility: definition of these functions in ISO C23. Existing binaries that use the versions returning intmax_t or uintmax_t will continue to work. +* The support for TX lock elision of pthread mutexes has been removed on all + architectures (powerpc, s390x, x86_64). + Changes to build and runtime requirements: [Add changes to build and runtime requirements here] diff --git a/elf/dl-tunables.list b/elf/dl-tunables.list index c03c9967f0..c7f12fbf9c 100644 --- a/elf/dl-tunables.list +++ b/elf/dl-tunables.list @@ -84,39 +84,6 @@ glibc { } } - elision { - enable { - type: INT_32 - minval: 0 - maxval: 1 - } - skip_lock_busy { - type: INT_32 - default: 3 - minval: 0 - } - skip_lock_internal_abort { - type: INT_32 - default: 3 - minval: 0 - } - skip_lock_after_retries { - type: INT_32 - default: 3 - minval: 0 - } - tries { - type: INT_32 - default: 3 - minval: 0 - } - skip_trylock_internal_abort { - type: INT_32 - default: 3 - minval: 0 - } - } - rtld { nns { type: SIZE_T diff --git a/elf/libc_early_init.c b/elf/libc_early_init.c index 4da170a7ca..8043559dce 100644 --- a/elf/libc_early_init.c +++ b/elf/libc_early_init.c @@ -17,7 +17,6 @@ . */ #include -#include #include #include #include @@ -47,10 +46,6 @@ __libc_early_init (_Bool initial) __getrandom_early_init (initial); -#if ENABLE_ELISION_SUPPORT - __lll_elision_init (); -#endif - /* Initialize system malloc (needs __libc_initial to be set). */ call_function_static_weak (__ptmalloc_init); } diff --git a/manual/README.pretty-printers b/manual/README.pretty-printers index a696406ce5..c8824c4c1d 100644 --- a/manual/README.pretty-printers +++ b/manual/README.pretty-printers @@ -14,7 +14,6 @@ $1 = { __nusers = 0, __kind = 576, __spins = 0, - __elision = 0, __list = { __prev = 0x0, __next = 0x0 diff --git a/manual/tunables.texi b/manual/tunables.texi index e4592e23cc..4d3c6e9933 100644 --- a/manual/tunables.texi +++ b/manual/tunables.texi @@ -35,20 +35,16 @@ tunables with minimum and maximum values: @example $ /lib64/ld-linux-x86-64.so.2 --list-tunables glibc.rtld.nns: 0x4 (min: 0x1, max: 0x10) -glibc.elision.skip_lock_after_retries: 3 (min: 0, max: 2147483647) glibc.malloc.trim_threshold: 0x0 (min: 0x0, max: 0xffffffffffffffff) glibc.malloc.perturb: 0 (min: 0, max: 255) glibc.cpu.x86_shared_cache_size: 0x100000 (min: 0x0, max: 0xffffffffffffffff) glibc.pthread.rseq: 1 (min: 0, max: 1) glibc.cpu.prefer_map_32bit_exec: 0 (min: 0, max: 1) glibc.mem.tagging: 0 (min: 0, max: 255) -glibc.elision.tries: 3 (min: 0, max: 2147483647) -glibc.elision.enable: 0 (min: 0, max: 1) glibc.malloc.hugetlb: 0x0 (min: 0x0, max: 0xffffffffffffffff) glibc.cpu.x86_rep_movsb_threshold: 0x2000 (min: 0x100, max: 0xffffffffffffffff) glibc.malloc.mxfast: 0x0 (min: 0x0, max: 0xffffffffffffffff) glibc.rtld.dynamic_sort: 2 (min: 1, max: 2) -glibc.elision.skip_lock_busy: 3 (min: 0, max: 2147483647) glibc.malloc.top_pad: 0x20000 (min: 0x0, max: 0xffffffffffffffff) glibc.cpu.x86_rep_stosb_threshold: 0x800 (min: 0x1, max: 0xffffffffffffffff) glibc.cpu.x86_non_temporal_threshold: 0xc0000 (min: 0x4040, max: 0xfffffffffffffff) @@ -56,12 +52,10 @@ glibc.cpu.x86_memset_non_temporal_threshold: 0xc0000 (min: 0x4040, max: 0xffffff glibc.cpu.x86_shstk: glibc.pthread.stack_cache_size: 0x2800000 (min: 0x0, max: 0xffffffffffffffff) glibc.malloc.mmap_max: 0 (min: 0, max: 2147483647) -glibc.elision.skip_trylock_internal_abort: 3 (min: 0, max: 2147483647) glibc.cpu.plt_rewrite: 0 (min: 0, max: 2) glibc.malloc.tcache_unsorted_limit: 0x0 (min: 0x0, max: 0xffffffffffffffff) glibc.cpu.x86_ibt: glibc.cpu.hwcaps: -glibc.elision.skip_lock_internal_abort: 3 (min: 0, max: 2147483647) glibc.malloc.arena_max: 0x0 (min: 0x1, max: 0xffffffffffffffff) glibc.malloc.mmap_threshold: 0x0 (min: 0x0, max: 0xffffffffffffffff) glibc.cpu.x86_data_cache_size: 0x8000 (min: 0x0, max: 0xffffffffffffffff) @@ -77,7 +71,6 @@ glibc.malloc.check: 0 (min: 0, max: 3) * Tunable names:: The structure of a tunable name * Memory Allocation Tunables:: Tunables in the memory allocation subsystem * Dynamic Linking Tunables:: Tunables in the dynamic linking subsystem -* Elision Tunables:: Tunables in elision subsystem * POSIX Thread Tunables:: Tunables in the POSIX thread subsystem * Hardware Capability Tunables:: Tunables that modify the hardware capabilities seen by @theglibc{} @@ -387,74 +380,6 @@ can be worked around by setting the tunable to @code{2}, where the stack is always executable. @end deftp -@node Elision Tunables -@section Elision Tunables -@cindex elision tunables -@cindex tunables, elision - -@deftp {Tunable namespace} glibc.elision -Contended locks are usually slow and can lead to performance and scalability -issues in multithread code. Lock elision will use memory transactions to under -certain conditions, to elide locks and improve performance. -Elision behavior can be modified by setting the following tunables in -the @code{elision} namespace: -@end deftp - -@deftp Tunable glibc.elision.enable -The @code{glibc.elision.enable} tunable enables lock elision if the feature is -supported by the hardware. If elision is not supported by the hardware this -tunable has no effect. - -Elision tunables are supported for 64-bit Intel, IBM POWER, and z System -architectures. -@end deftp - -@deftp Tunable glibc.elision.skip_lock_busy -The @code{glibc.elision.skip_lock_busy} tunable sets how many times to use a -non-transactional lock after a transactional failure has occurred because the -lock is already acquired. Expressed in number of lock acquisition attempts. - -The default value of this tunable is @samp{3}. -@end deftp - -@deftp Tunable glibc.elision.skip_lock_internal_abort -The @code{glibc.elision.skip_lock_internal_abort} tunable sets how many times -the thread should avoid using elision if a transaction aborted for any reason -other than a different thread's memory accesses. Expressed in number of lock -acquisition attempts. - -The default value of this tunable is @samp{3}. -@end deftp - -@deftp Tunable glibc.elision.skip_lock_after_retries -The @code{glibc.elision.skip_lock_after_retries} tunable sets how many times -to try to elide a lock with transactions, that only failed due to a different -thread's memory accesses, before falling back to regular lock. -Expressed in number of lock elision attempts. - -This tunable is supported only on IBM POWER, and z System architectures. - -The default value of this tunable is @samp{3}. -@end deftp - -@deftp Tunable glibc.elision.tries -The @code{glibc.elision.tries} sets how many times to retry elision if there is -chance for the transaction to finish execution e.g., it wasn't -aborted due to the lock being already acquired. If elision is not supported -by the hardware this tunable is set to @samp{0} to avoid retries. - -The default value of this tunable is @samp{3}. -@end deftp - -@deftp Tunable glibc.elision.skip_trylock_internal_abort -The @code{glibc.elision.skip_trylock_internal_abort} tunable sets how many -times the thread should avoid trying the lock if a transaction aborted due to -reasons other than a different thread's memory accesses. Expressed in number -of try lock attempts. - -The default value of this tunable is @samp{3}. -@end deftp - @node POSIX Thread Tunables @section POSIX Thread Tunables @cindex pthread mutex tunables diff --git a/nptl/Makefile b/nptl/Makefile index 881b9f98cc..4cb89fd5b5 100644 --- a/nptl/Makefile +++ b/nptl/Makefile @@ -42,11 +42,6 @@ routines = \ cleanup_defer \ cleanup_defer_compat \ cleanup_routine \ - elision-conf \ - elision-lock \ - elision-timed \ - elision-trylock \ - elision-unlock \ events \ futex-internal \ libc-cleanup \ @@ -724,8 +719,6 @@ endif $(objpfx)tst-compat-forwarder: $(objpfx)tst-compat-forwarder-mod.so -tst-mutex10-ENV = GLIBC_TUNABLES=glibc.elision.enable=1 - # Protect against a build using -Wl,-z,now. LDFLAGS-tst-audit-threads-mod1.so = -Wl,-z,lazy LDFLAGS-tst-audit-threads-mod2.so = -Wl,-z,lazy diff --git a/nptl/elision-conf.c b/nptl/elision-conf.c deleted file mode 100644 index a4230b5606..0000000000 --- a/nptl/elision-conf.c +++ /dev/null @@ -1,17 +0,0 @@ -/* elision-conf.c: Lock elision tunable parameters. Stub version. - Copyright (C) 2021-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ diff --git a/nptl/elision-conf.h b/nptl/elision-conf.h deleted file mode 100644 index 40a8c178f1..0000000000 --- a/nptl/elision-conf.h +++ /dev/null @@ -1 +0,0 @@ -/* empty */ diff --git a/nptl/elision-lock.c b/nptl/elision-lock.c deleted file mode 100644 index 7bed73a4af..0000000000 --- a/nptl/elision-lock.c +++ /dev/null @@ -1,17 +0,0 @@ -/* elision-lock.c: Lock elision locking. Stub version. - Copyright (C) 2021-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ diff --git a/nptl/elision-timed.c b/nptl/elision-timed.c deleted file mode 100644 index 98454978d8..0000000000 --- a/nptl/elision-timed.c +++ /dev/null @@ -1,17 +0,0 @@ -/* elision-lock.c: Lock elision timed locking. Stub version. - Copyright (C) 2021-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ diff --git a/nptl/elision-trylock.c b/nptl/elision-trylock.c deleted file mode 100644 index 392bf7bd98..0000000000 --- a/nptl/elision-trylock.c +++ /dev/null @@ -1,17 +0,0 @@ -/* elision-lock.c: Lock elision locking attempts. Stub version. - Copyright (C) 2021-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ diff --git a/nptl/elision-unlock.c b/nptl/elision-unlock.c deleted file mode 100644 index a3a1088321..0000000000 --- a/nptl/elision-unlock.c +++ /dev/null @@ -1,17 +0,0 @@ -/* elision-lock.c: Lock elision unlocking support. Stub version. - Copyright (C) 2021-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ diff --git a/nptl/lowlevellock.c b/nptl/lowlevellock.c index dd6ab154b2..637d7352bf 100644 --- a/nptl/lowlevellock.c +++ b/nptl/lowlevellock.c @@ -64,8 +64,3 @@ __lll_lock_wake (int *futex, int private) lll_futex_wake (futex, 1, private); } libc_hidden_def (__lll_lock_wake) - -#if ENABLE_ELISION_SUPPORT -int __pthread_force_elision; -libc_hidden_data_def (__pthread_force_elision) -#endif diff --git a/nptl/nptl-printers.py b/nptl/nptl-printers.py index 93aec34d74..db82890b67 100644 --- a/nptl/nptl-printers.py +++ b/nptl/nptl-printers.py @@ -99,12 +99,7 @@ class MutexPrinter(object): self.values.append(MUTEX_TYPES[int(mutex_type)]) def read_status(self): - """Read the mutex's status. - - Architectures that support lock elision might not record the mutex owner - ID in the __owner field. In that case, the owner will be reported as - "Unknown". - """ + """Read the mutex's status.""" if self.kind == PTHREAD_MUTEX_DESTROYED: self.values.append(('Status', 'Destroyed')) @@ -178,8 +173,6 @@ class MutexPrinter(object): if self.owner != 0: self.values.append(('Owner ID', owner)) else: - # Owner isn't recorded, probably because lock elision - # is enabled. self.values.append(('Owner ID', 'Unknown')) def read_attributes(self): @@ -275,8 +268,7 @@ class MutexAttributesPrinter(object): mutexattr_type = (self.mutexattr & 0xffffffff - & ~PTHREAD_MUTEXATTR_FLAG_BITS - & ~PTHREAD_MUTEX_NO_ELISION_NP) + & ~PTHREAD_MUTEXATTR_FLAG_BITS) # mutexattr_type must be casted to int because it's a gdb.Value self.values.append(MUTEX_TYPES[int(mutexattr_type)]) diff --git a/nptl/nptl_lock_constants.pysym b/nptl/nptl_lock_constants.pysym index ade4398e0c..7d2746b6a0 100644 --- a/nptl/nptl_lock_constants.pysym +++ b/nptl/nptl_lock_constants.pysym @@ -37,7 +37,6 @@ PTHREAD_MUTEXATTR_PRIO_CEILING_MASK PTHREAD_MUTEXATTR_FLAG_ROBUST PTHREAD_MUTEXATTR_FLAG_PSHARED PTHREAD_MUTEXATTR_FLAG_BITS -PTHREAD_MUTEX_NO_ELISION_NP -- Priority protocols PTHREAD_PRIO_NONE diff --git a/nptl/pthread_mutex_cond_lock.c b/nptl/pthread_mutex_cond_lock.c index f3af514305..3749864dee 100644 --- a/nptl/pthread_mutex_cond_lock.c +++ b/nptl/pthread_mutex_cond_lock.c @@ -4,13 +4,8 @@ lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)) #define LLL_MUTEX_LOCK_OPTIMIZED(mutex) LLL_MUTEX_LOCK (mutex) -/* Not actually elided so far. Needed? */ -#define LLL_MUTEX_LOCK_ELISION(mutex) \ - ({ lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)); 0; }) - #define LLL_MUTEX_TRYLOCK(mutex) \ lll_cond_trylock ((mutex)->__data.__lock) -#define LLL_MUTEX_TRYLOCK_ELISION(mutex) LLL_MUTEX_TRYLOCK(mutex) /* We need to assume that there are other threads blocked on the futex. See __pthread_mutex_lock_full for further details. */ diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c index 604941c9dd..84352fba20 100644 --- a/nptl/pthread_mutex_lock.c +++ b/nptl/pthread_mutex_lock.c @@ -54,12 +54,6 @@ lll_mutex_lock_optimized (pthread_mutex_t *mutex) # define LLL_MUTEX_TRYLOCK(mutex) \ lll_trylock ((mutex)->__data.__lock) # define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0 -# define LLL_MUTEX_LOCK_ELISION(mutex) \ - lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \ - PTHREAD_MUTEX_PSHARED (mutex)) -# define LLL_MUTEX_TRYLOCK_ELISION(mutex) \ - lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \ - PTHREAD_MUTEX_PSHARED (mutex)) # define PTHREAD_MUTEX_LOCK ___pthread_mutex_lock # define PTHREAD_MUTEX_VERSIONS 1 #endif @@ -77,39 +71,25 @@ PTHREAD_MUTEX_LOCK (pthread_mutex_t *mutex) { /* See concurrency notes regarding mutex type which is loaded from __kind in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */ - unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex); + unsigned int type = PTHREAD_MUTEX_TYPE (mutex); LIBC_PROBE (mutex_entry, 1, mutex); - if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP - | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0)) + if (__glibc_unlikely (type & ~PTHREAD_MUTEX_KIND_MASK_NP)) return __pthread_mutex_lock_full (mutex); + pid_t id = THREAD_GETMEM (THREAD_SELF, tid); + if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP)) { - FORCE_ELISION (mutex, goto elision); simple: /* Normal mutex. */ LLL_MUTEX_LOCK_OPTIMIZED (mutex); assert (mutex->__data.__owner == 0); } -#if ENABLE_ELISION_SUPPORT - else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP)) - { - elision: __attribute__((unused)) - /* This case can never happen on a system without elision, - as the mutex type initialization functions will not - allow to set the elision flags. */ - /* Don't record owner or users for elision case. This is a - tail call. */ - return LLL_MUTEX_LOCK_ELISION (mutex); - } -#endif - else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) - == PTHREAD_MUTEX_RECURSIVE_NP, 1)) + else if (__glibc_likely (type == PTHREAD_MUTEX_RECURSIVE_NP)) { /* Recursive mutex. */ - pid_t id = THREAD_GETMEM (THREAD_SELF, tid); /* Check whether we already hold the mutex. */ if (mutex->__data.__owner == id) @@ -130,8 +110,7 @@ PTHREAD_MUTEX_LOCK (pthread_mutex_t *mutex) assert (mutex->__data.__owner == 0); mutex->__data.__count = 1; } - else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) - == PTHREAD_MUTEX_ADAPTIVE_NP, 1)) + else if (__glibc_likely (type == PTHREAD_MUTEX_ADAPTIVE_NP)) { if (LLL_MUTEX_TRYLOCK (mutex) != 0) { @@ -168,16 +147,13 @@ PTHREAD_MUTEX_LOCK (pthread_mutex_t *mutex) } else { - pid_t id = THREAD_GETMEM (THREAD_SELF, tid); - assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP); + assert (type == PTHREAD_MUTEX_ERRORCHECK_NP); /* Check whether we already hold the mutex. */ if (__glibc_unlikely (mutex->__data.__owner == id)) return EDEADLK; goto simple; } - pid_t id = THREAD_GETMEM (THREAD_SELF, tid); - /* Record the ownership. */ mutex->__data.__owner = id; #ifndef NO_INCR diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c index 138cdcc985..598c6a0710 100644 --- a/nptl/pthread_mutex_timedlock.c +++ b/nptl/pthread_mutex_timedlock.c @@ -42,11 +42,10 @@ __pthread_mutex_clocklock_common (pthread_mutex_t *mutex, /* See concurrency notes regarding mutex type which is loaded from __kind in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */ - switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex), + switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex), PTHREAD_MUTEX_TIMED_NP)) { /* Recursive mutex. */ - case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP: case PTHREAD_MUTEX_RECURSIVE_NP: /* Check whether we already hold the mutex. */ if (mutex->__data.__owner == id) @@ -78,26 +77,14 @@ __pthread_mutex_clocklock_common (pthread_mutex_t *mutex, if (__glibc_unlikely (mutex->__data.__owner == id)) return EDEADLK; - /* Don't do lock elision on an error checking mutex. */ - goto simple; + /* FALLTHROUGH */ case PTHREAD_MUTEX_TIMED_NP: - FORCE_ELISION (mutex, goto elision); - simple: /* Normal mutex. */ result = __futex_clocklock64 (&mutex->__data.__lock, clockid, abstime, PTHREAD_MUTEX_PSHARED (mutex)); break; - case PTHREAD_MUTEX_TIMED_ELISION_NP: - elision: __attribute__((unused)) - /* Don't record ownership */ - return lll_clocklock_elision (mutex->__data.__lock, - mutex->__data.__spins, - clockid, abstime, - PTHREAD_MUTEX_PSHARED (mutex)); - - case PTHREAD_MUTEX_ADAPTIVE_NP: if (lll_trylock (mutex->__data.__lock) != 0) { diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c index c1ab3606b0..94621cc254 100644 --- a/nptl/pthread_mutex_trylock.c +++ b/nptl/pthread_mutex_trylock.c @@ -30,11 +30,10 @@ ___pthread_mutex_trylock (pthread_mutex_t *mutex) /* See concurrency notes regarding mutex type which is loaded from __kind in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */ - switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex), + switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex), PTHREAD_MUTEX_TIMED_NP)) { /* Recursive mutex. */ - case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP: case PTHREAD_MUTEX_RECURSIVE_NP: /* Check whether we already hold the mutex. */ if (mutex->__data.__owner == id) @@ -58,17 +57,7 @@ ___pthread_mutex_trylock (pthread_mutex_t *mutex) } break; - case PTHREAD_MUTEX_TIMED_ELISION_NP: - elision: __attribute__((unused)) - if (lll_trylock_elision (mutex->__data.__lock, - mutex->__data.__elision) != 0) - break; - /* Don't record the ownership. */ - return 0; - case PTHREAD_MUTEX_TIMED_NP: - FORCE_ELISION (mutex, goto elision); - [[fallthrough]]; case PTHREAD_MUTEX_ADAPTIVE_NP: case PTHREAD_MUTEX_ERRORCHECK_NP: if (lll_trylock (mutex->__data.__lock) != 0) diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c index 19cd2dcdb7..59eb8afce0 100644 --- a/nptl/pthread_mutex_unlock.c +++ b/nptl/pthread_mutex_unlock.c @@ -48,10 +48,8 @@ __pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr) { /* See concurrency notes regarding mutex type which is loaded from __kind in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */ - int type = PTHREAD_MUTEX_TYPE_ELISION (mutex); - if (__builtin_expect (type - & ~(PTHREAD_MUTEX_KIND_MASK_NP - |PTHREAD_MUTEX_ELISION_FLAGS_NP), 0)) + int type = PTHREAD_MUTEX_TYPE (mutex); + if (__glibc_unlikely (type & ~PTHREAD_MUTEX_KIND_MASK_NP)) return __pthread_mutex_unlock_full (mutex, decr); if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP) @@ -71,14 +69,7 @@ __pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr) return 0; } - else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP)) - { - /* Don't reset the owner/users fields for elision. */ - return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision, - PTHREAD_MUTEX_PSHARED (mutex)); - } - else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) - == PTHREAD_MUTEX_RECURSIVE_NP, 1)) + else if (__glibc_likely (type == PTHREAD_MUTEX_RECURSIVE_NP)) { /* Recursive mutex. */ if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)) @@ -89,8 +80,7 @@ __pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr) return 0; goto normal; } - else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) - == PTHREAD_MUTEX_ADAPTIVE_NP, 1)) + else if (__glibc_likely (type == PTHREAD_MUTEX_ADAPTIVE_NP)) goto normal; else { diff --git a/nptl/pthread_mutexattr_gettype.c b/nptl/pthread_mutexattr_gettype.c index 5448cbef15..7296578b0f 100644 --- a/nptl/pthread_mutexattr_gettype.c +++ b/nptl/pthread_mutexattr_gettype.c @@ -25,8 +25,7 @@ __pthread_mutexattr_gettype (const pthread_mutexattr_t *attr, int *kind) iattr = (const struct pthread_mutexattr *) attr; - *kind = (iattr->mutexkind & ~PTHREAD_MUTEXATTR_FLAG_BITS - & ~PTHREAD_MUTEX_NO_ELISION_NP); + *kind = iattr->mutexkind & ~PTHREAD_MUTEXATTR_FLAG_BITS; return 0; } diff --git a/nptl/pthread_mutexattr_settype.c b/nptl/pthread_mutexattr_settype.c index 7536800d94..b4cf62bad2 100644 --- a/nptl/pthread_mutexattr_settype.c +++ b/nptl/pthread_mutexattr_settype.c @@ -27,11 +27,6 @@ ___pthread_mutexattr_settype (pthread_mutexattr_t *attr, int kind) if (kind < PTHREAD_MUTEX_NORMAL || kind > PTHREAD_MUTEX_ADAPTIVE_NP) return EINVAL; - /* Cannot distinguish between DEFAULT and NORMAL. So any settype - call disables elision for now. */ - if (kind == PTHREAD_MUTEX_NORMAL) - kind |= PTHREAD_MUTEX_NO_ELISION_NP; - iattr = (struct pthread_mutexattr *) attr; iattr->mutexkind = (iattr->mutexkind & PTHREAD_MUTEXATTR_FLAG_BITS) | kind; diff --git a/nptl/pthread_rwlock_common.c b/nptl/pthread_rwlock_common.c index d49ea25fef..60afb7a9c3 100644 --- a/nptl/pthread_rwlock_common.c +++ b/nptl/pthread_rwlock_common.c @@ -206,13 +206,7 @@ POSIX allows but does not require rwlock acquisitions to be a cancellation - point. We do not support cancellation. - - TODO We do not try to elide any read or write lock acquisitions currently. - While this would be possible, it is unclear whether HTM performance is - currently predictable enough and our runtime tuning is good enough at - deciding when to use elision so that enabling it would lead to consistently - better performance. */ + point. We do not support cancellation. */ static int diff --git a/nptl/test-mutex-printers.py b/nptl/test-mutex-printers.py index 1d38dfe22a..153ecceea5 100644 --- a/nptl/test-mutex-printers.py +++ b/nptl/test-mutex-printers.py @@ -43,7 +43,6 @@ try: next_cmd() thread_id = get_current_thread_lwpid() # Owner ID might be reported either as the thread ID or as "Unknown" - # (if e.g. lock elision is enabled). test_printer(var, to_string, {'Status': 'Acquired, possibly with no waiters', 'Owner ID': r'({0}|Unknown)'.format(thread_id)}) diff --git a/nptl/tst-mutex8.c b/nptl/tst-mutex8.c index 0d9066a882..630e64c47b 100644 --- a/nptl/tst-mutex8.c +++ b/nptl/tst-mutex8.c @@ -21,8 +21,6 @@ #include #include #include -#include -#include static pthread_mutex_t *m; static pthread_barrier_t b; @@ -95,28 +93,6 @@ check_type (const char *mas, pthread_mutexattr_t *ma) { int e; - /* Check if a mutex will be elided. Lock elision can only be activated via - the tunables framework. By default, lock elision is disabled. */ - bool assume_elided_mutex = false; - int ma_type = PTHREAD_MUTEX_TIMED_NP; - if (ma != NULL) - { - e = pthread_mutexattr_gettype (ma, &ma_type); - if (e != 0) - { - printf ("pthread_mutexattr_gettype failed with %d (%m)\n", e); - return 1; - } - } - if (ma_type == PTHREAD_MUTEX_TIMED_NP) - { - /* This type of mutex can be elided if elision is enabled via the tunables - framework. Some tests below are failing if the mutex is elided. - Thus we only run those if we assume that the mutex won't be elided. */ - if (TUNABLE_GET_FULL (glibc, elision, enable, int32_t, NULL) == 1) - assume_elided_mutex = true; - } - e = pthread_mutex_init (m, ma); if (e != 0) { @@ -149,23 +125,17 @@ check_type (const char *mas, pthread_mutexattr_t *ma) return 1; } - /* Elided mutexes don't fail destroy, thus only test this if we don't assume - elision. */ - if (assume_elided_mutex == false) + e = pthread_mutex_destroy (m); + if (e == 0) { - e = pthread_mutex_destroy (m); - if (e == 0) - { - printf ("mutex_destroy of self-locked mutex succeeded for %s\n", mas); - return 1; - } - if (e != EBUSY) - { - printf ("\ -mutex_destroy of self-locked mutex did not return EBUSY %s\n", - mas); - return 1; - } + printf ("mutex_destroy of self-locked mutex succeeded for %s\n", mas); + return 1; + } + if (e != EBUSY) + { + printf ("mutex_destroy of self-locked mutex did not return EBUSY %s\n", + mas); + return 1; } if (pthread_mutex_unlock (m) != 0) @@ -180,23 +150,18 @@ mutex_destroy of self-locked mutex did not return EBUSY %s\n", return 1; } - /* Elided mutexes don't fail destroy. */ - if (assume_elided_mutex == false) + e = pthread_mutex_destroy (m); + if (e == 0) { - e = pthread_mutex_destroy (m); - if (e == 0) - { - printf ("mutex_destroy of self-trylocked mutex succeeded for %s\n", - mas); - return 1; - } - if (e != EBUSY) - { - printf ("\ -mutex_destroy of self-trylocked mutex did not return EBUSY %s\n", - mas); - return 1; - } + printf ("mutex_destroy of self-trylocked mutex succeeded for %s\n", + mas); + return 1; + } + if (e != EBUSY) + { + printf ("mutex_destroy of self-trylocked mutex did not return EBUSY %s\n", + mas); + return 1; } if (pthread_mutex_unlock (m) != 0) @@ -232,22 +197,18 @@ mutex_destroy of self-trylocked mutex did not return EBUSY %s\n", return 1; } - /* Elided mutexes don't fail destroy. */ - if (assume_elided_mutex == false) + e = pthread_mutex_destroy (m); + if (e == 0) { - e = pthread_mutex_destroy (m); - if (e == 0) - { - printf ("mutex_destroy of condvar-used mutex succeeded for %s\n", - mas); - return 1; - } - if (e != EBUSY) - { - printf ("\ + printf ("mutex_destroy of condvar-used mutex succeeded for %s\n", + mas); + return 1; + } + if (e != EBUSY) + { + printf ("\ mutex_destroy of condvar-used mutex did not return EBUSY for %s\n", mas); - return 1; - } + return 1; } done = true; @@ -307,23 +268,19 @@ mutex_destroy of condvar-used mutex did not return EBUSY for %s\n", mas); return 1; } - /* Elided mutexes don't fail destroy. */ - if (assume_elided_mutex == false) + e = pthread_mutex_destroy (m); + if (e == 0) { - e = pthread_mutex_destroy (m); - if (e == 0) - { - printf ("2nd mutex_destroy of condvar-used mutex succeeded for %s\n", - mas); - return 1; - } - if (e != EBUSY) - { - printf ("\ + printf ("2nd mutex_destroy of condvar-used mutex succeeded for %s\n", + mas); + return 1; + } + if (e != EBUSY) + { + printf ("\ 2nd mutex_destroy of condvar-used mutex did not return EBUSY for %s\n", - mas); - return 1; - } + mas); + return 1; } if (pthread_cancel (th) != 0) diff --git a/scripts/test_printers_common.py b/scripts/test_printers_common.py index e8c1ea0c81..afe0e19334 100644 --- a/scripts/test_printers_common.py +++ b/scripts/test_printers_common.py @@ -187,9 +187,6 @@ def init_test(test_bin, printer_files, printer_names): # Finally, load the test binary. test('file {0}'.format(test_bin)) - # Disable lock elision. - test('set environment GLIBC_TUNABLES glibc.elision.enable=0') - def go_to_main(): """Executes a gdb 'start' command, which takes us to main.""" diff --git a/sysdeps/generic/elide.h b/sysdeps/generic/elide.h deleted file mode 100644 index 8e616a0d1b..0000000000 --- a/sysdeps/generic/elide.h +++ /dev/null @@ -1,25 +0,0 @@ -/* elide.h: Fallback noop lock elision support. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ -#ifndef ELIDE_H -#define ELIDE_H 1 - -#define ELIDE_LOCK(adapt_count, is_lock_free) 0 -#define ELIDE_TRYLOCK(adapt_count, is_lock_free, write) 0 -#define ELIDE_UNLOCK(is_lock_free) 0 - -#endif diff --git a/sysdeps/nptl/bits/struct_mutex.h b/sysdeps/nptl/bits/struct_mutex.h index 60d7e747f4..40ce51502c 100644 --- a/sysdeps/nptl/bits/struct_mutex.h +++ b/sysdeps/nptl/bits/struct_mutex.h @@ -21,8 +21,8 @@ /* Generic struct for both POSIX and C11 mutexes. New ports are expected to use the default layout, however architecture can redefine it to - add arch-specific extension (such as lock-elision). The struct have - a size of 32 bytes on LP32 and 40 bytes on LP64 architectures. */ + add arch-specific extension. The struct have a size of 32 bytes on LP32 + and 40 bytes on LP64 architectures. */ struct __pthread_mutex_s { @@ -40,21 +40,7 @@ struct __pthread_mutex_s PTHREAD_MUTEX_INITIALIZER or by a call to pthread_mutex_init. After a mutex has been initialized, the __kind of a mutex is usually not - changed. BUT it can be set to -1 in pthread_mutex_destroy or elision can - be enabled. This is done concurrently in the pthread_mutex_*lock - functions by using the macro FORCE_ELISION. This macro is only defined - for architectures which supports lock elision. - - For elision, there are the flags PTHREAD_MUTEX_ELISION_NP and - PTHREAD_MUTEX_NO_ELISION_NP which can be set in addition to the already - set type of a mutex. Before a mutex is initialized, only - PTHREAD_MUTEX_NO_ELISION_NP can be set with pthread_mutexattr_settype. - - After a mutex has been initialized, the functions pthread_mutex_*lock can - enable elision - if the mutex-type and the machine supports it - by - setting the flag PTHREAD_MUTEX_ELISION_NP. This is done concurrently. - Afterwards the lock / unlock functions are using specific elision - code-paths. */ + changed. BUT it can be set to -1 in pthread_mutex_destroy. */ int __kind; #if __WORDSIZE != 64 unsigned int __nusers; diff --git a/sysdeps/nptl/bits/struct_rwlock.h b/sysdeps/nptl/bits/struct_rwlock.h index 005c5992e0..fe3b91f217 100644 --- a/sysdeps/nptl/bits/struct_rwlock.h +++ b/sysdeps/nptl/bits/struct_rwlock.h @@ -23,8 +23,8 @@ /* Generic struct for both POSIX read-write lock. New ports are expected to use the default layout, however archictetures can redefine it to add - arch-specific extensions (such as lock-elision). The struct have a size - of 32 bytes on both LP32 and LP64 architectures. */ + arch-specific extensions. The struct have a size of 32 bytes on both LP32 + and LP64 architectures. */ struct __pthread_rwlock_arch_t { diff --git a/sysdeps/nptl/lowlevellock.h b/sysdeps/nptl/lowlevellock.h index 393763f193..cb673b7e53 100644 --- a/sysdeps/nptl/lowlevellock.h +++ b/sysdeps/nptl/lowlevellock.h @@ -20,7 +20,6 @@ #define _LOWLEVELLOCK_H 1 #include -#include #include #include @@ -171,105 +170,4 @@ libc_hidden_proto (__lll_lock_wake) #define LLL_LOCK_INITIALIZER (0) #define LLL_LOCK_INITIALIZER_LOCKED (1) -/* Elision support. */ - -#if ENABLE_ELISION_SUPPORT -/* Force elision for all new locks. This is used to decide whether - existing DEFAULT locks should be automatically upgraded to elision - in pthread_mutex_lock. Disabled for suid programs. Only used when - elision is available. */ -extern int __pthread_force_elision; -libc_hidden_proto (__pthread_force_elision) - -extern void __lll_elision_init (void) attribute_hidden; -extern int __lll_clocklock_elision (int *futex, short *adapt_count, - clockid_t clockid, - const struct __timespec64 *timeout, - int private); -libc_hidden_proto (__lll_clocklock_elision) - -extern int __lll_lock_elision (int *futex, short *adapt_count, int private); -libc_hidden_proto (__lll_lock_elision) - -# if ELISION_UNLOCK_NEEDS_ADAPT_COUNT -extern int __lll_unlock_elision (int *lock, short *adapt_count, int private); -# else -extern int __lll_unlock_elision (int *lock, int private); -# endif -libc_hidden_proto (__lll_unlock_elision) - -extern int __lll_trylock_elision (int *lock, short *adapt_count); -libc_hidden_proto (__lll_trylock_elision) - -# define lll_clocklock_elision(futex, adapt_count, clockid, timeout, private) \ - __lll_clocklock_elision (&(futex), &(adapt_count), clockid, timeout, private) -# define lll_lock_elision(futex, adapt_count, private) \ - __lll_lock_elision (&(futex), &(adapt_count), private) -# define lll_trylock_elision(futex, adapt_count) \ - __lll_trylock_elision (&(futex), &(adapt_count)) -# if ELISION_UNLOCK_NEEDS_ADAPT_COUNT -# define lll_unlock_elision(futex, adapt_count, private) \ - __lll_unlock_elision (&(futex), &(adapt_count), private) -# else -# define lll_unlock_elision(futex, adapt_count, private) \ - __lll_unlock_elision (&(futex), private) -# endif - -/* Automatically enable elision for existing user lock kinds. */ -# define FORCE_ELISION(m, s) \ - if (__pthread_force_elision) \ - { \ - /* See concurrency notes regarding __kind in \ - struct __pthread_mutex_s in \ - sysdeps/nptl/bits/thread-shared-types.h. \ - \ - There are the following cases for the kind of a mutex \ - (The mask PTHREAD_MUTEX_ELISION_FLAGS_NP covers the flags \ - PTHREAD_MUTEX_ELISION_NP and PTHREAD_MUTEX_NO_ELISION_NP where \ - only one of both flags can be set): \ - - both flags are not set: \ - This is the first lock operation for this mutex. Enable \ - elision as it is not enabled so far. \ - Note: It can happen that multiple threads are calling e.g. \ - pthread_mutex_lock at the same time as the first lock \ - operation for this mutex. Then elision is enabled for this \ - mutex by multiple threads. Storing with relaxed MO is enough \ - as all threads will store the same new value for the kind of \ - the mutex. But we have to ensure that we always use the \ - elision path regardless if this thread has enabled elision or \ - another one. \ - \ - - PTHREAD_MUTEX_ELISION_NP flag is set: \ - Elision was already enabled for this mutex by a previous lock \ - operation. See case above. Just use the elision path. \ - \ - - PTHREAD_MUTEX_NO_ELISION_NP flag is set: \ - Elision was explicitly disabled by pthread_mutexattr_settype. \ - Do not use the elision path. \ - Note: The flag PTHREAD_MUTEX_NO_ELISION_NP will never be \ - changed after mutex initialization. */ \ - int mutex_kind = atomic_load_relaxed (&((m)->__data.__kind)); \ - if ((mutex_kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \ - { \ - mutex_kind |= PTHREAD_MUTEX_ELISION_NP; \ - atomic_store_relaxed (&((m)->__data.__kind), mutex_kind); \ - } \ - if ((mutex_kind & PTHREAD_MUTEX_ELISION_NP) != 0) \ - { \ - s; \ - } \ - } - -#else /* !ENABLE_ELISION_SUPPORT */ - -# define lll_clocklock_elision(futex, adapt_count, clockid, abstime, private) \ - __futex_clocklock64 (&(futex), clockid, abstime, private) -# define lll_lock_elision(lock, try_lock, private) \ - ({ lll_lock (lock, private); 0; }) -# define lll_trylock_elision(a,t) lll_trylock(a) -# define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; }) -# define FORCE_ELISION(m, s) - -#endif /* !ENABLE_ELISION_SUPPORT */ - #endif /* lowlevellock.h */ diff --git a/sysdeps/nptl/pthreadP.h b/sysdeps/nptl/pthreadP.h index 3c9a14c3d6..c9b5a2af11 100644 --- a/sysdeps/nptl/pthreadP.h +++ b/sysdeps/nptl/pthreadP.h @@ -59,10 +59,6 @@ static inline short max_adaptive_count (void) enum { PTHREAD_MUTEX_KIND_MASK_NP = 3, - - PTHREAD_MUTEX_ELISION_NP = 256, - PTHREAD_MUTEX_NO_ELISION_NP = 512, - PTHREAD_MUTEX_ROBUST_NORMAL_NP = 16, PTHREAD_MUTEX_ROBUST_RECURSIVE_NP = PTHREAD_MUTEX_ROBUST_NORMAL_NP | PTHREAD_MUTEX_RECURSIVE_NP, @@ -95,14 +91,7 @@ enum PTHREAD_MUTEX_PP_ERRORCHECK_NP = PTHREAD_MUTEX_PRIO_PROTECT_NP | PTHREAD_MUTEX_ERRORCHECK_NP, PTHREAD_MUTEX_PP_ADAPTIVE_NP - = PTHREAD_MUTEX_PRIO_PROTECT_NP | PTHREAD_MUTEX_ADAPTIVE_NP, - PTHREAD_MUTEX_ELISION_FLAGS_NP - = PTHREAD_MUTEX_ELISION_NP | PTHREAD_MUTEX_NO_ELISION_NP, - - PTHREAD_MUTEX_TIMED_ELISION_NP = - PTHREAD_MUTEX_TIMED_NP | PTHREAD_MUTEX_ELISION_NP, - PTHREAD_MUTEX_TIMED_NO_ELISION_NP = - PTHREAD_MUTEX_TIMED_NP | PTHREAD_MUTEX_NO_ELISION_NP, + = PTHREAD_MUTEX_PRIO_PROTECT_NP | PTHREAD_MUTEX_ADAPTIVE_NP }; #define PTHREAD_MUTEX_PSHARED_BIT 128 @@ -110,11 +99,6 @@ enum in sysdeps/nptl/bits/thread-shared-types.h. */ #define PTHREAD_MUTEX_TYPE(m) \ (atomic_load_relaxed (&((m)->__data.__kind)) & 127) -/* Don't include NO_ELISION, as that type is always the same - as the underlying lock type. */ -#define PTHREAD_MUTEX_TYPE_ELISION(m) \ - (atomic_load_relaxed (&((m)->__data.__kind)) \ - & (127 | PTHREAD_MUTEX_ELISION_NP)) #if LLL_PRIVATE == 0 && LLL_SHARED == 128 # define PTHREAD_MUTEX_PSHARED(m) \ diff --git a/sysdeps/powerpc/nptl/bits/struct_mutex.h b/sysdeps/powerpc/nptl/bits/struct_mutex.h index b61f49e7e2..570d9b288a 100644 --- a/sysdeps/powerpc/nptl/bits/struct_mutex.h +++ b/sysdeps/powerpc/nptl/bits/struct_mutex.h @@ -32,7 +32,7 @@ struct __pthread_mutex_s int __kind; #if __WORDSIZE == 64 short __spins; - short __elision; + short __unused; __pthread_list_t __list; # define __PTHREAD_MUTEX_HAVE_PREV 1 #else @@ -41,11 +41,10 @@ struct __pthread_mutex_s { struct { - short __espins; - short __elision; -# define __spins __elision_data.__espins -# define __elision __elision_data.__elision - } __elision_data; + short __data_spins; + short __data_unused; +# define __spins __data.__data_spins + } __data; __pthread_slist_t __list; }; # define __PTHREAD_MUTEX_HAVE_PREV 0 diff --git a/sysdeps/powerpc/nptl/bits/struct_rwlock.h b/sysdeps/powerpc/nptl/bits/struct_rwlock.h index f4ffb96223..d25fa5b215 100644 --- a/sysdeps/powerpc/nptl/bits/struct_rwlock.h +++ b/sysdeps/powerpc/nptl/bits/struct_rwlock.h @@ -31,31 +31,28 @@ struct __pthread_rwlock_arch_t #if __WORDSIZE == 64 int __cur_writer; int __shared; - unsigned char __rwelision; - unsigned char __pad1[7]; + unsigned long int __pad1; unsigned long int __pad2; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned int __flags; -# define __PTHREAD_RWLOCK_ELISION_EXTRA 0, {0, 0, 0, 0, 0, 0, 0 } #else - unsigned char __rwelision; + unsigned char __pad1; unsigned char __pad2; unsigned char __shared; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned char __flags; int __cur_writer; -# define __PTHREAD_RWLOCK_ELISION_EXTRA 0 #endif }; #if __WORDSIZE == 64 # define __PTHREAD_RWLOCK_INITIALIZER(__flags) \ - 0, 0, 0, 0, 0, 0, 0, 0, __PTHREAD_RWLOCK_ELISION_EXTRA, 0, __flags + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, __flags #else # define __PTHREAD_RWLOCK_INITIALIZER(__flags) \ - 0, 0, 0, 0, 0, 0, __PTHREAD_RWLOCK_ELISION_EXTRA, 0, 0, __flags, 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, __flags, 0 #endif #endif diff --git a/sysdeps/powerpc/nptl/elide.h b/sysdeps/powerpc/nptl/elide.h deleted file mode 100644 index 3b42fd03d2..0000000000 --- a/sysdeps/powerpc/nptl/elide.h +++ /dev/null @@ -1,116 +0,0 @@ -/* elide.h: Generic lock elision support for powerpc. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#ifndef ELIDE_PPC_H -# define ELIDE_PPC_H - -# include -# include - -/* Get the new value of adapt_count according to the elision - configurations. Returns true if the system should retry again or false - otherwise. */ -static inline bool -__get_new_count (uint8_t *adapt_count, int attempt) -{ - /* A persistent failure indicates that a retry will probably - result in another failure. Use normal locking now and - for the next couple of calls. */ - if (_TEXASRU_FAILURE_PERSISTENT (__builtin_get_texasru ())) - { - if (__elision_aconf.skip_lock_internal_abort > 0) - *adapt_count = __elision_aconf.skip_lock_internal_abort; - return false; - } - /* Same logic as above, but for a number of temporary failures in a - a row. */ - else if (attempt <= 1 && __elision_aconf.skip_lock_out_of_tbegin_retries > 0 - && __elision_aconf.try_tbegin > 0) - *adapt_count = __elision_aconf.skip_lock_out_of_tbegin_retries; - return true; -} - -/* CONCURRENCY NOTES: - - The evaluation of the macro expression is_lock_free encompasses one or - more loads from memory locations that are concurrently modified by other - threads. For lock elision to work, this evaluation and the rest of the - critical section protected by the lock must be atomic because an - execution with lock elision must be equivalent to an execution in which - the lock would have been actually acquired and released. Therefore, we - evaluate is_lock_free inside of the transaction that represents the - critical section for which we want to use lock elision, which ensures - the atomicity that we require. */ - -/* Returns 0 if the lock defined by is_lock_free was elided. - ADAPT_COUNT is a per-lock state variable. */ -# define ELIDE_LOCK(adapt_count, is_lock_free) \ - ({ \ - int ret = 0; \ - if (adapt_count > 0) \ - (adapt_count)--; \ - else \ - for (int i = __elision_aconf.try_tbegin; i > 0; i--) \ - { \ - if (__libc_tbegin (0)) \ - { \ - if (is_lock_free) \ - { \ - ret = 1; \ - break; \ - } \ - __libc_tabort (_ABORT_LOCK_BUSY); \ - } \ - else \ - if (!__get_new_count (&adapt_count,i)) \ - break; \ - } \ - ret; \ - }) - -# define ELIDE_TRYLOCK(adapt_count, is_lock_free, write) \ - ({ \ - int ret = 0; \ - if (__elision_aconf.try_tbegin > 0) \ - { \ - if (write) \ - __libc_tabort (_ABORT_NESTED_TRYLOCK); \ - ret = ELIDE_LOCK (adapt_count, is_lock_free); \ - } \ - ret; \ - }) - - -static inline bool -__elide_unlock (int is_lock_free) -{ - if (is_lock_free) - { - /* This code is expected to crash when trying to unlock a lock not - held by this thread. More information is available in the - __pthread_rwlock_unlock() implementation. */ - __libc_tend (0); - return true; - } - return false; -} - -# define ELIDE_UNLOCK(is_lock_free) \ - __elide_unlock (is_lock_free) - -#endif diff --git a/sysdeps/pthread/Makefile b/sysdeps/pthread/Makefile index 7572f62cc0..1a6e835efc 100644 --- a/sysdeps/pthread/Makefile +++ b/sysdeps/pthread/Makefile @@ -201,7 +201,6 @@ tests += \ tst-mutex6 \ tst-mutex7 \ tst-mutex9 \ - tst-mutex10 \ tst-mutex11 \ tst-once1 \ tst-once2 \ diff --git a/sysdeps/pthread/elision-conf.h b/sysdeps/pthread/elision-conf.h deleted file mode 100644 index a6b4b9a9c0..0000000000 --- a/sysdeps/pthread/elision-conf.h +++ /dev/null @@ -1,29 +0,0 @@ -/* elision-conf.h: Lock elision configuration. Stub version. - Copyright (C) 2021-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#ifndef _ELISION_CONF_H -#define _ELISION_CONF_H 1 - -/* No elision support by default. */ -#define ENABLE_ELISION_SUPPORT 0 - -/* Whether __lll_unlock_elision expects a pointer argument to the - adaptive counter. Here, an unused arbitrary value. */ -#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 0 - -#endif diff --git a/sysdeps/pthread/tst-mutex-errorcheck.c b/sysdeps/pthread/tst-mutex-errorcheck.c index 8a08ab9e79..94f6ab93f3 100644 --- a/sysdeps/pthread/tst-mutex-errorcheck.c +++ b/sysdeps/pthread/tst-mutex-errorcheck.c @@ -1,4 +1,4 @@ -/* Check that error checking mutexes are not subject to lock elision. +/* Check that already locked error checking mutexes return EDEADL. Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -37,10 +37,6 @@ do_test (void) TEST_COMPARE (pthread_mutex_init (&mutex, &mutexattr), 0); TEST_COMPARE (pthread_mutexattr_destroy (&mutexattr), 0); - /* The call to pthread_mutex_timedlock erroneously enabled lock elision - on the mutex, which then triggered an assertion failure in - pthread_mutex_unlock. It would also defeat the error checking nature - of the mutex. */ TEST_COMPARE (pthread_mutex_timedlock (&mutex, &tms), 0); TEST_COMPARE (pthread_mutex_timedlock (&mutex, &tms), EDEADLK); diff --git a/sysdeps/pthread/tst-mutex10.c b/sysdeps/pthread/tst-mutex10.c deleted file mode 100644 index 2346f85237..0000000000 --- a/sysdeps/pthread/tst-mutex10.c +++ /dev/null @@ -1,109 +0,0 @@ -/* Testing race while enabling lock elision. - Copyright (C) 2018-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ -#include -#include -#include -#include -#include -#include -#include -#include - -static pthread_barrier_t barrier; -static pthread_mutex_t mutex; -static long long int iteration_count = 1000000; -static unsigned int thread_count = 3; - -static void * -thr_func (void *arg) -{ - long long int i; - for (i = 0; i < iteration_count; i++) - { - if ((uintptr_t) arg == 0) - { - xpthread_mutex_destroy (&mutex); - xpthread_mutex_init (&mutex, NULL); - } - - xpthread_barrier_wait (&barrier); - - /* Test if enabling lock elision works if it is enabled concurrently. - There was a race in FORCE_ELISION macro which leads to either - pthread_mutex_destroy returning EBUSY as the owner was recorded - by pthread_mutex_lock - in "normal mutex" code path - but was not - reset in pthread_mutex_unlock - in "elision" code path. - Or it leads to the assertion in nptl/pthread_mutex_lock.c: - assert (mutex->__data.__owner == 0); - Please ensure that the test is run with lock elision: - export GLIBC_TUNABLES=glibc.elision.enable=1 */ - xpthread_mutex_lock (&mutex); - xpthread_mutex_unlock (&mutex); - - xpthread_barrier_wait (&barrier); - } - return NULL; -} - -static int -do_test (void) -{ - unsigned int i; - printf ("Starting %d threads to run %lld iterations.\n", - thread_count, iteration_count); - - pthread_t *threads = xmalloc (thread_count * sizeof (pthread_t)); - xpthread_barrier_init (&barrier, NULL, thread_count); - xpthread_mutex_init (&mutex, NULL); - - for (i = 0; i < thread_count; i++) - threads[i] = xpthread_create (NULL, thr_func, (void *) (uintptr_t) i); - - for (i = 0; i < thread_count; i++) - xpthread_join (threads[i]); - - xpthread_barrier_destroy (&barrier); - free (threads); - - return EXIT_SUCCESS; -} - -#define OPT_ITERATIONS 10000 -#define OPT_THREADS 10001 -#define CMDLINE_OPTIONS \ - { "iterations", required_argument, NULL, OPT_ITERATIONS }, \ - { "threads", required_argument, NULL, OPT_THREADS }, -static void -cmdline_process (int c) -{ - long long int arg = strtoll (optarg, NULL, 0); - switch (c) - { - case OPT_ITERATIONS: - if (arg > 0) - iteration_count = arg; - break; - case OPT_THREADS: - if (arg > 0 && arg < 100) - thread_count = arg; - break; - } -} -#define CMDLINE_PROCESS cmdline_process -#define TIMEOUT 50 -#include diff --git a/sysdeps/pthread/tst-mutex5.c b/sysdeps/pthread/tst-mutex5.c index 10b231dcf6..2682bb6ae8 100644 --- a/sysdeps/pthread/tst-mutex5.c +++ b/sysdeps/pthread/tst-mutex5.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include diff --git a/sysdeps/s390/configure b/sysdeps/s390/configure index 97f52524fb..e3766aa801 100644 --- a/sysdeps/s390/configure +++ b/sysdeps/s390/configure @@ -1,44 +1,6 @@ # This file is generated from configure.ac by Autoconf. DO NOT EDIT! # Local configure fragment for sysdeps/s390. -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __builtin_tbegin" >&5 -printf %s "checking for __builtin_tbegin... " >&6; } -if test ${libc_cv_gcc_builtin_tbegin+y} -then : - printf %s "(cached) " >&6 -else case e in #( - e) cat > conftest.c <<\EOF -#include -void testtransaction () -{ - if (__builtin_tbegin (0) == _HTM_TBEGIN_STARTED) - { - __builtin_tend (); - } -} -EOF -if { ac_try='${CC-cc} -mhtm -O2 -S conftest.c -o - | grep -w tbegin > /dev/null' - { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5 - (eval $ac_try) 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; } ; -then - libc_cv_gcc_builtin_tbegin=yes -else - libc_cv_gcc_builtin_tbegin=no -fi -rm -f conftest* ;; -esac -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libc_cv_gcc_builtin_tbegin" >&5 -printf "%s\n" "$libc_cv_gcc_builtin_tbegin" >&6; } - -if test "$libc_cv_gcc_builtin_tbegin" = no ; then - critic_missing="$critic_missing The used GCC has no support for __builtin_tbegin, which is needed for lock-elision on target S390." -fi - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for S390 vector instruction support" >&5 printf %s "checking for S390 vector instruction support... " >&6; } diff --git a/sysdeps/s390/configure.ac b/sysdeps/s390/configure.ac index 496866b850..4265bfd9f4 100644 --- a/sysdeps/s390/configure.ac +++ b/sysdeps/s390/configure.ac @@ -1,32 +1,6 @@ GLIBC_PROVIDES dnl See aclocal.m4 in the top level source directory. # Local configure fragment for sysdeps/s390. -AC_CACHE_CHECK(for __builtin_tbegin, libc_cv_gcc_builtin_tbegin, [dnl -cat > conftest.c <<\EOF -#include -void testtransaction () -{ - if (__builtin_tbegin (0) == _HTM_TBEGIN_STARTED) - { - __builtin_tend (); - } -} -EOF -dnl -dnl test, if the tbegin instruction is used by __builtin_tbegin -if AC_TRY_COMMAND([${CC-cc} -mhtm -O2 -S conftest.c -o - | grep -w tbegin > /dev/null]) ; -then - libc_cv_gcc_builtin_tbegin=yes -else - libc_cv_gcc_builtin_tbegin=no -fi -rm -f conftest* ]) - -if test "$libc_cv_gcc_builtin_tbegin" = no ; then - critic_missing="$critic_missing The used GCC has no support for __builtin_tbegin, which is needed for lock-elision on target S390." -fi - - AC_CACHE_CHECK([for S390 vector instruction support], libc_cv_asm_s390_vx, [ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ void testvecinsn () diff --git a/sysdeps/s390/nptl/bits/struct_mutex.h b/sysdeps/s390/nptl/bits/struct_mutex.h index af0e2e997e..5bc66d4a1e 100644 --- a/sysdeps/s390/nptl/bits/struct_mutex.h +++ b/sysdeps/s390/nptl/bits/struct_mutex.h @@ -32,7 +32,7 @@ struct __pthread_mutex_s int __kind; #if __WORDSIZE == 64 short __spins; - short __elision; + short __unused; __pthread_list_t __list; # define __PTHREAD_MUTEX_HAVE_PREV 1 #else @@ -41,11 +41,10 @@ struct __pthread_mutex_s { struct { - short __espins; - short __elision; - } _d; -# define __spins _d.__espins -# define __elision _d.__elision + short __data_spins; + short __data_unused; + } __data; +# define __spins __data.__data_spins __pthread_slist_t __list; }; # define __PTHREAD_MUTEX_HAVE_PREV 0 diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-conf.c b/sysdeps/unix/sysv/linux/powerpc/elision-conf.c deleted file mode 100644 index 451897bd3d..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-conf.c +++ /dev/null @@ -1,138 +0,0 @@ -/* elision-conf.c: Lock elision tunable parameters. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include "config.h" -#include -#include -#include -#include - -#define TUNABLE_NAMESPACE elision -#include - -/* Reasonable initial tuning values, may be revised in the future. - This is a conservative initial value. */ - -struct elision_config __elision_aconf = - { - /* How many times to use a non-transactional lock after a transactional - failure has occurred because the lock is already acquired. Expressed - in number of lock acquisition attempts. */ - .skip_lock_busy = 3, - /* How often to not attempt to use elision if a transaction aborted due - to reasons other than other threads' memory accesses. Expressed in - number of lock acquisition attempts. */ - .skip_lock_internal_abort = 3, - /* How often to not attempt to use elision if a lock used up all retries - without success. Expressed in number of lock acquisition attempts. */ - .skip_lock_out_of_tbegin_retries = 3, - /* How often we retry using elision if there is chance for the transaction - to finish execution (e.g., it wasn't aborted due to the lock being - already acquired. */ - .try_tbegin = 3, - /* Same as SKIP_LOCK_INTERNAL_ABORT but for trylock. */ - .skip_trylock_internal_abort = 3, - }; - -static inline void -__always_inline -do_set_elision_enable (int32_t elision_enable) -{ - /* Enable elision if it's available in hardware. It's not necessary to check - if __libc_enable_secure isn't enabled since elision_enable will be set - according to the default, which is disabled. */ - if (elision_enable == 1) - __pthread_force_elision = (GLRO (dl_hwcap2) - & PPC_FEATURE2_HAS_HTM) ? 1 : 0; -} - -/* The pthread->elision_enable tunable is 0 or 1 indicating that elision - should be disabled or enabled respectively. The feature will only be used - if it's supported by the hardware. */ - -void -TUNABLE_CALLBACK (set_elision_enable) (tunable_val_t *valp) -{ - int32_t elision_enable = (int32_t) valp->numval; - do_set_elision_enable (elision_enable); -} - -#define TUNABLE_CALLBACK_FNDECL(__name, __type) \ -static inline void \ -__always_inline \ -do_set_elision_ ## __name (__type value) \ -{ \ - __elision_aconf.__name = value; \ -} \ -void \ -TUNABLE_CALLBACK (set_elision_ ## __name) (tunable_val_t *valp) \ -{ \ - __type value = (__type) (valp)->numval; \ - do_set_elision_ ## __name (value); \ -} - -TUNABLE_CALLBACK_FNDECL (skip_lock_busy, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_lock_internal_abort, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_lock_out_of_tbegin_retries, int32_t); -TUNABLE_CALLBACK_FNDECL (try_tbegin, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_trylock_internal_abort, int32_t); - -/* Initialize elision. */ - -void -__lll_elision_init (void) -{ - /* Elision depends on tunables and must be explicitly turned on by setting - the appropriate tunable on a supported platform. */ - - TUNABLE_GET (enable, int32_t, - TUNABLE_CALLBACK (set_elision_enable)); - TUNABLE_GET (skip_lock_busy, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_busy)); - TUNABLE_GET (skip_lock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_internal_abort)); - TUNABLE_GET (skip_lock_after_retries, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_out_of_tbegin_retries)); - TUNABLE_GET (tries, int32_t, - TUNABLE_CALLBACK (set_elision_try_tbegin)); - TUNABLE_GET (skip_trylock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_trylock_internal_abort)); - - /* Linux from 3.9 through 4.2 do not abort HTM transaction on syscalls, - instead it suspends the transaction and resumes it when returning to - usercode. The side-effects of the syscall will always remain visible, - even if the transaction is aborted. This is an issue when a transaction - is used along with futex syscall, on pthread_cond_wait for instance, - where futex might succeed but the transaction is rolled back leading - the condition variable object in an inconsistent state. - - Glibc used to prevent it by always aborting a transaction before issuing - a syscall. Linux 4.2 also decided to abort active transaction in - syscalls which makes the glibc workaround superflours. Worse, glibc - transaction abortions leads to a performance issues on recent kernels. - - So Lock Elision is just enabled when it has been explicitly set (either - by tunables of by a configure switch) and if kernel aborts HTM - transactions on syscalls (PPC_FEATURE2_HTM_NOSC) */ - - __pthread_force_elision = (__pthread_force_elision - && GLRO (dl_hwcap2) & PPC_FEATURE2_HTM_NOSC); - - if (!__pthread_force_elision) - __elision_aconf.try_tbegin = 0; /* Disable elision on rwlocks. */ -} diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-conf.h b/sysdeps/unix/sysv/linux/powerpc/elision-conf.h deleted file mode 100644 index 11bc7edaff..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-conf.h +++ /dev/null @@ -1,40 +0,0 @@ -/* elision-conf.h: Lock elision tunable parameters. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#ifndef _ELISION_CONF_H -#define _ELISION_CONF_H 1 - -#include -#include - -#define ENABLE_ELISION_SUPPORT 1 -#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 1 - -/* Should make sure there is no false sharing on this. */ -struct elision_config -{ - int skip_lock_busy; - int skip_lock_internal_abort; - int skip_lock_out_of_tbegin_retries; - int try_tbegin; - int skip_trylock_internal_abort; -} __attribute__ ((__aligned__ (128))); - -extern struct elision_config __elision_aconf attribute_hidden; - -#endif diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-lock.c b/sysdeps/unix/sysv/linux/powerpc/elision-lock.c deleted file mode 100644 index 8eb7c3e547..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-lock.c +++ /dev/null @@ -1,81 +0,0 @@ -/* elision-lock.c: Elided pthread mutex lock. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include -#include -#include -#include "htm.h" - -#ifndef EXTRAARG -# define EXTRAARG -#endif -#ifndef LLL_LOCK -# define LLL_LOCK(a,b) lll_lock(a,b), 0 -#endif - -#define aconf __elision_aconf - -/* Adaptive lock using transactions. - By default the lock region is run as a transaction, and when it - aborts or the lock is busy the lock adapts itself. */ - -int -__lll_lock_elision (int *lock, short *adapt_count, EXTRAARG int pshared) -{ - /* adapt_count is accessed concurrently but is just a hint. Thus, - use atomic accesses but relaxed MO is sufficient. */ - if (atomic_load_relaxed (adapt_count) > 0) - { - goto use_lock; - } - - for (int i = aconf.try_tbegin; i > 0; i--) - { - if (__libc_tbegin (0)) - { - if (*lock == 0) - return 0; - /* Lock was busy. Fall back to normal locking. */ - __libc_tabort (_ABORT_LOCK_BUSY); - } - else - { - /* A persistent failure indicates that a retry will probably - result in another failure. Use normal locking now and - for the next couple of calls. */ - if (_TEXASRU_FAILURE_PERSISTENT (__builtin_get_texasru ())) - { - if (aconf.skip_lock_internal_abort > 0) - atomic_store_relaxed (adapt_count, - aconf.skip_lock_internal_abort); - goto use_lock; - } - } - } - - /* Fall back to locks for a bit if retries have been exhausted */ - if (aconf.try_tbegin > 0 && aconf.skip_lock_out_of_tbegin_retries > 0) - atomic_store_relaxed (adapt_count, - aconf.skip_lock_out_of_tbegin_retries); - -use_lock: - return LLL_LOCK ((*lock), pshared); -} -libc_hidden_def (__lll_lock_elision) diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-timed.c b/sysdeps/unix/sysv/linux/powerpc/elision-timed.c deleted file mode 100644 index 6442408a33..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-timed.c +++ /dev/null @@ -1,29 +0,0 @@ -/* elision-timed.c: Lock elision timed lock. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include "lowlevellock.h" -#include "futex-internal.h" - -#define __lll_lock_elision __lll_clocklock_elision -#define EXTRAARG clockid_t clockid, const struct __timespec64 *t, -#undef LLL_LOCK -#define LLL_LOCK(a, b) __futex_clocklock64 (&(a), clockid, t, b) - -#include "elision-lock.c" diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-trylock.c b/sysdeps/unix/sysv/linux/powerpc/elision-trylock.c deleted file mode 100644 index d252799c46..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-trylock.c +++ /dev/null @@ -1,70 +0,0 @@ -/* elision-trylock.c: Lock eliding trylock for pthreads. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include -#include -#include "htm.h" - -#define aconf __elision_aconf - -/* Try to elide a futex trylock. FUTEX is the futex variable. ADAPT_COUNT is - the adaptation counter in the mutex. */ - -int -__lll_trylock_elision (int *futex, short *adapt_count) -{ - /* Implement POSIX semantics by forbiding nesting elided trylocks. */ - __libc_tabort (_ABORT_NESTED_TRYLOCK); - - /* Only try a transaction if it's worth it. */ - if (atomic_load_relaxed (adapt_count) > 0) - { - goto use_lock; - } - - if (__libc_tbegin (0)) - { - if (*futex == 0) - return 0; - - /* Lock was busy. This is never a nested transaction. - End it, and set the adapt count. */ - __libc_tend (0); - - if (aconf.skip_lock_busy > 0) - atomic_store_relaxed (adapt_count, aconf.skip_lock_busy); - } - else - { - if (_TEXASRU_FAILURE_PERSISTENT (__builtin_get_texasru ())) - { - /* A persistent failure indicates that a retry will probably - result in another failure. Use normal locking now and - for the next couple of calls. */ - if (aconf.skip_trylock_internal_abort > 0) - atomic_store_relaxed (adapt_count, - aconf.skip_trylock_internal_abort); - } - } - -use_lock: - return lll_trylock (*futex); -} -libc_hidden_def (__lll_trylock_elision) diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-unlock.c b/sysdeps/unix/sysv/linux/powerpc/elision-unlock.c deleted file mode 100644 index 774151474e..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-unlock.c +++ /dev/null @@ -1,44 +0,0 @@ -/* elision-unlock.c: Commit an elided pthread lock. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include "pthreadP.h" -#include "lowlevellock.h" -#include "htm.h" - -int -__lll_unlock_elision (int *lock, short *adapt_count, int pshared) -{ - /* When the lock was free we're in a transaction. */ - if (*lock == 0) - __libc_tend (0); - else - { - /* Update adapt_count in the critical section to prevent a - write-after-destroy error as mentioned in BZ 20822. The - following update of adapt_count has to be contained within - the critical region of the fall-back lock in order to not violate - the mutex destruction requirements. */ - short __tmp = atomic_load_relaxed (adapt_count); - if (__tmp > 0) - atomic_store_relaxed (adapt_count, __tmp - 1); - - lll_unlock ((*lock), pshared); - } - return 0; -} -libc_hidden_def (__lll_unlock_elision) diff --git a/sysdeps/unix/sysv/linux/powerpc/htm.h b/sysdeps/unix/sysv/linux/powerpc/htm.h deleted file mode 100644 index 672b197a2c..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/htm.h +++ /dev/null @@ -1,171 +0,0 @@ -/* Shared HTM header. Emulate transactional execution facility intrinsics for - compilers and assemblers that do not support the intrinsics and instructions - yet. - - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#ifndef _HTM_H -#define _HTM_H 1 - -#ifdef __ASSEMBLER__ - -/* tbegin. */ -.macro TBEGIN - .long 0x7c00051d -.endm - -/* tend. 0 */ -.macro TEND - .long 0x7c00055d -.endm - -/* tabort. code */ -.macro TABORT code - .byte 0x7c - .byte \code - .byte 0x07 - .byte 0x1d -.endm - -/*"TEXASR - Transaction EXception And Summary Register" - mfspr %dst,130 */ -.macro TEXASR dst - mfspr \dst,130 -.endm - -#else - -#include - -/* Official HTM intrinsics interface matching GCC, but works - on older GCC compatible compilers and binutils. - We should somehow detect if the compiler supports it, because - it may be able to generate slightly better code. */ - -#define TBEGIN ".long 0x7c00051d" -#define TEND ".long 0x7c00055d" -#if __BYTE_ORDER == __LITTLE_ENDIAN -# define TABORT ".byte 0x1d,0x07,%1,0x7c" -#else -# define TABORT ".byte 0x7c,%1,0x07,0x1d" -#endif - -#define __force_inline inline __attribute__((__always_inline__)) - -#ifndef __HTM__ - -#define _TEXASRU_EXTRACT_BITS(TEXASR,BITNUM,SIZE) \ - (((TEXASR) >> (31-(BITNUM))) & ((1<<(SIZE))-1)) -#define _TEXASRU_FAILURE_PERSISTENT(TEXASRU) \ - _TEXASRU_EXTRACT_BITS(TEXASRU, 7, 1) - -#define _tbegin() \ - ({ unsigned int __ret; \ - asm volatile ( \ - TBEGIN "\t\n" \ - "mfcr %0\t\n" \ - "rlwinm %0,%0,3,1\t\n" \ - "xori %0,%0,1\t\n" \ - : "=r" (__ret) : \ - : "cr0", "memory"); \ - __ret; \ - }) - -#define _tend() \ - ({ unsigned int __ret; \ - asm volatile ( \ - TEND "\t\n" \ - "mfcr %0\t\n" \ - "rlwinm %0,%0,3,1\t\n" \ - "xori %0,%0,1\t\n" \ - : "=r" (__ret) : \ - : "cr0", "memory"); \ - __ret; \ - }) - -#define _tabort(__code) \ - ({ unsigned int __ret; \ - asm volatile ( \ - TABORT "\t\n" \ - "mfcr %0\t\n" \ - "rlwinm %0,%0,3,1\t\n" \ - "xori %0,%0,1\t\n" \ - : "=r" (__ret) : "r" (__code) \ - : "cr0", "memory"); \ - __ret; \ - }) - -#define _texasru() \ - ({ unsigned long __ret; \ - asm volatile ( \ - "mfspr %0,131\t\n" \ - : "=r" (__ret)); \ - __ret; \ - }) - -#define __libc_tbegin(tdb) _tbegin () -#define __libc_tend(nested) _tend () -#define __libc_tabort(abortcode) _tabort (abortcode) -#define __builtin_get_texasru() _texasru () - -#else -# include - -# ifdef __TM_FENCE__ - /* New GCC behavior. */ -# define __libc_tbegin(R) __builtin_tbegin (R) -# define __libc_tend(R) __builtin_tend (R) -# define __libc_tabort(R) __builtin_tabort (R) -# else - /* Workaround an old GCC behavior. Earlier releases of GCC 4.9 and 5.0, - didn't use to treat __builtin_tbegin, __builtin_tend and - __builtin_tabort as compiler barriers, moving instructions into and - out the transaction. - Remove this when glibc drops support for GCC 5.0. */ -# define __libc_tbegin(R) \ - ({ __asm__ volatile("" ::: "memory"); \ - unsigned int __ret = __builtin_tbegin (R); \ - __asm__ volatile("" ::: "memory"); \ - __ret; \ - }) -# define __libc_tabort(R) \ - ({ __asm__ volatile("" ::: "memory"); \ - unsigned int __ret = __builtin_tabort (R); \ - __asm__ volatile("" ::: "memory"); \ - __ret; \ - }) -# define __libc_tend(R) \ - ({ __asm__ volatile("" ::: "memory"); \ - unsigned int __ret = __builtin_tend (R); \ - __asm__ volatile("" ::: "memory"); \ - __ret; \ - }) -# endif /* __TM_FENCE__ */ -#endif /* __HTM__ */ - -#endif /* __ASSEMBLER__ */ - -/* Definitions used for TEXASR Failure code (bits 0:7). If the failure - should be persistent, the abort code must be odd. 0xd0 through 0xff - are reserved for the kernel and potential hypervisor. */ -#define _ABORT_PERSISTENT 0x01 /* An unspecified persistent abort. */ -#define _ABORT_LOCK_BUSY 0x34 /* Busy lock, not persistent. */ -#define _ABORT_NESTED_TRYLOCK (0x32 | _ABORT_PERSISTENT) -#define _ABORT_SYSCALL (0x30 | _ABORT_PERSISTENT) - -#endif diff --git a/sysdeps/unix/sysv/linux/s390/Makefile b/sysdeps/unix/sysv/linux/s390/Makefile index 3de9579e23..d5433f1dd9 100644 --- a/sysdeps/unix/sysv/linux/s390/Makefile +++ b/sysdeps/unix/sysv/linux/s390/Makefile @@ -11,14 +11,6 @@ ifeq ($(subdir),stdlib) gen-as-const-headers += ucontext_i.sym endif -ifeq ($(subdir),nptl) -elision-CFLAGS = -mhtm -msoft-float -CFLAGS-elision-lock.c = $(elision-CFLAGS) -CFLAGS-elision-timed.c = $(elision-CFLAGS) -CFLAGS-elision-trylock.c = $(elision-CFLAGS) -CFLAGS-elision-unlock.c = $(elision-CFLAGS) -endif - ifeq ($(subdir),misc) tests += tst-ptrace-singleblock endif diff --git a/sysdeps/unix/sysv/linux/s390/elision-conf.c b/sysdeps/unix/sysv/linux/s390/elision-conf.c deleted file mode 100644 index 32b9b3a17d..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-conf.c +++ /dev/null @@ -1,118 +0,0 @@ -/* Lock elision tunable parameters. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include -#include -#include -#include - -#define TUNABLE_NAMESPACE elision -#include - -/* Reasonable initial tuning values, may be revised in the future. - This is a conservative initial value. */ - -struct elision_config __elision_aconf = - { - /* How often to not attempt to use elision if a transaction aborted - because the lock is already acquired. Expressed in number of lock - acquisition attempts. */ - .skip_lock_busy = 3, - /* How often to not attempt to use elision if a transaction aborted due - to reasons other than other threads' memory accesses. Expressed in - number of lock acquisition attempts. */ - .skip_lock_internal_abort = 3, - /* How often to not attempt to use elision if a lock used up all retries - without success. Expressed in number of lock acquisition attempts. */ - .skip_lock_out_of_tbegin_retries = 3, - /* How often we try using elision if there is chance for the transaction - to finish execution (e.g., it wasn't aborted due to the lock being - already acquired. */ - .try_tbegin = 3, - /* Same as SKIP_LOCK_INTERNAL_ABORT but for trylock. */ - .skip_trylock_internal_abort = 3, - }; - -static inline void -__always_inline -do_set_elision_enable (int32_t elision_enable) -{ - /* Enable elision if it's available in hardware. It's not necessary to check - if __libc_enable_secure isn't enabled since elision_enable will be set - according to the default, which is disabled. */ - if (elision_enable == 1) - __pthread_force_elision = (GLRO (dl_hwcap) & HWCAP_S390_TE) ? 1 : 0; -} - -/* The pthread->elision_enable tunable is 0 or 1 indicating that elision - should be disabled or enabled respectively. The feature will only be used - if it's supported by the hardware. */ - -void -TUNABLE_CALLBACK (set_elision_enable) (tunable_val_t *valp) -{ - int32_t elision_enable = (int32_t) valp->numval; - do_set_elision_enable (elision_enable); -} - -#define TUNABLE_CALLBACK_FNDECL(__name, __type) \ -static inline void \ -__always_inline \ -do_set_elision_ ## __name (__type value) \ -{ \ - __elision_aconf.__name = value; \ -} \ -void \ -TUNABLE_CALLBACK (set_elision_ ## __name) (tunable_val_t *valp) \ -{ \ - __type value = (__type) (valp)->numval; \ - do_set_elision_ ## __name (value); \ -} - -TUNABLE_CALLBACK_FNDECL (skip_lock_busy, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_lock_internal_abort, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_lock_out_of_tbegin_retries, int32_t); -TUNABLE_CALLBACK_FNDECL (try_tbegin, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_trylock_internal_abort, int32_t); - -/* Initialize elison. */ - -void -__lll_elision_init (void) -{ - /* Elision depends on tunables and must be explicitly turned on by setting - the appropriate tunable on a supported platform. */ - - TUNABLE_GET (enable, int32_t, - TUNABLE_CALLBACK (set_elision_enable)); - TUNABLE_GET (skip_lock_busy, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_busy)); - TUNABLE_GET (skip_lock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_internal_abort)); - TUNABLE_GET (skip_lock_after_retries, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_out_of_tbegin_retries)); - TUNABLE_GET (tries, int32_t, - TUNABLE_CALLBACK (set_elision_try_tbegin)); - TUNABLE_GET (skip_trylock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_trylock_internal_abort)); - - if (!__pthread_force_elision) - __elision_aconf.try_tbegin = 0; /* Disable elision on rwlocks. */ -} diff --git a/sysdeps/unix/sysv/linux/s390/elision-conf.h b/sysdeps/unix/sysv/linux/s390/elision-conf.h deleted file mode 100644 index 90845de01e..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-conf.h +++ /dev/null @@ -1,40 +0,0 @@ -/* Lock elision tunable parameters. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ -#ifndef _ELISION_CONF_H -#define _ELISION_CONF_H 1 - -#include -#include - -#define ENABLE_ELISION_SUPPORT 1 -#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 1 - -/* Should make sure there is no false sharing on this. */ - -struct elision_config -{ - int skip_lock_busy; - int skip_lock_internal_abort; - int skip_lock_out_of_tbegin_retries; - int try_tbegin; - int skip_trylock_internal_abort; -}; - -extern struct elision_config __elision_aconf attribute_hidden; - -#endif diff --git a/sysdeps/unix/sysv/linux/s390/elision-lock.c b/sysdeps/unix/sysv/linux/s390/elision-lock.c deleted file mode 100644 index b22e6cd938..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-lock.c +++ /dev/null @@ -1,120 +0,0 @@ -/* Elided pthread mutex lock. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include -#include -#include -#include - -#ifndef EXTRAARG -#define EXTRAARG -#endif -#ifndef LLL_LOCK -#define LLL_LOCK(a,b) lll_lock(a,b), 0 -#endif - -#define aconf __elision_aconf - -/* Adaptive lock using transactions. - By default the lock region is run as a transaction, and when it - aborts or the lock is busy the lock adapts itself. */ - -int -__lll_lock_elision (int *futex, short *adapt_count, EXTRAARG int private) -{ - /* adapt_count can be accessed concurrently; these accesses can be both - inside of transactions (if critical sections are nested and the outer - critical section uses lock elision) and outside of transactions. Thus, - we need to use atomic accesses to avoid data races. However, the - value of adapt_count is just a hint, so relaxed MO accesses are - sufficient. */ - if (atomic_load_relaxed (adapt_count) <= 0 && aconf.try_tbegin > 0) - { - /* Start a transaction and retry it automatically if it aborts with - _HTM_TBEGIN_TRANSIENT. This macro calls tbegin at most retry_cnt - + 1 times. The second argument is considered as retry_cnt. */ - int status = __libc_tbegin_retry ((void *) 0, aconf.try_tbegin - 1); - if (__glibc_likely (status == _HTM_TBEGIN_STARTED)) - { - /* Check the futex to make sure nobody has touched it in the - mean time. This forces the futex into the cache and makes - sure the transaction aborts if another thread acquires the lock - concurrently. */ - if (__glibc_likely (atomic_load_relaxed (futex) == 0)) - /* Lock was free. Return to user code in a transaction. */ - return 0; - - /* Lock was busy. Fall back to normal locking. - This can be the case if e.g. adapt_count was decremented to zero - by a former release and another thread has been waken up and - acquired it. */ - if (__glibc_likely (__libc_tx_nesting_depth () <= 1)) - { - /* In a non-nested transaction there is no need to abort, - which is expensive. Simply end the started transaction. */ - __libc_tend (); - /* Don't try to use transactions for the next couple of times. - See above for why relaxed MO is sufficient. */ - if (aconf.skip_lock_busy > 0) - atomic_store_relaxed (adapt_count, aconf.skip_lock_busy); - } - else /* nesting depth is > 1 */ - { - /* A nested transaction will abort eventually because it - cannot make any progress before *futex changes back to 0. - So we may as well abort immediately. - This persistently aborts the outer transaction to force - the outer mutex use the default lock instead of retrying - with transactions until the try_tbegin of the outer mutex - is zero. - The adapt_count of this inner mutex is not changed, - because using the default lock with the inner mutex - would abort the outer transaction. */ - __libc_tabort (_HTM_FIRST_USER_ABORT_CODE | 1); - __builtin_unreachable (); - } - } - else if (status != _HTM_TBEGIN_TRANSIENT) - { - /* A persistent abort (cc 1 or 3) indicates that a retry is - probably futile. Use the normal locking now and for the - next couple of calls. - Be careful to avoid writing to the lock. See above for why - relaxed MO is sufficient. */ - if (aconf.skip_lock_internal_abort > 0) - atomic_store_relaxed (adapt_count, - aconf.skip_lock_internal_abort); - } - else - { - /* The transaction failed for some retries with - _HTM_TBEGIN_TRANSIENT. Use the normal locking now and for the - next couple of calls. */ - if (aconf.skip_lock_out_of_tbegin_retries > 0) - atomic_store_relaxed (adapt_count, - aconf.skip_lock_out_of_tbegin_retries); - } - } - - /* Use normal locking as fallback path if the transaction does not - succeed. */ - return LLL_LOCK ((*futex), private); -} -libc_hidden_def (__lll_lock_elision) diff --git a/sysdeps/unix/sysv/linux/s390/elision-timed.c b/sysdeps/unix/sysv/linux/s390/elision-timed.c deleted file mode 100644 index 2611e9cf9b..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-timed.c +++ /dev/null @@ -1,27 +0,0 @@ -/* Lock elision timed lock. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include -#include "futex-internal.h" -#define __lll_lock_elision __lll_clocklock_elision -#define EXTRAARG clockid_t clockid, const struct __timespec64 *t, -#undef LLL_LOCK -#define LLL_LOCK(a, b) __futex_clocklock64 (&(a), clockid, t, b) -#include "elision-lock.c" diff --git a/sysdeps/unix/sysv/linux/s390/elision-trylock.c b/sysdeps/unix/sysv/linux/s390/elision-trylock.c deleted file mode 100644 index 1aa99d806c..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-trylock.c +++ /dev/null @@ -1,98 +0,0 @@ -/* Elided pthread mutex trylock. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include -#include -#include - -#define aconf __elision_aconf - -/* Try to elide a futex trylock. FUTEX is the futex variable. ADAPT_COUNT is - the adaptation counter in the mutex. */ - -int -__lll_trylock_elision (int *futex, short *adapt_count) -{ - /* Implement POSIX semantics by forbiding nesting elided trylocks. - Sorry. After the abort the code is re-executed - non transactional and if the lock was already locked - return an error. */ - if (__libc_tx_nesting_depth () > 0) - { - /* Note that this abort may terminate an outermost transaction that - was created outside glibc. - This persistently aborts the current transactions to force - them to use the default lock instead of retrying transactions - until their try_tbegin is zero. - */ - __libc_tabort (_HTM_FIRST_USER_ABORT_CODE | 1); - __builtin_unreachable (); - } - - /* adapt_count can be accessed concurrently; these accesses can be both - inside of transactions (if critical sections are nested and the outer - critical section uses lock elision) and outside of transactions. Thus, - we need to use atomic accesses to avoid data races. However, the - value of adapt_count is just a hint, so relaxed MO accesses are - sufficient. */ - if (atomic_load_relaxed (adapt_count) <= 0 && aconf.try_tbegin > 0) - { - int status = __libc_tbegin ((void *) 0); - if (__glibc_likely (status == _HTM_TBEGIN_STARTED)) - { - /* Check the futex to make sure nobody has touched it in the - mean time. This forces the futex into the cache and makes - sure the transaction aborts if another thread acquires the lock - concurrently. */ - if (__glibc_likely (atomic_load_relaxed (futex) == 0)) - /* Lock was free. Return to user code in a transaction. */ - return 0; - - /* Lock was busy. Fall back to normal locking. - This can be the case if e.g. adapt_count was decremented to zero - by a former release and another thread has been waken up and - acquired it. - Since we are in a non-nested transaction there is no need to abort, - which is expensive. Simply end the started transaction. */ - __libc_tend (); - /* Note: Changing the adapt_count here might abort a transaction on a - different CPU, but that could happen anyway when the futex is - acquired, so there's no need to check the nesting depth here. - See above for why relaxed MO is sufficient. */ - if (aconf.skip_lock_busy > 0) - atomic_store_relaxed (adapt_count, aconf.skip_lock_busy); - } - else if (status != _HTM_TBEGIN_TRANSIENT) - { - /* A persistent abort (cc 1 or 3) indicates that a retry is - probably futile. Use the normal locking now and for the - next couple of calls. - Be careful to avoid writing to the lock. */ - if (aconf.skip_trylock_internal_abort > 0) - *adapt_count = aconf.skip_trylock_internal_abort; - } - /* Could do some retries here. */ - } - - /* Use normal locking as fallback path if the transaction does not - succeed. */ - return lll_trylock (*futex); -} -libc_hidden_def (__lll_trylock_elision) diff --git a/sysdeps/unix/sysv/linux/s390/elision-unlock.c b/sysdeps/unix/sysv/linux/s390/elision-unlock.c deleted file mode 100644 index 11e96abd5f..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-unlock.c +++ /dev/null @@ -1,62 +0,0 @@ -/* Commit an elided pthread lock. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include - -int -__lll_unlock_elision(int *futex, short *adapt_count, int private) -{ - /* If the lock is free, we elided the lock earlier. This does not - necessarily mean that we are in a transaction, because the user code may - have closed the transaction, but that is impossible to detect reliably. - Relaxed MO access to futex is sufficient because a correct program - will only release a lock it has acquired; therefore, it must either - changed the futex word's value to something !=0 or it must have used - elision; these are actions by the same thread, so these actions are - sequenced-before the relaxed load (and thus also happens-before the - relaxed load). Therefore, relaxed MO is sufficient. */ - if (atomic_load_relaxed (futex) == 0) - { - __libc_tend (); - } - else - { - /* Update the adapt_count while unlocking before completing the critical - section. adapt_count is accessed concurrently outside of a - transaction or a critical section (e.g. in elision-lock.c). So we need - to use atomic accesses. However, the value of adapt_count is just a - hint, so relaxed MO accesses are sufficient. - If adapt_count would be decremented while locking, multiple - CPUs, trying to lock the acquired mutex, will decrement adapt_count to - zero and another CPU will try to start a transaction, which will be - immediately aborted as the mutex is locked. - The update of adapt_count is done before releasing the lock as POSIX' - mutex destruction requirements disallow accesses to the mutex after it - has been released and thus could have been acquired or destroyed by - another thread. */ - short adapt_count_val = atomic_load_relaxed (adapt_count); - if (adapt_count_val > 0) - atomic_store_relaxed (adapt_count, adapt_count_val - 1); - - lll_unlock ((*futex), private); - } - return 0; -} -libc_hidden_def (__lll_unlock_elision) diff --git a/sysdeps/unix/sysv/linux/s390/htm.h b/sysdeps/unix/sysv/linux/s390/htm.h deleted file mode 100644 index bf1a89c1a5..0000000000 --- a/sysdeps/unix/sysv/linux/s390/htm.h +++ /dev/null @@ -1,187 +0,0 @@ -/* Shared HTM header. Work around false transactional execution facility - intrinsics. - - Copyright (C) 2016-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#ifndef _HTM_H -#define _HTM_H 1 - -#include - -#ifdef __s390x__ -# define TX_FPRS_BYTES 64 -# define TX_SAVE_FPRS \ - " std %%f8, 0(%[R_FPRS])\n\t" \ - " std %%f9, 8(%[R_FPRS])\n\t" \ - " std %%f10, 16(%[R_FPRS])\n\t" \ - " std %%f11, 24(%[R_FPRS])\n\t" \ - " std %%f12, 32(%[R_FPRS])\n\t" \ - " std %%f13, 40(%[R_FPRS])\n\t" \ - " std %%f14, 48(%[R_FPRS])\n\t" \ - " std %%f15, 56(%[R_FPRS])\n\t" - -# define TX_RESTORE_FPRS \ - " ld %%f8, 0(%[R_FPRS])\n\t" \ - " ld %%f9, 8(%[R_FPRS])\n\t" \ - " ld %%f10, 16(%[R_FPRS])\n\t" \ - " ld %%f11, 24(%[R_FPRS])\n\t" \ - " ld %%f12, 32(%[R_FPRS])\n\t" \ - " ld %%f13, 40(%[R_FPRS])\n\t" \ - " ld %%f14, 48(%[R_FPRS])\n\t" \ - " ld %%f15, 56(%[R_FPRS])\n\t" - -#else - -# define TX_FPRS_BYTES 16 -# define TX_SAVE_FPRS \ - " std %%f4, 0(%[R_FPRS])\n\t" \ - " std %%f6, 8(%[R_FPRS])\n\t" - -# define TX_RESTORE_FPRS \ - " ld %%f4, 0(%[R_FPRS])\n\t" \ - " ld %%f6, 8(%[R_FPRS])\n\t" - -#endif /* ! __s390x__ */ - -/* Use own inline assembly instead of __builtin_tbegin, as tbegin - has to filter program interruptions which can't be done with the builtin. - Now the fprs have to be saved / restored here, too. - The fpc is also not saved / restored with the builtin. - The used inline assembly does not clobber the volatile fprs / vrs! - Clobbering the latter ones would force the compiler to save / restore - the call saved fprs as those overlap with the vrs, but they only need to be - restored if the transaction fails but not if the transaction is successfully - started. Thus the user of the tbegin macros in this header file has to - compile the file / function with -msoft-float. It prevents gcc from using - fprs / vrs. */ -#define __libc_tbegin(tdb) __libc_tbegin_base(tdb,,,) - -#define __libc_tbegin_retry_output_regs , [R_TX_CNT] "+&d" (__tx_cnt) -#define __libc_tbegin_retry_input_regs(retry_cnt) , [R_RETRY] "d" (retry_cnt) -#define __libc_tbegin_retry_abort_path_insn \ - /* If tbegin returned _HTM_TBEGIN_TRANSIENT, retry immediately so \ - that max tbegin_cnt transactions are tried. Otherwise return and \ - let the caller of this macro do the fallback path. */ \ - " jnh 1f\n\t" /* cc 1/3: jump to fallback path. */ \ - /* tbegin returned _HTM_TBEGIN_TRANSIENT: retry with transaction. */ \ - " crje %[R_TX_CNT], %[R_RETRY], 1f\n\t" /* Reached max retries? */ \ - " ahi %[R_TX_CNT], 1\n\t" \ - " ppa %[R_TX_CNT], 0, 1\n\t" /* Transaction-Abort Assist. */ \ - " j 2b\n\t" /* Loop to tbegin. */ - -/* Same as __libc_tbegin except if tbegin aborts with _HTM_TBEGIN_TRANSIENT. - Then this macros restores the fpc, fprs and automatically retries up to - retry_cnt tbegins. Further saving of the state is omitted as it is already - saved. This macro calls tbegin at most as retry_cnt + 1 times. */ -#define __libc_tbegin_retry(tdb, retry_cnt) \ - ({ int __ret; \ - int __tx_cnt = 0; \ - __ret = __libc_tbegin_base(tdb, \ - __libc_tbegin_retry_abort_path_insn, \ - __libc_tbegin_retry_output_regs, \ - __libc_tbegin_retry_input_regs(retry_cnt)); \ - __ret; \ - }) - -#define __libc_tbegin_base(tdb, abort_path_insn, output_regs, input_regs) \ - ({ int __ret; \ - int __fpc; \ - char __fprs[TX_FPRS_BYTES]; \ - __asm__ __volatile__ (".machine push\n\t" \ - ".machinemode \"zarch_nohighgprs\"\n\t" \ - ".machine \"all\"\n\t" \ - /* Save state at the outermost transaction. \ - As extracting nesting depth is expensive \ - on at least zEC12, save fprs at inner \ - transactions, too. \ - The fpc and fprs are saved here as they \ - are not saved by tbegin. There exist no \ - call-saved vrs, thus they are not saved \ - here. */ \ - " efpc %[R_FPC]\n\t" \ - TX_SAVE_FPRS \ - /* Begin transaction: save all gprs, allow \ - ar modification and fp operations. Some \ - program-interruptions (e.g. a null \ - pointer access) are filtered and the \ - transaction will abort. In this case \ - the normal lock path will execute it \ - again and result in a core dump which does \ - now show at tbegin but the real executed \ - instruction. \ - However it is not guaranteed that this \ - retry operate on the same data and thus \ - may not end in an program-interruption. \ - Note: This could also be used to probe \ - memory for being accessible! */ \ - "2: tbegin 0, 0xFF0E\n\t" \ - /* Branch away in abort case (this is the \ - preferred sequence. See PoP in chapter 5 \ - Transactional-Execution Facility \ - Operation). */ \ - " jnz 0f\n\t" \ - /* Transaction has successfully started. */ \ - " lhi %[R_RET], 0\n\t" \ - " j 1f\n\t" \ - /* Transaction has aborted. Now we are at \ - the outermost transaction. Restore fprs \ - and fpc. */ \ - "0: ipm %[R_RET]\n\t" \ - " srl %[R_RET], 28\n\t" \ - " sfpc %[R_FPC]\n\t" \ - TX_RESTORE_FPRS \ - abort_path_insn \ - "1:\n\t" \ - ".machine pop\n" \ - : [R_RET] "=&d" (__ret), \ - [R_FPC] "=&d" (__fpc) \ - output_regs \ - : [R_FPRS] "a" (__fprs) \ - input_regs \ - : "cc", "memory"); \ - __ret; \ - }) - -/* These builtins are usable in context of glibc lock elision code without any - changes. Use them. */ -#define __libc_tend() \ - ({ __asm__ __volatile__ (".machine push\n\t" \ - ".machinemode \"zarch_nohighgprs\"\n\t" \ - ".machine \"all\"\n\t"); \ - int __ret = __builtin_tend (); \ - __asm__ __volatile__ (".machine pop"); \ - __ret; \ - }) - -#define __libc_tabort(abortcode) \ - __asm__ __volatile__ (".machine push\n\t" \ - ".machinemode \"zarch_nohighgprs\"\n\t" \ - ".machine \"all\"\n\t"); \ - __builtin_tabort (abortcode); \ - __asm__ __volatile__ (".machine pop") - -#define __libc_tx_nesting_depth() \ - ({ __asm__ __volatile__ (".machine push\n\t" \ - ".machinemode \"zarch_nohighgprs\"\n\t" \ - ".machine \"all\"\n\t"); \ - int __ret = __builtin_tx_nesting_depth (); \ - __asm__ __volatile__ (".machine pop"); \ - __ret; \ - }) - -#endif diff --git a/sysdeps/unix/sysv/linux/x86/Makefile b/sysdeps/unix/sysv/linux/x86/Makefile index 743b633b65..92a5e03f64 100644 --- a/sysdeps/unix/sysv/linux/x86/Makefile +++ b/sysdeps/unix/sysv/linux/x86/Makefile @@ -11,13 +11,6 @@ ifeq ($(subdir),misc) sysdep_headers += sys/elf.h sys/perm.h sys/reg.h sys/vm86.h sys/debugreg.h sys/io.h endif -ifeq ($(subdir),nptl) -CFLAGS-elision-lock.c += -mrtm -CFLAGS-elision-unlock.c += -mrtm -CFLAGS-elision-timed.c += -mrtm -CFLAGS-elision-trylock.c += -mrtm -endif - ifeq ($(subdir),setjmp) tests += tst-saved_mask-1 endif diff --git a/sysdeps/unix/sysv/linux/x86/elision-conf.c b/sysdeps/unix/sysv/linux/x86/elision-conf.c deleted file mode 100644 index cf120a9ec6..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-conf.c +++ /dev/null @@ -1,109 +0,0 @@ -/* elision-conf.c: Lock elision tunable parameters. - Copyright (C) 2013-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include "config.h" -#include -#include -#include -#include - -#define TUNABLE_NAMESPACE elision -#include - -/* Reasonable initial tuning values, may be revised in the future. - This is a conservative initial value. */ - -struct elision_config __elision_aconf = - { - /* How often to not attempt to use elision if a transaction aborted - because the lock is already acquired. Expressed in number of lock - acquisition attempts. */ - .skip_lock_busy = 3, - /* How often to not attempt to use elision if a transaction aborted due - to reasons other than other threads' memory accesses. Expressed in - number of lock acquisition attempts. */ - .skip_lock_internal_abort = 3, - /* How often we retry using elision if there is chance for the transaction - to finish execution (e.g., it wasn't aborted due to the lock being - already acquired. */ - .retry_try_xbegin = 3, - /* Same as SKIP_LOCK_INTERNAL_ABORT but for trylock. */ - .skip_trylock_internal_abort = 3, - }; - -static __always_inline void -do_set_elision_enable (int32_t elision_enable) -{ - /* Enable elision if it's available in hardware. It's not necessary to check - if __libc_enable_secure isn't enabled since elision_enable will be set - according to the default, which is disabled. */ - if (elision_enable == 1) - __pthread_force_elision = CPU_FEATURE_USABLE (RTM) ? 1 : 0; -} - -/* The pthread->elision_enable tunable is 0 or 1 indicating that elision - should be disabled or enabled respectively. The feature will only be used - if it's supported by the hardware. */ - -void -TUNABLE_CALLBACK (set_elision_enable) (tunable_val_t *valp) -{ - int32_t elision_enable = (int32_t) valp->numval; - do_set_elision_enable (elision_enable); -} - -#define TUNABLE_CALLBACK_FNDECL(__name, __type) \ -static __always_inline void \ -do_set_elision_ ## __name (__type value) \ -{ \ - __elision_aconf.__name = value; \ -} \ -void \ -TUNABLE_CALLBACK (set_elision_ ## __name) (tunable_val_t *valp) \ -{ \ - __type value = (__type) (valp)->numval; \ - do_set_elision_ ## __name (value); \ -} - -TUNABLE_CALLBACK_FNDECL (skip_lock_busy, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_lock_internal_abort, int32_t); -TUNABLE_CALLBACK_FNDECL (retry_try_xbegin, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_trylock_internal_abort, int32_t); - -/* Initialize elision. */ - -void -__lll_elision_init (void) -{ - /* Elision depends on tunables and must be explicitly turned on by setting - the appropriate tunable on a supported platform. */ - - TUNABLE_GET (enable, int32_t, - TUNABLE_CALLBACK (set_elision_enable)); - TUNABLE_GET (skip_lock_busy, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_busy)); - TUNABLE_GET (skip_lock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_internal_abort)); - TUNABLE_GET (tries, int32_t, - TUNABLE_CALLBACK (set_elision_retry_try_xbegin)); - TUNABLE_GET (skip_trylock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_trylock_internal_abort)); - - if (!__pthread_force_elision) - __elision_aconf.retry_try_xbegin = 0; /* Disable elision on rwlocks. */ -} diff --git a/sysdeps/unix/sysv/linux/x86/elision-conf.h b/sysdeps/unix/sysv/linux/x86/elision-conf.h deleted file mode 100644 index 6ba8810654..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-conf.h +++ /dev/null @@ -1,39 +0,0 @@ -/* elision-conf.h: Lock elision tunable parameters. - Copyright (C) 2013-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ -#ifndef _ELISION_CONF_H -#define _ELISION_CONF_H 1 - -#include -#include - -#define ENABLE_ELISION_SUPPORT 1 -#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 0 - -/* Should make sure there is no false sharing on this. */ - -struct elision_config -{ - int skip_lock_busy; - int skip_lock_internal_abort; - int retry_try_xbegin; - int skip_trylock_internal_abort; -}; - -extern struct elision_config __elision_aconf attribute_hidden; - -#endif diff --git a/sysdeps/unix/sysv/linux/x86/elision-lock.c b/sysdeps/unix/sysv/linux/x86/elision-lock.c deleted file mode 100644 index 74e748266d..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-lock.c +++ /dev/null @@ -1,102 +0,0 @@ -/* elision-lock.c: Elided pthread mutex lock. - Copyright (C) 2011-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include "pthreadP.h" -#include "lowlevellock.h" -#include "hle.h" -#include - -#ifndef EXTRAARG -#define EXTRAARG -#endif -#ifndef LLL_LOCK -#define LLL_LOCK(a,b) lll_lock(a,b), 0 -#endif - -#define aconf __elision_aconf - -/* Adaptive lock using transactions. - By default the lock region is run as a transaction, and when it - aborts or the lock is busy the lock adapts itself. */ - -int -__lll_lock_elision (int *futex, short *adapt_count, EXTRAARG int private) -{ - /* adapt_count can be accessed concurrently; these accesses can be both - inside of transactions (if critical sections are nested and the outer - critical section uses lock elision) and outside of transactions. Thus, - we need to use atomic accesses to avoid data races. However, the - value of adapt_count is just a hint, so relaxed MO accesses are - sufficient. */ - if (atomic_load_relaxed (adapt_count) <= 0) - { - unsigned status; - int try_xbegin; - - for (try_xbegin = aconf.retry_try_xbegin; - try_xbegin > 0; - try_xbegin--) - { - if ((status = _xbegin()) == _XBEGIN_STARTED) - { - if (*futex == 0) - return 0; - - /* Lock was busy. Fall back to normal locking. - Could also _xend here but xabort with 0xff code - is more visible in the profiler. */ - _xabort (_ABORT_LOCK_BUSY); - } - - if (!(status & _XABORT_RETRY)) - { - if ((status & _XABORT_EXPLICIT) - && _XABORT_CODE (status) == _ABORT_LOCK_BUSY) - { - /* Right now we skip here. Better would be to wait a bit - and retry. This likely needs some spinning. See - above for why relaxed MO is sufficient. */ - if (atomic_load_relaxed (adapt_count) - != aconf.skip_lock_busy) - atomic_store_relaxed (adapt_count, aconf.skip_lock_busy); - } - /* Internal abort. There is no chance for retry. - Use the normal locking and next time use lock. - Be careful to avoid writing to the lock. See above for why - relaxed MO is sufficient. */ - else if (atomic_load_relaxed (adapt_count) - != aconf.skip_lock_internal_abort) - atomic_store_relaxed (adapt_count, - aconf.skip_lock_internal_abort); - break; - } - } - } - else - { - /* Use a normal lock until the threshold counter runs out. - Lost updates possible. */ - atomic_store_relaxed (adapt_count, - atomic_load_relaxed (adapt_count) - 1); - } - - /* Use a normal lock here. */ - return LLL_LOCK ((*futex), private); -} -libc_hidden_def (__lll_lock_elision) diff --git a/sysdeps/unix/sysv/linux/x86/elision-timed.c b/sysdeps/unix/sysv/linux/x86/elision-timed.c deleted file mode 100644 index 50925f4a60..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-timed.c +++ /dev/null @@ -1,27 +0,0 @@ -/* elision-timed.c: Lock elision timed lock. - Copyright (C) 2013-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include "lowlevellock.h" -#include "futex-internal.h" -#define __lll_lock_elision __lll_clocklock_elision -#define EXTRAARG clockid_t clockid, const struct __timespec64 *t, -#undef LLL_LOCK -#define LLL_LOCK(a, b) __futex_clocklock64 (&(a), clockid, t, b) -#include "elision-lock.c" diff --git a/sysdeps/unix/sysv/linux/x86/elision-trylock.c b/sysdeps/unix/sysv/linux/x86/elision-trylock.c deleted file mode 100644 index 60c4311798..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-trylock.c +++ /dev/null @@ -1,76 +0,0 @@ -/* elision-trylock.c: Lock eliding trylock for pthreads. - Copyright (C) 2013-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include -#include "hle.h" -#include - -#define aconf __elision_aconf - -/* Try to elide a futex trylock. FUTEX is the futex variable. ADAPT_COUNT is - the adaptation counter in the mutex. */ - -int -__lll_trylock_elision (int *futex, short *adapt_count) -{ - /* Implement POSIX semantics by forbiding nesting - trylock. Sorry. After the abort the code is re-executed - non transactional and if the lock was already locked - return an error. */ - _xabort (_ABORT_NESTED_TRYLOCK); - - /* Only try a transaction if it's worth it. See __lll_lock_elision for - why we need atomic accesses. Relaxed MO is sufficient because this is - just a hint. */ - if (atomic_load_relaxed (adapt_count) <= 0) - { - unsigned status; - - if ((status = _xbegin()) == _XBEGIN_STARTED) - { - if (*futex == 0) - return 0; - - /* Lock was busy. Fall back to normal locking. - Could also _xend here but xabort with 0xff code - is more visible in the profiler. */ - _xabort (_ABORT_LOCK_BUSY); - } - - if (!(status & _XABORT_RETRY)) - { - /* Internal abort. No chance for retry. For future - locks don't try speculation for some time. See above for MO. */ - if (atomic_load_relaxed (adapt_count) - != aconf.skip_lock_internal_abort) - atomic_store_relaxed (adapt_count, aconf.skip_lock_internal_abort); - } - /* Could do some retries here. */ - } - else - { - /* Lost updates are possible but harmless (see above). */ - atomic_store_relaxed (adapt_count, - atomic_load_relaxed (adapt_count) - 1); - } - - return lll_trylock (*futex); -} -libc_hidden_def (__lll_trylock_elision) diff --git a/sysdeps/unix/sysv/linux/x86/elision-unlock.c b/sysdeps/unix/sysv/linux/x86/elision-unlock.c deleted file mode 100644 index 67dd1e6949..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-unlock.c +++ /dev/null @@ -1,34 +0,0 @@ -/* elision-unlock.c: Commit an elided pthread lock. - Copyright (C) 2013-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include "pthreadP.h" -#include "lowlevellock.h" -#include "hle.h" - -int -__lll_unlock_elision(int *lock, int private) -{ - /* When the lock was free we're in a transaction. - When you crash here you unlocked a free lock. */ - if (*lock == 0) - _xend(); - else - lll_unlock ((*lock), private); - return 0; -} -libc_hidden_def (__lll_unlock_elision) diff --git a/sysdeps/unix/sysv/linux/x86/hle.h b/sysdeps/unix/sysv/linux/x86/hle.h deleted file mode 100644 index 0449026839..0000000000 --- a/sysdeps/unix/sysv/linux/x86/hle.h +++ /dev/null @@ -1,11 +0,0 @@ -/* Shared RTM header. */ -#ifndef _HLE_H -#define _HLE_H 1 - -#include - -#define _ABORT_LOCK_BUSY 0xff -#define _ABORT_LOCK_IS_LOCKED 0xfe -#define _ABORT_NESTED_TRYLOCK 0xfd - -#endif diff --git a/sysdeps/x86/elide.h b/sysdeps/x86/elide.h deleted file mode 100644 index 7c4e66d141..0000000000 --- a/sysdeps/x86/elide.h +++ /dev/null @@ -1,119 +0,0 @@ -/* elide.h: Generic lock elision support. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ -#ifndef ELIDE_H -#define ELIDE_H 1 - -#include -#include -#include - - -/* Adapt elision with ADAPT_COUNT and STATUS and decide retries. */ - -static inline bool -elision_adapt(signed char *adapt_count, unsigned int status) -{ - if (status & _XABORT_RETRY) - return false; - if ((status & _XABORT_EXPLICIT) - && _XABORT_CODE (status) == _ABORT_LOCK_BUSY) - { - /* Right now we skip here. Better would be to wait a bit - and retry. This likely needs some spinning. Be careful - to avoid writing the lock. - Using relaxed MO and separate atomic accesses is sufficient because - adapt_count is just a hint. */ - if (atomic_load_relaxed (adapt_count) != __elision_aconf.skip_lock_busy) - atomic_store_relaxed (adapt_count, __elision_aconf.skip_lock_busy); - } - /* Internal abort. There is no chance for retry. - Use the normal locking and next time use lock. - Be careful to avoid writing to the lock. See above for MO. */ - else if (atomic_load_relaxed (adapt_count) - != __elision_aconf.skip_lock_internal_abort) - atomic_store_relaxed (adapt_count, - __elision_aconf.skip_lock_internal_abort); - return true; -} - -/* is_lock_free must be executed inside the transaction */ - -/* Returns true if lock defined by IS_LOCK_FREE was elided. - ADAPT_COUNT is a per-lock state variable; it must be accessed atomically - to avoid data races but is just a hint, so using relaxed MO and separate - atomic loads and stores instead of atomic read-modify-write operations is - sufficient. */ - -#define ELIDE_LOCK(adapt_count, is_lock_free) \ - ({ \ - int ret = 0; \ - \ - if (atomic_load_relaxed (&(adapt_count)) <= 0) \ - { \ - for (int i = __elision_aconf.retry_try_xbegin; i > 0; i--) \ - { \ - unsigned int status; \ - if ((status = _xbegin ()) == _XBEGIN_STARTED) \ - { \ - if (is_lock_free) \ - { \ - ret = 1; \ - break; \ - } \ - _xabort (_ABORT_LOCK_BUSY); \ - } \ - if (!elision_adapt (&(adapt_count), status)) \ - break; \ - } \ - } \ - else \ - atomic_store_relaxed (&(adapt_count), \ - atomic_load_relaxed (&(adapt_count)) - 1); \ - ret; \ - }) - -/* Returns true if lock defined by IS_LOCK_FREE was try-elided. - ADAPT_COUNT is a per-lock state variable. */ - -#define ELIDE_TRYLOCK(adapt_count, is_lock_free, write) ({ \ - int ret = 0; \ - if (__elision_aconf.retry_try_xbegin > 0) \ - { \ - if (write) \ - _xabort (_ABORT_NESTED_TRYLOCK); \ - ret = ELIDE_LOCK (adapt_count, is_lock_free); \ - } \ - ret; \ - }) - -/* Returns true if lock defined by IS_LOCK_FREE was elided. The call - to _xend crashes if the application incorrectly tries to unlock a - lock which has not been locked. */ - -#define ELIDE_UNLOCK(is_lock_free) \ - ({ \ - int ret = 0; \ - if (is_lock_free) \ - { \ - _xend (); \ - ret = 1; \ - } \ - ret; \ - }) - -#endif diff --git a/sysdeps/x86/nptl/bits/struct_mutex.h b/sysdeps/x86/nptl/bits/struct_mutex.h index 0784a21c06..4a18b30aac 100644 --- a/sysdeps/x86/nptl/bits/struct_mutex.h +++ b/sysdeps/x86/nptl/bits/struct_mutex.h @@ -32,7 +32,7 @@ struct __pthread_mutex_s int __kind; #ifdef __x86_64__ short __spins; - short __elision; + short __unused; __pthread_list_t __list; # define __PTHREAD_MUTEX_HAVE_PREV 1 #else @@ -41,11 +41,10 @@ struct __pthread_mutex_s { struct { - short __espins; - short __eelision; -# define __spins __elision_data.__espins -# define __elision __elision_data.__eelision - } __elision_data; + short __data_spins; + short __data_unused; +# define __spins __data.__data_spins + } __data; __pthread_slist_t __list; }; # define __PTHREAD_MUTEX_HAVE_PREV 0 diff --git a/sysdeps/x86/nptl/bits/struct_rwlock.h b/sysdeps/x86/nptl/bits/struct_rwlock.h index da526a1603..43782a9562 100644 --- a/sysdeps/x86/nptl/bits/struct_rwlock.h +++ b/sysdeps/x86/nptl/bits/struct_rwlock.h @@ -31,14 +31,7 @@ struct __pthread_rwlock_arch_t #ifdef __x86_64__ int __cur_writer; int __shared; - signed char __rwelision; -# ifdef __ILP32__ - unsigned char __pad1[3]; -# define __PTHREAD_RWLOCK_ELISION_EXTRA 0, { 0, 0, 0 } -# else - unsigned char __pad1[7]; -# define __PTHREAD_RWLOCK_ELISION_EXTRA 0, { 0, 0, 0, 0, 0, 0, 0 } -# endif + unsigned long int __pad1; unsigned long int __pad2; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ @@ -48,7 +41,7 @@ struct __pthread_rwlock_arch_t binary compatibility. */ unsigned char __flags; unsigned char __shared; - signed char __rwelision; + unsigned char __pad1; unsigned char __pad2; int __cur_writer; #endif @@ -56,7 +49,7 @@ struct __pthread_rwlock_arch_t #ifdef __x86_64__ # define __PTHREAD_RWLOCK_INITIALIZER(__flags) \ - 0, 0, 0, 0, 0, 0, 0, 0, __PTHREAD_RWLOCK_ELISION_EXTRA, 0, __flags + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, __flags #else # define __PTHREAD_RWLOCK_INITIALIZER(__flags) \ 0, 0, 0, 0, 0, 0, __flags, 0, 0, 0, 0