diff --git a/malloc/malloc.c b/malloc/malloc.c index bcb6e5b83c..27dfd1eb90 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -1,5 +1,5 @@ /* Malloc implementation for multiple threads without lock contention. - Copyright (C) 1996-2024 Free Software Foundation, Inc. + Copyright (C) 1996-2025 Free Software Foundation, Inc. Copyright The GNU Toolchain Authors. This file is part of the GNU C Library. @@ -369,7 +369,7 @@ #include "morecore.c" #define MORECORE (*__glibc_morecore) -#define MORECORE_FAILURE 0 +#define MORECORE_FAILURE NULL /* Memory tagging. */ @@ -1086,7 +1086,9 @@ typedef struct malloc_chunk* mchunkptr; /* Internal routines. */ static void* _int_malloc(mstate, size_t); -static void _int_free(mstate, mchunkptr, int); +static void _int_free (mstate, mchunkptr, int); +static void _int_free_check (mstate, mchunkptr, INTERNAL_SIZE_T); +static void _int_free_chunk (mstate, mchunkptr, INTERNAL_SIZE_T, int); static void _int_free_merge_chunk (mstate, mchunkptr, INTERNAL_SIZE_T); static INTERNAL_SIZE_T _int_free_create_chunk (mstate, mchunkptr, INTERNAL_SIZE_T, @@ -2418,7 +2420,7 @@ sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags, mstate av) if ((unsigned long) (size) <= (unsigned long) (nb)) return MAP_FAILED; - char *mm = (char *) MMAP (0, size, + char *mm = (char *) MMAP (NULL, size, mtag_mmap_flags | PROT_READ | PROT_WRITE, extra_flags); if (mm == MAP_FAILED) @@ -2505,7 +2507,7 @@ sysmalloc_mmap_fallback (long int *s, INTERNAL_SIZE_T nb, if ((unsigned long) (size) <= (unsigned long) (nb)) return MORECORE_FAILURE; - char *mbrk = (char *) (MMAP (0, size, + char *mbrk = (char *) (MMAP (NULL, size, mtag_mmap_flags | PROT_READ | PROT_WRITE, extra_flags)); if (mbrk == MAP_FAILED) @@ -2581,7 +2583,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) /* There are no usable arenas and mmap also failed. */ if (av == NULL) - return 0; + return NULL; /* Record incoming configuration of top */ @@ -2741,7 +2743,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) if (brk != (char *) (MORECORE_FAILURE)) { - if (mp_.sbrk_base == 0) + if (mp_.sbrk_base == NULL) mp_.sbrk_base = brk; av->system_mem += size; @@ -2940,7 +2942,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) /* catch all failure paths */ __set_errno (ENOMEM); - return 0; + return NULL; } @@ -3078,7 +3080,7 @@ mremap_chunk (mchunkptr p, size_t new_size) MREMAP_MAYMOVE); if (cp == MAP_FAILED) - return 0; + return NULL; madvise_thp (cp, new_size); @@ -3140,8 +3142,8 @@ static void tcache_key_initialize (void) { /* We need to use the _nostatus version here, see BZ 29624. */ - if (__getrandom_nocancel_nostatus (&tcache_key, sizeof(tcache_key), - GRND_NONBLOCK) + if (__getrandom_nocancel_nostatus_direct (&tcache_key, sizeof(tcache_key), + GRND_NONBLOCK) != sizeof (tcache_key)) { tcache_key = random_bits (); @@ -3206,6 +3208,69 @@ tcache_next (tcache_entry *e) return (tcache_entry *) REVEAL_PTR (e->next); } +/* Check if tcache is available for alloc by corresponding tc_idx. */ +static __always_inline bool +tcache_available (size_t tc_idx) +{ + if (tc_idx < mp_.tcache_bins + && tcache != NULL + && tcache->counts[tc_idx] > 0) + return true; + else + return false; +} + +/* Verify if the suspicious tcache_entry is double free. + It's not expected to execute very often, mark it as noinline. */ +static __attribute__ ((noinline)) void +tcache_double_free_verify (tcache_entry *e, size_t tc_idx) +{ + tcache_entry *tmp; + size_t cnt = 0; + LIBC_PROBE (memory_tcache_double_free, 2, e, tc_idx); + for (tmp = tcache->entries[tc_idx]; + tmp; + tmp = REVEAL_PTR (tmp->next), ++cnt) + { + if (cnt >= mp_.tcache_count) + malloc_printerr ("free(): too many chunks detected in tcache"); + if (__glibc_unlikely (!aligned_OK (tmp))) + malloc_printerr ("free(): unaligned chunk detected in tcache 2"); + if (tmp == e) + malloc_printerr ("free(): double free detected in tcache 2"); + /* If we get here, it was a coincidence. We've wasted a + few cycles, but don't abort. */ + } +} + +/* Try to free chunk to the tcache, if success return true. + Caller must ensure that chunk and size are valid. */ +static inline bool +tcache_free (mchunkptr p, INTERNAL_SIZE_T size) +{ + bool done = false; + size_t tc_idx = csize2tidx (size); + if (tcache != NULL && tc_idx < mp_.tcache_bins) + { + /* Check to see if it's already in the tcache. */ + tcache_entry *e = (tcache_entry *) chunk2mem (p); + + /* This test succeeds on double free. However, we don't 100% + trust it (it also matches random payload data at a 1 in + 2^ chance), so verify it's not an unlikely + coincidence before aborting. */ + if (__glibc_unlikely (e->key == tcache_key)) + tcache_double_free_verify (e, tc_idx); + + if (tcache->counts[tc_idx] < mp_.tcache_count) + { + tcache_put (p, tc_idx); + done = true; + } + } + return done; +} + static void tcache_thread_shutdown (void) { @@ -3242,7 +3307,7 @@ static void tcache_init(void) { mstate ar_ptr; - void *victim = 0; + void *victim = NULL; const size_t bytes = sizeof (tcache_perthread_struct); if (tcache_shutting_down) @@ -3277,6 +3342,32 @@ tcache_init(void) if (__glibc_unlikely (tcache == NULL)) \ tcache_init(); +/* Trying to alloc BYTES from tcache. If tcache is available, chunk + is allocated and stored to MEMPTR, otherwise, MEMPTR is NULL. + It returns true if error occurs, else false. */ +static __always_inline bool +tcache_try_malloc (size_t bytes, void **memptr) +{ + /* int_free also calls request2size, be careful to not pad twice. */ + size_t tbytes = checked_request2size (bytes); + if (tbytes == 0) + { + __set_errno (ENOMEM); + return true; + } + + size_t tc_idx = csize2tidx (tbytes); + + MAYBE_INIT_TCACHE (); + + if (tcache_available (tc_idx)) + *memptr = tcache_get (tc_idx); + else + *memptr = NULL; + + return false; +} + #else /* !USE_TCACHE */ # define MAYBE_INIT_TCACHE() @@ -3301,26 +3392,13 @@ __libc_malloc (size_t bytes) if (!__malloc_initialized) ptmalloc_init (); #if USE_TCACHE - /* int_free also calls request2size, be careful to not pad twice. */ - size_t tbytes = checked_request2size (bytes); - if (tbytes == 0) - { - __set_errno (ENOMEM); - return NULL; - } - size_t tc_idx = csize2tidx (tbytes); + bool err = tcache_try_malloc (bytes, &victim); - MAYBE_INIT_TCACHE (); + if (err) + return NULL; - DIAG_PUSH_NEEDS_COMMENT; - if (tc_idx < mp_.tcache_bins - && tcache != NULL - && tcache->counts[tc_idx] > 0) - { - victim = tcache_get (tc_idx); + if (victim) return tag_new_usable (victim); - } - DIAG_POP_NEEDS_COMMENT; #endif if (SINGLE_THREAD_P) @@ -3360,7 +3438,7 @@ __libc_free (void *mem) mstate ar_ptr; mchunkptr p; /* chunk corresponding to mem */ - if (mem == 0) /* free(0) has no effect */ + if (mem == NULL) /* free(0) has no effect */ return; /* Quickly check that the freed pointer matches the tag for the memory. @@ -3416,12 +3494,12 @@ __libc_realloc (void *oldmem, size_t bytes) #if REALLOC_ZERO_BYTES_FREES if (bytes == 0 && oldmem != NULL) { - __libc_free (oldmem); return 0; + __libc_free (oldmem); return NULL; } #endif /* realloc of null is supposed to be same as malloc */ - if (oldmem == 0) + if (oldmem == NULL) return __libc_malloc (bytes); /* Perform a quick check to ensure that the pointer's tag matches the @@ -3495,8 +3573,8 @@ __libc_realloc (void *oldmem, size_t bytes) /* Must alloc, copy, free. */ newmem = __libc_malloc (bytes); - if (newmem == 0) - return 0; /* propagate failure */ + if (newmem == NULL) + return NULL; /* propagate failure */ memcpy (newmem, oldmem, oldsize - CHUNK_HDR_SZ); munmap_chunk (oldp); @@ -3564,7 +3642,7 @@ aligned_alloc (size_t alignment, size_t bytes) if (!powerof2 (alignment) || alignment == 0) { __set_errno (EINVAL); - return 0; + return NULL; } void *address = RETURN_ADDRESS (0); @@ -3590,7 +3668,7 @@ _mid_memalign (size_t alignment, size_t bytes, void *address) if (alignment > SIZE_MAX / 2 + 1) { __set_errno (EINVAL); - return 0; + return NULL; } @@ -3614,9 +3692,7 @@ _mid_memalign (size_t alignment, size_t bytes, void *address) } size_t tc_idx = csize2tidx (tbytes); - if (tc_idx < mp_.tcache_bins - && tcache != NULL - && tcache->counts[tc_idx] > 0) + if (tcache_available (tc_idx)) { /* The tcache itself isn't encoded, but the chain is. */ tcache_entry **tep = & tcache->entries[tc_idx]; @@ -3687,7 +3763,7 @@ __libc_pvalloc (size_t bytes) &rounded_bytes))) { __set_errno (ENOMEM); - return 0; + return NULL; } rounded_bytes = rounded_bytes & -(pagesize - 1); @@ -3698,12 +3774,10 @@ void * __libc_calloc (size_t n, size_t elem_size) { mstate av; - mchunkptr oldtop; - INTERNAL_SIZE_T sz, oldtopsize; + mchunkptr oldtop, p; + INTERNAL_SIZE_T sz, oldtopsize, csz; void *mem; unsigned long clearsize; - unsigned long nclears; - INTERNAL_SIZE_T *d; ptrdiff_t bytes; if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes))) @@ -3717,7 +3791,23 @@ __libc_calloc (size_t n, size_t elem_size) if (!__malloc_initialized) ptmalloc_init (); - MAYBE_INIT_TCACHE (); +#if USE_TCACHE + bool err = tcache_try_malloc (bytes, &mem); + + if (err) + return NULL; + + if (mem) + { + p = mem2chunk (mem); + if (__glibc_unlikely (mtag_enabled)) + return tag_new_zero_region (mem, memsize (p)); + + csz = chunksize (p); + clearsize = csz - SIZE_SZ; + return clear_memory ((INTERNAL_SIZE_T *) mem, clearsize); + } +#endif if (SINGLE_THREAD_P) av = &main_arena; @@ -3748,7 +3838,7 @@ __libc_calloc (size_t n, size_t elem_size) else { /* No usable arenas. */ - oldtop = 0; + oldtop = NULL; oldtopsize = 0; } mem = _int_malloc (av, sz); @@ -3758,7 +3848,7 @@ __libc_calloc (size_t n, size_t elem_size) if (!SINGLE_THREAD_P) { - if (mem == 0 && av != NULL) + if (mem == NULL && av != NULL) { LIBC_PROBE (memory_calloc_retry, 1, sz); av = arena_get_retry (av, sz); @@ -3770,10 +3860,10 @@ __libc_calloc (size_t n, size_t elem_size) } /* Allocation failed even after a retry. */ - if (mem == 0) - return 0; + if (mem == NULL) + return NULL; - mchunkptr p = mem2chunk (mem); + p = mem2chunk (mem); /* If we are using memory tagging, then we need to set the tags regardless of MORECORE_CLEARS, so we zero the whole block while @@ -3781,7 +3871,7 @@ __libc_calloc (size_t n, size_t elem_size) if (__glibc_unlikely (mtag_enabled)) return tag_new_zero_region (mem, memsize (p)); - INTERNAL_SIZE_T csz = chunksize (p); + csz = chunksize (p); /* Two optional cases in which clearing not necessary */ if (chunk_is_mmapped (p)) @@ -3800,40 +3890,8 @@ __libc_calloc (size_t n, size_t elem_size) } #endif - /* Unroll clear of <= 36 bytes (72 if 8byte sizes). We know that - contents have an odd number of INTERNAL_SIZE_T-sized words; - minimally 3. */ - d = (INTERNAL_SIZE_T *) mem; clearsize = csz - SIZE_SZ; - nclears = clearsize / sizeof (INTERNAL_SIZE_T); - assert (nclears >= 3); - - if (nclears > 9) - return memset (d, 0, clearsize); - - else - { - *(d + 0) = 0; - *(d + 1) = 0; - *(d + 2) = 0; - if (nclears > 4) - { - *(d + 3) = 0; - *(d + 4) = 0; - if (nclears > 6) - { - *(d + 5) = 0; - *(d + 6) = 0; - if (nclears > 8) - { - *(d + 7) = 0; - *(d + 8) = 0; - } - } - } - } - - return mem; + return clear_memory ((INTERNAL_SIZE_T *) mem, clearsize); } #endif /* IS_IN (libc) */ @@ -4003,7 +4061,7 @@ _int_malloc (mstate av, size_t bytes) while (tcache->counts[tc_idx] < mp_.tcache_count && (tc_victim = last (bin)) != bin) { - if (tc_victim != 0) + if (tc_victim != NULL) { bck = tc_victim->bk; set_inuse_bit_at_offset (tc_victim, nb); @@ -4156,9 +4214,9 @@ _int_malloc (mstate av, size_t bytes) #endif } - /* place chunk in bin */ - - if (in_smallbin_range (size)) + /* Place chunk in bin. Only malloc_consolidate() and splitting can put + small chunks into the unsorted bin. */ + if (__glibc_unlikely (in_smallbin_range (size))) { victim_index = smallbin_index (size); bck = bin_at (av, victim_index); @@ -4490,14 +4548,9 @@ _int_malloc (mstate av, size_t bytes) ------------------------------ free ------------------------------ */ -static void -_int_free (mstate av, mchunkptr p, int have_lock) +static inline void +_int_free_check (mstate av, mchunkptr p, INTERNAL_SIZE_T size) { - INTERNAL_SIZE_T size; /* its size */ - mfastbinptr *fb; /* associated fastbin */ - - size = chunksize (p); - /* Little security check which won't hurt performance: the allocator never wraps around at the end of the address space. Therefore we can exclude some size values which might appear @@ -4510,48 +4563,16 @@ _int_free (mstate av, mchunkptr p, int have_lock) if (__glibc_unlikely (size < MINSIZE || !aligned_OK (size))) malloc_printerr ("free(): invalid size"); - check_inuse_chunk(av, p); - -#if USE_TCACHE - { - size_t tc_idx = csize2tidx (size); - if (tcache != NULL && tc_idx < mp_.tcache_bins) - { - /* Check to see if it's already in the tcache. */ - tcache_entry *e = (tcache_entry *) chunk2mem (p); - - /* This test succeeds on double free. However, we don't 100% - trust it (it also matches random payload data at a 1 in - 2^ chance), so verify it's not an unlikely - coincidence before aborting. */ - if (__glibc_unlikely (e->key == tcache_key)) - { - tcache_entry *tmp; - size_t cnt = 0; - LIBC_PROBE (memory_tcache_double_free, 2, e, tc_idx); - for (tmp = tcache->entries[tc_idx]; - tmp; - tmp = REVEAL_PTR (tmp->next), ++cnt) - { - if (cnt >= mp_.tcache_count) - malloc_printerr ("free(): too many chunks detected in tcache"); - if (__glibc_unlikely (!aligned_OK (tmp))) - malloc_printerr ("free(): unaligned chunk detected in tcache 2"); - if (tmp == e) - malloc_printerr ("free(): double free detected in tcache 2"); - /* If we get here, it was a coincidence. We've wasted a - few cycles, but don't abort. */ - } - } + check_inuse_chunk (av, p); +} - if (tcache->counts[tc_idx] < mp_.tcache_count) - { - tcache_put (p, tc_idx); - return; - } - } - } -#endif +/* Free chunk P of SIZE bytes to the arena. HAVE_LOCK indicates where + the arena for P has already been locked. Caller must ensure chunk + and size are valid. */ +static void +_int_free_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size, int have_lock) +{ + mfastbinptr *fb; /* associated fastbin */ /* If eligible, place chunk on a fastbin so it can be found @@ -4657,6 +4678,27 @@ _int_free (mstate av, mchunkptr p, int have_lock) } } +/* Free chunk P to its arena AV. HAVE_LOCK indicates where the arena for + P has already been locked. It will perform sanity check, then try the + fast path to free into tcache. If the attempt not success, free the + chunk to arena. */ +static inline void +_int_free (mstate av, mchunkptr p, int have_lock) +{ + INTERNAL_SIZE_T size; /* its size */ + + size = chunksize (p); + + _int_free_check (av, p, size); + +#if USE_TCACHE + if (tcache_free (p, size)) + return; +#endif + + _int_free_chunk (av, p, size, have_lock); +} + /* Try to merge chunk P of SIZE bytes with its neighbors. Put the resulting chunk on the appropriate bin list. P must not be on a bin list yet, and it can be in use. */ @@ -4723,23 +4765,39 @@ _int_free_create_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size, } else clear_inuse_bit_at_offset(nextchunk, 0); - /* - Place the chunk in unsorted chunk list. Chunks are - not placed into regular bins until after they have - been given one chance to be used in malloc. - */ + mchunkptr bck, fwd; + + if (!in_smallbin_range (size)) + { + /* Place large chunks in unsorted chunk list. Large chunks are + not placed into regular bins until after they have + been given one chance to be used in malloc. + + This branch is first in the if-statement to help branch + prediction on consecutive adjacent frees. */ + bck = unsorted_chunks (av); + fwd = bck->fd; + if (__glibc_unlikely (fwd->bk != bck)) + malloc_printerr ("free(): corrupted unsorted chunks"); + p->fd_nextsize = NULL; + p->bk_nextsize = NULL; + } + else + { + /* Place small chunks directly in their smallbin, so they + don't pollute the unsorted bin. */ + int chunk_index = smallbin_index (size); + bck = bin_at (av, chunk_index); + fwd = bck->fd; + + if (__glibc_unlikely (fwd->bk != bck)) + malloc_printerr ("free(): chunks in smallbin corrupted"); + + mark_bin (av, chunk_index); + } - mchunkptr bck = unsorted_chunks (av); - mchunkptr fwd = bck->fd; - if (__glibc_unlikely (fwd->bk != bck)) - malloc_printerr ("free(): corrupted unsorted chunks"); - p->fd = fwd; p->bk = bck; - if (!in_smallbin_range(size)) - { - p->fd_nextsize = NULL; - p->bk_nextsize = NULL; - } + p->fd = fwd; bck->fd = p; fwd->bk = p; @@ -4748,7 +4806,6 @@ _int_free_create_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size, check_free_chunk(av, p); } - else { /* If the chunk borders the current high end of memory, @@ -4839,7 +4896,7 @@ static void malloc_consolidate(mstate av) fb = &fastbin (av, 0); do { p = atomic_exchange_acquire (fb, NULL); - if (p != 0) { + if (p != NULL) { do { { if (__glibc_unlikely (misaligned_chunk (p))) @@ -4898,7 +4955,7 @@ static void malloc_consolidate(mstate av) av->top = p; } - } while ( (p = nextp) != 0); + } while ( (p = nextp) != NULL); } } while (fb++ != maxfb); @@ -4973,8 +5030,8 @@ _int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, else { newmem = _int_malloc (av, nb - MALLOC_ALIGN_MASK); - if (newmem == 0) - return 0; /* propagate failure */ + if (newmem == NULL) + return NULL; /* propagate failure */ newp = mem2chunk (newmem); newsize = chunksize (newp); @@ -5068,8 +5125,8 @@ _int_memalign (mstate av, size_t alignment, size_t bytes) /* Call malloc with worst case padding to hit alignment. */ m = (char *) (_int_malloc (av, nb + alignment + MINSIZE)); - if (m == 0) - return 0; /* propagate failure */ + if (m == NULL) + return NULL; /* propagate failure */ p = mem2chunk (m); @@ -5281,7 +5338,7 @@ int_mallinfo (mstate av, struct mallinfo2 *m) for (i = 0; i < NFASTBINS; ++i) { for (p = fastbin (av, i); - p != 0; + p != NULL; p = REVEAL_PTR (p->fd)) { if (__glibc_unlikely (misaligned_chunk (p)))