diff --git a/malloc/malloc.c b/malloc/malloc.c index ee87ddbbf9..1f4bbd8edf 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -1,5 +1,5 @@ /* Malloc implementation for multiple threads without lock contention. - Copyright (C) 1996-2020 Free Software Foundation, Inc. + Copyright (C) 1996-2021 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Wolfram Gloger and Doug Lea , 2001. @@ -242,11 +242,16 @@ /* For DIAG_PUSH/POP_NEEDS_COMMENT et al. */ #include +/* For memory tagging. */ +#include + #include /* For SINGLE_THREAD_P. */ #include +#include + /* Debugging: @@ -378,6 +383,96 @@ __malloc_assert (const char *assertion, const char *file, unsigned int line, void * __default_morecore (ptrdiff_t); void *(*__morecore)(ptrdiff_t) = __default_morecore; +/* Memory tagging. */ + +/* Some systems support the concept of tagging (sometimes known as + coloring) memory locations on a fine grained basis. Each memory + location is given a color (normally allocated randomly) and + pointers are also colored. When the pointer is dereferenced, the + pointer's color is checked against the memory's color and if they + differ the access is faulted (sometimes lazily). + + We use this in glibc by maintaining a single color for the malloc + data structures that are interleaved with the user data and then + assigning separate colors for each block allocation handed out. In + this way simple buffer overruns will be rapidly detected. When + memory is freed, the memory is recolored back to the glibc default + so that simple use-after-free errors can also be detected. + + If memory is reallocated the buffer is recolored even if the + address remains the same. This has a performance impact, but + guarantees that the old pointer cannot mistakenly be reused (code + that compares old against new will see a mismatch and will then + need to behave as though realloc moved the data to a new location). + + Internal API for memory tagging support. + + The aim is to keep the code for memory tagging support as close to + the normal APIs in glibc as possible, so that if tagging is not + enabled in the library, or is disabled at runtime then standard + operations can continue to be used. Support macros are used to do + this: + + void *TAG_NEW_MEMSET (void *ptr, int, val, size_t size) + + Has the same interface as memset(), but additionally allocates a + new tag, colors the memory with that tag and returns a pointer that + is correctly colored for that location. The non-tagging version + will simply call memset. + + void *TAG_REGION (void *ptr, size_t size) + + Color the region of memory pointed to by PTR and size SIZE with + the color of PTR. Returns the original pointer. + + void *TAG_NEW_USABLE (void *ptr) + + Allocate a new random color and use it to color the user region of + a chunk; this may include data from the subsequent chunk's header + if tagging is sufficiently fine grained. Returns PTR suitably + recolored for accessing the memory there. + + void *TAG_AT (void *ptr) + + Read the current color of the memory at the address pointed to by + PTR (ignoring it's current color) and return PTR recolored to that + color. PTR must be valid address in all other respects. When + tagging is not enabled, it simply returns the original pointer. +*/ + +#ifdef USE_MTAG + +/* Default implementaions when memory tagging is supported, but disabled. */ +static void * +__default_tag_region (void *ptr, size_t size) +{ + return ptr; +} + +static void * +__default_tag_nop (void *ptr) +{ + return ptr; +} + +static int __mtag_mmap_flags = 0; +static size_t __mtag_granule_mask = ~(size_t)0; + +static void *(*__tag_new_memset)(void *, int, size_t) = memset; +static void *(*__tag_region)(void *, size_t) = __default_tag_region; +static void *(*__tag_new_usable)(void *) = __default_tag_nop; +static void *(*__tag_at)(void *) = __default_tag_nop; + +# define TAG_NEW_MEMSET(ptr, val, size) __tag_new_memset (ptr, val, size) +# define TAG_REGION(ptr, size) __tag_region (ptr, size) +# define TAG_NEW_USABLE(ptr) __tag_new_usable (ptr) +# define TAG_AT(ptr) __tag_at (ptr) +#else +# define TAG_NEW_MEMSET(ptr, val, size) memset (ptr, val, size) +# define TAG_REGION(ptr, size) (ptr) +# define TAG_NEW_USABLE(ptr) (ptr) +# define TAG_AT(ptr) (ptr) +#endif #include @@ -505,7 +600,7 @@ void *(*__morecore)(ptrdiff_t) = __default_morecore; if no space is available. Additionally, on failure, errno is set to ENOMEM on ANSI C systems. - If n is zero, malloc returns a minumum-sized chunk. (The minimum + If n is zero, malloc returns a minimum-sized chunk. (The minimum size is 16 bytes on most 32bit systems, and 24 or 32 bytes on 64bit systems.) On most systems, size_t is an unsigned type, so calls with negative arguments are interpreted as requests for huge amounts @@ -638,6 +733,9 @@ libc_hidden_proto (__libc_mallopt) be kept as longs, the reported values may wrap around zero and thus be inaccurate. */ +struct mallinfo2 __libc_mallinfo2(void); +libc_hidden_proto (__libc_mallinfo2) + struct mallinfo __libc_mallinfo(void); @@ -1182,10 +1280,31 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ---------- Size and alignment checks and conversions ---------- */ -/* conversion from malloc headers to user pointers, and back */ +/* Conversion from malloc headers to user pointers, and back. When + using memory tagging the user data and the malloc data structure + headers have distinct tags. Converting fully from one to the other + involves extracting the tag at the other address and creating a + suitable pointer using it. That can be quite expensive. There are + many occasions, though when the pointer will not be dereferenced + (for example, because we only want to assert that the pointer is + correctly aligned). In these cases it is more efficient not + to extract the tag, since the answer will be the same either way. + chunk2rawmem() can be used in these cases. + */ + +/* The chunk header is two SIZE_SZ elements, but this is used widely, so + we define it here for clarity later. */ +#define CHUNK_HDR_SZ (2 * SIZE_SZ) -#define chunk2mem(p) ((void*)((char*)(p) + 2*SIZE_SZ)) -#define mem2chunk(mem) ((mchunkptr)((char*)(mem) - 2*SIZE_SZ)) +/* Convert a user mem pointer to a chunk address without correcting + the tag. */ +#define chunk2rawmem(p) ((void*)((char*)(p) + CHUNK_HDR_SZ)) + +/* Convert between user mem pointers and chunk pointers, updating any + memory tags on the pointer to respect the tag value at that + location. */ +#define chunk2mem(p) ((void*)TAG_AT (((char*)(p) + CHUNK_HDR_SZ))) +#define mem2chunk(mem) ((mchunkptr)TAG_AT (((char*)(mem) - CHUNK_HDR_SZ))) /* The smallest possible chunk */ #define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize)) @@ -1200,16 +1319,28 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ #define aligned_OK(m) (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0) #define misaligned_chunk(p) \ - ((uintptr_t)(MALLOC_ALIGNMENT == 2 * SIZE_SZ ? (p) : chunk2mem (p)) \ + ((uintptr_t)(MALLOC_ALIGNMENT == CHUNK_HDR_SZ ? (p) : chunk2mem (p)) \ & MALLOC_ALIGN_MASK) /* pad request bytes into a usable size -- internal version */ - +/* Note: This must be a macro that evaluates to a compile time constant + if passed a literal constant. */ #define request2size(req) \ (((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \ MINSIZE : \ ((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK) +/* Available size of chunk. This is the size of the real usable data + in the chunk, plus the chunk header. */ +#ifdef USE_MTAG +#define CHUNK_AVAILABLE_SIZE(p) \ + ((chunksize (p) + (chunk_is_mmapped (p) ? 0 : SIZE_SZ)) \ + & __mtag_granule_mask) +#else +#define CHUNK_AVAILABLE_SIZE(p) \ + (chunksize (p) + (chunk_is_mmapped (p) ? 0 : SIZE_SZ)) +#endif + /* Check if REQ overflows when padded and aligned and if the resulting value is less than PTRDIFF_T. Returns TRUE and the requested size or MINSIZE in case the value is less than MINSIZE on SZ or false if any of the previous @@ -1219,6 +1350,18 @@ checked_request2size (size_t req, size_t *sz) __nonnull (1) { if (__glibc_unlikely (req > PTRDIFF_MAX)) return false; + +#ifdef USE_MTAG + /* When using tagged memory, we cannot share the end of the user + block with the header for the next chunk, so ensure that we + allocate blocks that are rounded up to the granule size. Take + care not to overflow from close to MAX_SIZE_T to a small + number. Ideally, this would be part of request2size(), but that + must be a macro that produces a compile time constant if passed + a constant literal. */ + req = (req + ~__mtag_granule_mask) & __mtag_granule_mask; +#endif + *sz = request2size (req); return true; } @@ -1317,7 +1460,6 @@ checked_request2size (size_t req, size_t *sz) __nonnull (1) /* Set size at footer (only when chunk is not in use) */ #define set_foot(p, s) (((mchunkptr) ((char *) (p) + (s)))->mchunk_prev_size = (s)) - #pragma GCC poison mchunk_size #pragma GCC poison mchunk_prev_size @@ -1413,7 +1555,7 @@ typedef struct malloc_chunk *mbinptr; #define NBINS 128 #define NSMALLBINS 64 #define SMALLBIN_WIDTH MALLOC_ALIGNMENT -#define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > 2 * SIZE_SZ) +#define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > CHUNK_HDR_SZ) #define MIN_LARGE_SIZE ((NSMALLBINS - SMALLBIN_CORRECTION) * SMALLBIN_WIDTH) #define in_smallbin_range(sz) \ @@ -1964,7 +2106,7 @@ do_check_chunk (mstate av, mchunkptr p) /* chunk is page-aligned */ assert (((prev_size (p) + sz) & (GLRO (dl_pagesize) - 1)) == 0); /* mem is aligned */ - assert (aligned_OK (chunk2mem (p))); + assert (aligned_OK (chunk2rawmem (p))); } } @@ -1988,7 +2130,7 @@ do_check_free_chunk (mstate av, mchunkptr p) if ((unsigned long) (sz) >= MINSIZE) { assert ((sz & MALLOC_ALIGN_MASK) == 0); - assert (aligned_OK (chunk2mem (p))); + assert (aligned_OK (chunk2rawmem (p))); /* ... matching footer field */ assert (prev_size (next_chunk (p)) == sz); /* ... and is fully consolidated */ @@ -2067,7 +2209,7 @@ do_check_remalloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s) assert ((sz & MALLOC_ALIGN_MASK) == 0); assert ((unsigned long) (sz) >= MINSIZE); /* ... and alignment */ - assert (aligned_OK (chunk2mem (p))); + assert (aligned_OK (chunk2rawmem (p))); /* chunk is less than MINSIZE more than request */ assert ((long) (sz) - (long) (s) >= 0); assert ((long) (sz) - (long) (s + MINSIZE) < 0); @@ -2313,7 +2455,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) See the front_misalign handling below, for glibc there is no need for further alignments unless we have have high alignment. */ - if (MALLOC_ALIGNMENT == 2 * SIZE_SZ) + if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ) size = ALIGN_UP (nb + SIZE_SZ, pagesize); else size = ALIGN_UP (nb + SIZE_SZ + MALLOC_ALIGN_MASK, pagesize); @@ -2322,7 +2464,8 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) /* Don't try if size wraps around 0 */ if ((unsigned long) (size) > (unsigned long) (nb)) { - mm = (char *) (MMAP (0, size, PROT_READ | PROT_WRITE, 0)); + mm = (char *) (MMAP (0, size, + MTAG_MMAP_FLAGS | PROT_READ | PROT_WRITE, 0)); if (mm != MAP_FAILED) { @@ -2334,16 +2477,18 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) address argument for later munmap in free() and realloc(). */ - if (MALLOC_ALIGNMENT == 2 * SIZE_SZ) + if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ) { - /* For glibc, chunk2mem increases the address by 2*SIZE_SZ and - MALLOC_ALIGN_MASK is 2*SIZE_SZ-1. Each mmap'ed area is page - aligned and therefore definitely MALLOC_ALIGN_MASK-aligned. */ - assert (((INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK) == 0); + /* For glibc, chunk2rawmem increases the address by + CHUNK_HDR_SZ and MALLOC_ALIGN_MASK is + CHUNK_HDR_SZ-1. Each mmap'ed area is page + aligned and therefore definitely + MALLOC_ALIGN_MASK-aligned. */ + assert (((INTERNAL_SIZE_T) chunk2rawmem (mm) & MALLOC_ALIGN_MASK) == 0); front_misalign = 0; } else - front_misalign = (INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK; + front_misalign = (INTERNAL_SIZE_T) chunk2rawmem (mm) & MALLOC_ALIGN_MASK; if (front_misalign > 0) { correction = MALLOC_ALIGNMENT - front_misalign; @@ -2431,18 +2576,20 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) become the top chunk again later. Note that a footer is set up, too, although the chunk is marked in use. */ old_size = (old_size - MINSIZE) & ~MALLOC_ALIGN_MASK; - set_head (chunk_at_offset (old_top, old_size + 2 * SIZE_SZ), 0 | PREV_INUSE); + set_head (chunk_at_offset (old_top, old_size + CHUNK_HDR_SZ), + 0 | PREV_INUSE); if (old_size >= MINSIZE) { - set_head (chunk_at_offset (old_top, old_size), (2 * SIZE_SZ) | PREV_INUSE); - set_foot (chunk_at_offset (old_top, old_size), (2 * SIZE_SZ)); + set_head (chunk_at_offset (old_top, old_size), + CHUNK_HDR_SZ | PREV_INUSE); + set_foot (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ); set_head (old_top, old_size | PREV_INUSE | NON_MAIN_ARENA); _int_free (av, old_top, 1); } else { - set_head (old_top, (old_size + 2 * SIZE_SZ) | PREV_INUSE); - set_foot (old_top, (old_size + 2 * SIZE_SZ)); + set_head (old_top, (old_size + CHUNK_HDR_SZ) | PREV_INUSE); + set_foot (old_top, (old_size + CHUNK_HDR_SZ)); } } else if (!tried_mmap) @@ -2515,7 +2662,9 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) /* Don't try if size wraps around 0 */ if ((unsigned long) (size) > (unsigned long) (nb)) { - char *mbrk = (char *) (MMAP (0, size, PROT_READ | PROT_WRITE, 0)); + char *mbrk = (char *) (MMAP (0, size, + MTAG_MMAP_FLAGS | PROT_READ | PROT_WRITE, + 0)); if (mbrk != MAP_FAILED) { @@ -2586,7 +2735,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) /* Guarantee alignment of first new chunk made from this space */ - front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK; + front_misalign = (INTERNAL_SIZE_T) chunk2rawmem (brk) & MALLOC_ALIGN_MASK; if (front_misalign > 0) { /* @@ -2642,12 +2791,12 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) /* handle non-contiguous cases */ else { - if (MALLOC_ALIGNMENT == 2 * SIZE_SZ) + if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ) /* MORECORE/mmap must correctly align */ - assert (((unsigned long) chunk2mem (brk) & MALLOC_ALIGN_MASK) == 0); + assert (((unsigned long) chunk2rawmem (brk) & MALLOC_ALIGN_MASK) == 0); else { - front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK; + front_misalign = (INTERNAL_SIZE_T) chunk2rawmem (brk) & MALLOC_ALIGN_MASK; if (front_misalign > 0) { /* @@ -2692,7 +2841,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) multiple of MALLOC_ALIGNMENT. We know there is at least enough space in old_top to do this. */ - old_size = (old_size - 4 * SIZE_SZ) & ~MALLOC_ALIGN_MASK; + old_size = (old_size - 2 * CHUNK_HDR_SZ) & ~MALLOC_ALIGN_MASK; set_head (old_top, old_size | PREV_INUSE); /* @@ -2702,9 +2851,10 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) lost. */ set_head (chunk_at_offset (old_top, old_size), - (2 * SIZE_SZ) | PREV_INUSE); - set_head (chunk_at_offset (old_top, old_size + 2 * SIZE_SZ), - (2 * SIZE_SZ) | PREV_INUSE); + CHUNK_HDR_SZ | PREV_INUSE); + set_head (chunk_at_offset (old_top, + old_size + CHUNK_HDR_SZ), + CHUNK_HDR_SZ | PREV_INUSE); /* If possible, release the rest. */ if (old_size >= MINSIZE) @@ -2832,7 +2982,7 @@ munmap_chunk (mchunkptr p) if (DUMPED_MAIN_ARENA_CHUNK (p)) return; - uintptr_t mem = (uintptr_t) chunk2mem (p); + uintptr_t mem = (uintptr_t) chunk2rawmem (p); uintptr_t block = (uintptr_t) p - prev_size (p); size_t total_size = prev_size (p) + size; /* Unfortunately we have to do the compilers job by hand here. Normally @@ -2887,7 +3037,7 @@ mremap_chunk (mchunkptr p, size_t new_size) p = (mchunkptr) (cp + offset); - assert (aligned_OK (chunk2mem (p))); + assert (aligned_OK (chunk2rawmem (p))); assert (prev_size (p) == offset); set_head (p, (new_size - offset) | IS_MMAPPED); @@ -3068,14 +3218,15 @@ __libc_malloc (size_t bytes) && tcache && tcache->counts[tc_idx] > 0) { - return tcache_get (tc_idx); + victim = tcache_get (tc_idx); + return TAG_NEW_USABLE (victim); } DIAG_POP_NEEDS_COMMENT; #endif if (SINGLE_THREAD_P) { - victim = _int_malloc (&main_arena, bytes); + victim = TAG_NEW_USABLE (_int_malloc (&main_arena, bytes)); assert (!victim || chunk_is_mmapped (mem2chunk (victim)) || &main_arena == arena_for_chunk (mem2chunk (victim))); return victim; @@ -3096,6 +3247,8 @@ __libc_malloc (size_t bytes) if (ar_ptr != NULL) __libc_lock_unlock (ar_ptr->mutex); + victim = TAG_NEW_USABLE (victim); + assert (!victim || chunk_is_mmapped (mem2chunk (victim)) || ar_ptr == arena_for_chunk (mem2chunk (victim))); return victim; @@ -3119,8 +3272,19 @@ __libc_free (void *mem) if (mem == 0) /* free(0) has no effect */ return; +#ifdef USE_MTAG + /* Quickly check that the freed pointer matches the tag for the memory. + This gives a useful double-free detection. */ + *(volatile char *)mem; +#endif + + int err = errno; + p = mem2chunk (mem); + /* Mark the chunk as belonging to the library again. */ + (void)TAG_REGION (chunk2rawmem (p), CHUNK_AVAILABLE_SIZE (p) - CHUNK_HDR_SZ); + if (chunk_is_mmapped (p)) /* release mmapped memory. */ { /* See if the dynamic brk/mmap threshold needs adjusting. @@ -3136,13 +3300,16 @@ __libc_free (void *mem) mp_.mmap_threshold, mp_.trim_threshold); } munmap_chunk (p); - return; } + else + { + MAYBE_INIT_TCACHE (); - MAYBE_INIT_TCACHE (); + ar_ptr = arena_for_chunk (p); + _int_free (ar_ptr, p, 0); + } - ar_ptr = arena_for_chunk (p); - _int_free (ar_ptr, p, 0); + __set_errno (err); } libc_hidden_def (__libc_free) @@ -3170,6 +3337,12 @@ __libc_realloc (void *oldmem, size_t bytes) if (oldmem == 0) return __libc_malloc (bytes); +#ifdef USE_MTAG + /* Perform a quick check to ensure that the pointer's tag matches the + memory's tag. */ + *(volatile char*) oldmem; +#endif + /* chunk corresponding to oldmem */ const mchunkptr oldp = mem2chunk (oldmem); /* its size */ @@ -3212,7 +3385,7 @@ __libc_realloc (void *oldmem, size_t bytes) return NULL; /* Copy as many bytes as are available from the old chunk and fit into the new size. NB: The overhead for faked - mmapped chunks is only SIZE_SZ, not 2 * SIZE_SZ as for + mmapped chunks is only SIZE_SZ, not CHUNK_HDR_SZ as for regular mmapped chunks. */ if (bytes > oldsize - SIZE_SZ) bytes = oldsize - SIZE_SZ; @@ -3225,7 +3398,15 @@ __libc_realloc (void *oldmem, size_t bytes) #if HAVE_MREMAP newp = mremap_chunk (oldp, nb); if (newp) - return chunk2mem (newp); + { + void *newmem = chunk2rawmem (newp); + /* Give the new block a different tag. This helps to ensure + that stale handles to the previous mapping are not + reused. There's a performance hit for both us and the + caller for doing this, so we might want to + reconsider. */ + return TAG_NEW_USABLE (newmem); + } #endif /* Note the extra SIZE_SZ overhead. */ if (oldsize - SIZE_SZ >= nb) @@ -3236,7 +3417,7 @@ __libc_realloc (void *oldmem, size_t bytes) if (newmem == 0) return 0; /* propagate failure */ - memcpy (newmem, oldmem, oldsize - 2 * SIZE_SZ); + memcpy (newmem, oldmem, oldsize - CHUNK_HDR_SZ); munmap_chunk (oldp); return newmem; } @@ -3323,8 +3504,7 @@ _mid_memalign (size_t alignment, size_t bytes, void *address) p = _int_memalign (&main_arena, alignment, bytes); assert (!p || chunk_is_mmapped (mem2chunk (p)) || &main_arena == arena_for_chunk (mem2chunk (p))); - - return p; + return TAG_NEW_USABLE (p); } arena_get (ar_ptr, bytes + alignment + MINSIZE); @@ -3342,7 +3522,7 @@ _mid_memalign (size_t alignment, size_t bytes, void *address) assert (!p || chunk_is_mmapped (mem2chunk (p)) || ar_ptr == arena_for_chunk (mem2chunk (p))); - return p; + return TAG_NEW_USABLE (p); } /* For ISO C11. */ weak_alias (__libc_memalign, aligned_alloc) @@ -3351,17 +3531,22 @@ libc_hidden_def (__libc_memalign) void * __libc_valloc (size_t bytes) { + void *p; + if (__malloc_initialized < 0) ptmalloc_init (); void *address = RETURN_ADDRESS (0); size_t pagesize = GLRO (dl_pagesize); - return _mid_memalign (pagesize, bytes, address); + p = _mid_memalign (pagesize, bytes, address); + return TAG_NEW_USABLE (p); } void * __libc_pvalloc (size_t bytes) { + void *p; + if (__malloc_initialized < 0) ptmalloc_init (); @@ -3378,19 +3563,22 @@ __libc_pvalloc (size_t bytes) } rounded_bytes = rounded_bytes & -(pagesize - 1); - return _mid_memalign (pagesize, rounded_bytes, address); + p = _mid_memalign (pagesize, rounded_bytes, address); + return TAG_NEW_USABLE (p); } void * __libc_calloc (size_t n, size_t elem_size) { mstate av; - mchunkptr oldtop, p; - INTERNAL_SIZE_T sz, csz, oldtopsize; + mchunkptr oldtop; + INTERNAL_SIZE_T sz, oldtopsize; void *mem; +#ifndef USE_MTAG unsigned long clearsize; unsigned long nclears; INTERNAL_SIZE_T *d; +#endif ptrdiff_t bytes; if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes))) @@ -3398,6 +3586,7 @@ __libc_calloc (size_t n, size_t elem_size) __set_errno (ENOMEM); return NULL; } + sz = bytes; void *(*hook) (size_t, const void *) = @@ -3467,7 +3656,14 @@ __libc_calloc (size_t n, size_t elem_size) if (mem == 0) return 0; - p = mem2chunk (mem); + mchunkptr p = mem2chunk (mem); + /* If we are using memory tagging, then we need to set the tags + regardless of MORECORE_CLEARS, so we zero the whole block while + doing so. */ +#ifdef USE_MTAG + return TAG_NEW_MEMSET (mem, 0, CHUNK_AVAILABLE_SIZE (p) - CHUNK_HDR_SZ); +#else + INTERNAL_SIZE_T csz = chunksize (p); /* Two optional cases in which clearing not necessary */ if (chunk_is_mmapped (p)) @@ -3478,8 +3674,6 @@ __libc_calloc (size_t n, size_t elem_size) return mem; } - csz = chunksize (p); - #if MORECORE_CLEARS if (perturb_byte == 0 && (p == oldtop && csz > oldtopsize)) { @@ -3522,6 +3716,7 @@ __libc_calloc (size_t n, size_t elem_size) } return mem; +#endif } /* @@ -3759,10 +3954,10 @@ _int_malloc (mstate av, size_t bytes) size = chunksize (victim); mchunkptr next = chunk_at_offset (victim, size); - if (__glibc_unlikely (size <= 2 * SIZE_SZ) + if (__glibc_unlikely (size <= CHUNK_HDR_SZ) || __glibc_unlikely (size > av->system_mem)) malloc_printerr ("malloc(): invalid size (unsorted)"); - if (__glibc_unlikely (chunksize_nomask (next) < 2 * SIZE_SZ) + if (__glibc_unlikely (chunksize_nomask (next) < CHUNK_HDR_SZ) || __glibc_unlikely (chunksize_nomask (next) > av->system_mem)) malloc_printerr ("malloc(): invalid next size (unsorted)"); if (__glibc_unlikely ((prev_size (next) & ~(SIZE_BITS)) != size)) @@ -4221,11 +4416,14 @@ _int_free (mstate av, mchunkptr p, int have_lock) if (__glibc_unlikely (e->key == tcache)) { tcache_entry *tmp; + size_t cnt = 0; LIBC_PROBE (memory_tcache_double_free, 2, e, tc_idx); for (tmp = tcache->entries[tc_idx]; tmp; - tmp = REVEAL_PTR (tmp->next)) + tmp = REVEAL_PTR (tmp->next), ++cnt) { + if (cnt >= mp_.tcache_count) + malloc_printerr ("free(): too many chunks detected in tcache"); if (__glibc_unlikely (!aligned_OK (tmp))) malloc_printerr ("free(): unaligned chunk detected in tcache 2"); if (tmp == e) @@ -4261,7 +4459,7 @@ _int_free (mstate av, mchunkptr p, int have_lock) ) { if (__builtin_expect (chunksize_nomask (chunk_at_offset (p, size)) - <= 2 * SIZE_SZ, 0) + <= CHUNK_HDR_SZ, 0) || __builtin_expect (chunksize (chunk_at_offset (p, size)) >= av->system_mem, 0)) { @@ -4272,7 +4470,7 @@ _int_free (mstate av, mchunkptr p, int have_lock) if (!have_lock) { __libc_lock_lock (av->mutex); - fail = (chunksize_nomask (chunk_at_offset (p, size)) <= 2 * SIZE_SZ + fail = (chunksize_nomask (chunk_at_offset (p, size)) <= CHUNK_HDR_SZ || chunksize (chunk_at_offset (p, size)) >= av->system_mem); __libc_lock_unlock (av->mutex); } @@ -4281,7 +4479,7 @@ _int_free (mstate av, mchunkptr p, int have_lock) malloc_printerr ("free(): invalid next size (fast)"); } - free_perturb (chunk2mem(p), size - 2 * SIZE_SZ); + free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ); atomic_store_relaxed (&av->have_fastchunks, true); unsigned int idx = fastbin_index(size); @@ -4350,11 +4548,11 @@ _int_free (mstate av, mchunkptr p, int have_lock) malloc_printerr ("double free or corruption (!prev)"); nextsize = chunksize(nextchunk); - if (__builtin_expect (chunksize_nomask (nextchunk) <= 2 * SIZE_SZ, 0) + if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, 0) || __builtin_expect (nextsize >= av->system_mem, 0)) malloc_printerr ("free(): invalid next size (normal)"); - free_perturb (chunk2mem(p), size - 2 * SIZE_SZ); + free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ); /* consolidate backward */ if (!prev_inuse(p)) { @@ -4585,7 +4783,7 @@ _int_realloc(mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, unsigned long remainder_size; /* its size */ /* oldmem size */ - if (__builtin_expect (chunksize_nomask (oldp) <= 2 * SIZE_SZ, 0) + if (__builtin_expect (chunksize_nomask (oldp) <= CHUNK_HDR_SZ, 0) || __builtin_expect (oldsize >= av->system_mem, 0)) malloc_printerr ("realloc(): invalid old size"); @@ -4596,7 +4794,7 @@ _int_realloc(mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, next = chunk_at_offset (oldp, oldsize); INTERNAL_SIZE_T nextsize = chunksize (next); - if (__builtin_expect (chunksize_nomask (next) <= 2 * SIZE_SZ, 0) + if (__builtin_expect (chunksize_nomask (next) <= CHUNK_HDR_SZ, 0) || __builtin_expect (nextsize >= av->system_mem, 0)) malloc_printerr ("realloc(): invalid next size"); @@ -4618,7 +4816,7 @@ _int_realloc(mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, av->top = chunk_at_offset (oldp, nb); set_head (av->top, (newsize - nb) | PREV_INUSE); check_inuse_chunk (av, oldp); - return chunk2mem (oldp); + return TAG_NEW_USABLE (chunk2rawmem (oldp)); } /* Try to expand forward into next chunk; split off remainder below */ @@ -4651,7 +4849,11 @@ _int_realloc(mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, } else { - memcpy (newmem, chunk2mem (oldp), oldsize - SIZE_SZ); + void *oldmem = chunk2mem (oldp); + newmem = TAG_NEW_USABLE (newmem); + memcpy (newmem, oldmem, + CHUNK_AVAILABLE_SIZE (oldp) - CHUNK_HDR_SZ); + (void) TAG_REGION (chunk2rawmem (oldp), oldsize); _int_free (av, oldp, 1); check_inuse_chunk (av, newp); return chunk2mem (newp); @@ -4673,6 +4875,8 @@ _int_realloc(mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, else /* split remainder */ { remainder = chunk_at_offset (newp, nb); + /* Clear any user-space tags before writing the header. */ + remainder = TAG_REGION (remainder, remainder_size); set_head_size (newp, nb | (av != &main_arena ? NON_MAIN_ARENA : 0)); set_head (remainder, remainder_size | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0)); @@ -4682,7 +4886,7 @@ _int_realloc(mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, } check_inuse_chunk (av, newp); - return chunk2mem (newp); + return TAG_NEW_USABLE (chunk2rawmem (newp)); } /* @@ -4760,7 +4964,7 @@ _int_memalign (mstate av, size_t alignment, size_t bytes) p = newp; assert (newsize >= nb && - (((unsigned long) (chunk2mem (p))) % alignment) == 0); + (((unsigned long) (chunk2rawmem (p))) % alignment) == 0); } /* Also give back spare room at the end */ @@ -4814,7 +5018,8 @@ mtrim (mstate av, size_t pad) + sizeof (struct malloc_chunk) + psm1) & ~psm1); - assert ((char *) chunk2mem (p) + 4 * SIZE_SZ <= paligned_mem); + assert ((char *) chunk2rawmem (p) + 2 * CHUNK_HDR_SZ + <= paligned_mem); assert ((char *) p + size > paligned_mem); /* This is the size we could potentially free. */ @@ -4877,20 +5082,30 @@ musable (void *mem) mchunkptr p; if (mem != 0) { + size_t result = 0; + p = mem2chunk (mem); if (__builtin_expect (using_malloc_checking == 1, 0)) - return malloc_check_get_size (p); + return malloc_check_get_size (p); if (chunk_is_mmapped (p)) { if (DUMPED_MAIN_ARENA_CHUNK (p)) - return chunksize (p) - SIZE_SZ; + result = chunksize (p) - SIZE_SZ; else - return chunksize (p) - 2 * SIZE_SZ; + result = chunksize (p) - CHUNK_HDR_SZ; } else if (inuse (p)) - return chunksize (p) - SIZE_SZ; + result = chunksize (p) - SIZE_SZ; + +#ifdef USE_MTAG + /* The usable space may be reduced if memory tagging is needed, + since we cannot share the user-space data with malloc's internal + data structure. */ + result &= __mtag_granule_mask; +#endif + return result; } return 0; } @@ -4911,7 +5126,7 @@ __malloc_usable_size (void *m) */ static void -int_mallinfo (mstate av, struct mallinfo *m) +int_mallinfo (mstate av, struct mallinfo2 *m) { size_t i; mbinptr b; @@ -4974,10 +5189,10 @@ int_mallinfo (mstate av, struct mallinfo *m) } -struct mallinfo -__libc_mallinfo (void) +struct mallinfo2 +__libc_mallinfo2 (void) { - struct mallinfo m; + struct mallinfo2 m; mstate ar_ptr; if (__malloc_initialized < 0) @@ -4997,6 +5212,28 @@ __libc_mallinfo (void) return m; } +libc_hidden_def (__libc_mallinfo2) + +struct mallinfo +__libc_mallinfo (void) +{ + struct mallinfo m; + struct mallinfo2 m2 = __libc_mallinfo2 (); + + m.arena = m2.arena; + m.ordblks = m2.ordblks; + m.smblks = m2.smblks; + m.hblks = m2.hblks; + m.hblkhd = m2.hblkhd; + m.usmblks = m2.usmblks; + m.fsmblks = m2.fsmblks; + m.uordblks = m2.uordblks; + m.fordblks = m2.fordblks; + m.keepcost = m2.keepcost; + + return m; +} + /* ------------------------------ malloc_stats ------------------------------ @@ -5016,7 +5253,7 @@ __malloc_stats (void) stderr->_flags2 |= _IO_FLAGS2_NOTCANCEL; for (i = 0, ar_ptr = &main_arena;; i++) { - struct mallinfo mi; + struct mallinfo2 mi; memset (&mi, 0, sizeof (mi)); __libc_lock_lock (ar_ptr->mutex); @@ -5632,6 +5869,8 @@ strong_alias (__libc_valloc, __valloc) weak_alias (__libc_valloc, valloc) strong_alias (__libc_pvalloc, __pvalloc) weak_alias (__libc_pvalloc, pvalloc) strong_alias (__libc_mallinfo, __mallinfo) weak_alias (__libc_mallinfo, mallinfo) +strong_alias (__libc_mallinfo2, __mallinfo2) +weak_alias (__libc_mallinfo2, mallinfo2) strong_alias (__libc_mallopt, __mallopt) weak_alias (__libc_mallopt, mallopt) weak_alias (__malloc_stats, malloc_stats)