Message ID | 20211214185806.4109231-6-adhemerval.zanella@linaro.org |
---|---|
State | New |
Headers | show |
Series | malloc: Improve Huge Page support | expand |
Have you run any benchmarks against this series? We're replacing compile-time constants with variables, which may have a tiny affect on performance. LGTM Reviewed-by: DJ Delorie <dj@redhat.com> Adhemerval Zanella via Libc-alpha <libc-alpha@sourceware.org> writes: > tst-malloc-usable \ > tst-malloc-usable-tunables \ > tst-mallocstate > +# The tst-free-errno relies on the used malloc page size to mmap an > +# overlapping region. > +tests-exclude-hugetlb2 = \ > + $(tests-exclude-hugetlb1) \ > + tst-free-errno Ok. > tests-malloc-hugetlb2 = \ > - $(filter-out $(tests-exclude-hugetlb1), $(tests)) > + $(filter-out $(tests-exclude-hugetlb2), $(tests)) Ok. > diff --git a/malloc/arena.c b/malloc/arena.c > +/* When huge pages are used to create new arenas, the maximum and minumum > + size are based on the runtime defined huge page size. */ > + > +static inline size_t > +heap_min_size (void) > +{ > +#if HAVE_TUNABLES > + return mp_.hp_pagesize == 0 ? HEAP_MIN_SIZE : mp_.hp_pagesize; > +#else > + return HEAP_MIN_SIZE; > +#endif > +} > + > +static inline size_t > +heap_max_size (void) > +{ > +#if HAVE_TUNABLES > + return mp_.hp_pagesize == 0 ? HEAP_MAX_SIZE : mp_.hp_pagesize * 4; > +#else > + return HEAP_MAX_SIZE; > +#endif > +} Ok. > @@ -56,10 +79,11 @@ typedef struct _heap_info > size_t size; /* Current size in bytes. */ > size_t mprotect_size; /* Size in bytes that has been mprotected > PROT_READ|PROT_WRITE. */ > + size_t pagesize; /* Page size used when allocating the arena. */ > /* Make sure the following data is properly aligned, particularly > that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of > MALLOC_ALIGNMENT. */ > - char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK]; > + char pad[-3 * SIZE_SZ & MALLOC_ALIGN_MASK]; > } heap_info; Ok. > @@ -125,10 +149,18 @@ static bool __malloc_initialized = false; > > /* find the heap and corresponding arena for a given ptr */ > > -#define heap_for_ptr(ptr) \ > - ((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1))) > -#define arena_for_chunk(ptr) \ > - (chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr) > +static inline heap_info * > +heap_for_ptr (void *ptr) > +{ > + size_t max_size = heap_max_size (); > + return PTR_ALIGN_DOWN (ptr, max_size); > +} > + > +static inline struct malloc_state * > +arena_for_chunk (mchunkptr ptr) > +{ > + return chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr; > +} Ok. > @@ -443,71 +475,72 @@ static char *aligned_heap_area; > of the page size. */ > > static heap_info * > -new_heap (size_t size, size_t top_pad) > +alloc_new_heap (size_t size, size_t top_pad, size_t pagesize, > + int mmap_flags) > { > - size_t pagesize = GLRO (dl_pagesize); Moved to args, ok. > char *p1, *p2; > unsigned long ul; > heap_info *h; > + size_t min_size = heap_min_size (); > + size_t max_size = heap_max_size (); Ok. > - if (size + top_pad < HEAP_MIN_SIZE) > - size = HEAP_MIN_SIZE; > - else if (size + top_pad <= HEAP_MAX_SIZE) > + if (size + top_pad < min_size) > + size = min_size; > + else if (size + top_pad <= max_size) > size += top_pad; > - else if (size > HEAP_MAX_SIZE) > + else if (size > max_size) > return 0; > else > - size = HEAP_MAX_SIZE; > + size = max_size; > size = ALIGN_UP (size, pagesize); Ok. > - /* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed. > + /* A memory region aligned to a multiple of max_size is needed. > No swap space needs to be reserved for the following large > mapping (on Linux, this is the case for all non-writable mappings > anyway). */ Ok. > p2 = MAP_FAILED; > if (aligned_heap_area) > { > - p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE, > - MAP_NORESERVE); > + p2 = (char *) MMAP (aligned_heap_area, max_size, PROT_NONE, mmap_flags); > aligned_heap_area = NULL; > - if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1))) > + if (p2 != MAP_FAILED && ((unsigned long) p2 & (max_size - 1))) > { > - __munmap (p2, HEAP_MAX_SIZE); > + __munmap (p2, max_size); > p2 = MAP_FAILED; > } Ok. > if (p2 == MAP_FAILED) > { > - p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE); > + p1 = (char *) MMAP (0, max_size << 1, PROT_NONE, mmap_flags); > if (p1 != MAP_FAILED) > { > - p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1)) > - & ~(HEAP_MAX_SIZE - 1)); > + p2 = (char *) (((unsigned long) p1 + (max_size - 1)) > + & ~(max_size - 1)); > ul = p2 - p1; > if (ul) > __munmap (p1, ul); > else > - aligned_heap_area = p2 + HEAP_MAX_SIZE; > - __munmap (p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul); > + aligned_heap_area = p2 + max_size; > + __munmap (p2 + max_size, max_size - ul); > } Ok. > else > { > - /* Try to take the chance that an allocation of only HEAP_MAX_SIZE > + /* Try to take the chance that an allocation of only max_size > is already aligned. */ > - p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE); > + p2 = (char *) MMAP (0, max_size, PROT_NONE, mmap_flags); > if (p2 == MAP_FAILED) > return 0; > > - if ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)) > + if ((unsigned long) p2 & (max_size - 1)) > { > - __munmap (p2, HEAP_MAX_SIZE); > + __munmap (p2, max_size); > return 0; > } > } > } > if (__mprotect (p2, size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0) > { > - __munmap (p2, HEAP_MAX_SIZE); > + __munmap (p2, max_size); > return 0; > } Ok. > @@ -516,22 +549,42 @@ new_heap (size_t size, size_t top_pad) > + h->pagesize = pagesize; Ok. > +static heap_info * > +new_heap (size_t size, size_t top_pad) > +{ > +#if HAVE_TUNABLES > + if (__glibc_unlikely (mp_.hp_pagesize != 0)) > + { > + /* MAP_NORESERVE is not used for huge pages because some kernel may > + not reserve the mmap region and a subsequent access may trigger > + a SIGBUS if there is no free pages in the pool. */ > + heap_info *h = alloc_new_heap (size, top_pad, mp_.hp_pagesize, > + mp_.hp_flags); > + if (h != NULL) > + return h; > + } > +#endif > + return alloc_new_heap (size, top_pad, GLRO (dl_pagesize), MAP_NORESERVE); > +} > + Ok. > /* Grow a heap. size is automatically rounded up to a > multiple of the page size. */ > > static int > grow_heap (heap_info *h, long diff) > { > - size_t pagesize = GLRO (dl_pagesize); > + size_t pagesize = h->pagesize; > + size_t max_size = heap_max_size (); > long new_size; > > diff = ALIGN_UP (diff, pagesize); > new_size = (long) h->size + diff; > - if ((unsigned long) new_size > (unsigned long) HEAP_MAX_SIZE) > + if ((unsigned long) new_size > (unsigned long) max_size) > return -1; Ok. > @@ -581,21 +634,14 @@ shrink_heap (heap_info *h, long diff) > > /* Delete a heap. */ > > -#define delete_heap(heap) \ > - do { \ > - if ((char *) (heap) + HEAP_MAX_SIZE == aligned_heap_area) \ > - aligned_heap_area = NULL; \ > - __munmap ((char *) (heap), HEAP_MAX_SIZE); \ > - } while (0) > - > static int > heap_trim (heap_info *heap, size_t pad) > { > mstate ar_ptr = heap->ar_ptr; > - unsigned long pagesz = GLRO (dl_pagesize); > mchunkptr top_chunk = top (ar_ptr), p; > heap_info *prev_heap; > long new_size, top_size, top_area, extra, prev_size, misalign; > + size_t max_size = heap_max_size (); > > /* Can this heap go away completely? */ > while (top_chunk == chunk_at_offset (heap, sizeof (*heap))) > @@ -612,19 +658,23 @@ heap_trim (heap_info *heap, size_t pad) > assert (new_size > 0 && new_size < (long) (2 * MINSIZE)); > if (!prev_inuse (p)) > new_size += prev_size (p); > - assert (new_size > 0 && new_size < HEAP_MAX_SIZE); > - if (new_size + (HEAP_MAX_SIZE - prev_heap->size) < pad + MINSIZE + pagesz) > + assert (new_size > 0 && new_size < max_size); > + if (new_size + (max_size - prev_heap->size) < pad + MINSIZE > + + heap->pagesize) > break; Ok. > ar_ptr->system_mem -= heap->size; > LIBC_PROBE (memory_heap_free, 2, heap, heap->size); > - delete_heap (heap); > + if ((char *) heap + max_size == aligned_heap_area) > + aligned_heap_area = NULL; > + __munmap (heap, max_size); Ok. > - assert (((unsigned long) ((char *) p + new_size) & (pagesz - 1)) == 0); > + assert (((unsigned long) ((char *) p + new_size) & (heap->pagesize - 1)) > + == 0); Ok. > @@ -644,7 +694,7 @@ heap_trim (heap_info *heap, size_t pad) > return 0; > > /* Release in pagesize units and round down to the nearest page. */ > - extra = ALIGN_DOWN(top_area - pad, pagesz); > + extra = ALIGN_DOWN(top_area - pad, heap->pagesize); > if (extra == 0) > return 0; Ok. > diff --git a/malloc/malloc.c b/malloc/malloc.c > index 3e2f427d94..37fbd7e51d 100644 > --- a/malloc/malloc.c > +++ b/malloc/malloc.c > @@ -5301,7 +5301,7 @@ static __always_inline int > do_set_mmap_threshold (size_t value) > { > /* Forbid setting the threshold too high. */ > - if (value <= HEAP_MAX_SIZE / 2) > + if (value <= heap_max_size () / 2) Ok.
On 15/12/2021 01:51, DJ Delorie wrote: > > Have you run any benchmarks against this series? We're replacing > compile-time constants with variables, which may have a tiny affect on > performance. With our own benchtests, the bench-malloc-simple shows a lot of noise, with some values better other worse (which in the end equalizes, so I think it is pretty much the same). The bench-malloc-thread shows: master patched diff 1 time_per_iteration 41.5724 43.1116 3.70 8 time_per_iteration 48.5594 48.0252 -1.10 16 time_per_iteration 62.5156 63.1405 1.00 24 time_per_iteration 84.7967 85.5267 0.86 So I think it might also be just noise or some OS jitter. > > LGTM > Reviewed-by: DJ Delorie <dj@redhat.com> > > Adhemerval Zanella via Libc-alpha <libc-alpha@sourceware.org> writes: >> tst-malloc-usable \ >> tst-malloc-usable-tunables \ >> tst-mallocstate >> +# The tst-free-errno relies on the used malloc page size to mmap an >> +# overlapping region. >> +tests-exclude-hugetlb2 = \ >> + $(tests-exclude-hugetlb1) \ >> + tst-free-errno > > Ok. > >> tests-malloc-hugetlb2 = \ >> - $(filter-out $(tests-exclude-hugetlb1), $(tests)) >> + $(filter-out $(tests-exclude-hugetlb2), $(tests)) > > Ok. > >> diff --git a/malloc/arena.c b/malloc/arena.c >> +/* When huge pages are used to create new arenas, the maximum and minumum >> + size are based on the runtime defined huge page size. */ >> + >> +static inline size_t >> +heap_min_size (void) >> +{ >> +#if HAVE_TUNABLES >> + return mp_.hp_pagesize == 0 ? HEAP_MIN_SIZE : mp_.hp_pagesize; >> +#else >> + return HEAP_MIN_SIZE; >> +#endif >> +} >> + >> +static inline size_t >> +heap_max_size (void) >> +{ >> +#if HAVE_TUNABLES >> + return mp_.hp_pagesize == 0 ? HEAP_MAX_SIZE : mp_.hp_pagesize * 4; >> +#else >> + return HEAP_MAX_SIZE; >> +#endif >> +} > > Ok. > >> @@ -56,10 +79,11 @@ typedef struct _heap_info >> size_t size; /* Current size in bytes. */ >> size_t mprotect_size; /* Size in bytes that has been mprotected >> PROT_READ|PROT_WRITE. */ >> + size_t pagesize; /* Page size used when allocating the arena. */ >> /* Make sure the following data is properly aligned, particularly >> that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of >> MALLOC_ALIGNMENT. */ >> - char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK]; >> + char pad[-3 * SIZE_SZ & MALLOC_ALIGN_MASK]; >> } heap_info; > > Ok. > >> @@ -125,10 +149,18 @@ static bool __malloc_initialized = false; >> >> /* find the heap and corresponding arena for a given ptr */ >> >> -#define heap_for_ptr(ptr) \ >> - ((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1))) >> -#define arena_for_chunk(ptr) \ >> - (chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr) >> +static inline heap_info * >> +heap_for_ptr (void *ptr) >> +{ >> + size_t max_size = heap_max_size (); >> + return PTR_ALIGN_DOWN (ptr, max_size); >> +} >> + >> +static inline struct malloc_state * >> +arena_for_chunk (mchunkptr ptr) >> +{ >> + return chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr; >> +} > > Ok. > > >> @@ -443,71 +475,72 @@ static char *aligned_heap_area; >> of the page size. */ >> >> static heap_info * >> -new_heap (size_t size, size_t top_pad) >> +alloc_new_heap (size_t size, size_t top_pad, size_t pagesize, >> + int mmap_flags) >> { >> - size_t pagesize = GLRO (dl_pagesize); > > Moved to args, ok. > >> char *p1, *p2; >> unsigned long ul; >> heap_info *h; >> + size_t min_size = heap_min_size (); >> + size_t max_size = heap_max_size (); > > Ok. > >> - if (size + top_pad < HEAP_MIN_SIZE) >> - size = HEAP_MIN_SIZE; >> - else if (size + top_pad <= HEAP_MAX_SIZE) >> + if (size + top_pad < min_size) >> + size = min_size; >> + else if (size + top_pad <= max_size) >> size += top_pad; >> - else if (size > HEAP_MAX_SIZE) >> + else if (size > max_size) >> return 0; >> else >> - size = HEAP_MAX_SIZE; >> + size = max_size; >> size = ALIGN_UP (size, pagesize); > > Ok. > >> - /* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed. >> + /* A memory region aligned to a multiple of max_size is needed. >> No swap space needs to be reserved for the following large >> mapping (on Linux, this is the case for all non-writable mappings >> anyway). */ > > Ok. > >> p2 = MAP_FAILED; >> if (aligned_heap_area) >> { >> - p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE, >> - MAP_NORESERVE); >> + p2 = (char *) MMAP (aligned_heap_area, max_size, PROT_NONE, mmap_flags); >> aligned_heap_area = NULL; >> - if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1))) >> + if (p2 != MAP_FAILED && ((unsigned long) p2 & (max_size - 1))) >> { >> - __munmap (p2, HEAP_MAX_SIZE); >> + __munmap (p2, max_size); >> p2 = MAP_FAILED; >> } > > Ok. > >> if (p2 == MAP_FAILED) >> { >> - p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE); >> + p1 = (char *) MMAP (0, max_size << 1, PROT_NONE, mmap_flags); >> if (p1 != MAP_FAILED) >> { >> - p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1)) >> - & ~(HEAP_MAX_SIZE - 1)); >> + p2 = (char *) (((unsigned long) p1 + (max_size - 1)) >> + & ~(max_size - 1)); >> ul = p2 - p1; >> if (ul) >> __munmap (p1, ul); >> else >> - aligned_heap_area = p2 + HEAP_MAX_SIZE; >> - __munmap (p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul); >> + aligned_heap_area = p2 + max_size; >> + __munmap (p2 + max_size, max_size - ul); >> } > > Ok. > >> else >> { >> - /* Try to take the chance that an allocation of only HEAP_MAX_SIZE >> + /* Try to take the chance that an allocation of only max_size >> is already aligned. */ >> - p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE); >> + p2 = (char *) MMAP (0, max_size, PROT_NONE, mmap_flags); >> if (p2 == MAP_FAILED) >> return 0; >> >> - if ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)) >> + if ((unsigned long) p2 & (max_size - 1)) >> { >> - __munmap (p2, HEAP_MAX_SIZE); >> + __munmap (p2, max_size); >> return 0; >> } >> } >> } >> if (__mprotect (p2, size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0) >> { >> - __munmap (p2, HEAP_MAX_SIZE); >> + __munmap (p2, max_size); >> return 0; >> } > > Ok. > >> @@ -516,22 +549,42 @@ new_heap (size_t size, size_t top_pad) >> + h->pagesize = pagesize; > > Ok. > >> +static heap_info * >> +new_heap (size_t size, size_t top_pad) >> +{ >> +#if HAVE_TUNABLES >> + if (__glibc_unlikely (mp_.hp_pagesize != 0)) >> + { >> + /* MAP_NORESERVE is not used for huge pages because some kernel may >> + not reserve the mmap region and a subsequent access may trigger >> + a SIGBUS if there is no free pages in the pool. */ >> + heap_info *h = alloc_new_heap (size, top_pad, mp_.hp_pagesize, >> + mp_.hp_flags); >> + if (h != NULL) >> + return h; >> + } >> +#endif >> + return alloc_new_heap (size, top_pad, GLRO (dl_pagesize), MAP_NORESERVE); >> +} >> + > > Ok. > >> /* Grow a heap. size is automatically rounded up to a >> multiple of the page size. */ >> >> static int >> grow_heap (heap_info *h, long diff) >> { >> - size_t pagesize = GLRO (dl_pagesize); >> + size_t pagesize = h->pagesize; >> + size_t max_size = heap_max_size (); >> long new_size; >> >> diff = ALIGN_UP (diff, pagesize); >> new_size = (long) h->size + diff; >> - if ((unsigned long) new_size > (unsigned long) HEAP_MAX_SIZE) >> + if ((unsigned long) new_size > (unsigned long) max_size) >> return -1; > > Ok. > >> @@ -581,21 +634,14 @@ shrink_heap (heap_info *h, long diff) >> >> /* Delete a heap. */ >> >> -#define delete_heap(heap) \ >> - do { \ >> - if ((char *) (heap) + HEAP_MAX_SIZE == aligned_heap_area) \ >> - aligned_heap_area = NULL; \ >> - __munmap ((char *) (heap), HEAP_MAX_SIZE); \ >> - } while (0) >> - >> static int >> heap_trim (heap_info *heap, size_t pad) >> { >> mstate ar_ptr = heap->ar_ptr; >> - unsigned long pagesz = GLRO (dl_pagesize); >> mchunkptr top_chunk = top (ar_ptr), p; >> heap_info *prev_heap; >> long new_size, top_size, top_area, extra, prev_size, misalign; >> + size_t max_size = heap_max_size (); >> >> /* Can this heap go away completely? */ >> while (top_chunk == chunk_at_offset (heap, sizeof (*heap))) >> @@ -612,19 +658,23 @@ heap_trim (heap_info *heap, size_t pad) >> assert (new_size > 0 && new_size < (long) (2 * MINSIZE)); >> if (!prev_inuse (p)) >> new_size += prev_size (p); >> - assert (new_size > 0 && new_size < HEAP_MAX_SIZE); >> - if (new_size + (HEAP_MAX_SIZE - prev_heap->size) < pad + MINSIZE + pagesz) >> + assert (new_size > 0 && new_size < max_size); >> + if (new_size + (max_size - prev_heap->size) < pad + MINSIZE >> + + heap->pagesize) >> break; > > Ok. > >> ar_ptr->system_mem -= heap->size; >> LIBC_PROBE (memory_heap_free, 2, heap, heap->size); >> - delete_heap (heap); >> + if ((char *) heap + max_size == aligned_heap_area) >> + aligned_heap_area = NULL; >> + __munmap (heap, max_size); > > Ok. > >> - assert (((unsigned long) ((char *) p + new_size) & (pagesz - 1)) == 0); >> + assert (((unsigned long) ((char *) p + new_size) & (heap->pagesize - 1)) >> + == 0); > > Ok. > >> @@ -644,7 +694,7 @@ heap_trim (heap_info *heap, size_t pad) >> return 0; >> >> /* Release in pagesize units and round down to the nearest page. */ >> - extra = ALIGN_DOWN(top_area - pad, pagesz); >> + extra = ALIGN_DOWN(top_area - pad, heap->pagesize); >> if (extra == 0) >> return 0; > > Ok. > >> diff --git a/malloc/malloc.c b/malloc/malloc.c >> index 3e2f427d94..37fbd7e51d 100644 >> --- a/malloc/malloc.c >> +++ b/malloc/malloc.c >> @@ -5301,7 +5301,7 @@ static __always_inline int >> do_set_mmap_threshold (size_t value) >> { >> /* Forbid setting the threshold too high. */ >> - if (value <= HEAP_MAX_SIZE / 2) >> + if (value <= heap_max_size () / 2) > > Ok. >
diff --git a/malloc/Makefile b/malloc/Makefile index 83de7f2a35..abd0f811d4 100644 --- a/malloc/Makefile +++ b/malloc/Makefile @@ -91,10 +91,15 @@ tests-exclude-hugetlb1 = \ tst-malloc-usable \ tst-malloc-usable-tunables \ tst-mallocstate +# The tst-free-errno relies on the used malloc page size to mmap an +# overlapping region. +tests-exclude-hugetlb2 = \ + $(tests-exclude-hugetlb1) \ + tst-free-errno tests-malloc-hugetlb1 = \ $(filter-out $(tests-exclude-hugetlb1), $(tests)) tests-malloc-hugetlb2 = \ - $(filter-out $(tests-exclude-hugetlb1), $(tests)) + $(filter-out $(tests-exclude-hugetlb2), $(tests)) # -lmcheck needs __malloc_initialize_hook, which was deprecated in 2.24. ifeq ($(have-GLIBC_2.23)$(build-shared),yesyes) diff --git a/malloc/arena.c b/malloc/arena.c index cd00c7bef4..c6d811ff3b 100644 --- a/malloc/arena.c +++ b/malloc/arena.c @@ -41,6 +41,29 @@ mmap threshold, so that requests with a size just below that threshold can be fulfilled without creating too many heaps. */ +/* When huge pages are used to create new arenas, the maximum and minumum + size are based on the runtime defined huge page size. */ + +static inline size_t +heap_min_size (void) +{ +#if HAVE_TUNABLES + return mp_.hp_pagesize == 0 ? HEAP_MIN_SIZE : mp_.hp_pagesize; +#else + return HEAP_MIN_SIZE; +#endif +} + +static inline size_t +heap_max_size (void) +{ +#if HAVE_TUNABLES + return mp_.hp_pagesize == 0 ? HEAP_MAX_SIZE : mp_.hp_pagesize * 4; +#else + return HEAP_MAX_SIZE; +#endif +} + /***************************************************************************/ #define top(ar_ptr) ((ar_ptr)->top) @@ -56,10 +79,11 @@ typedef struct _heap_info size_t size; /* Current size in bytes. */ size_t mprotect_size; /* Size in bytes that has been mprotected PROT_READ|PROT_WRITE. */ + size_t pagesize; /* Page size used when allocating the arena. */ /* Make sure the following data is properly aligned, particularly that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of MALLOC_ALIGNMENT. */ - char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK]; + char pad[-3 * SIZE_SZ & MALLOC_ALIGN_MASK]; } heap_info; /* Get a compile-time error if the heap_info padding is not correct @@ -125,10 +149,18 @@ static bool __malloc_initialized = false; /* find the heap and corresponding arena for a given ptr */ -#define heap_for_ptr(ptr) \ - ((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1))) -#define arena_for_chunk(ptr) \ - (chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr) +static inline heap_info * +heap_for_ptr (void *ptr) +{ + size_t max_size = heap_max_size (); + return PTR_ALIGN_DOWN (ptr, max_size); +} + +static inline struct malloc_state * +arena_for_chunk (mchunkptr ptr) +{ + return chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr; +} /**************************************************************************/ @@ -443,71 +475,72 @@ static char *aligned_heap_area; of the page size. */ static heap_info * -new_heap (size_t size, size_t top_pad) +alloc_new_heap (size_t size, size_t top_pad, size_t pagesize, + int mmap_flags) { - size_t pagesize = GLRO (dl_pagesize); char *p1, *p2; unsigned long ul; heap_info *h; + size_t min_size = heap_min_size (); + size_t max_size = heap_max_size (); - if (size + top_pad < HEAP_MIN_SIZE) - size = HEAP_MIN_SIZE; - else if (size + top_pad <= HEAP_MAX_SIZE) + if (size + top_pad < min_size) + size = min_size; + else if (size + top_pad <= max_size) size += top_pad; - else if (size > HEAP_MAX_SIZE) + else if (size > max_size) return 0; else - size = HEAP_MAX_SIZE; + size = max_size; size = ALIGN_UP (size, pagesize); - /* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed. + /* A memory region aligned to a multiple of max_size is needed. No swap space needs to be reserved for the following large mapping (on Linux, this is the case for all non-writable mappings anyway). */ p2 = MAP_FAILED; if (aligned_heap_area) { - p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE, - MAP_NORESERVE); + p2 = (char *) MMAP (aligned_heap_area, max_size, PROT_NONE, mmap_flags); aligned_heap_area = NULL; - if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1))) + if (p2 != MAP_FAILED && ((unsigned long) p2 & (max_size - 1))) { - __munmap (p2, HEAP_MAX_SIZE); + __munmap (p2, max_size); p2 = MAP_FAILED; } } if (p2 == MAP_FAILED) { - p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE); + p1 = (char *) MMAP (0, max_size << 1, PROT_NONE, mmap_flags); if (p1 != MAP_FAILED) { - p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1)) - & ~(HEAP_MAX_SIZE - 1)); + p2 = (char *) (((unsigned long) p1 + (max_size - 1)) + & ~(max_size - 1)); ul = p2 - p1; if (ul) __munmap (p1, ul); else - aligned_heap_area = p2 + HEAP_MAX_SIZE; - __munmap (p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul); + aligned_heap_area = p2 + max_size; + __munmap (p2 + max_size, max_size - ul); } else { - /* Try to take the chance that an allocation of only HEAP_MAX_SIZE + /* Try to take the chance that an allocation of only max_size is already aligned. */ - p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE); + p2 = (char *) MMAP (0, max_size, PROT_NONE, mmap_flags); if (p2 == MAP_FAILED) return 0; - if ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)) + if ((unsigned long) p2 & (max_size - 1)) { - __munmap (p2, HEAP_MAX_SIZE); + __munmap (p2, max_size); return 0; } } } if (__mprotect (p2, size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0) { - __munmap (p2, HEAP_MAX_SIZE); + __munmap (p2, max_size); return 0; } @@ -516,22 +549,42 @@ new_heap (size_t size, size_t top_pad) h = (heap_info *) p2; h->size = size; h->mprotect_size = size; + h->pagesize = pagesize; LIBC_PROBE (memory_heap_new, 2, h, h->size); return h; } +static heap_info * +new_heap (size_t size, size_t top_pad) +{ +#if HAVE_TUNABLES + if (__glibc_unlikely (mp_.hp_pagesize != 0)) + { + /* MAP_NORESERVE is not used for huge pages because some kernel may + not reserve the mmap region and a subsequent access may trigger + a SIGBUS if there is no free pages in the pool. */ + heap_info *h = alloc_new_heap (size, top_pad, mp_.hp_pagesize, + mp_.hp_flags); + if (h != NULL) + return h; + } +#endif + return alloc_new_heap (size, top_pad, GLRO (dl_pagesize), MAP_NORESERVE); +} + /* Grow a heap. size is automatically rounded up to a multiple of the page size. */ static int grow_heap (heap_info *h, long diff) { - size_t pagesize = GLRO (dl_pagesize); + size_t pagesize = h->pagesize; + size_t max_size = heap_max_size (); long new_size; diff = ALIGN_UP (diff, pagesize); new_size = (long) h->size + diff; - if ((unsigned long) new_size > (unsigned long) HEAP_MAX_SIZE) + if ((unsigned long) new_size > (unsigned long) max_size) return -1; if ((unsigned long) new_size > h->mprotect_size) @@ -581,21 +634,14 @@ shrink_heap (heap_info *h, long diff) /* Delete a heap. */ -#define delete_heap(heap) \ - do { \ - if ((char *) (heap) + HEAP_MAX_SIZE == aligned_heap_area) \ - aligned_heap_area = NULL; \ - __munmap ((char *) (heap), HEAP_MAX_SIZE); \ - } while (0) - static int heap_trim (heap_info *heap, size_t pad) { mstate ar_ptr = heap->ar_ptr; - unsigned long pagesz = GLRO (dl_pagesize); mchunkptr top_chunk = top (ar_ptr), p; heap_info *prev_heap; long new_size, top_size, top_area, extra, prev_size, misalign; + size_t max_size = heap_max_size (); /* Can this heap go away completely? */ while (top_chunk == chunk_at_offset (heap, sizeof (*heap))) @@ -612,19 +658,23 @@ heap_trim (heap_info *heap, size_t pad) assert (new_size > 0 && new_size < (long) (2 * MINSIZE)); if (!prev_inuse (p)) new_size += prev_size (p); - assert (new_size > 0 && new_size < HEAP_MAX_SIZE); - if (new_size + (HEAP_MAX_SIZE - prev_heap->size) < pad + MINSIZE + pagesz) + assert (new_size > 0 && new_size < max_size); + if (new_size + (max_size - prev_heap->size) < pad + MINSIZE + + heap->pagesize) break; ar_ptr->system_mem -= heap->size; LIBC_PROBE (memory_heap_free, 2, heap, heap->size); - delete_heap (heap); + if ((char *) heap + max_size == aligned_heap_area) + aligned_heap_area = NULL; + __munmap (heap, max_size); heap = prev_heap; if (!prev_inuse (p)) /* consolidate backward */ { p = prev_chunk (p); unlink_chunk (ar_ptr, p); } - assert (((unsigned long) ((char *) p + new_size) & (pagesz - 1)) == 0); + assert (((unsigned long) ((char *) p + new_size) & (heap->pagesize - 1)) + == 0); assert (((char *) p + new_size) == ((char *) heap + heap->size)); top (ar_ptr) = top_chunk = p; set_head (top_chunk, new_size | PREV_INUSE); @@ -644,7 +694,7 @@ heap_trim (heap_info *heap, size_t pad) return 0; /* Release in pagesize units and round down to the nearest page. */ - extra = ALIGN_DOWN(top_area - pad, pagesz); + extra = ALIGN_DOWN(top_area - pad, heap->pagesize); if (extra == 0) return 0; diff --git a/malloc/malloc.c b/malloc/malloc.c index 3e2f427d94..37fbd7e51d 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -5301,7 +5301,7 @@ static __always_inline int do_set_mmap_threshold (size_t value) { /* Forbid setting the threshold too high. */ - if (value <= HEAP_MAX_SIZE / 2) + if (value <= heap_max_size () / 2) { LIBC_PROBE (memory_mallopt_mmap_threshold, 3, value, mp_.mmap_threshold, mp_.no_dyn_threshold);