Add BionicAllocator::memalign

Bionic needs this functionality to allocate a TLS segment with greater
than 16-byte alignment. For simplicity, this allocator only supports up
to one page of alignment.

The memory layout changes slightly when allocating an object of exactly
PAGE_SIZE alignment. Instead of allocating the page_info header at the
start of the page containing the pointer, it is allocated at the start
of the preceding page.

Bug: http://b/78026329
Test: linker-unit-tests{32,64}
Change-Id: I1c8d1cd7ca72d113bced5ee15ba8d831426b0081

1 files changed, 43 insertions, 14 deletions

diff --git a/libc/bionic/bionic_allocator.cpp b/libc/bionic/bionic_allocator.cpp
index a93321282..d9302ad1b 100644
--- a/libc/bionic/bionic_allocator.cpp
+++ b/libc/bionic/bionic_allocator.cpp
@@ -31,6 +31,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/mman.h>
+#include <sys/param.h>
 #include <sys/prctl.h>
 #include <unistd.h>
 
@@ -38,6 +39,7 @@
 
 #include <async_safe/log.h>
 
+#include "private/bionic_macros.h"
 #include "private/bionic_page.h"
 
 //
@@ -262,8 +264,14 @@ void BionicAllocator::initialize_allocators() {
   allocators_ = allocators;
 }
 
-void* BionicAllocator::alloc_mmap(size_t size) {
-  size_t allocated_size = PAGE_END(size + kPageInfoSize);
+void* BionicAllocator::alloc_mmap(size_t align, size_t size) {
+  size_t header_size = __BIONIC_ALIGN(kPageInfoSize, align);
+  size_t allocated_size;
+  if (__builtin_add_overflow(header_size, size, &allocated_size) ||
+      PAGE_END(allocated_size) < allocated_size) {
+    async_safe_fatal("overflow trying to alloc %zu bytes", size);
+  }
+  allocated_size = PAGE_END(allocated_size);
   void* map_ptr = mmap(nullptr, allocated_size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
                        -1, 0);
 
@@ -273,23 +281,19 @@ void* BionicAllocator::alloc_mmap(size_t size) {
 
   prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, map_ptr, allocated_size, "bionic_alloc_lob");
 
-  page_info* info = reinterpret_cast<page_info*>(map_ptr);
+  void* result = static_cast<char*>(map_ptr) + header_size;
+  page_info* info = get_page_info_unchecked(result);
   memcpy(info->signature, kSignature, sizeof(kSignature));
   info->type = kLargeObject;
   info->allocated_size = allocated_size;
 
-  return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(info) +
-                                 kPageInfoSize);
+  return result;
 }
 
-void* BionicAllocator::alloc(size_t size) {
-  // treat alloc(0) as alloc(1)
-  if (size == 0) {
-    size = 1;
-  }
 
+inline void* BionicAllocator::alloc_impl(size_t align, size_t size) {
   if (size > kSmallObjectMaxSize) {
-    return alloc_mmap(size);
+    return alloc_mmap(align, size);
   }
 
   uint16_t log2_size = log2(size);
@@ -301,8 +305,33 @@ void* BionicAllocator::alloc(size_t size) {
   return get_small_object_allocator(log2_size)->alloc();
 }
 
-page_info* BionicAllocator::get_page_info(void* ptr) {
-  page_info* info = reinterpret_cast<page_info*>(PAGE_START(reinterpret_cast<size_t>(ptr)));
+void* BionicAllocator::alloc(size_t size) {
+  // treat alloc(0) as alloc(1)
+  if (size == 0) {
+    size = 1;
+  }
+  return alloc_impl(16, size);
+}
+
+void* BionicAllocator::memalign(size_t align, size_t size) {
+  // The Bionic allocator only supports alignment up to one page, which is good
+  // enough for ELF TLS.
+  align = MIN(align, PAGE_SIZE);
+  align = MAX(align, 16);
+  if (!powerof2(align)) {
+    align = BIONIC_ROUND_UP_POWER_OF_2(align);
+  }
+  size = MAX(size, align);
+  return alloc_impl(align, size);
+}
+
+inline page_info* BionicAllocator::get_page_info_unchecked(void* ptr) {
+  uintptr_t header_page = PAGE_START(reinterpret_cast<size_t>(ptr) - kPageInfoSize);
+  return reinterpret_cast<page_info*>(header_page);
+}
+
+inline page_info* BionicAllocator::get_page_info(void* ptr) {
+  page_info* info = get_page_info_unchecked(ptr);
   if (memcmp(info->signature, kSignature, sizeof(kSignature)) != 0) {
     async_safe_fatal("invalid pointer %p (page signature mismatch)", ptr);
   }
@@ -325,7 +354,7 @@ void* BionicAllocator::realloc(void* ptr, size_t size) {
   size_t old_size = 0;
 
   if (info->type == kLargeObject) {
-    old_size = info->allocated_size - kPageInfoSize;
+    old_size = info->allocated_size - (static_cast<char*>(ptr) - reinterpret_cast<char*>(info));
   } else {
     BionicSmallObjectAllocator* allocator = get_small_object_allocator(info->type);
     if (allocator != info->allocator_addr) {