/* * Copyright (C) 2019 The Android Open Source Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #if defined(LIBC_STATIC) #error This file should not be compiled for static targets. #endif #include #include #include #include #include #include #include #include #include #include #include "gwp_asan_wrappers.h" #include "malloc_common.h" #include "malloc_common_dynamic.h" #include "malloc_heapprofd.h" #include "malloc_limit.h" // Installing heapprofd hooks is a multi step process, as outlined below. // // The incremental hooking and a dedicated task thread are used since we cannot // do heavy work within a signal handler, or when blocking a malloc invocation. // // +--->+-------------+------------------+ // | +->+kInitialState+----------------+ | malloc functions are not intercepted in any way. // | | +-------+-----+ | | // | | | HandleHeapprofd | | // | | v Signal() | | // | | +-------+----------------+ | | currently installing the ephemeral hooks. // | | |kInstallingEphemeralHook|<--+ | | // | | +-------+----------------+ | | | // | | | | | | // | | v | | | // | | +-------+---------------+ | | | ephemeral hooks are installed. on the first call to // | | |kEphemeralHookInstalled| | | | malloc these hooks spawn a thread that installs the // | | +-------+---------------+ A B C heapprofd hooks. // | | | MallocInit | | | // | | v HeapprofdHook () | | | // | | +-------+--------------+ | | | first call to malloc happened. the hooks are reset to // | +--|kRemovingEphemeralHook| | | | kInitialState. // | +----------------------+ | | | // | | | | // | | | | // | +---------------+ | | | currently installing the heapprofd hook // | |kInstallingHook|<-----------|-+ | // | +-------+-------+ | | // | | | | // | v | | // | +-------+------+ | | heapprofd hooks are installed. these forward calls to // | |kHookInstalled|-------------+ | malloc / free / etc. to heapprofd_client.so. // | +-------+------+ | // | | DispatchReset() | // | v | // | +-------+---------+ | currently resetting the hooks to default. // |----+kUninstallingHook| | // +-----------------+ | // | // | // +------------------+ | malloc debug / malloc hooks are active. these take // |kIncompatibleHooks+<------------+ precendence over heapprofd, so heapprofd will not get // +------------------+ enabled. this is a terminal state. // // // A) HandleHeapprofdSignal() // B) HeapprofdInstallHooksAtInit() / InitHeapprofd() // C) HeapprofdRememberHookConflict() enum MallocHeapprofdState : uint8_t { kInitialState, kInstallingEphemeralHook, kEphemeralHookInstalled, kRemovingEphemeralHook, kInstallingHook, kHookInstalled, kUninstallingHook, kIncompatibleHooks }; enum ModifyGlobalsMode { kWithLock, // all calls to MaybeModifyGlobals with kWithLock will serialise. they can fail // due to a concurrent call with kWithoutLock. kWithoutLock // calls to MaybeModifyGlobals with kWithoutLock do not serialise. they can fail // due to concurrent calls with kWithoutLock or kWithLock. }; // Provide mutual exclusion so no two threads try to modify the globals at the same time. template bool MaybeModifyGlobals(ModifyGlobalsMode mode, Fn f) { bool success = false; if (mode == kWithLock) { pthread_mutex_lock(&gGlobalsMutateLock); } // As we have grabbed the mutex, the following condition should always hold, except // if we are currently running HandleHeapprofdSignal. if (!atomic_exchange(&gGlobalsMutating, true)) { f(); success = true; atomic_store(&gGlobalsMutating, false); } else { error_log("%s: heapprofd client: concurrent modification.", getprogname()); } if (mode == kWithLock) { pthread_mutex_unlock(&gGlobalsMutateLock); } return success; } extern "C" void* MallocInitHeapprofdHook(size_t); static constexpr char kHeapprofdSharedLib[] = "heapprofd_client.so"; static constexpr char kHeapprofdPrefix[] = "heapprofd"; static constexpr char kHeapprofdPropertyEnable[] = "heapprofd.enable"; constexpr char kHeapprofdProgramPropertyPrefix[] = "heapprofd.enable."; constexpr size_t kHeapprofdProgramPropertyPrefixSize = sizeof(kHeapprofdProgramPropertyPrefix) - 1; constexpr size_t kMaxCmdlineSize = 512; // The handle returned by dlopen when previously loading the heapprofd // hooks. nullptr if shared library has not been already been loaded. static _Atomic (void*) gHeapprofdHandle = nullptr; static _Atomic MallocHeapprofdState gHeapprofdState = kInitialState; static bool GetHeapprofdProgramProperty(char* data, size_t size) { if (size < kHeapprofdProgramPropertyPrefixSize) { error_log("%s: Overflow constructing heapprofd property", getprogname()); return false; } memcpy(data, kHeapprofdProgramPropertyPrefix, kHeapprofdProgramPropertyPrefixSize); int fd = open("/proc/self/cmdline", O_RDONLY | O_CLOEXEC); if (fd == -1) { error_log("%s: Failed to open /proc/self/cmdline", getprogname()); return false; } char cmdline[kMaxCmdlineSize]; ssize_t rd = read(fd, cmdline, sizeof(cmdline) - 1); close(fd); if (rd == -1) { error_log("%s: Failed to read /proc/self/cmdline", getprogname()); return false; } cmdline[rd] = '\0'; char* first_arg = static_cast(memchr(cmdline, '\0', rd)); if (first_arg == nullptr) { error_log("%s: Overflow reading cmdline", getprogname()); return false; } // For consistency with what we do with Java app cmdlines, trim everything // after the @ sign of the first arg. char* first_at = static_cast(memchr(cmdline, '@', rd)); if (first_at != nullptr && first_at < first_arg) { *first_at = '\0'; first_arg = first_at; } char* start = static_cast(memrchr(cmdline, '/', first_arg - cmdline)); if (start == first_arg) { // The first argument ended in a slash. error_log("%s: cmdline ends in /", getprogname()); return false; } else if (start == nullptr) { start = cmdline; } else { // Skip the /. start++; } size_t name_size = static_cast(first_arg - start); if (name_size >= size - kHeapprofdProgramPropertyPrefixSize) { error_log("%s: overflow constructing heapprofd property.", getprogname()); return false; } // + 1 to also copy the trailing null byte. memcpy(data + kHeapprofdProgramPropertyPrefixSize, start, name_size + 1); return true; } // Runtime triggering entry-point. Two possible call sites: // * when receiving a profiling signal with a si_value indicating heapprofd. // * when a Zygote child is marking itself as profileable, and there's a // matching profiling request for this process (in which case heapprofd client // is loaded synchronously). // In both cases, the caller is responsible for verifying that the process is // considered profileable. // Previously installed default dispatch table, if it exists. This is used to // load heapprofd properly when GWP-ASan was already installed. If GWP-ASan was // already installed, heapprofd will take over the dispatch table, but will use // GWP-ASan as the backing dispatch. Writes to this variable is atomically // protected by MaybeModifyGlobals. // Reads are not protected, so this is atomic. We cannot fail the call in // MallocInitHeapprofdHook. static _Atomic (const MallocDispatch*) gPreviousDefaultDispatchTable = nullptr; static MallocDispatch gEphemeralDispatch; void HandleHeapprofdSignal() { if (atomic_load(&gHeapprofdState) == kIncompatibleHooks) { error_log("%s: not enabling heapprofd, malloc_debug/malloc_hooks are enabled.", getprogname()); return; } // We cannot grab the mutex here, as this is used in a signal handler. MaybeModifyGlobals(kWithoutLock, [] { MallocHeapprofdState expected = kInitialState; // If hooks are already installed, we still want to install ephemeral hooks to retrigger // heapprofd client initialization. MallocHeapprofdState expected2 = kHookInstalled; if (atomic_compare_exchange_strong(&gHeapprofdState, &expected, kInstallingEphemeralHook)) { const MallocDispatch* default_dispatch = GetDefaultDispatchTable(); // Below, we initialize heapprofd lazily by redirecting libc's malloc() to // call MallocInitHeapprofdHook, which spawns off a thread and initializes // heapprofd. During the short period between now and when heapprofd is // initialized, allocations may need to be serviced. There are three // possible configurations: if (DispatchIsGwpAsan(default_dispatch)) { // 1. GWP-ASan was installed. We should use GWP-ASan for everything but // malloc() in the interim period before heapprofd is properly // installed. After heapprofd is finished installing, we will use // GWP-ASan as heapprofd's backing allocator to allow heapprofd and // GWP-ASan to coexist. atomic_store(&gPreviousDefaultDispatchTable, default_dispatch); gEphemeralDispatch = *default_dispatch; } else { // Either, // 2. No malloc hooking has been done (heapprofd, GWP-ASan, etc.). In // this case, everything but malloc() should come from the system // allocator. // // or, // // 3. It may be possible at this point in time that heapprofd is // *already* the default dispatch, and when it was initialized there // was no default dispatch installed. As such we don't want to use // heapprofd as the backing store for itself (otherwise infinite // recursion occurs). We will use the system allocator functions. Note: // We've checked that no other malloc interceptors are being used by // validating `gHeapprofdIncompatibleHooks` above, so we don't need to // worry about that case here. atomic_store(&gPreviousDefaultDispatchTable, nullptr); gEphemeralDispatch = *NativeAllocatorDispatch(); } } else if (expected == kEphemeralHookInstalled) { // Nothing to do here. The ephemeral hook was installed, but // MallocInitHeapprofdHook() was never called. Since the ephemeral hook // is already there, no need to reinstall it. return; } else if (atomic_compare_exchange_strong(&gHeapprofdState, &expected2, kInstallingEphemeralHook)) { // if we still have hook installed, we can reuse the previous // decision. THIS IS REQUIRED FOR CORRECTNESS, because otherwise the // following can happen // 1. Assume DispatchIsGwpAsan(default_dispatch) // 2. This function is ran, sets gPreviousDefaultDispatchTable to // GWP ASan. // 3. The sessions ends, DispatchReset FAILS due to a race. Now // heapprofd hooks are default dispatch. // 4. We re-enter this function later. If we did NOT look at the // previously recorded gPreviousDefaultDispatchTable, we would // incorrectly reach case 3. below. // 5. The session ends, DispatchReset now resets the hooks to the // system allocator. This is incorrect. const MallocDispatch* prev_dispatch = atomic_load(&gPreviousDefaultDispatchTable); gEphemeralDispatch = prev_dispatch ? *prev_dispatch : *NativeAllocatorDispatch(); } else { error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingEphemeralHook. " "current state (possible race): %d", getprogname(), expected2); return; } // Now, replace the malloc function so that the next call to malloc() will // initialize heapprofd. gEphemeralDispatch.malloc = MallocInitHeapprofdHook; // And finally, install these new malloc-family interceptors. __libc_globals.mutate([](libc_globals* globals) { atomic_store(&globals->default_dispatch_table, &gEphemeralDispatch); if (!MallocLimitInstalled()) { atomic_store(&globals->current_dispatch_table, &gEphemeralDispatch); } }); atomic_store(&gHeapprofdState, kEphemeralHookInstalled); }); // Otherwise, we're racing against malloc_limit's enable logic (at most once // per process, and a niche feature). This is highly unlikely, so simply give // up if it does happen. } bool HeapprofdShouldLoad() { // First check for heapprofd.enable. If it is set to "all", enable // heapprofd for all processes. Otherwise, check heapprofd.enable.${prog}, // if it is set and not 0, enable heap profiling for this process. char property_value[PROP_VALUE_MAX]; if (__system_property_get(kHeapprofdPropertyEnable, property_value) == 0) { return false; } if (strcmp(property_value, "all") == 0) { return true; } char program_property[kHeapprofdProgramPropertyPrefixSize + kMaxCmdlineSize]; if (!GetHeapprofdProgramProperty(program_property, sizeof(program_property))) { return false; } if (__system_property_get(program_property, property_value) == 0) { return false; } return property_value[0] != '\0'; } void HeapprofdRememberHookConflict() { atomic_store(&gHeapprofdState, kIncompatibleHooks); } static void CommonInstallHooks(libc_globals* globals) { void* impl_handle = atomic_load(&gHeapprofdHandle); if (impl_handle == nullptr) { impl_handle = LoadSharedLibrary(kHeapprofdSharedLib, kHeapprofdPrefix, &globals->malloc_dispatch_table); if (impl_handle == nullptr) { return; } atomic_store(&gHeapprofdHandle, impl_handle); } else if (!InitSharedLibrary(impl_handle, kHeapprofdSharedLib, kHeapprofdPrefix, &globals->malloc_dispatch_table)) { return; } FinishInstallHooks(globals, nullptr, kHeapprofdPrefix); } void HeapprofdInstallHooksAtInit(libc_globals *globals) { // Before we set the new default_dispatch_table in FinishInstallHooks, save // the previous dispatch table. If DispatchReset() gets called later, we want // to be able to restore the dispatch. We're still under // MaybeModifyGlobals locks at this point. atomic_store(&gPreviousDefaultDispatchTable, GetDefaultDispatchTable()); MaybeModifyGlobals(kWithoutLock, [globals] { MallocHeapprofdState expected = kInitialState; if (atomic_compare_exchange_strong(&gHeapprofdState, &expected, kInstallingHook)) { CommonInstallHooks(globals); atomic_store(&gHeapprofdState, kHookInstalled); } else { error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingHook. " "current state (possible race): %d", getprogname(), expected); } }); } static void* InitHeapprofd(void*) { MaybeModifyGlobals(kWithLock, [] { MallocHeapprofdState expected = kInitialState; if (atomic_compare_exchange_strong(&gHeapprofdState, &expected, kInstallingHook)) { __libc_globals.mutate([](libc_globals* globals) { CommonInstallHooks(globals); }); atomic_store(&gHeapprofdState, kHookInstalled); } else { error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingHook. " "current state (possible race): %d", getprogname(), expected); } }); return nullptr; } extern "C" void* MallocInitHeapprofdHook(size_t bytes) { MaybeModifyGlobals(kWithLock, [] { MallocHeapprofdState expected = kEphemeralHookInstalled; if (atomic_compare_exchange_strong(&gHeapprofdState, &expected, kRemovingEphemeralHook)) { __libc_globals.mutate([](libc_globals* globals) { const MallocDispatch* previous_dispatch = atomic_load(&gPreviousDefaultDispatchTable); atomic_store(&globals->default_dispatch_table, previous_dispatch); if (!MallocLimitInstalled()) { atomic_store(&globals->current_dispatch_table, previous_dispatch); } }); atomic_store(&gHeapprofdState, kInitialState); pthread_t thread_id; if (pthread_create(&thread_id, nullptr, InitHeapprofd, nullptr) != 0) { error_log("%s: heapprofd: failed to pthread_create.", getprogname()); } else if (pthread_setname_np(thread_id, "heapprofdinit") != 0) { error_log("%s: heapprod: failed to pthread_setname_np", getprogname()); } else if (pthread_detach(thread_id) != 0) { error_log("%s: heapprofd: failed to pthread_detach", getprogname()); } } else { warning_log("%s: heapprofd: could not transition kEphemeralHookInstalled -> " "kRemovingEphemeralHook. current state (possible race): %d. this can be benign " "if two threads try this transition at the same time", getprogname(), expected); } }); // If we had a previous dispatch table, use that to service the allocation, // otherwise fall back to the native allocator. // This could be modified by a concurrent HandleHeapprofdSignal, but that is // benign as we will dispatch to the ephemeral handler, which will then dispatch // to the underlying one. const MallocDispatch* previous_dispatch = atomic_load(&gPreviousDefaultDispatchTable); if (previous_dispatch) { return previous_dispatch->malloc(bytes); } return NativeAllocatorDispatch()->malloc(bytes); } bool HeapprofdInitZygoteChildProfiling() { // Conditionally start "from startup" profiling. if (HeapprofdShouldLoad()) { // Directly call the signal handler codepath (properly protects against // concurrent invocations). HandleHeapprofdSignal(); } return true; } static bool DispatchReset() { if (atomic_load(&gHeapprofdState) == kInitialState) { return true; } bool success = false; MaybeModifyGlobals(kWithLock, [&success] { MallocHeapprofdState expected = kHookInstalled; if(atomic_compare_exchange_strong(&gHeapprofdState, &expected, kUninstallingHook)){ __libc_globals.mutate([](libc_globals* globals) { const MallocDispatch* previous_dispatch = atomic_load(&gPreviousDefaultDispatchTable); atomic_store(&globals->default_dispatch_table, previous_dispatch); if (!MallocLimitInstalled()) { atomic_store(&globals->current_dispatch_table, previous_dispatch); } }); atomic_store(&gHeapprofdState, kInitialState); success = true; } else { error_log("%s: heapprofd: failed to transition kHookInstalled -> kUninstallingHook. " "current state (possible race): %d", getprogname(), expected); } }); if (!success) { errno = EAGAIN; } return success; } bool HeapprofdMallopt(int opcode, void* arg, size_t arg_size) { if (opcode == M_RESET_HOOKS) { if (arg != nullptr || arg_size != 0) { errno = EINVAL; return false; } return DispatchReset(); } errno = ENOTSUP; return false; }