From 395978fcfd6ce25aae9383cb54179ec7dc86ce4c Mon Sep 17 00:00:00 2001 From: Roopesh Rajashekharaiah Nataraja Date: Tue, 23 Jan 2018 15:31:48 -0800 Subject: [PATCH 1/9] Add inaddr.h header file Change-Id: Iad92c39fb729538cf51bf9d9037b15515104b453 (cherry picked from commit 4c5bcdb9b015651c85cec5c9a87521837b30a5c4) Signed-off-by: SayantamGiri --- libc/include/arpa/inet.h | 1 + libc/include/bits/in_addr.h | 3 +-- libc/include/inaddr.h | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 libc/include/inaddr.h diff --git a/libc/include/arpa/inet.h b/libc/include/arpa/inet.h index db054c9e1d..7716b94457 100644 --- a/libc/include/arpa/inet.h +++ b/libc/include/arpa/inet.h @@ -33,6 +33,7 @@ #include #include #include +#include __BEGIN_DECLS diff --git a/libc/include/bits/in_addr.h b/libc/include/bits/in_addr.h index 30eb04b668..3e46dad2b0 100644 --- a/libc/include/bits/in_addr.h +++ b/libc/include/bits/in_addr.h @@ -36,8 +36,7 @@ #include #include -/** An integral type representing an IPv4 address. */ -typedef uint32_t in_addr_t; +#include /** A structure representing an IPv4 address. */ struct in_addr { diff --git a/libc/include/inaddr.h b/libc/include/inaddr.h new file mode 100644 index 0000000000..524addabf6 --- /dev/null +++ b/libc/include/inaddr.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _INADDR_H_ +#define _INADDR_H_ + +#include + +typedef uint32_t in_addr_t; + +#endif From 2de18b7707eb9876bec16f52a7f617a2dbee5b13 Mon Sep 17 00:00:00 2001 From: Rashed Abdel-Tawab Date: Sun, 15 Apr 2018 10:18:26 -0700 Subject: [PATCH 2/9] libc: Mark libstdc++ as vendor available A lot of blobs still link this even on 8.1, so allow devices to build a vendor copy of it. Change-Id: I2349478ec0507e3a5136fe89f15e7dc4bfc1a03e (cherry picked from commit a7b9a1b792e42ecc2c9846209bb56f370c73e6cd) Signed-off-by: SayantamGiri --- libc/Android.bp | 1 + 1 file changed, 1 insertion(+) diff --git a/libc/Android.bp b/libc/Android.bp index 97146aa6f8..7304aa2a2e 100644 --- a/libc/Android.bp +++ b/libc/Android.bp @@ -1998,6 +1998,7 @@ cc_library { name: "libstdc++", static_ndk_lib: true, static_libs: ["libasync_safe"], + vendor_available: true, static: { system_shared_libs: [], From 330c9ab3cf05bdca5599ce0caa7ba2299aef2ef6 Mon Sep 17 00:00:00 2001 From: "Christopher R. Palmer" Date: Sat, 8 Apr 2017 22:40:01 +0200 Subject: [PATCH 3/9] linker: Add support for dynamic SHIM libraries Author: Christopher R. Palmer Date: Tue Nov 3 16:44:44 2015 -0500 linker: Add support for dynamic "shim" libs Add a new environment variable LD_SHIM_LIBS that is a colon (":") separated list of vertical bar ("|") separated pairs. The pairs are the name for a soinfo reference (executable or shared library) followed by the name of the shim library to load. For example: LD_SHIM_LIBS=rmt_storage|libshim_ioprio.so:/system/lib/libicuuv.so|libshim_icu53.so will instruct the linker to load the dynamic library libshim_ioprio.so whenver rmt_storage is executed [*] and will load libshim_icu53.so whenever any executable or other shared library links against /system/lib/libicuuv.so. There are no restrictions against circular references. In this example, libshim_icu53.so can link against libicuuv.so which provides a simple and convenient means of adding compatibility symbols. [*] Note that the absolute path is not available to the linker and therefore using the name of executables does depend on the invocation and therefore should only be used if absolutely necessary. That is, running /system/bin/rmt_storage would not load any shim libs in this example because it does not match the name of the invocation of the command. If you have trouble determining the sonames being loaded, you can also set the environment variable LD_DEBUG=1 which will cause additional information to be logged to help trace the detection of the shim libs. Change-Id: I0ef80fa466167f7bcb7dac90842bef1c3cf879b6 Author: Christopher R. Palmer Date: Sun Nov 15 14:26:32 2015 -0500 linker: Fix the fact that shim libs do not properly call constructors Change-Id: I34333e13443a154e675b853fa41442351bc4243a Author: Christopher R. Palmer Date: Tue Dec 1 07:10:36 2015 -0500 linker: Don't try to walk the g_active_shim_libs when doing dlsym This is a bug in the original shim_lib implementation which was doing the shim lib resolution both when loading the libraries and when doing the dynamic symbol resolution. Change-Id: Ib2df0498cf551b3bbd37d7c351410b9908eb1795 Author: Christopher R. Palmer Date: Sun Nov 29 08:28:10 2015 -0500 linker: Reset the active shim libs each time we do a dlopen We use the active libs to avoid recursively trying to load the same library: A -> shimlibs add B -> depends on A -> shimlibs add B -> ... However, when we repeatedly dlopen the same library we need to reset the active shim libs to avoid failing to add B the second time we dlopen A. Change-Id: I27580e3d6a53858e8bca025d6c85f981cffbea06 Author: Danny Baumann Date: Fri Dec 11 10:29:16 2015 +0100 Make shim lib load failure non-fatal. Instead, print an appropriate warning message. Aborting symbol resolution on shim lib load failure leads to weird symbol lookup failures, because symbols in libraries referenced after the one loading the shim won't be loaded anymore without a log message stating why that happened. Change-Id: Ic3ad7095ddae7ea1039cb6a18603d5cde8a16143 Author: Christopher R. Palmer Date: Sat Dec 12 06:10:09 2015 -0500 bionic: Do not allow LD_SHIM_LIBS for setuid executables That's really not safe... Change-Id: If79af951830966fc21812cd0f60a8998a752a941 Author: Christopher R. Palmer Date: Sun Feb 14 11:38:44 2016 -0500 bionic: linker: Load shim libs *before* the self-linked libs By loading them earlier, this allows us to override a symbol in a library that is being directly linked. I believe this explains why some people have had problems shimming one lib but when the changet he shim to be against a different lib it magically works. It also makes it possible to override some symbols that were nearly impossible to override before this change. For example, it is pretty much impossible to override a symbol in libutils without this change because it's loaded almost everywhere so no matter where you try to place the shimming, it will be too late and the other symbol will have priority. In particularly, this is necessary to be able to correctly shim the VectorImpl symbols for dlx. Change-Id: I461ca416bc288e28035352da00fde5f34f8d9ffa Author: Chirayu Desai Date: Thu Aug 25 19:02:41 2016 +0530 linker: Update find_library call for shimlibs commits 0cdef7e7f3c6837b56a969120d9098463d1df8d8 "Respect caller DT_RUNPATH in dlopen()." and 42d5fcb9f494eb45de3b6bf759f4a18076e84728 "Introducing linker namespaces" added new arguments to find_library, add them here. Change-Id: I8f35a45b00d14f8b2ce01a0a96d2dc7759be04a6 Author: Chippa-a Date: Sat Aug 27 14:56:30 2016 +0200 linker: Update LD_SHIM_LIBS parser function * Upgrade the code using the same changes as 42d5fcb9f494eb45de3b6bf759f4a18076e84728 bda20e78f0f314dbbf0f0bbcf0740cf2d6a4b85e Change-Id: Ic8be0871945bd9feccd0f94a6770f3cc78a70a0f Author: Danny Baumann Date: Wed Sep 7 16:54:06 2016 +0200 Inject shim libs as if they were DT_NEEDED. The previous separate approach had one flaw: If the shim lib requires another lib that's already loaded, find_library_internal() would return the previously loaded copy, but the later load action would fail as the ELF reader map of the initial loading round was already discarded and thus a new ElfReader instance for the soinfo instance was created, which didn't know about the previous reading/loading state. Change-Id: Ib224dbd35d114197097e3dee14a077cc9130fedb Author: jrior001 Date: Fri Oct 7 19:36:51 2016 -0400 linker: load shims prior to DT_NEEDED check This allows shims to override existing symbols, not just inject new symbols. Change-Id: Ib9216bcc651d8d38999c593babb94d76dc1dbc95 Author: Adrian DC Date: Sat, 8 Apr 2017 22:40:01 +0200 * Adapt to latest AOSP Oreo bionic linker changes * Additional header to avoid unused function Change-Id: Ib9216bcc651d8d38999c593babb94d76dc1dbc95 Author: Paul Keith Date: Thu Feb 15 21:57:33 2018 +0100 linker: Move shims to TARGET_LD_SHIM_LIBS * To reduce security exposure, let's set this at compile time, and block off all the code unless the board flag is set Change-Id: Ieec5f5d9e0f39a798fd48eae037ecffe9502474c Author: Nich Date: Fri Jun 8 09:48:17 2018 +0800 linker: Provide soinfo path of the shimmed binary This is a forward port of part of the original change that was missed out since the initial port of the shim logic to O. Change-Id: I1f7ff98472cfef5cb2d2bcb303082784898cd0c6 Author: Nich Date: Tue Jun 5 13:36:43 2018 +0800 linker: Remove unused find_libraries declaration commit "Inject shim libs as if they were DT_NEEDED." removed references to the forward declaration. Change-Id: I5f1aaa3a96f2af3edef07d4ea4e204b586424631 Author: Nich Date: Sun Jun 10 00:45:51 2018 +0800 linker: Make shim reference path absolute This way, we can filter out non-existent binaries, and ensure we get its absolute path before matching with get_realpath(). This for one allows the use of symlinks in TARGET_LD_SHIM_LIBS. Change-Id: I823815271b3257965534b6b87d8ea36ffb68bc08 Author: Nich Date: Fri Jun 15 03:59:05 2018 +0800 linker: Ensure active matching pairs Change-Id: I54c666b4560dbfb40839b0bf9132a7fd8d3ed2dd Author: Nich Date: Thu Jun 21 01:58:10 2018 +0800 linker: Don't involve shim in for_each_dt_needed for_each_dt_needed may have other usages that shouldn't involve the shim, for example, in the unloading of soinfos. Change-Id: Id38de183d90c3f707767bdca032a5ea2bc82fde8 Author: Jiyong Park Date: Fri Jan 25 18:18:01 2019 +0900 Call realpath(3) only when the path is accessible for read Suppress the SELinux denial log spam by not calling realpath(3) when the path does not exist or is not accessible for read, and then not auditing access(2) failure. Change-Id: I729ecb8ea0bb581069eb849bae7cd28e6ab636cc Change-Id: Ic3ad7095ddae7ea1039cb6a18603d5cde8a16152 Signed-off-by: Wang Han <416810799@qq.com> Signed-off-by: SayantamGiri --- linker/Android.bp | 3 ++ linker/linker.cpp | 71 ++++++++++++++++++++++++++++++++++++++++++ linker/linker.h | 20 ++++++++++++ linker/linker_main.cpp | 11 +++++++ 4 files changed, 105 insertions(+) diff --git a/linker/Android.bp b/linker/Android.bp index d5e7367f4f..63d4daee83 100644 --- a/linker/Android.bp +++ b/linker/Android.bp @@ -70,6 +70,9 @@ cc_object { // Configuration for the linker binary and any of its static libraries. cc_defaults { name: "linker_defaults", + defaults: [ + "shim_libs_defaults", + ], arch: { arm: { cflags: ["-D__work_around_b_24465209__"], diff --git a/linker/linker.cpp b/linker/linker.cpp index c6588d2cd4..5df379936e 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -655,6 +655,68 @@ enum walk_action_result_t : uint32_t { kWalkSkip = 2 }; +#ifdef LD_SHIM_LIBS +// g_ld_all_shim_libs maintains the references to memory as it used +// in the soinfo structures and in the g_active_shim_libs list. + +static std::vector g_ld_all_shim_libs; + +// g_active_shim_libs are all shim libs that are still eligible +// to be loaded. We must remove a shim lib from the list before +// we load the library to avoid recursive loops (load shim libA +// for libB where libA also links against libB). +static linked_list_t g_active_shim_libs; + +static void reset_g_active_shim_libs(void) { + g_active_shim_libs.clear(); + for (const auto& pair : g_ld_all_shim_libs) { + g_active_shim_libs.push_back(&pair); + } +} + +void parse_LD_SHIM_LIBS(const char* path) { + g_ld_all_shim_libs.clear(); + if (path != nullptr) { + for (const auto& pair : android::base::Split(path, ":")) { + std::vector pieces = android::base::Split(pair, "|"); + if (pieces.size() != 2) continue; + // If the path can be resolved, resolve it + char buf[PATH_MAX]; + std::string resolved_path = pieces[0]; + if (access(pieces[0].c_str(), R_OK) != 0) { + if (errno == ENOENT) { + // no need to test for non-existing path. skip. + continue; + } + // If not accessible, don't call realpath as it will just cause + // SELinux denial spam. Use the path unresolved. + } else if (realpath(pieces[0].c_str(), buf) != nullptr) { + resolved_path = buf; + } + auto desc = std::pair(resolved_path, pieces[1]); + g_ld_all_shim_libs.push_back(desc); + } + } + reset_g_active_shim_libs(); +} + +std::vector shim_matching_pairs(const char* path) { + std::vector matched_pairs; + + g_active_shim_libs.for_each([&](const ShimDescriptor* a_pair) { + if (a_pair->first == path) { + matched_pairs.push_back(a_pair); + } + }); + + g_active_shim_libs.remove_if([&](const ShimDescriptor* a_pair) { + return a_pair->first == path; + }); + + return matched_pairs; +} +#endif + // This function walks down the tree of soinfo dependencies // in breadth-first order and // * calls action(soinfo* si) for each node, and @@ -1269,6 +1331,12 @@ static bool load_library(android_namespace_t* ns, } #endif +#ifdef LD_SHIM_LIBS + for_each_matching_shim(realpath.c_str(), [&](const char* name) { + load_tasks->push_back(LoadTask::create(name, si, ns, task->get_readers_map())); + }); +#endif + for_each_dt_needed(task->get_elf_reader(), [&](const char* name) { LD_LOG(kLogDlopen, "load_library(ns=%s, task=%s): Adding DT_NEEDED task: %s", ns->get_name(), task->get_name(), name); @@ -2158,6 +2226,9 @@ void* do_dlopen(const char* name, int flags, } ProtectedDataGuard guard; +#ifdef LD_SHIM_LIBS + reset_g_active_shim_libs(); +#endif soinfo* si = find_library(ns, translated_name, flags, extinfo, caller); loading_trace.End(); diff --git a/linker/linker.h b/linker/linker.h index a80342479c..17126c6dde 100644 --- a/linker/linker.h +++ b/linker/linker.h @@ -42,6 +42,10 @@ #include "linker_logger.h" #include "linker_soinfo.h" +#ifdef LD_SHIM_LIBS +#include "linker_debug.h" +#endif + #include #include @@ -81,6 +85,22 @@ soinfo* find_containing_library(const void* p); int open_executable(const char* path, off64_t* file_offset, std::string* realpath); +#ifdef LD_SHIM_LIBS +typedef std::pair ShimDescriptor; +void parse_LD_SHIM_LIBS(const char* path); +std::vector shim_matching_pairs(const char* path); + +template +void for_each_matching_shim(const char* path, F action) { + if (path == nullptr) return; + INFO("Finding shim libs for \"%s\"", path); + for (const auto& one_pair : shim_matching_pairs(path)) { + INFO("Injecting shim lib \"%s\" as needed for %s", one_pair->second.c_str(), path); + action(one_pair->second.c_str()); + } +} +#endif + void do_android_get_LD_LIBRARY_PATH(char*, size_t); void do_android_update_LD_LIBRARY_PATH(const char* ld_library_path); void* do_dlopen(const char* name, diff --git a/linker/linker_main.cpp b/linker/linker_main.cpp index 9e5be345db..73947e161f 100644 --- a/linker/linker_main.cpp +++ b/linker/linker_main.cpp @@ -447,6 +447,11 @@ static ElfW(Addr) linker_main(KernelArgumentBlock& args, const char* exe_to_load parse_LD_LIBRARY_PATH(ldpath_env); parse_LD_PRELOAD(ldpreload_env); +#ifdef LD_SHIM_LIBS + // Read from TARGET_LD_SHIM_LIBS + parse_LD_SHIM_LIBS(LD_SHIM_LIBS); +#endif + std::vector namespaces = init_default_namespaces(exe_info.path.c_str()); if (!si->prelink_image()) __linker_cannot_link(g_argv[0]); @@ -472,6 +477,12 @@ static ElfW(Addr) linker_main(KernelArgumentBlock& args, const char* exe_to_load ++ld_preloads_count; } +#ifdef LD_SHIM_LIBS + for_each_matching_shim(si->get_realpath(), [&](const char* name) { + needed_library_name_list.push_back(name); + }); +#endif + for_each_dt_needed(si, [&](const char* name) { needed_library_name_list.push_back(name); }); From c354ffe5ff718855368d8b8f25ecece3491c9999 Mon Sep 17 00:00:00 2001 From: Quallenauge Date: Tue, 20 Oct 2020 21:19:01 +0200 Subject: [PATCH 4/9] Add ability to build scudo-free 32-bit libc variant. Scudo seems to have issues with camera blobs, which also the case with google devices. https://android-review.googlesource.com/q/topic:%22disable-camera24-memory-mitigations%22+(status:open%20OR%20status:merged) These changes are applied upstream and needed much more commits in various components which makes it difficult to backport. Therefore add option to disable scudo only for 32 bit components, like camera modules. Change-Id: Ie4e62477b0801413827007c511e547b12ea2f46d Signed-off-by: SayantamGiri --- libc/Android.bp | 54 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/libc/Android.bp b/libc/Android.bp index 7304aa2a2e..544a1e0b46 100644 --- a/libc/Android.bp +++ b/libc/Android.bp @@ -119,15 +119,32 @@ cc_defaults { // warning since this is intended right now. ldflags: ["-Wl,-z,muldefs"], - product_variables: { - malloc_zero_contents: { - cflags: ["-DSCUDO_ZERO_CONTENTS"], - }, - malloc_pattern_fill_contents: { - cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"], + multilib: { + lib64: { + product_variables: { + malloc_zero_contents: { + cflags: ["-DSCUDO_ZERO_CONTENTS"], + }, + malloc_pattern_fill_contents: { + cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"], + }, + malloc_not_svelte: { + cflags: ["-DUSE_SCUDO"], + }, + }, }, - malloc_not_svelte: { - cflags: ["-DUSE_SCUDO"], + lib32: { + product_variables: { + malloc_zero_contents: { + cflags: ["-DSCUDO_ZERO_CONTENTS"], + }, + malloc_pattern_fill_contents: { + cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"], + }, + malloc_not_svelte_libc32: { + cflags: ["-DUSE_SCUDO"], + }, + }, }, }, @@ -147,6 +164,18 @@ libc_scudo_product_variables = { }, } +libc32_scudo_product_variables = { + malloc_not_svelte_libc32: { + cflags: ["-DUSE_SCUDO"], + whole_static_libs: ["libscudo"], + exclude_static_libs: [ + "libjemalloc5", + "libc_jemalloc_wrapper", + ], + }, +} + + // Defaults for native allocator libs/includes to make it // easier to change. // To disable scudo for the non-svelte config remove the line: @@ -161,7 +190,14 @@ cc_defaults { "libc_jemalloc_wrapper", ], header_libs: ["gwp_asan_headers"], - product_variables: libc_scudo_product_variables, + multilib: { + lib64: { + product_variables: libc_scudo_product_variables, + }, + lib32: { + product_variables: libc32_scudo_product_variables, + } + }, } // Functions not implemented by jemalloc directly, or that need to From d314594a9db868ac8f0bbd8ad2ba6f0bc0ef731a Mon Sep 17 00:00:00 2001 From: Tom Marshall Date: Fri, 17 Jun 2016 16:38:12 -0700 Subject: [PATCH 5/9] bionic: Sort and cache hosts file data for fast lookup The hosts file is normally searched linearly. This is very slow when the file is large. To mitigate this, read the hosts file and sort the entries in an in-memory cache. When an address is requested via gethostbyname or getaddrinfo, binary search the cache. In case where the cache is not available, return a suitable error code and fall back to the existing lookup code. This has been written to behave as much like the existing lookup code as possible. But note bionic and glibc differ in behavior for some corner cases. Choose the most standard compliant behavior for these where possible. Otherwise choose the behavior that seems most reasonable. RM-290 Change-Id: I3b322883cbc48b0d76a0ce9d149b59faaac1dc58 (cherry picked from commit ed4c3a6bd449a4ed70645071a440ae146f194116) Signed-off-by: SayantamGiri --- libc/dns/net/getaddrinfo.c | 10 + libc/dns/net/hosts_cache.c | 520 +++++++++++++++++++++++++++++++++++++ libc/dns/net/hosts_cache.h | 23 ++ libc/dns/net/sethostent.c | 7 + 4 files changed, 560 insertions(+) create mode 100644 libc/dns/net/hosts_cache.c create mode 100644 libc/dns/net/hosts_cache.h diff --git a/libc/dns/net/getaddrinfo.c b/libc/dns/net/getaddrinfo.c index d0c11d2b03..cc94b21e27 100644 --- a/libc/dns/net/getaddrinfo.c +++ b/libc/dns/net/getaddrinfo.c @@ -109,6 +109,8 @@ #include "nsswitch.h" #include "private/bionic_defs.h" +#include "hosts_cache.h" + typedef union sockaddr_union { struct sockaddr generic; struct sockaddr_in in; @@ -2125,6 +2127,14 @@ _files_getaddrinfo(void *rv, void *cb_data, va_list ap) name = va_arg(ap, char *); pai = va_arg(ap, struct addrinfo *); + memset(&sentinel, 0, sizeof(sentinel)); + cur = &sentinel; + int gai_error = hc_getaddrinfo(name, NULL, pai, &cur); + if (gai_error != EAI_SYSTEM) { + *((struct addrinfo **)rv) = sentinel.ai_next; + return (gai_error == 0 ? NS_SUCCESS : NS_NOTFOUND); + } + // fprintf(stderr, "_files_getaddrinfo() name = '%s'\n", name); memset(&sentinel, 0, sizeof(sentinel)); cur = &sentinel; diff --git a/libc/dns/net/hosts_cache.c b/libc/dns/net/hosts_cache.c new file mode 100644 index 0000000000..52d29e0320 --- /dev/null +++ b/libc/dns/net/hosts_cache.c @@ -0,0 +1,520 @@ +/* + * Copyright (C) 2016 The CyanogenMod Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "hostent.h" +#include "resolv_private.h" + +#define MAX_ADDRLEN (INET6_ADDRSTRLEN - (1 + 5)) +#define MAX_HOSTLEN MAXHOSTNAMELEN + +#define ESTIMATED_LINELEN 32 +#define HCFILE_ALLOC_SIZE 256 + +/* + * Host cache entry for hcfile.c_data. + * Offsets are into hcfile.h_data. + * Strings are *not* terminated by NULL, but by whitespace (isspace) or '#'. + * Use hstr* functions with these. + */ +struct hcent +{ + uint32_t addr; + uint32_t name; +}; + +/* + * Overall host cache file state. + */ +struct hcfile +{ + int h_fd; + struct stat h_st; + char *h_data; + + uint32_t c_alloc; + uint32_t c_len; + struct hcent *c_data; +}; +static struct hcfile hcfile; +static pthread_mutex_t hclock = PTHREAD_MUTEX_INITIALIZER; + +static size_t hstrlen(const char *s) +{ + const char *p = s; + while (*p && *p != '#' && !isspace(*p)) + ++p; + return p - s; +} + +static int hstrcmp(const char *a, const char *b) +{ + size_t alen = hstrlen(a); + size_t blen = hstrlen(b); + int res = strncmp(a, b, MIN(alen, blen)); + if (res == 0) + res = alen - blen; + return res; +} + +static char *hstrcpy(char *dest, const char *src) +{ + size_t len = hstrlen(src); + memcpy(dest, src, len); + dest[len] = '\0'; + return dest; +} + +static char *hstrdup(const char *s) +{ + size_t len = hstrlen(s); + char *dest = (char *)malloc(len + 1); + if (!dest) + return NULL; + memcpy(dest, s, len); + dest[len] = '\0'; + return dest; +} + +static int cmp_hcent_name(const void *a, const void *b) +{ + struct hcent *ea = (struct hcent *)a; + const char *na = hcfile.h_data + ea->name; + struct hcent *eb = (struct hcent *)b; + const char *nb = hcfile.h_data + eb->name; + + return hstrcmp(na, nb); +} + +static struct hcent *_hcfindname(const char *name) +{ + size_t first, last, mid; + struct hcent *cur = NULL; + int cmp; + + if (hcfile.c_len == 0) + return NULL; + + first = 0; + last = hcfile.c_len - 1; + mid = (first + last) / 2; + while (first <= last) { + cur = hcfile.c_data + mid; + cmp = hstrcmp(hcfile.h_data + cur->name, name); + if (cmp == 0) + goto found; + if (cmp < 0) + first = mid + 1; + else { + if (mid > 0) + last = mid - 1; + else + return NULL; + } + mid = (first + last) / 2; + } + return NULL; + +found: + while (cur > hcfile.c_data) { + struct hcent *prev = cur - 1; + cmp = cmp_hcent_name(cur, prev); + if (cmp) + break; + cur = prev; + } + + return cur; +} + +/* + * Find next name on line, if any. + * + * Assumes that line is terminated by LF. + */ +static const char *_hcnextname(const char *name) +{ + while (!isspace(*name)) { + if (*name == '#') + return NULL; + ++name; + } + while (isspace(*name)) { + if (*name == '\n') + return NULL; + ++name; + } + if (*name == '#') + return NULL; + return name; +} + +static int _hcfilemmap(void) +{ + struct stat st; + int h_fd; + char *h_addr; + const char *p, *pend; + uint32_t c_alloc; + + h_fd = open(_PATH_HOSTS, O_RDONLY); + if (h_fd < 0) + return -1; + if (flock(h_fd, LOCK_EX) != 0) { + close(h_fd); + return -1; + } + + if (hcfile.h_data) { + memset(&st, 0, sizeof(st)); + if (fstat(h_fd, &st) == 0) { + if (st.st_size == hcfile.h_st.st_size && + st.st_mtime == hcfile.h_st.st_mtime) { + flock(h_fd, LOCK_UN); + close(h_fd); + return 0; + } + } + free(hcfile.c_data); + munmap(hcfile.h_data, hcfile.h_st.st_size); + close(hcfile.h_fd); + memset(&hcfile, 0, sizeof(struct hcfile)); + } + + if (fstat(h_fd, &st) != 0) { + flock(h_fd, LOCK_UN); + close(h_fd); + return -1; + } + h_addr = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, h_fd, 0); + if (h_addr == MAP_FAILED) { + flock(h_fd, LOCK_UN); + close(h_fd); + return -1; + } + + hcfile.h_fd = h_fd; + hcfile.h_st = st; + hcfile.h_data = h_addr; + + c_alloc = 0; + /* + * Do an initial allocation if the file is "large". Estimate + * 32 bytes per line and define "large" as more than half of + * the alloc growth size (256 entries). + */ + if (st.st_size >= ESTIMATED_LINELEN * HCFILE_ALLOC_SIZE / 2) { + c_alloc = st.st_size / ESTIMATED_LINELEN; + hcfile.c_data = malloc(c_alloc * sizeof(struct hcent)); + if (!hcfile.c_data) { + goto oom; + } + } + + p = (const char *)h_addr; + pend = p + st.st_size; + while (p < pend) { + const char *eol, *addr, *name; + size_t len; + addr = p; + eol = memchr(p, '\n', pend - p); + if (!eol) + break; + p = eol + 1; + if (*addr == '#' || *addr == '\n') + continue; + len = hstrlen(addr); + if (len > MAX_ADDRLEN) + continue; + name = addr + len; + while (name < eol && isspace(*name)) + ++name; + while (name < eol) { + len = hstrlen(name); + if (len == 0) + break; + if (len < MAX_HOSTLEN) { + struct hcent *ent; + if (c_alloc <= hcfile.c_len) { + struct hcent *c_data; + c_alloc += HCFILE_ALLOC_SIZE; + c_data = realloc(hcfile.c_data, c_alloc * sizeof(struct hcent)); + if (!c_data) { + goto oom; + } + hcfile.c_data = c_data; + } + ent = hcfile.c_data + hcfile.c_len; + ent->addr = addr - h_addr; + ent->name = name - h_addr; + ++hcfile.c_len; + } + name += len; + while (name < eol && isspace(*name)) + ++name; + } + } + + qsort(hcfile.c_data, hcfile.c_len, + sizeof(struct hcent), cmp_hcent_name); + + flock(h_fd, LOCK_UN); + + return 0; + +oom: + free(hcfile.c_data); + munmap(hcfile.h_data, hcfile.h_st.st_size); + flock(hcfile.h_fd, LOCK_UN); + close(hcfile.h_fd); + memset(&hcfile, 0, sizeof(struct hcfile)); + return -1; +} + +/* + * Caching version of getaddrinfo. + * + * If we find the requested host name in the cache, use getaddrinfo to + * populate the result for each address we find. + * + * Note glibc and bionic differ in the handling of ai_canonname. POSIX + * says that ai_canonname is only populated in the first result entry. + * glibc does this. bionic populates ai_canonname in all result entries. + * We choose the POSIX/glibc way here. + */ +int hc_getaddrinfo(const char *host, const char *service, + const struct addrinfo *hints, + struct addrinfo **result) +{ + int ret = 0; + struct hcent *ent, *cur; + struct addrinfo *ai; + struct addrinfo rhints; + struct addrinfo *last; + int canonname = 0; + int cmp; + + if (getenv("ANDROID_HOSTS_CACHE_DISABLE") != NULL) + return EAI_SYSTEM; + + /* Avoid needless work and recursion */ + if (hints && (hints->ai_flags & AI_NUMERICHOST)) + return EAI_SYSTEM; + if (!host) + return EAI_SYSTEM; + + pthread_mutex_lock(&hclock); + + if (_hcfilemmap() != 0) { + ret = EAI_SYSTEM; + goto out; + } + ent = _hcfindname(host); + if (!ent) { + ret = EAI_NONAME; + goto out; + } + + if (hints) { + canonname = (hints->ai_flags & AI_CANONNAME); + memcpy(&rhints, hints, sizeof(rhints)); + rhints.ai_flags &= ~AI_CANONNAME; + } + else { + memset(&rhints, 0, sizeof(rhints)); + } + rhints.ai_flags |= AI_NUMERICHOST; + + last = NULL; + cur = ent; + do { + char addrstr[MAX_ADDRLEN]; + struct addrinfo *res; + + hstrcpy(addrstr, hcfile.h_data + cur->addr); + + if (getaddrinfo(addrstr, service, &rhints, &res) == 0) { + if (!last) + (*result)->ai_next = res; + else + last->ai_next = res; + last = res; + while (last->ai_next) + last = last->ai_next; + } + + if(cur + 1 >= hcfile.c_data + hcfile.c_len) + break; + cmp = cmp_hcent_name(cur, cur + 1); + cur = cur + 1; + } + while (!cmp); + + if (last == NULL) { + /* This check is equivalent to (*result)->ai_next == NULL */ + ret = EAI_NODATA; + goto out; + } + + if (canonname) { + ai = (*result)->ai_next; + free(ai->ai_canonname); + ai->ai_canonname = hstrdup(hcfile.h_data + ent->name); + } + +out: + pthread_mutex_unlock(&hclock); + return ret; +} + +/* + * Caching version of gethtbyname. + * + * Note glibc and bionic differ in the handling of aliases. glibc returns + * all aliases for all entries, regardless of whether they match h_addrtype. + * bionic returns only the aliases for the first hosts entry. We return all + * aliases for all IPv4 entries. + * + * Additionally, if an alias is IPv6 and the primary name for an alias also + * has an IPv4 entry, glibc will return the IPv4 address(es), but bionic + * will not. Neither do we. + */ +int hc_gethtbyname(const char *host, int af, struct getnamaddr *info) +{ + int ret = NETDB_SUCCESS; + struct hcent *ent, *cur; + int cmp; + size_t addrlen; + unsigned int naliases = 0; + char *aliases[MAXALIASES]; + unsigned int naddrs = 0; + char *addr_ptrs[MAXADDRS]; + unsigned int n; + + if (getenv("ANDROID_HOSTS_CACHE_DISABLE") != NULL) + return NETDB_INTERNAL; + + switch (af) { + case AF_INET: addrlen = NS_INADDRSZ; break; + case AF_INET6: addrlen = NS_IN6ADDRSZ; break; + default: + return NETDB_INTERNAL; + } + + pthread_mutex_lock(&hclock); + + if (_hcfilemmap() != 0) { + ret = NETDB_INTERNAL; + goto out; + } + + ent = _hcfindname(host); + if (!ent) { + ret = HOST_NOT_FOUND; + goto out; + } + + cur = ent; + do { + char addr[16]; + char addrstr[MAX_ADDRLEN]; + char namestr[MAX_HOSTLEN]; + const char *name; + + hstrcpy(addrstr, hcfile.h_data + cur->addr); + if (inet_pton(af, addrstr, &addr) == 1) { + char *aligned; + /* First match is considered the official hostname */ + if (naddrs == 0) { + hstrcpy(namestr, hcfile.h_data + cur->name); + HENT_SCOPY(info->hp->h_name, namestr, info->buf, info->buflen); + } + for (name = hcfile.h_data + cur->name; name; name = _hcnextname(name)) { + if (!hstrcmp(name, host)) + continue; + hstrcpy(namestr, name); + HENT_SCOPY(aliases[naliases], namestr, info->buf, info->buflen); + ++naliases; + if (naliases >= MAXALIASES) + goto nospc; + } + aligned = (char *)ALIGN(info->buf); + if (info->buf != aligned) { + if ((ptrdiff_t)info->buflen < (aligned - info->buf)) + goto nospc; + info->buflen -= (aligned - info->buf); + info->buf = aligned; + } + HENT_COPY(addr_ptrs[naddrs], addr, addrlen, info->buf, info->buflen); + ++naddrs; + if (naddrs >= MAXADDRS) + goto nospc; + } + + if(cur + 1 >= hcfile.c_data + hcfile.c_len) + break; + cmp = cmp_hcent_name(cur, cur + 1); + cur = cur + 1; + } + while (!cmp); + + if (naddrs == 0) { + ret = HOST_NOT_FOUND; + goto out; + } + + addr_ptrs[naddrs++] = NULL; + aliases[naliases++] = NULL; + + /* hp->h_name already populated */ + HENT_ARRAY(info->hp->h_aliases, naliases, info->buf, info->buflen); + for (n = 0; n < naliases; ++n) { + info->hp->h_aliases[n] = aliases[n]; + } + info->hp->h_addrtype = af; + info->hp->h_length = addrlen; + HENT_ARRAY(info->hp->h_addr_list, naddrs, info->buf, info->buflen); + for (n = 0; n < naddrs; ++n) { + info->hp->h_addr_list[n] = addr_ptrs[n]; + } + +out: + pthread_mutex_unlock(&hclock); + *info->he = ret; + return ret; + +nospc: + ret = NETDB_INTERNAL; + goto out; +} diff --git a/libc/dns/net/hosts_cache.h b/libc/dns/net/hosts_cache.h new file mode 100644 index 0000000000..fa5488f515 --- /dev/null +++ b/libc/dns/net/hosts_cache.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2016 The CyanogenMod Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +struct getnamaddr; + +int hc_getaddrinfo(const char *host, const char *service, + const struct addrinfo *hints, + struct addrinfo **result); + +int hc_gethtbyname(const char *host, int af, struct getnamaddr *info); diff --git a/libc/dns/net/sethostent.c b/libc/dns/net/sethostent.c index 483105a95f..1399378cd1 100644 --- a/libc/dns/net/sethostent.c +++ b/libc/dns/net/sethostent.c @@ -55,6 +55,8 @@ __RCSID("$NetBSD: sethostent.c,v 1.20 2014/03/17 13:24:23 christos Exp $"); #include "hostent.h" #include "resolv_private.h" +#include "hosts_cache.h" + #ifndef _REENTRANT void res_close(void); #endif @@ -109,6 +111,11 @@ _hf_gethtbyname(void *rv, void *cb_data, va_list ap) /* NOSTRICT skip string len */(void)va_arg(ap, int); af = va_arg(ap, int); + int rc = hc_gethtbyname(name, af, info); + if (rc != NETDB_INTERNAL) { + return (rc == NETDB_SUCCESS ? NS_SUCCESS : NS_NOTFOUND); + } + #if 0 { res_state res = __res_get_state(); From 14c50c30966b2104f21c5a044d5b733979b2c7e1 Mon Sep 17 00:00:00 2001 From: Tom Marshall Date: Thu, 16 Jan 2020 13:07:04 -0800 Subject: [PATCH 6/9] bionic: Support wildcards in cached hosts file If an exact name is not found in the hosts file and the host name contains at least one dot, search for entries of the form "*.domain", where domain is the portion of the host name after the first dot. If that is not found, repeat using the domain. Example: a.b.c.example.com would search for the following in turn: a.b.c.example.com *.b.c.example.com *.c.example.com *.example.com *.com Change-Id: I4b0bb81699151d5b371850daebf785e35ec9b180 Signed-off-by: SayantamGiri --- libc/dns/net/hosts_cache.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/libc/dns/net/hosts_cache.c b/libc/dns/net/hosts_cache.c index 52d29e0320..fc6370d0c1 100644 --- a/libc/dns/net/hosts_cache.c +++ b/libc/dns/net/hosts_cache.c @@ -117,7 +117,7 @@ static int cmp_hcent_name(const void *a, const void *b) return hstrcmp(na, nb); } -static struct hcent *_hcfindname(const char *name) +static struct hcent *_hcfindname_exact(const char *name) { size_t first, last, mid; struct hcent *cur = NULL; @@ -158,6 +158,33 @@ static struct hcent *_hcfindname(const char *name) return cur; } +static struct hcent *_hcfindname(const char *name) +{ + struct hcent *ent; + char namebuf[MAX_HOSTLEN]; + char *p; + char *dot; + + ent = _hcfindname_exact(name); + if (!ent && strlen(name) < sizeof(namebuf)) { + strcpy(namebuf, name); + p = namebuf; + do { + dot = strchr(p, '.'); + if (!dot) + break; + if (dot > p) { + *(dot - 1) = '*'; + ent = _hcfindname_exact(dot - 1); + } + p = dot + 1; + } + while (!ent); + } + + return ent; +} + /* * Find next name on line, if any. * From e4d7836ed3058b98861ac4b3d02d5ea6e991e3fe Mon Sep 17 00:00:00 2001 From: SayantamGiri Date: Sat, 22 Apr 2023 10:30:52 +0200 Subject: [PATCH 7/9] Revert "Add ability to build scudo-free 32-bit libc variant." This reverts commit 5cd482a14a172fcfc426b1b4ea9879f31a7da38c. --- libc/Android.bp | 54 +++++++++---------------------------------------- 1 file changed, 9 insertions(+), 45 deletions(-) diff --git a/libc/Android.bp b/libc/Android.bp index 544a1e0b46..7304aa2a2e 100644 --- a/libc/Android.bp +++ b/libc/Android.bp @@ -119,32 +119,15 @@ cc_defaults { // warning since this is intended right now. ldflags: ["-Wl,-z,muldefs"], - multilib: { - lib64: { - product_variables: { - malloc_zero_contents: { - cflags: ["-DSCUDO_ZERO_CONTENTS"], - }, - malloc_pattern_fill_contents: { - cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"], - }, - malloc_not_svelte: { - cflags: ["-DUSE_SCUDO"], - }, - }, + product_variables: { + malloc_zero_contents: { + cflags: ["-DSCUDO_ZERO_CONTENTS"], }, - lib32: { - product_variables: { - malloc_zero_contents: { - cflags: ["-DSCUDO_ZERO_CONTENTS"], - }, - malloc_pattern_fill_contents: { - cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"], - }, - malloc_not_svelte_libc32: { - cflags: ["-DUSE_SCUDO"], - }, - }, + malloc_pattern_fill_contents: { + cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"], + }, + malloc_not_svelte: { + cflags: ["-DUSE_SCUDO"], }, }, @@ -164,18 +147,6 @@ libc_scudo_product_variables = { }, } -libc32_scudo_product_variables = { - malloc_not_svelte_libc32: { - cflags: ["-DUSE_SCUDO"], - whole_static_libs: ["libscudo"], - exclude_static_libs: [ - "libjemalloc5", - "libc_jemalloc_wrapper", - ], - }, -} - - // Defaults for native allocator libs/includes to make it // easier to change. // To disable scudo for the non-svelte config remove the line: @@ -190,14 +161,7 @@ cc_defaults { "libc_jemalloc_wrapper", ], header_libs: ["gwp_asan_headers"], - multilib: { - lib64: { - product_variables: libc_scudo_product_variables, - }, - lib32: { - product_variables: libc32_scudo_product_variables, - } - }, + product_variables: libc_scudo_product_variables, } // Functions not implemented by jemalloc directly, or that need to From fca69399fbb71c863e7014cbda0d7bdbbd32a8f4 Mon Sep 17 00:00:00 2001 From: Danny Lin Date: Sat, 20 Mar 2021 22:02:38 -0700 Subject: [PATCH 8/9] Switch to jemalloc memory allocator Overall, jemalloc performs significantly better than Scudo in Bionic's real-world memory_replay traces (all times in milliseconds): +----------------+-------+----------+ | Trace | Scudo | jemalloc | +----------------+-------+----------+ | SQL | 27 | 21 | | Angry Birds 2 | 2236 | 1501 | | Camera | 4251 | 979 | | Candy Crush | 2197 | 1325 | | Gmail | 594 | 463 | | Maps | 434 | 344 | | Photos | 1330 | 477 | | PUBG | 666 | 416 | | surfaceflinger | 221 | 192 | | system_server | 1921 | 1416 | | SystemUI | 102 | 79 | | YouTube | 363 | 294 | +----------------+-------+----------+ jemalloc also tends to use slightly less memory than Scudo for most traces. These tests were conducted on desktop x86 Linux with glibc and the latest stable version of each allocator, but they should still be relevant. RSS values in KiB: +----------------+--------+----------+ | Trace | Scudo | jemalloc | +----------------+--------+----------+ | Angry Birds 2 | 793948 | 746992 | | Camera | 219372 | 251888 | | Candy Crush | 548288 | 550148 | | Gmail | 195236 | 193048 | | Maps | 159860 | 159816 | | Photos | 175436 | 171872 | | PUBG | 233752 | 223572 | | surfaceflinger | 94736 | 107068 | | system_server | 471048 | 484392 | | SystemUI | 54432 | 60740 | | YouTube | 139376 | 142252 | +----------------+--------+----------+ While not representative of real-world usage, jemalloc also performs fairly well in synthetic benchmarks (all times in nanoseconds): +-----------------+---------+----------+ | Benchmark | Scudo | jemalloc | +-----------------+---------+----------+ | alloc 8 | 87.9 | 60.1 | | alloc 16 | 87.9 | 60 | | alloc 32 | 88.6 | 60.7 | | alloc 64 | 88.6 | 59.7 | | alloc 512 | 89.2 | 60 | | alloc 1024 | 89.4 | 59.8 | | alloc 8192 | 89.8 | 65.2 | | alloc 16384 | 92.7 | 69.1 | | alloc 32768 | 97.2 | 74 | | alloc 65536 | 109 | 83.8 | | alloc 131072 | 41536 | 42720 | | alloc40x 8 | 2156 | 2556 | | alloc40x 16 | 2155 | 2244 | | alloc40x 32 | 2234 | 2312 | | alloc40x 64 | 2234 | 2289 | | alloc40x 512 | 2274 | 8171 | | alloc40x 1024 | 2397 | 2162 | | alloc40x 8192 | 3550 | 78880 | | alloc40x 16384 | 3732 | 124454 | | alloc40x 32768 | 3849 | 275460 | | alloc40x 65536 | 4987 | 727598 | | alloc40x 131072 | 2745207 | 3067980 | | alloc8192 1x | 464 | 454 | | alloc8192 2x | 510 | 488 | | alloc8192 3x | 587 | 523 | | alloc8192 4x | 665 | 557 | | alloc8192 5x | 742 | 598 | | alloc8192 6x | 818 | 633 | | alloc8192 7x | 884 | 669 | | alloc8192 8x | 960 | 699 | | alloc8192 9x | 1045 | 734 | | alloc8192 10x | 1131 | 770 | | alloc8192 11x | 1207 | 806 | | alloc8192 12x | 1282 | 841 | | alloc8192 13x | 1363 | 877 | | alloc8192 14x | 1442 | 912 | | alloc8192 15x | 1512 | 944 | | alloc8192 16x | 1587 | 978 | | alloc8192 24x | 2256 | 21195 | | alloc8192 32x | 2867 | 45446 | | alloc8192 40x | 3522 | 71618 | | alloc8192 48x | 4126 | 89740 | | alloc8192 56x | 4786 | 114990 | | alloc8192 64x | 5412 | 141082 | | alloc8192 72x | 6049 | 170742 | | alloc8192 80x | 6712 | 198480 | | alloc8192 88x | 7331 | 221557 | | alloc8192 96x | 7976 | 251462 | | alloc8192 104x | 8581 | 281626 | | alloc8192 112x | 9245 | 313164 | | alloc8192 120x | 9914 | 353147 | | alloc8192 128x | 10514 | 376625 | | alloc8192 136x | 11187 | 408194 | | alloc8192 144x | 11802 | 445694 | | alloc8192 160x | 13083 | 514547 | | alloc8192 176x | 14414 | 582501 | | alloc8192 192x | 15746 | 654346 | | alloc8192 208x | 17044 | 712620 | | alloc8192 224x | 18405 | 769963 | | alloc8192 240x | 19744 | 843969 | | alloc8192 256x | 21160 | 917803 | +-----------------+---------+----------+ Scudo performs fairly well for a hardened memory allocator, but we're optimizing for performance. Full benchmark data with graphs: https://docs.google.com/spreadsheets/d/1LG_kxaK5cI14gGtnyM-nNNmfpMdV9Vh-LtYoq7H5J4s/edit Change-Id: Ia4901eedfaa2c9779678c5b6532979de4919ee01 Signed-off-by: Pranav Vashi Signed-off-by: Anush02198 Signed-off-by: SayantamGiri --- libc/Android.bp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libc/Android.bp b/libc/Android.bp index 7304aa2a2e..b6551f5513 100644 --- a/libc/Android.bp +++ b/libc/Android.bp @@ -126,9 +126,6 @@ cc_defaults { malloc_pattern_fill_contents: { cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"], }, - malloc_not_svelte: { - cflags: ["-DUSE_SCUDO"], - }, }, lto: { @@ -161,7 +158,6 @@ cc_defaults { "libc_jemalloc_wrapper", ], header_libs: ["gwp_asan_headers"], - product_variables: libc_scudo_product_variables, } // Functions not implemented by jemalloc directly, or that need to From 608ac5cffbb4347f8732e5af34b34105c1773383 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Thu, 2 Jun 2022 15:20:23 -0700 Subject: [PATCH 9/9] Verify that allocated memory is always zeroed. Test: New unit tests pass on scudo. Test: New unit tests fail on original jemalloc. Test: New unit tests pass on modified jemalloc. Change-Id: Id62ab74f649c0f7c14a951235fe79972926ecec0 Signed-off-by: minaripenguin37 Signed-off-by: Anush02198 Signed-off-by: SayantamGiri --- libc/Android.bp | 19 +++--- tests/malloc_test.cpp | 131 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 8 deletions(-) diff --git a/libc/Android.bp b/libc/Android.bp index b6551f5513..0313993a2c 100644 --- a/libc/Android.bp +++ b/libc/Android.bp @@ -126,6 +126,9 @@ cc_defaults { malloc_pattern_fill_contents: { cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"], }, + //malloc_not_svelte: { + //cflags: ["-DUSE_SCUDO"], + //}, }, lto: { @@ -134,14 +137,14 @@ cc_defaults { } libc_scudo_product_variables = { - malloc_not_svelte: { - cflags: ["-DUSE_SCUDO"], - whole_static_libs: ["libscudo"], - exclude_static_libs: [ - "libjemalloc5", - "libc_jemalloc_wrapper", - ], - }, + //malloc_not_svelte: { + //cflags: ["-DUSE_SCUDO"], + //whole_static_libs: ["libscudo"], + //exclude_static_libs: [ + //"libjemalloc5", + //"libc_jemalloc_wrapper", + //], + //}, } // Defaults for native allocator libs/includes to make it diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp index 69f8506fdf..7f0e9a6c24 100644 --- a/tests/malloc_test.cpp +++ b/tests/malloc_test.cpp @@ -35,6 +35,7 @@ #include #include +#include #include #include @@ -1519,3 +1520,133 @@ TEST(malloc, realloc_mte_crash_b206701345) { } } } + +void VerifyAllocationsAreZero(std::function alloc_func, std::string function_name, + std::vector& test_sizes, size_t max_allocations) { + // Vector of zero'd data used for comparisons. Make it twice the larges size. + std::vector zero(test_sizes.back() * 2, 0); + + SCOPED_TRACE(testing::Message() << function_name << " failed to zero memory"); + + for (size_t test_size : test_sizes) { + std::vector ptrs(max_allocations); + for (size_t i = 0; i < ptrs.size(); i++) { + SCOPED_TRACE(testing::Message() << "size " << test_size << " at iteration " << i); + ptrs[i] = alloc_func(test_size); + ASSERT_TRUE(ptrs[i] != nullptr); + size_t alloc_size = malloc_usable_size(ptrs[i]); + ASSERT_LE(alloc_size, zero.size()); + ASSERT_EQ(0, memcmp(ptrs[i], zero.data(), alloc_size)); + + // Set the memory to non-zero to make sure if the pointer + // is reused it's still zero. + memset(ptrs[i], 0xab, alloc_size); + } + // Free the pointers. + for (size_t i = 0; i < ptrs.size(); i++) { + free(ptrs[i]); + } + for (size_t i = 0; i < ptrs.size(); i++) { + SCOPED_TRACE(testing::Message() << "size " << test_size << " at iteration " << i); + ptrs[i] = malloc(test_size); + ASSERT_TRUE(ptrs[i] != nullptr); + size_t alloc_size = malloc_usable_size(ptrs[i]); + ASSERT_LE(alloc_size, zero.size()); + ASSERT_EQ(0, memcmp(ptrs[i], zero.data(), alloc_size)); + } + // Free all of the pointers later to maximize the chance of reusing from + // the first loop. + for (size_t i = 0; i < ptrs.size(); i++) { + free(ptrs[i]); + } + } +} + +// Verify that small and medium allocations are always zero. +TEST(malloc, zeroed_allocations_small_medium_sizes) { + constexpr size_t kMaxAllocations = 1024; + std::vector test_sizes = {16, 48, 128, 1024, 4096, 65536}; + VerifyAllocationsAreZero([](size_t size) -> void* { return malloc(size); }, "malloc", test_sizes, + kMaxAllocations); + + VerifyAllocationsAreZero([](size_t size) -> void* { return memalign(64, size); }, "memalign", + test_sizes, kMaxAllocations); + + VerifyAllocationsAreZero( + [](size_t size) -> void* { + void* ptr; + if (posix_memalign(&ptr, 64, size) == 0) { + return ptr; + } + return nullptr; + }, + "posix_memalign", test_sizes, kMaxAllocations); +} + +// Verify that large allocations are always zero. +TEST(malloc, zeroed_allocations_large_sizes) { + constexpr size_t kMaxAllocations = 20; + std::vector test_sizes = {1000000, 2000000, 3000000, 4000000}; + VerifyAllocationsAreZero([](size_t size) -> void* { return malloc(size); }, "malloc", test_sizes, + kMaxAllocations); + + VerifyAllocationsAreZero([](size_t size) -> void* { return memalign(64, size); }, "memalign", + test_sizes, kMaxAllocations); + + VerifyAllocationsAreZero( + [](size_t size) -> void* { + void* ptr; + if (posix_memalign(&ptr, 64, size) == 0) { + return ptr; + } + return nullptr; + }, + "posix_memalign", test_sizes, kMaxAllocations); +} + +TEST(malloc, zeroed_allocations_realloc) { + // Vector of zero'd data used for comparisons. + constexpr size_t kMaxMemorySize = 131072; + std::vector zero(kMaxMemorySize, 0); + + constexpr size_t kMaxAllocations = 1024; + std::vector test_sizes = {16, 48, 128, 1024, 4096, 65536}; + // Do a number of allocations and set them to non-zero. + for (size_t test_size : test_sizes) { + std::vector ptrs(kMaxAllocations); + for (size_t i = 0; i < kMaxAllocations; i++) { + ptrs[i] = malloc(test_size); + ASSERT_TRUE(ptrs[i] != nullptr); + + // Set the memory to non-zero to make sure if the pointer + // is reused it's still zero. + memset(ptrs[i], 0xab, malloc_usable_size(ptrs[i])); + } + // Free the pointers. + for (size_t i = 0; i < kMaxAllocations; i++) { + free(ptrs[i]); + } + } + + // Do the reallocs to a larger size and verify the rest of the allocation + // is zero. + constexpr size_t kInitialSize = 8; + for (size_t test_size : test_sizes) { + std::vector ptrs(kMaxAllocations); + for (size_t i = 0; i < kMaxAllocations; i++) { + ptrs[i] = malloc(kInitialSize); + ASSERT_TRUE(ptrs[i] != nullptr); + size_t orig_alloc_size = malloc_usable_size(ptrs[i]); + + ptrs[i] = realloc(ptrs[i], test_size); + ASSERT_TRUE(ptrs[i] != nullptr); + size_t new_alloc_size = malloc_usable_size(ptrs[i]); + char* ptr = reinterpret_cast(ptrs[i]); + ASSERT_EQ(0, memcmp(&ptr[orig_alloc_size], zero.data(), new_alloc_size - orig_alloc_size)) + << "realloc from " << kInitialSize << " to size " << test_size << " at iteration " << i; + } + for (size_t i = 0; i < kMaxAllocations; i++) { + free(ptrs[i]); + } + } +}