From 9364367881c3344ea6f77243f6c4f2a9b9b883fa Mon Sep 17 00:00:00 2001 From: zijunzhao Date: Wed, 2 Feb 2022 19:15:39 +0000 Subject: [PATCH] Tsan builder Bugs: https://github.com/android/ndk/issues/1041#issuecomment-511045039 Tests: Run build.py and verified that shared libs, like libclang_rt.tsan-aarch64-android.so , are present in out/stage2-install/runtimes_ndk_cxx Change-Id: I9612e1be4783c2fd17f6b32fef93869d9fdf7061 --- base_builders.py | 3 ++ builders.py | 82 ++++++++++++++++++++++++++++++++++++------------ configs.py | 15 +++++++++ do_build.py | 3 +- mapfile.py | 6 ++-- 5 files changed, 85 insertions(+), 24 deletions(-) diff --git a/base_builders.py b/base_builders.py index 47f0574..aea3f5a 100644 --- a/base_builders.py +++ b/base_builders.py @@ -147,6 +147,9 @@ def _build_config(self) -> None: def _is_cross_compiling(self) -> bool: return self._config.target_os != hosts.build_host() + def _is_64bit(self) -> bool: + return self._config.target_arch in (hosts.Arch.AARCH64, hosts.Arch.X86_64) + @property def _cc(self) -> Path: return self._config.get_c_compiler(self.toolchain) diff --git a/builders.py b/builders.py index 13f8f08..0027ae7 100644 --- a/builders.py +++ b/builders.py @@ -33,32 +33,27 @@ import paths import utils -class AsanMapFileBuilder(base_builders.Builder): - name: str = 'asan-mapfile' +class SanitizerMapFileBuilder(base_builders.Builder): + name: str = 'sanitizer-mapfile' config_list: List[configs.Config] = configs.android_configs() def _build_config(self) -> None: arch = self._config.target_arch - # We can not build asan_test using current CMake building system. Since - # those files are not used to build AOSP, we just simply touch them so that - # we can pass the build checks. - asan_test_path = self.output_toolchain.path / 'test' / arch.llvm_arch / 'bin' - asan_test_path.mkdir(parents=True, exist_ok=True) - asan_test_bin_path = asan_test_path / 'asan_test' - asan_test_bin_path.touch(exist_ok=True) lib_dir = self.output_toolchain.resource_dir - self._build_sanitizer_map_file('asan', arch, lib_dir) - self._build_sanitizer_map_file('ubsan_standalone', arch, lib_dir) + self._build_sanitizer_map_file('asan', arch, lib_dir, 'ASAN') + self._build_sanitizer_map_file('ubsan_standalone', arch, lib_dir, 'ASAN') + if super()._is_64bit(): + self._build_sanitizer_map_file('tsan', arch, lib_dir, 'TSAN') if arch == hosts.Arch.AARCH64: - self._build_sanitizer_map_file('hwasan', arch, lib_dir) + self._build_sanitizer_map_file('hwasan', arch, lib_dir, 'ASAN') @staticmethod - def _build_sanitizer_map_file(san: str, arch: hosts.Arch, lib_dir: Path) -> None: + def _build_sanitizer_map_file(san: str, arch: hosts.Arch, lib_dir: Path, section_name: str) -> None: lib_file = lib_dir / f'libclang_rt.{san}-{arch.llvm_arch}-android.so' map_file = lib_dir / f'libclang_rt.{san}-{arch.llvm_arch}-android.map.txt' - mapfile.create_map_file(lib_file, map_file) + mapfile.create_map_file(lib_file, map_file, section_name) class Stage1Builder(base_builders.LLVMBuilder): @@ -335,7 +330,6 @@ def install_dir(self) -> Path: @property def cmake_defines(self) -> Dict[str, str]: defines = super().cmake_defines - arch = self._config.target_arch defines['COMPILER_RT_BUILD_BUILTINS'] = 'OFF' defines['COMPILER_RT_USE_BUILTINS_LIBRARY'] = 'ON' # FIXME: Disable WError build until upstream fixed the compiler-rt @@ -866,11 +860,8 @@ def cmake_defines(self) -> Dict[str, str]: defines['LIBCXX_ENABLE_STATIC_ABI_LIBRARY'] = 'ON' return defines - def _is_64bit(self) -> bool: - return self._config.target_arch in (hosts.Arch.AARCH64, hosts.Arch.X86_64) - def _build_config(self) -> None: - if self._is_64bit(): + if super()._is_64bit(): # For arm64 and x86_64, build static cxxabi library from # toolchain/libcxxabi and use it when building runtimes. This # should affect all compiler-rt runtimes that use libcxxabi @@ -884,7 +875,7 @@ def install_config(self) -> None: lib_name = 'lib64' if arch == hosts.Arch.X86_64 else 'lib' install_dir = self._config.sysroot / 'usr' / lib_name - if self._is_64bit(): + if super()._is_64bit(): src_path = self.output_dir / 'lib' / 'libc++abi.a' shutil.copy2(src_path, install_dir / 'libc++abi.a') else: @@ -1037,3 +1028,54 @@ def install_config(self) -> None: %~dp0lldb.exe %* EXIT /B %ERRORLEVEL% """)) + + +class TsanBuilder(base_builders.LLVMRuntimeBuilder): + name: str = 'tsan' + src_dir: Path = paths.LLVM_PATH / 'compiler-rt' + config_list: List[configs.Config] = configs.android_ndk_tsan_configs() + + @property + def install_dir(self) -> Path: + # Installs to a temporary dir and copies to runtimes_ndk_cxx manually. + output_dir = self.output_dir + return output_dir.parent / (output_dir.name + '-install') + + @property + def cmake_defines(self) -> Dict[str, str]: + defines = super().cmake_defines + defines['COMPILER_RT_BUILD_BUILTINS'] = 'OFF' + defines['COMPILER_RT_USE_BUILTINS_LIBRARY'] = 'ON' + defines['COMPILER_RT_SANITIZERS_TO_BUILD'] = 'tsan' + defines['COMPILER_RT_TEST_COMPILER_CFLAGS'] = defines['CMAKE_C_FLAGS'] + defines['COMPILER_RT_DEFAULT_TARGET_TRIPLE'] = self._config.llvm_triple + defines['COMPILER_RT_INCLUDE_TESTS'] = 'OFF' + defines['SANITIZER_CXX_ABI'] = 'libcxxabi' + # With CMAKE_SYSTEM_NAME='Android', compiler-rt will be installed to + # lib/android instead of lib/linux. + del defines['CMAKE_SYSTEM_NAME'] + libs: List[str] = [] + # Currently, -rtlib=compiler-rt (even with -unwindlib=libunwind) does + # not automatically link libunwind.a on Android. + libs += ['-lunwind'] + defines['SANITIZER_COMMON_LINK_LIBS'] = ' '.join(libs) + # compiler-rt's CMakeLists.txt file deletes -Wl,-z,defs from + # CMAKE_SHARED_LINKER_FLAGS when COMPILER_RT_USE_BUILTINS_LIBRARY is + # set. We want this flag on instead to catch unresolved references + # early. + defines['SANITIZER_COMMON_LINK_FLAGS'] = '-Wl,-z,defs' + return defines + + @property + def cflags(self) -> List[str]: + cflags = super().cflags + cflags.append('-funwind-tables') + return cflags + + def install_config(self) -> None: + # Still run `ninja install`. + super().install_config() + + lib_dir = self.install_dir / 'lib' / 'linux' + dst_dir = self.output_toolchain.path / 'runtimes_ndk_cxx' + shutil.copytree(lib_dir, dst_dir, dirs_exist_ok=True) diff --git a/configs.py b/configs.py index 5b7f399..c8e5d83 100644 --- a/configs.py +++ b/configs.py @@ -287,6 +287,7 @@ class AndroidConfig(_BaseConfig): static: bool = False platform: bool = False suppress_libcxx_headers: bool = False + override_api_level: Optional[int] = None @property def base_llvm_triple(self) -> str: @@ -370,6 +371,8 @@ def cxxflags(self) -> List[str]: @property def api_level(self) -> int: + if self.override_api_level: + return self.override_api_level if self.static or self.platform: # Set API level for platform to to 29 since these runtimes can be # used for apexes targeting that API level. @@ -467,3 +470,15 @@ def android_configs(platform: bool=True, config.extra_config = extra_config # List is not covariant. Explicit convert is required to make it List[Config]. return list(configs) + + +def android_ndk_tsan_configs() -> List[Config]: + """Returns a list of configs for android builds.""" + configs = [ + AndroidAArch64Config(), + AndroidX64Config(), + ] + for config in configs: + config.override_api_level = 24 + # List is not covariant. Explicit convert is required to make it List[Config]. + return list(configs) \ No newline at end of file diff --git a/do_build.py b/do_build.py index 1001a0f..49eda55 100755 --- a/do_build.py +++ b/do_build.py @@ -115,6 +115,7 @@ def build_runtimes(build_lldb_server: bool): builders.LibUnwindBuilder().build() builders.PlatformLibcxxAbiBuilder().build() builders.CompilerRTBuilder().build() + builders.TsanBuilder().build() # 32-bit host crts are not needed for Darwin if hosts.build_host().is_linux: builders.CompilerRTHostI386Builder().build() @@ -124,7 +125,7 @@ def build_runtimes(build_lldb_server: bool): # Bug: http://b/64037266. `strtod_l` is missing in NDK r15. This will break # libcxx build. # build_libcxx(toolchain, version) - builders.AsanMapFileBuilder().build() + builders.SanitizerMapFileBuilder().build() def install_wrappers(llvm_install_path: Path, llvm_next=False) -> None: diff --git a/mapfile.py b/mapfile.py index de05ee4..16840d9 100755 --- a/mapfile.py +++ b/mapfile.py @@ -20,13 +20,13 @@ import sys import subprocess -def create_map_file(lib_file: Path, map_file: Path) -> None: +def create_map_file(lib_file: Path, map_file: Path, section_name: str) -> None: """Creates a map_file for lib_file.""" symbols = subprocess.check_output(['nm', '-g', '--defined-only', str(lib_file)], text=True) with map_file.open('w') as output: output.write('# AUTO-GENERATED by mapfile.py. DO NOT EDIT.\n') - output.write('LIBCLANG_RT_ASAN {\n') + output.write(f'LIBCLANG_RT_{section_name} {{\n') output.write(' global:\n') for line in symbols.splitlines(): _, symbol_type, symbol_name = line.split(' ', 2) @@ -38,4 +38,4 @@ def create_map_file(lib_file: Path, map_file: Path) -> None: # for testing and standalone usage. if __name__ == '__main__': - create_map_file(Path(sys.argv[1]), Path(sys.argv[2])) + create_map_file(Path(sys.argv[1]), Path(sys.argv[2]), str(sys.argv[3]))