Tsan builder

Bugs: android/ndk#1041 (comment)
Tests: Run build.py and verified that shared libs, like libclang_rt.tsan-aarch64-android.so
, are present in out/stage2-install/runtimes_ndk_cxx
Change-Id: I9612e1be4783c2fd17f6b32fef93869d9fdf7061

base_builders.py

@@ -147,6 +147,9 @@ def _build_config(self) -> None:
     def _is_cross_compiling(self) -> bool:
         return self._config.target_os != hosts.build_host()
 
+    def _is_64bit(self) -> bool:
+        return self._config.target_arch in (hosts.Arch.AARCH64, hosts.Arch.X86_64)
+
     @property
     def _cc(self) -> Path:
         return self._config.get_c_compiler(self.toolchain)

builders.py

@@ -33,32 +33,27 @@
 import paths
 import utils
 
-class AsanMapFileBuilder(base_builders.Builder):
-    name: str = 'asan-mapfile'
+class SanitizerMapFileBuilder(base_builders.Builder):
+    name: str = 'sanitizer-mapfile'
     config_list: List[configs.Config] = configs.android_configs()
 
     def _build_config(self) -> None:
         arch = self._config.target_arch
-        # We can not build asan_test using current CMake building system. Since
-        # those files are not used to build AOSP, we just simply touch them so that
-        # we can pass the build checks.
-        asan_test_path = self.output_toolchain.path / 'test' / arch.llvm_arch / 'bin'
-        asan_test_path.mkdir(parents=True, exist_ok=True)
-        asan_test_bin_path = asan_test_path / 'asan_test'
-        asan_test_bin_path.touch(exist_ok=True)
 
         lib_dir = self.output_toolchain.resource_dir
-        self._build_sanitizer_map_file('asan', arch, lib_dir)
-        self._build_sanitizer_map_file('ubsan_standalone', arch, lib_dir)
+        self._build_sanitizer_map_file('asan', arch, lib_dir, 'ASAN')
+        self._build_sanitizer_map_file('ubsan_standalone', arch, lib_dir, 'ASAN')
+        if super()._is_64bit():
+           self._build_sanitizer_map_file('tsan', arch, lib_dir, 'TSAN')
 
         if arch == hosts.Arch.AARCH64:
-            self._build_sanitizer_map_file('hwasan', arch, lib_dir)
+            self._build_sanitizer_map_file('hwasan', arch, lib_dir, 'ASAN')
 
     @staticmethod
-    def _build_sanitizer_map_file(san: str, arch: hosts.Arch, lib_dir: Path) -> None:
+    def _build_sanitizer_map_file(san: str, arch: hosts.Arch, lib_dir: Path, section_name: str) -> None:
         lib_file = lib_dir / f'libclang_rt.{san}-{arch.llvm_arch}-android.so'
         map_file = lib_dir / f'libclang_rt.{san}-{arch.llvm_arch}-android.map.txt'
-        mapfile.create_map_file(lib_file, map_file)
+        mapfile.create_map_file(lib_file, map_file, section_name)
 
 
 class Stage1Builder(base_builders.LLVMBuilder):
@@ -335,7 +330,6 @@ def install_dir(self) -> Path:
     @property
     def cmake_defines(self) -> Dict[str, str]:
         defines = super().cmake_defines
-        arch = self._config.target_arch
         defines['COMPILER_RT_BUILD_BUILTINS'] = 'OFF'
         defines['COMPILER_RT_USE_BUILTINS_LIBRARY'] = 'ON'
         # FIXME: Disable WError build until upstream fixed the compiler-rt
@@ -866,11 +860,8 @@ def cmake_defines(self) -> Dict[str, str]:
         defines['LIBCXX_ENABLE_STATIC_ABI_LIBRARY'] = 'ON'
         return defines
 
-    def _is_64bit(self) -> bool:
-        return self._config.target_arch in (hosts.Arch.AARCH64, hosts.Arch.X86_64)
-
     def _build_config(self) -> None:
-        if self._is_64bit():
+        if super()._is_64bit():
             # For arm64 and x86_64, build static cxxabi library from
             # toolchain/libcxxabi and use it when building runtimes.  This
             # should affect all compiler-rt runtimes that use libcxxabi
@@ -884,7 +875,7 @@ def install_config(self) -> None:
         lib_name = 'lib64' if arch == hosts.Arch.X86_64 else 'lib'
         install_dir = self._config.sysroot / 'usr' / lib_name
 
-        if self._is_64bit():
+        if super()._is_64bit():
             src_path = self.output_dir / 'lib' / 'libc++abi.a'
             shutil.copy2(src_path, install_dir / 'libc++abi.a')
         else:
@@ -1037,3 +1028,54 @@ def install_config(self) -> None:
             %~dp0lldb.exe %*
             EXIT /B %ERRORLEVEL%
         """))
+
+
+class TsanBuilder(base_builders.LLVMRuntimeBuilder):
+    name: str = 'tsan'
+    src_dir: Path = paths.LLVM_PATH / 'compiler-rt'
+    config_list: List[configs.Config] = configs.android_ndk_tsan_configs()
+
+    @property
+    def install_dir(self) -> Path:
+        # Installs to a temporary dir and copies to runtimes_ndk_cxx manually.
+        output_dir = self.output_dir
+        return output_dir.parent / (output_dir.name + '-install')
+
+    @property
+    def cmake_defines(self) -> Dict[str, str]:
+        defines = super().cmake_defines
+        defines['COMPILER_RT_BUILD_BUILTINS'] = 'OFF'
+        defines['COMPILER_RT_USE_BUILTINS_LIBRARY'] = 'ON'
+        defines['COMPILER_RT_SANITIZERS_TO_BUILD'] = 'tsan'
+        defines['COMPILER_RT_TEST_COMPILER_CFLAGS'] = defines['CMAKE_C_FLAGS']
+        defines['COMPILER_RT_DEFAULT_TARGET_TRIPLE'] = self._config.llvm_triple
+        defines['COMPILER_RT_INCLUDE_TESTS'] = 'OFF'
+        defines['SANITIZER_CXX_ABI'] = 'libcxxabi'
+        # With CMAKE_SYSTEM_NAME='Android', compiler-rt will be installed to
+        # lib/android instead of lib/linux.
+        del defines['CMAKE_SYSTEM_NAME']
+        libs: List[str] = []
+        # Currently, -rtlib=compiler-rt (even with -unwindlib=libunwind) does
+        # not automatically link libunwind.a on Android.
+        libs += ['-lunwind']
+        defines['SANITIZER_COMMON_LINK_LIBS'] = ' '.join(libs)
+        # compiler-rt's CMakeLists.txt file deletes -Wl,-z,defs from
+        # CMAKE_SHARED_LINKER_FLAGS when COMPILER_RT_USE_BUILTINS_LIBRARY is
+        # set. We want this flag on instead to catch unresolved references
+        # early.
+        defines['SANITIZER_COMMON_LINK_FLAGS'] = '-Wl,-z,defs'
+        return defines
+
+    @property
+    def cflags(self) -> List[str]:
+        cflags = super().cflags
+        cflags.append('-funwind-tables')
+        return cflags
+
+    def install_config(self) -> None:
+        # Still run `ninja install`.
+        super().install_config()
+
+        lib_dir = self.install_dir / 'lib' / 'linux'
+        dst_dir = self.output_toolchain.path / 'runtimes_ndk_cxx'
+        shutil.copytree(lib_dir, dst_dir, dirs_exist_ok=True)

configs.py

@@ -287,6 +287,7 @@ class AndroidConfig(_BaseConfig):
     static: bool = False
     platform: bool = False
     suppress_libcxx_headers: bool = False
+    override_api_level: Optional[int] = None
 
     @property
     def base_llvm_triple(self) -> str:
@@ -370,6 +371,8 @@ def cxxflags(self) -> List[str]:
 
     @property
     def api_level(self) -> int:
+        if self.override_api_level:
+            return self.override_api_level
         if self.static or self.platform:
             # Set API level for platform to to 29 since these runtimes can be
             # used for apexes targeting that API level.
@@ -467,3 +470,15 @@ def android_configs(platform: bool=True,
         config.extra_config = extra_config
     # List is not covariant. Explicit convert is required to make it List[Config].
     return list(configs)
+
+
+def android_ndk_tsan_configs() -> List[Config]:
+    """Returns a list of configs for android builds."""
+    configs = [
+        AndroidAArch64Config(),
+        AndroidX64Config(),
+    ]
+    for config in configs:
+        config.override_api_level = 24
+    # List is not covariant. Explicit convert is required to make it List[Config].
+    return list(configs)

do_build.py

@@ -115,6 +115,7 @@ def build_runtimes(build_lldb_server: bool):
     builders.LibUnwindBuilder().build()
     builders.PlatformLibcxxAbiBuilder().build()
     builders.CompilerRTBuilder().build()
+    builders.TsanBuilder().build()
     # 32-bit host crts are not needed for Darwin
     if hosts.build_host().is_linux:
         builders.CompilerRTHostI386Builder().build()
@@ -124,7 +125,7 @@ def build_runtimes(build_lldb_server: bool):
     # Bug: http://b/64037266. `strtod_l` is missing in NDK r15. This will break
     # libcxx build.
     # build_libcxx(toolchain, version)
-    builders.AsanMapFileBuilder().build()
+    builders.SanitizerMapFileBuilder().build()
 
 
 def install_wrappers(llvm_install_path: Path, llvm_next=False) -> None:

mapfile.py

@@ -20,13 +20,13 @@
 import sys
 import subprocess
 
-def create_map_file(lib_file: Path, map_file: Path) -> None:
+def create_map_file(lib_file: Path, map_file: Path, section_name: str) -> None:
     """Creates a map_file for lib_file."""
     symbols = subprocess.check_output(['nm', '-g', '--defined-only', str(lib_file)],
                                       text=True)
     with map_file.open('w') as output:
         output.write('# AUTO-GENERATED by mapfile.py. DO NOT EDIT.\n')
-        output.write('LIBCLANG_RT_ASAN {\n')
+        output.write(f'LIBCLANG_RT_{section_name} {{\n')
         output.write('  global:\n')
         for line in symbols.splitlines():
             _, symbol_type, symbol_name = line.split(' ', 2)
@@ -38,4 +38,4 @@ def create_map_file(lib_file: Path, map_file: Path) -> None:
 
 # for testing and standalone usage.
 if __name__ == '__main__':
-    create_map_file(Path(sys.argv[1]), Path(sys.argv[2]))
+    create_map_file(Path(sys.argv[1]), Path(sys.argv[2]), str(sys.argv[3]))