--- a/src/3rdparty/chromium/tools/clang/scripts/update.py 2016-05-26 04:58:54.000000000 -0800 +++ b/src/3rdparty/chromium/tools/clang/scripts/update.py 2016-11-04 08:35:34.956154012 -0800 @@ -3,12 +3,12 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Windows can't run .sh files, so this is a Python implementation of -update.sh. This script should replace update.sh on all platforms eventually.""" +"""This script is used to download prebuilt clang binaries. + +It is also used by package.py to build the prebuilt clang binaries.""" import argparse -import contextlib -import cStringIO +import distutils.spawn import glob import os import pipes @@ -18,6 +18,7 @@ import stat import sys import tarfile +import tempfile import time import urllib2 import zipfile @@ -25,19 +26,16 @@ # Do NOT CHANGE this if you don't know what you're doing -- see # https://code.google.com/p/chromium/wiki/UpdatingClang # Reverting problematic clang rolls is safe, though. -# Note: this revision is only used for Windows. Other platforms use update.sh. -# TODO(thakis): Use the same revision on Windows and non-Windows. -# TODO(thakis): Remove update.sh, use update.py everywhere. -LLVM_WIN_REVISION = '239674' +CLANG_REVISION = '239674' use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ if use_head_revision: - LLVM_WIN_REVISION = 'HEAD' + CLANG_REVISION = 'HEAD' # This is incremented when pushing a new build of Clang at the same revision. CLANG_SUB_REVISION=1 -PACKAGE_VERSION = "%s-%s" % (LLVM_WIN_REVISION, CLANG_SUB_REVISION) +PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION) # Path constants. (All of these should be absolute paths.) THIS_DIR = os.path.abspath(os.path.dirname(__file__)) @@ -50,17 +48,26 @@ CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools') LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build', 'Release+Asserts') -COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt') +COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'compiler-rt') CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang') LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld') -COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt') +# compiler-rt is built as part of the regular LLVM build on Windows to get +# the 64-bit runtime, and out-of-tree elsewhere. +# TODO(thakis): Try to unify this. +if sys.platform == 'win32': + COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt') +else: + COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'compiler-rt') LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx') LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi') LLVM_BUILD_TOOLS_DIR = os.path.abspath( os.path.join(LLVM_DIR, '..', 'llvm-build-tools')) -STAMP_FILE = os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision') +STAMP_FILE = os.path.normpath( + os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision')) BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils') -VERSION = '3.7.0' +VERSION = '3.8.0' +ANDROID_NDK_DIR = os.path.join( + CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk') # URL for pre-built binaries. CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang' @@ -74,40 +81,75 @@ """Download url into output_file.""" CHUNK_SIZE = 4096 TOTAL_DOTS = 10 - sys.stdout.write('Downloading %s ' % url) - sys.stdout.flush() - response = urllib2.urlopen(url) - total_size = int(response.info().getheader('Content-Length').strip()) - bytes_done = 0 - dots_printed = 0 + num_retries = 3 + retry_wait_s = 5 # Doubled at each retry. + while True: - chunk = response.read(CHUNK_SIZE) - if not chunk: - break - output_file.write(chunk) - bytes_done += len(chunk) - num_dots = TOTAL_DOTS * bytes_done / total_size - sys.stdout.write('.' * (num_dots - dots_printed)) - sys.stdout.flush() - dots_printed = num_dots - print ' Done.' + try: + sys.stdout.write('Downloading %s ' % url) + sys.stdout.flush() + response = urllib2.urlopen(url) + total_size = int(response.info().getheader('Content-Length').strip()) + bytes_done = 0 + dots_printed = 0 + while True: + chunk = response.read(CHUNK_SIZE) + if not chunk: + break + output_file.write(chunk) + bytes_done += len(chunk) + num_dots = TOTAL_DOTS * bytes_done / total_size + sys.stdout.write('.' * (num_dots - dots_printed)) + sys.stdout.flush() + dots_printed = num_dots + if bytes_done != total_size: + raise urllib2.URLError("only got %d of %d bytes" % + (bytes_done, total_size)) + print ' Done.' + return + except urllib2.URLError as e: + sys.stdout.write('\n') + print e + if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404: + raise e + num_retries -= 1 + print 'Retrying in %d s ...' % retry_wait_s + time.sleep(retry_wait_s) + retry_wait_s *= 2 + + +def EnsureDirExists(path): + if not os.path.exists(path): + print "Creating directory %s" % path + os.makedirs(path) + + +def DownloadAndUnpack(url, output_dir): + with tempfile.TemporaryFile() as f: + DownloadUrl(url, f) + f.seek(0) + EnsureDirExists(output_dir) + if url.endswith('.zip'): + zipfile.ZipFile(f).extractall(path=output_dir) + else: + tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir) def ReadStampFile(): """Return the contents of the stamp file, or '' if it doesn't exist.""" try: with open(STAMP_FILE, 'r') as f: - return f.read() + return f.read().rstrip() except IOError: return '' def WriteStampFile(s): """Write s to the stamp file.""" - if not os.path.exists(os.path.dirname(STAMP_FILE)): - os.makedirs(os.path.dirname(STAMP_FILE)) + EnsureDirExists(os.path.dirname(STAMP_FILE)) with open(STAMP_FILE, 'w') as f: f.write(s) + f.write('\n') def GetSvnRevision(svn_repo): @@ -129,6 +171,13 @@ shutil.rmtree(dir, onerror=ChmodAndRetry) +def RmCmakeCache(dir): + """Delete CMakeCache.txt files under dir recursively.""" + for dirpath, _, files in os.walk(dir): + if 'CMakeCache.txt' in files: + os.remove(os.path.join(dirpath, 'CMakeCache.txt')) + + def RunCommand(command, msvc_arch=None, env=None, fail_hard=True): """Run command and return success (True) or failure; or if fail_hard is True, exit on failure. If msvc_arch is set, runs the command in a @@ -170,8 +219,8 @@ def CopyDirectoryContents(src, dst, filename_filter=None): """Copy the files from directory src to dst with an optional filename filter.""" - if not os.path.exists(dst): - os.makedirs(dst) + dst = os.path.realpath(dst) # realpath() in case dst ends in /.. + EnsureDirExists(dst) for root, _, files in os.walk(src): for f in files: if filename_filter and not re.match(filename_filter, f): @@ -181,9 +230,9 @@ def Checkout(name, url, dir): """Checkout the SVN module at url into dir. Use name for the log message.""" - print "Checking out %s r%s into '%s'" % (name, LLVM_WIN_REVISION, dir) + print "Checking out %s r%s into '%s'" % (name, CLANG_REVISION, dir) - command = ['svn', 'checkout', '--force', url + '@' + LLVM_WIN_REVISION, dir] + command = ['svn', 'checkout', '--force', url + '@' + CLANG_REVISION, dir] if RunCommand(command, fail_hard=False): return @@ -195,120 +244,9 @@ RunCommand(command) -def RevertPreviouslyPatchedFiles(): - print 'Reverting previously patched files' - files = [ - '%(clang)s/test/Index/crash-recovery-modules.m', - '%(clang)s/unittests/libclang/LibclangTest.cpp', - '%(compiler_rt)s/lib/asan/asan_rtl.cc', - '%(compiler_rt)s/test/asan/TestCases/Linux/new_array_cookie_test.cc', - '%(llvm)s/test/DebugInfo/gmlt.ll', - '%(llvm)s/lib/CodeGen/SpillPlacement.cpp', - '%(llvm)s/lib/CodeGen/SpillPlacement.h', - '%(llvm)s/lib/Transforms/Instrumentation/MemorySanitizer.cpp', - '%(clang)s/test/Driver/env.c', - '%(clang)s/lib/Frontend/InitPreprocessor.cpp', - '%(clang)s/test/Frontend/exceptions.c', - '%(clang)s/test/Preprocessor/predefined-exceptions.m', - '%(llvm)s/test/Bindings/Go/go.test', - '%(clang)s/lib/Parse/ParseExpr.cpp', - '%(clang)s/lib/Parse/ParseTemplate.cpp', - '%(clang)s/lib/Sema/SemaDeclCXX.cpp', - '%(clang)s/lib/Sema/SemaExprCXX.cpp', - '%(clang)s/test/SemaCXX/default2.cpp', - '%(clang)s/test/SemaCXX/typo-correction-delayed.cpp', - '%(compiler_rt)s/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc', - '%(compiler_rt)s/test/tsan/signal_segv_handler.cc', - '%(compiler_rt)s/lib/sanitizer_common/sanitizer_coverage_libcdep.cc', - '%(compiler_rt)s/cmake/config-ix.cmake', - '%(compiler_rt)s/CMakeLists.txt', - '%(compiler_rt)s/lib/ubsan/ubsan_platform.h', - ] - for f in files: - f = f % { - 'clang': CLANG_DIR, - 'compiler_rt': COMPILER_RT_DIR, - 'llvm': LLVM_DIR, - } - if os.path.exists(f): - os.remove(f) # For unversioned files. - RunCommand(['svn', 'revert', f]) - - -def ApplyLocalPatches(): - # There's no patch program on Windows by default. We don't need patches on - # Windows yet, and maybe this not working on Windows will motivate us to - # remove patches over time. - assert sys.platform != 'win32' - - # Apply patch for tests failing with --disable-pthreads (llvm.org/PR11974) - clang_patches = [ r"""\ ---- test/Index/crash-recovery-modules.m (revision 202554) -+++ test/Index/crash-recovery-modules.m (working copy) -@@ -12,6 +12,8 @@ - - // REQUIRES: crash-recovery - // REQUIRES: shell -+// XFAIL: * -+// (PR11974) - - @import Crash; -""", r"""\ ---- unittests/libclang/LibclangTest.cpp (revision 215949) -+++ unittests/libclang/LibclangTest.cpp (working copy) -@@ -431,7 +431,7 @@ - EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU)); - } - --TEST_F(LibclangReparseTest, ReparseWithModule) { -+TEST_F(LibclangReparseTest, DISABLED_ReparseWithModule) { - const char *HeaderTop = "#ifndef H\n#define H\nstruct Foo { int bar;"; - const char *HeaderBottom = "\n};\n#endif\n"; - const char *MFile = "#include \"HeaderFile.h\"\nint main() {" -""" - ] - - # This Go bindings test doesn't work after bootstrap on Linux, PR21552. - llvm_patches = [ r"""\ ---- test/Bindings/Go/go.test (revision 223109) -+++ test/Bindings/Go/go.test (working copy) -@@ -1,3 +1,3 @@ --; RUN: llvm-go test llvm.org/llvm/bindings/go/llvm -+; RUN: true - - ; REQUIRES: shell -""" - ] - - # The UBSan run-time, which is now bundled with the ASan run-time, doesn't - # work on Mac OS X 10.8 (PR23539). - compiler_rt_patches = [ r"""\ ---- CMakeLists.txt (revision 241602) -+++ CMakeLists.txt (working copy) -@@ -305,6 +305,7 @@ - list(APPEND SANITIZER_COMMON_SUPPORTED_OS iossim) - endif() - endif() -+ set(SANITIZER_MIN_OSX_VERSION "10.7") - if(SANITIZER_MIN_OSX_VERSION VERSION_LESS "10.7") - message(FATAL_ERROR "Too old OS X version: ${SANITIZER_MIN_OSX_VERSION}") - endif() -""" - ] - - for path, patches in [(LLVM_DIR, llvm_patches), - (CLANG_DIR, clang_patches), - (COMPILER_RT_DIR, compiler_rt_patches)]: - print 'Applying patches in', path - for patch in patches: - print patch - p = subprocess.Popen( ['patch', '-p0', '-d', path], stdin=subprocess.PIPE) - (stdout, stderr) = p.communicate(input=patch) - if p.returncode != 0: - raise RuntimeError('stdout %s, stderr %s' % (stdout, stderr)) - - def DeleteChromeToolsShim(): + OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools') + shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True) shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True) @@ -337,6 +275,25 @@ f.write('endif (CHROMIUM_TOOLS_SRC)\n') +def MaybeDownloadHostGcc(args): + """Downloads gcc 4.8.2 if needed and makes sure args.gcc_toolchain is set.""" + if not sys.platform.startswith('linux') or args.gcc_toolchain: + return + + if subprocess.check_output(['gcc', '-dumpversion']).rstrip() < '4.7.0': + # We need a newer gcc version. + gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc482precise') + if not os.path.exists(gcc_dir): + print 'Downloading pre-built GCC 4.8.2...' + DownloadAndUnpack( + CDS_URL + '/tools/gcc482precise.tgz', LLVM_BUILD_TOOLS_DIR) + args.gcc_toolchain = gcc_dir + else: + # Always set gcc_toolchain; llvm-symbolizer needs the bundled libstdc++. + args.gcc_toolchain = \ + os.path.dirname(os.path.dirname(distutils.spawn.find_executable('gcc'))) + + def AddCMakeToPath(): """Download CMake and add it to PATH.""" if sys.platform == 'win32': @@ -345,20 +302,10 @@ 'cmake-3.2.2-win32-x86', 'bin') else: suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux' - zip_name = 'cmake310_%s.tgz' % suffix - cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake310', 'bin') + zip_name = 'cmake322_%s.tgz' % suffix + cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake322', 'bin') if not os.path.exists(cmake_dir): - if not os.path.exists(LLVM_BUILD_TOOLS_DIR): - os.makedirs(LLVM_BUILD_TOOLS_DIR) - # The cmake archive is smaller than 20 MB, small enough to keep in memory: - with contextlib.closing(cStringIO.StringIO()) as f: - DownloadUrl(CDS_URL + '/tools/' + zip_name, f) - f.seek(0) - if zip_name.endswith('.zip'): - zipfile.ZipFile(f).extractall(path=LLVM_BUILD_TOOLS_DIR) - else: - tarfile.open(mode='r:gz', fileobj=f).extractall(path= - LLVM_BUILD_TOOLS_DIR) + DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR) os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '') vs_version = None @@ -383,37 +330,61 @@ def UpdateClang(args): print 'Updating Clang to %s...' % PACKAGE_VERSION - if ReadStampFile() == PACKAGE_VERSION: - print 'Already up to date.' - return 0 + + need_gold_plugin = 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ or ( + sys.platform.startswith('linux') and + 'buildtype=Official' in os.environ.get('GYP_DEFINES', '') and + 'branding=Chrome' in os.environ.get('GYP_DEFINES', '')) + + if ReadStampFile() == PACKAGE_VERSION and not args.force_local_build: + print 'Clang is already up to date.' + if not need_gold_plugin or os.path.exists( + os.path.join(LLVM_BUILD_DIR, "lib/LLVMgold.so")): + return 0 # Reset the stamp file in case the build is unsuccessful. WriteStampFile('') if not args.force_local_build: cds_file = "clang-%s.tgz" % PACKAGE_VERSION - cds_full_url = CDS_URL + '/Win/' + cds_file + if sys.platform == 'win32': + cds_full_url = CDS_URL + '/Win/' + cds_file + elif sys.platform == 'darwin': + cds_full_url = CDS_URL + '/Mac/' + cds_file + else: + assert sys.platform.startswith('linux') + cds_full_url = CDS_URL + '/Linux_x64/' + cds_file - # Check if there's a prebuilt binary and if so just fetch that. That's - # faster, and goma relies on having matching binary hashes on client and - # server too. - print 'Trying to download prebuilt clang' - - # clang packages are smaller than 50 MB, small enough to keep in memory. - with contextlib.closing(cStringIO.StringIO()) as f: - try: - DownloadUrl(cds_full_url, f) - f.seek(0) - tarfile.open(mode='r:gz', fileobj=f).extractall(path=LLVM_BUILD_DIR) - print 'clang %s unpacked' % PACKAGE_VERSION - WriteStampFile(PACKAGE_VERSION) - return 0 - except urllib2.HTTPError: - print 'Did not find prebuilt clang %s, building locally' % cds_file + print 'Downloading prebuilt clang' + if os.path.exists(LLVM_BUILD_DIR): + RmTree(LLVM_BUILD_DIR) + try: + DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR) + print 'clang %s unpacked' % PACKAGE_VERSION + # Download the gold plugin if requested to by an environment variable. + # This is used by the CFI ClusterFuzz bot, and it's required for official + # builds on linux. + if need_gold_plugin: + RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py']) + WriteStampFile(PACKAGE_VERSION) + return 0 + except urllib2.URLError: + print 'Failed to download prebuilt clang %s' % cds_file + print 'Use --force-local-build if you want to build locally.' + print 'Exiting.' + return 1 + + if args.with_android and not os.path.exists(ANDROID_NDK_DIR): + print 'Android NDK not found at ' + ANDROID_NDK_DIR + print 'The Android NDK is needed to build a Clang whose -fsanitize=address' + print 'works on Android. See ' + print 'http://code.google.com/p/chromium/wiki/AndroidBuildInstructions' + print 'for how to install the NDK, or pass --without-android.' + return 1 + MaybeDownloadHostGcc(args) AddCMakeToPath() - RevertPreviouslyPatchedFiles() DeleteChromeToolsShim() Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR) @@ -429,10 +400,24 @@ # into it too (since OS X 10.6 doesn't have libc++abi.dylib either). Checkout('libcxxabi', LLVM_REPO_URL + '/libcxxabi/trunk', LIBCXXABI_DIR) - if args.with_patches and sys.platform != 'win32': - ApplyLocalPatches() - cc, cxx = None, None + libstdcpp = None + if args.gcc_toolchain: # This option is only used on Linux. + # Use the specified gcc installation for building. + cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc') + cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++') + + if not os.access(cc, os.X_OK): + print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain + print '"%s" does not appear to be valid.' % cc + return 1 + + # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap + # compiler, etc.) find the .so. + libstdcpp = subprocess.check_output( + [cxx, '-print-file-name=libstdc++.so.6']).rstrip() + os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp) + cflags = cxxflags = ldflags = [] # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is @@ -462,8 +447,7 @@ if args.bootstrap: print 'Building bootstrap compiler' - if not os.path.exists(LLVM_BOOTSTRAP_DIR): - os.makedirs(LLVM_BOOTSTRAP_DIR) + EnsureDirExists(LLVM_BOOTSTRAP_DIR) os.chdir(LLVM_BOOTSTRAP_DIR) bootstrap_args = base_cmake_args + [ '-DLLVM_TARGETS_TO_BUILD=host', @@ -473,11 +457,16 @@ ] if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc) if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx) + RmCmakeCache('.') RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64') RunCommand(['ninja'], msvc_arch='x64') if args.run_tests: RunCommand(['ninja', 'check-all'], msvc_arch='x64') RunCommand(['ninja', 'install'], msvc_arch='x64') + if args.gcc_toolchain: + # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap + # compiler can start. + CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib')) if sys.platform == 'win32': cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe') @@ -489,6 +478,12 @@ else: cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang') cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++') + + if args.gcc_toolchain: + # Tell the bootstrap compiler to use a specific gcc prefix to search + # for standard library headers and shared object files. + cflags = ['--gcc-toolchain=' + args.gcc_toolchain] + cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain] print 'Building final compiler' if sys.platform == 'darwin': @@ -543,7 +538,7 @@ binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include') # If building at head, define a macro that plugins can use for #ifdefing - # out code that builds at head, but not at LLVM_WIN_REVISION or vice versa. + # out code that builds at head, but not at CLANG_REVISION or vice versa. if use_head_revision: cflags += ['-DLLVM_FORCE_HEAD_REVISION'] cxxflags += ['-DLLVM_FORCE_HEAD_REVISION'] @@ -555,8 +550,15 @@ deployment_env = os.environ.copy() deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target - cmake_args = base_cmake_args + [ + cmake_args = [] + # TODO(thakis): Unconditionally append this to base_cmake_args instead once + # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698) + cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args + if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc) + if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx) + cmake_args += base_cmake_args + [ '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, + '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly', '-DCMAKE_C_FLAGS=' + ' '.join(cflags), '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags), @@ -565,35 +567,44 @@ '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR, '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'), '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)] - # TODO(thakis): Unconditionally append this to base_cmake_args instead once - # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698) - cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args - if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc) - if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx) - if not os.path.exists(LLVM_BUILD_DIR): - os.makedirs(LLVM_BUILD_DIR) + EnsureDirExists(LLVM_BUILD_DIR) os.chdir(LLVM_BUILD_DIR) + RmCmakeCache('.') RunCommand(['cmake'] + cmake_args + [LLVM_DIR], msvc_arch='x64', env=deployment_env) - RunCommand(['ninja'], msvc_arch='x64') + + if args.gcc_toolchain: + # Copy in the right stdlibc++.so.6 so clang can start. + if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')): + os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib')) + libstdcpp = subprocess.check_output( + [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip() + CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib')) + + # TODO(thakis): Remove "-d explain" once http://crbug.com/569337 is fixed. + RunCommand(['ninja', '-d', 'explain'], msvc_arch='x64') if args.tools: # If any Chromium tools were built, install those now. RunCommand(['ninja', 'cr-install'], msvc_arch='x64') if sys.platform == 'darwin': - CopyFile(os.path.join(LLVM_BUILD_DIR, 'libc++.1.dylib'), + CopyFile(os.path.join(libcxxbuild, 'libc++.1.dylib'), os.path.join(LLVM_BUILD_DIR, 'bin')) # See http://crbug.com/256342 RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')]) elif sys.platform.startswith('linux'): RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')]) - # Do an x86 build of compiler-rt to get the 32-bit ASan run-time. + # Do an out-of-tree build of compiler-rt. + # On Windows, this is used to get the 32-bit ASan run-time. # TODO(hans): Remove once the regular build above produces this. - if not os.path.exists(COMPILER_RT_BUILD_DIR): - os.makedirs(COMPILER_RT_BUILD_DIR) + # On Mac and Linux, this is used to get the regular 64-bit run-time. + # Do a clobbered build due to cmake changes. + if os.path.isdir(COMPILER_RT_BUILD_DIR): + RmTree(COMPILER_RT_BUILD_DIR) + os.makedirs(COMPILER_RT_BUILD_DIR) os.chdir(COMPILER_RT_BUILD_DIR) # TODO(thakis): Add this once compiler-rt can build with clang-cl (see # above). @@ -606,11 +617,17 @@ '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)] if sys.platform != 'win32': compiler_rt_args += ['-DLLVM_CONFIG_PATH=' + - os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config')] - RunCommand(['cmake'] + compiler_rt_args + [LLVM_DIR], - msvc_arch='x86', env=deployment_env) + os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'), + '-DSANITIZER_MIN_OSX_VERSION="10.7"'] + # compiler-rt is part of the llvm checkout on Windows but a stand-alone + # directory elsewhere, see the TODO above COMPILER_RT_DIR. + RmCmakeCache('.') + RunCommand(['cmake'] + compiler_rt_args + + [LLVM_DIR if sys.platform == 'win32' else COMPILER_RT_DIR], + msvc_arch='x86', env=deployment_env) RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86') + # Copy select output to the main tree. # TODO(hans): Make this (and the .gypi and .isolate files) version number # independent. if sys.platform == 'win32': @@ -620,17 +637,35 @@ else: assert sys.platform.startswith('linux') platform = 'linux' - asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang', - VERSION, 'lib', platform) + asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', platform) + if sys.platform == 'win32': + # TODO(thakis): This too is due to compiler-rt being part of the checkout + # on Windows, see TODO above COMPILER_RT_DIR. + asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang', + VERSION, 'lib', platform) asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', VERSION, 'lib', platform) - CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir, - r'^.*-i386\.lib$') - CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir, - r'^.*-i386\.dll$') + # Blacklists: + CopyDirectoryContents(os.path.join(asan_rt_lib_src_dir, '..', '..'), + os.path.join(asan_rt_lib_dst_dir, '..', '..'), + r'^.*blacklist\.txt$') + # Headers: + if sys.platform != 'win32': + CopyDirectoryContents( + os.path.join(COMPILER_RT_BUILD_DIR, 'include/sanitizer'), + os.path.join(LLVM_BUILD_DIR, 'lib/clang', VERSION, 'include/sanitizer')) + # Static and dynamic libraries: + CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir) + if sys.platform == 'darwin': + for dylib in glob.glob(os.path.join(asan_rt_lib_dst_dir, '*.dylib')): + # Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to + # @executable_path. + # TODO(glider): this is transitional. We'll need to fix the dylib + # name either in our build system, or in Clang. See also + # http://crbug.com/344836. + subprocess.call(['install_name_tool', '-id', + '@executable_path/' + os.path.basename(dylib), dylib]) - CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'), - os.path.join(asan_rt_lib_dst_dir, '..', '..')) if sys.platform == 'win32': # Make an extra copy of the sanitizer headers, to be put on the include path @@ -640,22 +675,67 @@ aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', VERSION, 'include_sanitizer', 'sanitizer') - if not os.path.exists(aux_sanitizer_include_dir): - os.makedirs(aux_sanitizer_include_dir) + EnsureDirExists(aux_sanitizer_include_dir) for _, _, files in os.walk(sanitizer_include_dir): for f in files: CopyFile(os.path.join(sanitizer_include_dir, f), aux_sanitizer_include_dir) + if args.with_android: + make_toolchain = os.path.join( + ANDROID_NDK_DIR, 'build', 'tools', 'make-standalone-toolchain.sh') + for target_arch in ['aarch64', 'arm', 'i686']: + # Make standalone Android toolchain for target_arch. + toolchain_dir = os.path.join( + LLVM_BUILD_DIR, 'android-toolchain-' + target_arch) + RunCommand([ + make_toolchain, + '--platform=android-' + ('21' if target_arch == 'aarch64' else '19'), + '--install-dir="%s"' % toolchain_dir, + '--system=linux-x86_64', + '--stl=stlport', + '--toolchain=' + { + 'aarch64': 'aarch64-linux-android-4.9', + 'arm': 'arm-linux-androideabi-4.9', + 'i686': 'x86-4.9', + }[target_arch]]) + # Android NDK r9d copies a broken unwind.h into the toolchain, see + # http://crbug.com/357890 + for f in glob.glob(os.path.join(toolchain_dir, 'include/c++/*/unwind.h')): + os.remove(f) + + # Build ASan runtime for Android in a separate build tree. + build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch) + if not os.path.exists(build_dir): + os.mkdir(os.path.join(build_dir)) + os.chdir(build_dir) + cflags = ['--target=%s-linux-androideabi' % target_arch, + '--sysroot=%s/sysroot' % toolchain_dir, + '-B%s' % toolchain_dir] + android_args = base_cmake_args + [ + '-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'), + '-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'), + '-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'), + '-DCMAKE_C_FLAGS=' + ' '.join(cflags), + '-DCMAKE_CXX_FLAGS=' + ' '.join(cflags), + '-DANDROID=1'] + RmCmakeCache('.') + RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR]) + RunCommand(['ninja', 'libclang_rt.asan-%s-android.so' % target_arch]) + + # And copy it into the main build tree. + runtime = 'libclang_rt.asan-%s-android.so' % target_arch + for root, _, files in os.walk(build_dir): + if runtime in files: + shutil.copy(os.path.join(root, runtime), asan_rt_lib_dst_dir) + # Run tests. if args.run_tests or use_head_revision: os.chdir(LLVM_BUILD_DIR) - RunCommand(GetVSVersion().SetupScript('x64') + - ['&&', 'ninja', 'cr-check-all']) + RunCommand(['ninja', 'cr-check-all'], msvc_arch='x64') if args.run_tests: os.chdir(LLVM_BUILD_DIR) - RunCommand(GetVSVersion().SetupScript('x64') + - ['&&', 'ninja', 'check-all']) + RunCommand(['ninja', 'check-all'], msvc_arch='x64') WriteStampFile(PACKAGE_VERSION) print 'Clang update was successful.' @@ -663,31 +743,6 @@ def main(): - if not sys.platform in ['win32', 'cygwin']: - # For non-Windows, fall back to update.sh. - # TODO(hans): Make update.py replace update.sh completely. - - # This script is called by gclient. gclient opens its hooks subprocesses - # with (stdout=subprocess.PIPE, stderr=subprocess.STDOUT) and then does - # custom output processing that breaks printing '\r' characters for - # single-line updating status messages as printed by curl and wget. - # Work around this by setting stderr of the update.sh process to stdin (!): - # gclient doesn't redirect stdin, and while stdin itself is read-only, a - # dup()ed sys.stdin is writable, try - # fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi') - # TODO: Fix gclient instead, http://crbug.com/95350 - if '--no-stdin-hack' in sys.argv: - sys.argv.remove('--no-stdin-hack') - stderr = None - else: - try: - stderr = os.fdopen(os.dup(sys.stdin.fileno())) - except: - stderr = sys.stderr - return subprocess.call( - [os.path.join(os.path.dirname(__file__), 'update.sh')] + sys.argv[1:], - stderr=stderr) - parser = argparse.ArgumentParser(description='Build Clang.') parser.add_argument('--bootstrap', action='store_true', help='first build clang with CC, then with itself.') @@ -695,26 +750,24 @@ help="run only if the script thinks clang is needed") parser.add_argument('--force-local-build', action='store_true', help="don't try to download prebuild binaries") + parser.add_argument('--gcc-toolchain', help='set the version for which gcc ' + 'version be used for building; --gcc-toolchain=/opt/foo ' + 'picks /opt/foo/bin/gcc') parser.add_argument('--print-revision', action='store_true', help='print current clang revision and exit.') + parser.add_argument('--print-clang-version', action='store_true', + help='print current clang version (e.g. x.y.z) and exit.') parser.add_argument('--run-tests', action='store_true', help='run tests after building; only for local builds') parser.add_argument('--tools', nargs='*', help='select which chrome tools to build', default=['plugins', 'blink_gc_plugin']) - parser.add_argument('--without-patches', action='store_false', - help="don't apply patches (default)", dest='with_patches', - default=True) - - # For now, these flags are only used for the non-Windows flow, but argparser - # gets mad if it sees a flag it doesn't recognize. - parser.add_argument('--no-stdin-hack', action='store_true') - + parser.add_argument('--without-android', action='store_false', + help='don\'t build Android ASan runtime (linux only)', + dest='with_android', + default=sys.platform.startswith('linux')) args = parser.parse_args() - if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')): - print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).' - return 0 if args.if_needed: is_clang_required = False # clang is always used on Mac and Linux. @@ -730,8 +783,16 @@ is_clang_required = True if not is_clang_required: return 0 + if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')): + print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).' + return 0 + + if use_head_revision: + # TODO(hans): Remove after the next roll. + global VERSION + VERSION = '3.9.0' - global LLVM_WIN_REVISION, PACKAGE_VERSION + global CLANG_REVISION, PACKAGE_VERSION if args.print_revision: if use_head_revision: print GetSvnRevision(LLVM_DIR) @@ -739,6 +800,10 @@ print PACKAGE_VERSION return 0 + if args.print_clang_version: + sys.stdout.write(VERSION) + return 0 + # Don't buffer stdout, so that print statements are immediately flushed. # Do this only after --print-revision has been handled, else we'll get # an error message when this script is run from gn for some reason. @@ -747,12 +812,13 @@ if use_head_revision: # Use a real revision number rather than HEAD to make sure that the stamp # file logic works. - LLVM_WIN_REVISION = GetSvnRevision(LLVM_REPO_URL) - PACKAGE_VERSION = LLVM_WIN_REVISION + '-0' + CLANG_REVISION = GetSvnRevision(LLVM_REPO_URL) + PACKAGE_VERSION = CLANG_REVISION + '-0' args.force_local_build = True - # Skip local patches when using HEAD: they probably don't apply anymore. - args.with_patches = False + if 'OS=android' not in os.environ.get('GYP_DEFINES', ''): + # Only build the Android ASan rt on ToT bots when targetting Android. + args.with_android = False return UpdateClang(args)