[x265] [PATCH 1 of 5] cmake: be more aggressive with compiler flags in checked builds

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1424887732 21600
#      Wed Feb 25 12:08:52 2015 -0600
# Node ID 515a229245f687815b0f1c9d6973daa56219ba6b
# Parent  e2d079803d1d0359258221b47b4b8b74c6d8f8e8
cmake: be more aggressive with compiler flags in checked builds

This is going to introduce a lot of EXC_BAD_INSTRUCTION exceptions in checked
builds until we work through everything it finds. All issues found by these
flags so far have been valid but perhaps questionable code.

diff -r e2d079803d1d -r 515a229245f6 source/CMakeLists.txt
--- a/source/CMakeLists.txt	Wed Feb 25 11:27:27 2015 -0600
+++ b/source/CMakeLists.txt	Wed Feb 25 12:08:52 2015 -0600
@@ -29,11 +29,6 @@
 
 SET(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" "${CMAKE_MODULE_PATH}")
 
-option(CHECKED_BUILD "Enable run-time sanity checks (debugging)" OFF)
-if(CHECKED_BUILD)
-    add_definitions(-DCHECKED_BUILD=1)
-endif()
-
 # System architecture detection
 string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" SYSPROC)
 set(X86_ALIASES x86 i386 i686 x86_64 amd64)
@@ -177,6 +172,25 @@
     endif()
 endif()
 
+option(CHECKED_BUILD "Enable run-time sanity checks (debugging)" OFF)
+if(CHECKED_BUILD)
+    add_definitions(-DCHECKED_BUILD=1)
+    if(GCC)
+        check_cxx_compiler_flag(-fsanitize=address CC_HAS_FSANITIZE)              # clang and gcc
+        check_cxx_compiler_flag(-fsanitize=undefined-trap CC_HAS_CATCH_UNDEFINED) # clang
+        check_cxx_compiler_flag(-ftrapv CC_HAS_FTRAPV)                            # gcc
+        if(CC_HAS_FSANITIZE)
+            add_definitions(-fsanitize=address)
+        endif()
+        if(CC_HAS_FTRAPV)
+            add_definitions(-ftrapv)
+        endif()
+        if(CC_HAS_CATCH_UNDEFINED)
+            add_definitions(-fsanitize=undefined-trap -fsanitize-undefined-trap-on-error)
+        endif()
+    endif(GCC)
+endif()
+
 # Build options
 set(LIB_INSTALL_DIR lib CACHE STRING "Install location of libraries")
 set(BIN_INSTALL_DIR bin CACHE STRING "Install location of executables")