[x264-devel] input: Add a workaround for swscale overread bugs
Henrik Gramner
git at videolan.org
Mon Dec 25 20:40:09 CET 2017
x264 | branch: master | Henrik Gramner <henrik at gramner.com> | Sun Oct 22 09:59:28 2017 +0200| [61e8b5cc482b08d51e18b336081073736d963e7e] | committer: Anton Mitrofanov
input: Add a workaround for swscale overread bugs
swscale can read past the end of the input buffer, which may result in
crashes if such a read crosses a page boundary into an invalid page.
Work around this by adding some padding space at the end of the buffer when
using memory-mapped input frames. This may sometimes require copying the
last frame into a new buffer on Windows since the Microsoft memory-mapping
implementation has very limited capabilities compared to POSIX systems.
> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=61e8b5cc482b08d51e18b336081073736d963e7e
---
input/input.c | 80 +++++++++++++++++++++++++++++++++++++++++++++--------------
input/input.h | 2 ++
2 files changed, 63 insertions(+), 19 deletions(-)
diff --git a/input/input.c b/input/input.c
index 546ebd0c..b7e5e0a1 100644
--- a/input/input.c
+++ b/input/input.c
@@ -148,35 +148,71 @@ const x264_cli_csp_t *x264_cli_get_csp( int csp )
/* Functions for handling memory-mapped input frames */
int x264_cli_mmap_init( cli_mmap_t *h, FILE *fh )
{
-#ifdef _WIN32
- HANDLE osfhandle = (HANDLE)_get_osfhandle( _fileno( fh ) );
- if( osfhandle != INVALID_HANDLE_VALUE )
+#if defined(_WIN32) || HAVE_MMAP
+ int fd = fileno( fh );
+ x264_struct_stat file_stat;
+ if( !x264_fstat( fd, &file_stat ) )
{
- SYSTEM_INFO si;
- GetSystemInfo( &si );
- h->align_mask = si.dwAllocationGranularity - 1;
- h->prefetch_virtual_memory = (void*)GetProcAddress( GetModuleHandleW( L"kernel32.dll" ), "PrefetchVirtualMemory" );
- h->process_handle = GetCurrentProcess();
- h->map_handle = CreateFileMappingW( osfhandle, NULL, PAGE_READONLY, 0, 0, NULL );
- return !h->map_handle;
- }
+ h->file_size = file_stat.st_size;
+#ifdef _WIN32
+ HANDLE osfhandle = (HANDLE)_get_osfhandle( fd );
+ if( osfhandle != INVALID_HANDLE_VALUE )
+ {
+ SYSTEM_INFO si;
+ GetSystemInfo( &si );
+ h->page_mask = si.dwPageSize - 1;
+ h->align_mask = si.dwAllocationGranularity - 1;
+ h->prefetch_virtual_memory = (void*)GetProcAddress( GetModuleHandleW( L"kernel32.dll" ), "PrefetchVirtualMemory" );
+ h->process_handle = GetCurrentProcess();
+ h->map_handle = CreateFileMappingW( osfhandle, NULL, PAGE_READONLY, 0, 0, NULL );
+ return !h->map_handle;
+ }
#elif HAVE_MMAP && defined(_SC_PAGESIZE)
- h->align_mask = sysconf( _SC_PAGESIZE ) - 1;
- h->fd = fileno( fh );
- return h->align_mask < 0 || h->fd < 0;
+ h->align_mask = sysconf( _SC_PAGESIZE ) - 1;
+ h->fd = fd;
+ return h->align_mask < 0 || fd < 0;
+#endif
+ }
#endif
return -1;
}
+/* Third-party filters such as swscale can overread the input buffer which may result
+ * in segfaults. We have to pad the buffer size as a workaround to avoid that. */
+#define MMAP_PADDING 64
+
void *x264_cli_mmap( cli_mmap_t *h, int64_t offset, size_t size )
{
#if defined(_WIN32) || HAVE_MMAP
+ uint8_t *base;
int align = offset & h->align_mask;
offset -= align;
size += align;
#ifdef _WIN32
- uint8_t *base = MapViewOfFile( h->map_handle, FILE_MAP_READ, offset >> 32, offset, size );
- if( base )
+ /* If the padding crosses a page boundary we need to increase the mapping size. */
+ size_t padded_size = (-size & h->page_mask) < MMAP_PADDING ? size + MMAP_PADDING : size;
+ if( offset + padded_size > h->file_size )
+ {
+ /* It's not possible to do the POSIX mmap() remapping trick on Windows, so if the padding crosses a
+ * page boundary past the end of the file we have to copy the entire frame into a padded buffer. */
+ if( (base = MapViewOfFile( h->map_handle, FILE_MAP_READ, offset >> 32, offset, size )) )
+ {
+ uint8_t *buf = NULL;
+ HANDLE anon_map = CreateFileMappingW( INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, padded_size, NULL );
+ if( anon_map )
+ {
+ if( (buf = MapViewOfFile( anon_map, FILE_MAP_WRITE, 0, 0, 0 )) )
+ {
+ buf += align;
+ memcpy( buf, base + align, size - align );
+ }
+ CloseHandle( anon_map );
+ }
+ UnmapViewOfFile( base );
+ return buf;
+ }
+ }
+ else if( (base = MapViewOfFile( h->map_handle, FILE_MAP_READ, offset >> 32, offset, padded_size )) )
{
/* PrefetchVirtualMemory() is only available on Windows 8 and newer. */
if( h->prefetch_virtual_memory )
@@ -187,8 +223,8 @@ void *x264_cli_mmap( cli_mmap_t *h, int64_t offset, size_t size )
return base + align;
}
#else
- uint8_t *base = mmap( NULL, size, PROT_READ, MAP_PRIVATE, h->fd, offset );
- if( base != MAP_FAILED )
+ size_t padded_size = size + MMAP_PADDING;
+ if( (base = mmap( NULL, padded_size, PROT_READ, MAP_PRIVATE, h->fd, offset )) != MAP_FAILED )
{
/* Ask the OS to readahead pages. This improves performance whereas
* forcing page faults by manually accessing every page does not.
@@ -199,6 +235,12 @@ void *x264_cli_mmap( cli_mmap_t *h, int64_t offset, size_t size )
#elif defined(POSIX_MADV_WILLNEED)
posix_madvise( base, size, POSIX_MADV_WILLNEED );
#endif
+ /* Remap the file mapping of any padding that crosses a page boundary past the end of
+ * the file into a copy of the last valid page to prevent reads from invalid memory. */
+ size_t aligned_size = (padded_size - 1) & ~h->align_mask;
+ if( offset + aligned_size >= h->file_size )
+ mmap( base + aligned_size, padded_size - aligned_size, PROT_READ, MAP_PRIVATE|MAP_FIXED, h->fd, (offset + size - 1) & ~h->align_mask );
+
return base + align;
}
#endif
@@ -213,7 +255,7 @@ int x264_cli_munmap( cli_mmap_t *h, void *addr, size_t size )
#ifdef _WIN32
return !UnmapViewOfFile( base );
#else
- return munmap( base, size + (intptr_t)addr - (intptr_t)base );
+ return munmap( base, size + MMAP_PADDING + (intptr_t)addr - (intptr_t)base );
#endif
#endif
return -1;
diff --git a/input/input.h b/input/input.h
index 3b33be4a..2a759220 100644
--- a/input/input.h
+++ b/input/input.h
@@ -138,8 +138,10 @@ const x264_cli_csp_t *x264_cli_get_csp( int csp );
typedef struct
{
+ int64_t file_size;
int align_mask;
#ifdef _WIN32
+ int page_mask;
BOOL (WINAPI *prefetch_virtual_memory)( HANDLE, ULONG_PTR, PVOID, ULONG );
HANDLE process_handle;
HANDLE map_handle;
More information about the x264-devel
mailing list