[x265] [Patch] Enable sharing cutree data among encoders | encoding only the focused frames in the crf 2 pass
Dakshinya T R S
dakshinya at multicorewareinc.com
Sun Sep 19 16:19:27 UTC 2021
*Enable sharing cutree data among encoders*
>From 69d41f124d14347ec71cc61f3290d3eae7369631 Mon Sep 17 00:00:00 2001
From: lwWang <liwei at multicorewareinc.com>
Date: Wed, 8 Sep 2021 13:38:37 +0800
Subject: [PATCH] Enable sharing cutree data among encoders
---
source/common/CMakeLists.txt | 3 +-
source/common/ringmem.cpp | 357 +++++++++++++++++++
source/common/ringmem.h | 85 +++++
source/common/threading.h | 137 ++++++++
source/encoder/encoder.cpp | 4 +-
source/encoder/encoder.h | 2 +-
source/encoder/ratecontrol.cpp | 603 +++++++++++++++++++++------------
source/encoder/ratecontrol.h | 20 +-
8 files changed, 987 insertions(+), 224 deletions(-)
create mode 100644 source/common/ringmem.cpp
create mode 100644 source/common/ringmem.h
diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt
index 12b643ad5..8752c6199 100644
--- a/source/common/CMakeLists.txt
+++ b/source/common/CMakeLists.txt
@@ -169,4 +169,5 @@ add_library(common OBJECT
scalinglist.cpp scalinglist.h
quant.cpp quant.h contexts.h
deblock.cpp deblock.h
- scaler.cpp scaler.h)
+ scaler.cpp scaler.h
+ ringmem.cpp ringmem.h)
diff --git a/source/common/ringmem.cpp b/source/common/ringmem.cpp
new file mode 100644
index 000000000..a4f191c90
--- /dev/null
+++ b/source/common/ringmem.cpp
@@ -0,0 +1,357 @@
+/*****************************************************************************
+ * Copyright (C) 2013-2017 MulticoreWare, Inc
+ *
+ * Authors: liwei <liwei at multicorewareinc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111,
USA.
+ *
+ * This program is also available under a commercial proprietary license.
+ * For more information, contact us at license @ x265.com
+
*****************************************************************************/
+
+#include "ringmem.h"
+
+#ifndef _WIN32
+#include <sys/mman.h>
+#endif ////< _WIN32
+
+#ifdef _WIN32
+#define X265_SHARED_MEM_NAME "Local\\_x265_shr_mem_"
+#define X265_SEMAPHORE_RINGMEM_WRITER_NAME "_x265_semW_"
+#define X265_SEMAPHORE_RINGMEM_READER_NAME "_x265_semR_"
+#else /* POSIX / pthreads */
+#define X265_SHARED_MEM_NAME "/tmp/_x265_shr_mem_"
+#define X265_SEMAPHORE_RINGMEM_WRITER_NAME "/tmp/_x265_semW_"
+#define X265_SEMAPHORE_RINGMEM_READER_NAME "/tmp/_x265_semR_"
+#endif
+
+#define RINGMEM_ALLIGNMENT 64
+
+namespace X265_NS {
+ RingMem::RingMem()
+ : m_initialized(false)
+ , m_protectRW(false)
+ , m_itemSize(0)
+ , m_itemCnt(0)
+ , m_dataPool(NULL)
+ , m_shrMem(NULL)
+#ifdef _WIN32
+ , m_handle(NULL)
+#else //_WIN32
+ , m_filepath(NULL)
+#endif //_WIN32
+ , m_writeSem(NULL)
+ , m_readSem(NULL)
+ {
+ }
+
+
+ RingMem::~RingMem()
+ {
+ }
+
+ bool RingMem::skipRead(int32_t cnt) {
+ if (!m_initialized)
+ {
+ return false;
+ }
+
+ if (m_protectRW)
+ {
+ for (int i = 0; i < cnt; i++)
+ {
+ m_readSem->take();
+ }
+ }
+
+ ATOMIC_ADD(&m_shrMem->m_read, cnt);
+
+ if (m_protectRW)
+ {
+ m_writeSem->give(cnt);
+ }
+
+ return true;
+ }
+
+ bool RingMem::skipWrite(int32_t cnt) {
+ if (!m_initialized)
+ {
+ return false;
+ }
+
+ if (m_protectRW)
+ {
+ for (int i = 0; i < cnt; i++)
+ {
+ m_writeSem->take();
+ }
+ }
+
+ ATOMIC_ADD(&m_shrMem->m_write, cnt);
+
+ if (m_protectRW)
+ {
+ m_readSem->give(cnt);
+ }
+
+ return true;
+ }
+
+ ///< initialize
+ bool RingMem::init(int32_t itemSize, int32_t itemCnt, const char
*name, bool protectRW)
+ {
+ ///< check parameters
+ if (itemSize <= 0 || itemCnt <= 0 || NULL == name)
+ {
+ ///< invalid parameters
+ return false;
+ }
+
+ if (!m_initialized)
+ {
+ ///< formating names
+ char nameBuf[MAX_SHR_NAME_LEN] = { 0 };
+
+ ///< shared memory name
+ snprintf(nameBuf, sizeof(nameBuf) - 1, "%s%s",
X265_SHARED_MEM_NAME, name);
+
+ ///< create or open shared memory
+ bool newCreated = false;
+
+ ///< calculate the size of the shared memory
+ int32_t shrMemSize = (itemSize * itemCnt + sizeof(ShrMemCtrl)
+ RINGMEM_ALLIGNMENT - 1) & ~(RINGMEM_ALLIGNMENT - 1);
+
+#ifdef _WIN32
+ HANDLE h = OpenFileMappingA(FILE_MAP_WRITE | FILE_MAP_READ,
FALSE, nameBuf);
+ if (!h)
+ {
+ h = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL,
PAGE_READWRITE, 0, shrMemSize, nameBuf);
+
+ if (!h)
+ {
+ return false;
+ }
+
+ newCreated = true;
+ }
+
+ void *pool = MapViewOfFile(h, FILE_MAP_ALL_ACCESS, 0, 0, 0);
+
+ ///< should not close the handle here, otherwise the
OpenFileMapping would fail
+ //CloseHandle(h);
+ m_handle = h;
+
+ if (!pool)
+ {
+ return false;
+ }
+
+#else /* POSIX / pthreads */
+ mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH
| S_IWOTH;
+ int flag = O_RDWR;
+ int shrfd = -1;
+ if ((shrfd = open(nameBuf, flag, mode)) < 0)
+ {
+ flag |= O_CREAT;
+
+ shrfd = open(nameBuf, flag, mode);
+ if (shrfd < 0)
+ {
+ return false;
+ }
+ newCreated = true;
+
+ lseek(shrfd, shrMemSize - 1, SEEK_SET);
+
+ if (-1 == write(shrfd, "\0", 1))
+ {
+ close(shrfd);
+ return false;
+ }
+
+ if (lseek(shrfd, 0, SEEK_END) < shrMemSize)
+ {
+ close(shrfd);
+ return false;
+ }
+ }
+
+ void *pool = mmap(0,
+ shrMemSize,
+ PROT_READ | PROT_WRITE,
+ MAP_SHARED,
+ shrfd,
+ 0);
+
+ close(shrfd);
+ if (pool == MAP_FAILED)
+ {
+ return false;
+ }
+
+ m_filepath = strdup(nameBuf);
+#endif ///< _WIN32
+
+ if (newCreated)
+ {
+ memset(pool, 0, shrMemSize);
+ }
+
+ m_shrMem = reinterpret_cast<ShrMemCtrl *>(pool);
+ m_dataPool = reinterpret_cast<uint8_t *>(pool) +
sizeof(ShrMemCtrl);
+ m_itemSize = itemSize;
+ m_itemCnt = itemCnt;
+ m_initialized = true;
+
+ if (protectRW)
+ {
+ m_protectRW = true;
+ m_writeSem = new NamedSemaphore();
+ if (!m_writeSem)
+ {
+ release();
+ return false;
+ }
+
+ ///< shared memory name
+ snprintf(nameBuf, sizeof(nameBuf - 1), "%s%s",
X265_SEMAPHORE_RINGMEM_WRITER_NAME, name);
+ if (!m_writeSem->create(nameBuf, m_itemCnt, m_itemCnt))
+ {
+ release();
+ return false;
+ }
+
+ m_readSem = new NamedSemaphore();
+ if (!m_readSem)
+ {
+ release();
+ return false;
+ }
+
+ ///< shared memory name
+ snprintf(nameBuf, sizeof(nameBuf - 1), "%s%s",
X265_SEMAPHORE_RINGMEM_READER_NAME, name);
+ if (!m_readSem->create(nameBuf, 0, m_itemCnt))
+ {
+ release();
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+ ///< finalize
+ void RingMem::release()
+ {
+ if (m_initialized)
+ {
+ m_initialized = false;
+
+ if (m_shrMem)
+ {
+#ifdef _WIN32
+ UnmapViewOfFile(m_shrMem);
+ CloseHandle(m_handle);
+ m_handle = NULL;
+#else /* POSIX / pthreads */
+ int32_t shrMemSize = (m_itemSize * m_itemCnt +
sizeof(ShrMemCtrl) + RINGMEM_ALLIGNMENT - 1) & (~RINGMEM_ALLIGNMENT - 1);
+ munmap(m_shrMem, shrMemSize);
+ unlink(m_filepath);
+ free(m_filepath);
+ m_filepath = NULL;
+#endif ///< _WIN32
+ m_shrMem = NULL;
+ m_dataPool = NULL;
+ m_itemSize = 0;
+ m_itemCnt = 0;
+ }
+
+ if (m_protectRW)
+ {
+ m_protectRW = false;
+ if (m_writeSem)
+ {
+ m_writeSem->release();
+
+ delete m_writeSem;
+ m_writeSem = NULL;
+ }
+
+ if (m_readSem)
+ {
+ m_readSem->release();
+
+ delete m_readSem;
+ m_readSem = NULL;
+ }
+ }
+
+ }
+ }
+
+ ///< data read
+ bool RingMem::readNext(void* dst, fnRWSharedData callback)
+ {
+ if (!m_initialized || !callback || !dst)
+ {
+ return false;
+ }
+
+ if (m_protectRW)
+ {
+ if (!m_readSem->take())
+ {
+ return false;
+ }
+ }
+
+ int32_t index = ATOMIC_ADD(&m_shrMem->m_read, 1) % m_itemCnt;
+ (*callback)(dst, reinterpret_cast<uint8_t *>(m_dataPool) + index *
m_itemSize, m_itemSize);
+
+ if (m_protectRW)
+ {
+ m_writeSem->give(1);
+ }
+
+ return true;
+ }
+ ///< data write
+ bool RingMem::writeData(void *data, fnRWSharedData callback)
+ {
+ if (!m_initialized || !data || !callback)
+ {
+ return false;
+ }
+
+ if (m_protectRW)
+ {
+ if (!m_writeSem->take())
+ {
+ return false;
+ }
+ }
+
+ int32_t index = ATOMIC_ADD(&m_shrMem->m_write, 1) % m_itemCnt;
+ (*callback)(reinterpret_cast<uint8_t *>(m_dataPool) + index *
m_itemSize, data, m_itemSize);
+
+ if (m_protectRW)
+ {
+ m_readSem->give(1);
+ }
+
+ return true;
+ }
+}
diff --git a/source/common/ringmem.h b/source/common/ringmem.h
new file mode 100644
index 000000000..76f54bb19
--- /dev/null
+++ b/source/common/ringmem.h
@@ -0,0 +1,85 @@
+/*****************************************************************************
+ * Copyright (C) 2013-2017 MulticoreWare, Inc
+ *
+ * Authors: liwei <liwei at multicorewareinc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111,
USA.
+ *
+ * This program is also available under a commercial proprietary license.
+ * For more information, contact us at license @ x265.com
+
*****************************************************************************/
+
+#ifndef X265_RINGMEM_H
+#define X265_RINGMEM_H
+
+#include "common.h"
+#include "threading.h"
+
+namespace X265_NS {
+
+#define MAX_SHR_NAME_LEN 256
+
+ class RingMem {
+ public:
+ RingMem();
+ ~RingMem();
+
+ bool skipRead(int32_t cnt);
+
+ bool skipWrite(int32_t cnt);
+
+ ///< initialize
+ ///< protectRW: if use the semaphore the protect the write and
read operation.
+ bool init(int32_t itemSize, int32_t itemCnt, const char *name,
bool protectRW = false);
+ ///< finalize
+ void release();
+
+ typedef void(*fnRWSharedData)(void *dst, void *src, int32_t size);
+
+ ///< data read
+ bool readNext(void* dst, fnRWSharedData callback);
+ ///< data write
+ bool writeData(void *data, fnRWSharedData callback);
+
+ private:
+ bool m_initialized;
+ bool m_protectRW;
+
+ int32_t m_itemSize;
+ int32_t m_itemCnt;
+ ///< data pool
+ void *m_dataPool;
+ typedef struct {
+ ///< index to write
+ int32_t m_write;
+ ///< index to read
+ int32_t m_read;
+
+ }ShrMemCtrl;
+
+ ShrMemCtrl *m_shrMem;
+#ifdef _WIN32
+ void *m_handle;
+#else // _WIN32
+ char *m_filepath;
+#endif // _WIN32
+
+ ///< Semaphores
+ NamedSemaphore *m_writeSem;
+ NamedSemaphore *m_readSem;
+ };
+};
+
+#endif // ifndef X265_RINGMEM_H
diff --git a/source/common/threading.h b/source/common/threading.h
index 53a63beaf..dcf6081e3 100644
--- a/source/common/threading.h
+++ b/source/common/threading.h
@@ -3,6 +3,7 @@
*
* Authors: Steve Borho <steve at borho.org>
* Min Chen <chenm003 at 163.com>
+ liwei <liwei at multicorewareinc.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -253,6 +254,47 @@ protected:
int m_val;
};
+class NamedSemaphore
+{
+public:
+ NamedSemaphore() : m_sem(NULL)
+ {
+ }
+
+ ~NamedSemaphore()
+ {
+ }
+
+ bool create(const char* name, const int initcnt, const int maxcnt)
+ {
+ if(!m_sem)
+ {
+ m_sem = CreateSemaphoreA(NULL, initcnt, maxcnt, name);
+ }
+ return m_sem != NULL;
+ }
+
+ bool give(const int32_t cnt)
+ {
+ return ReleaseSemaphore(m_sem, (LONG)cnt, NULL) != FALSE;
+ }
+
+ bool take(const uint32_t time_out = INFINITE)
+ {
+ int32_t rt = WaitForSingleObject(m_sem, time_out);
+ return rt != WAIT_TIMEOUT && rt != WAIT_FAILED;
+ }
+
+ void release()
+ {
+ CloseHandle(m_sem);
+ m_sem = NULL;
+ }
+
+private:
+ HANDLE m_sem;
+};
+
#else /* POSIX / pthreads */
typedef pthread_t ThreadHandle;
@@ -459,6 +501,101 @@ protected:
int m_val;
};
+#define TIMEOUT_INFINITE 0xFFFFFFFF
+
+class NamedSemaphore
+{
+public:
+ NamedSemaphore()
+ : m_sem(NULL)
+ , m_name(NULL)
+ {
+ }
+
+ ~NamedSemaphore()
+ {
+ }
+
+ bool create(const char* name, const int initcnt, const int maxcnt)
+ {
+ bool ret = false;
+
+ if (initcnt >= maxcnt)
+ {
+ return false;
+ }
+
+ m_sem = sem_open(name, O_CREAT | O_EXCL, 0666, initcnt);
+ if (m_sem != SEM_FAILED)
+ {
+ m_name = strdup(name);
+ ret = true;
+ }
+ else
+ {
+ if (EEXIST == errno)
+ {
+ m_sem = sem_open(name, 0);
+ if (m_sem != SEM_FAILED)
+ {
+ m_name = strdup(name);
+ ret = true;
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ bool give(const int32_t cnt)
+ {
+ int ret = 0;
+ int32_t curCnt = cnt;
+ while (curCnt-- && !ret) {
+ ret = sem_post(m_sem);
+ }
+
+ return 0 == ret;
+ }
+
+ bool take(const uint32_t time_out = TIMEOUT_INFINITE)
+ {
+ if (TIMEOUT_INFINITE == time_out) {
+ return 0 == sem_wait(m_sem);
+ }
+ else
+ {
+ if (0 == time_out)
+ {
+ return 0 == sem_trywait(m_sem);
+ }
+ else
+ {
+ struct timespec ts;
+ ts.tv_sec = time_out / 1000L;
+ ts.tv_nsec = (time_out * 1000000L) - ts.tv_sec * 1000 *
1000 * 1000;
+ return 0 == sem_timedwait(m_sem, &ts);
+ }
+ }
+ }
+
+ void release()
+ {
+ if (m_sem)
+ {
+ sem_close(m_sem);
+ sem_unlink(m_name);
+ m_sem = NULL;
+ free(m_name);
+ m_name = NULL;
+ }
+ }
+
+private:
+ sem_t *m_sem;
+ char *m_name;
+};
+
#endif // ifdef _WIN32
class ScopedLock
diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp
index cc014a740..19551b1f2 100644
--- a/source/encoder/encoder.cpp
+++ b/source/encoder/encoder.cpp
@@ -187,7 +187,7 @@ inline char *strcatFilename(const char *input, const
char *suffix)
return output;
}
-void Encoder::create()
+void Encoder::create(char *dataShr)
{
if (!primitives.pu[0].sad)
{
@@ -389,7 +389,7 @@ void Encoder::create()
lookAheadThreadPool[i].start();
m_lookahead->m_numPools = pools;
m_dpb = new DPB(m_param);
- m_rateControl = new RateControl(*m_param, this);
+ m_rateControl = new RateControl(*m_param, this, dataShr);
if (!m_param->bResetZoneConfig)
{
zoneReadCount = new ThreadSafeInteger[m_param->rc.zonefileCount];
diff --git a/source/encoder/encoder.h b/source/encoder/encoder.h
index 2ee5bdaee..4b8478f16 100644
--- a/source/encoder/encoder.h
+++ b/source/encoder/encoder.h
@@ -305,7 +305,7 @@ public:
#endif
};
- void create();
+ void create(char *dataShr = NULL);
void stopJobs();
void destroy();
diff --git a/source/encoder/ratecontrol.cpp b/source/encoder/ratecontrol.cpp
index 71f08a73e..e2f514172 100644
--- a/source/encoder/ratecontrol.cpp
+++ b/source/encoder/ratecontrol.cpp
@@ -41,6 +41,10 @@
#define BR_SHIFT 6
#define CPB_SHIFT 4
+#define SHARED_DATA_ALIGNMENT 4 ///< 4btye, 32bit
+#define CUTREE_SHARED_MEM_NAME "cutree"
+#define GOP_CNT_CU_TREE 3
+
using namespace X265_NS;
/* Amortize the partial cost of I frames over the next N frames */
@@ -104,6 +108,37 @@ inline char *strcatFilename(const char *input, const
char *suffix)
return output;
}
+typedef struct CUTreeSharedDataItem
+{
+ uint8_t *type;
+ uint16_t *stats;
+}CUTreeSharedDataItem;
+
+void ReadSharedCUTreeData(void *dst, void *src, int32_t size)
+{
+ CUTreeSharedDataItem *statsDst = reinterpret_cast<CUTreeSharedDataItem
*>(dst);
+ uint8_t *typeSrc = reinterpret_cast<uint8_t *>(src);
+ *statsDst->type = *typeSrc;
+
+ ///< for memory alignment, the type will take 32bit in the shared
memory
+ int32_t offset = (sizeof(*statsDst->type) + SHARED_DATA_ALIGNMENT - 1)
& ~(SHARED_DATA_ALIGNMENT - 1);
+ uint16_t *statsSrc = reinterpret_cast<uint16_t *>(typeSrc + offset);
+ memcpy(statsDst->stats, statsSrc, size - offset);
+}
+
+void WriteSharedCUTreeData(void *dst, void *src, int32_t size)
+{
+ CUTreeSharedDataItem *statsSrc = reinterpret_cast<CUTreeSharedDataItem
*>(src);
+ uint8_t *typeDst = reinterpret_cast<uint8_t *>(dst);
+ *typeDst = *statsSrc->type;
+
+ ///< for memory alignment, the type will take 32bit in the shared
memory
+ int32_t offset = (sizeof(*statsSrc->type) + SHARED_DATA_ALIGNMENT - 1)
& ~(SHARED_DATA_ALIGNMENT - 1);
+ uint16_t *statsDst = reinterpret_cast<uint16_t *>(typeDst + offset);
+ memcpy(statsDst, statsSrc->stats, size - offset);
+}
+
+
inline double qScale2bits(RateControlEntry *rce, double qScale)
{
if (qScale < 0.1)
@@ -146,7 +181,7 @@ x265_zone* RateControl::getZone()
return NULL;
}
-RateControl::RateControl(x265_param& p, Encoder *top)
+RateControl::RateControl(x265_param& p, Encoder *top, char * dataShr)
{
m_param = &p;
m_top = top;
@@ -209,6 +244,14 @@ RateControl::RateControl(x265_param& p, Encoder *top)
m_lastAbrResetPoc = -1;
m_statFileOut = NULL;
m_cutreeStatFileOut = m_cutreeStatFileIn = NULL;
+ ///< store the cutree data in file by default
+ m_cutreeStorageMode = !dataShr ? SHARED_MODE_FILE : SHARED_MODE_MEM;
+ m_shrname = NULL;
+ if (dataShr)
+ {
+ m_shrname = strdup(dataShr);
+ }
+ m_cutreeShrMem = NULL;
m_rce2Pass = NULL;
m_encOrder = NULL;
m_lastBsliceSatdCost = 0;
@@ -320,6 +363,42 @@ RateControl::RateControl(x265_param& p, Encoder *top)
m_cuTreeStats.qpBuffer[i] = NULL;
}
+bool RateControl::initCUTreeSharedMem()
+{
+ if (!m_cutreeShrMem) {
+ m_cutreeShrMem = new RingMem();
+ if (!m_cutreeShrMem)
+ {
+ return false;
+ }
+
+ ///< now cutree data form at most 3 gops would be stored in the
shared memory at the same time
+ int32_t itemSize = (sizeof(uint8_t) + SHARED_DATA_ALIGNMENT - 1) &
~(SHARED_DATA_ALIGNMENT - 1);
+ if (m_param->rc.qgSize == 8)
+ {
+ itemSize += sizeof(uint16_t) * m_ncu * 4;
+ }
+ else
+ {
+ itemSize += sizeof(uint16_t) * m_ncu;
+ }
+
+ int32_t itemCnt = X265_MIN(m_param->keyframeMax, (int)(m_fps +
0.5));
+ itemCnt *= GOP_CNT_CU_TREE;
+
+ char shrname[MAX_SHR_NAME_LEN] = { 0 };
+ strcpy(shrname, m_shrname);
+ strcat(shrname, CUTREE_SHARED_MEM_NAME);
+
+ if (!m_cutreeShrMem->init(itemSize, itemCnt, shrname))
+ {
+ return false;
+ }
+ }
+
+ return true;
+}
+
bool RateControl::init(const SPS& sps)
{
if (m_isVbv && !m_initVbv)
@@ -421,244 +500,261 @@ bool RateControl::init(const SPS& sps)
/* Load stat file and init 2pass algo */
if (m_param->rc.bStatRead)
{
- m_expectedBitsSum = 0;
- char *p, *statsIn, *statsBuf;
- /* read 1st pass stats */
- statsIn = statsBuf = x265_slurp_file(fileName);
- if (!statsBuf)
- return false;
- if (m_param->rc.cuTree)
+ if (SHARED_MODE_FILE == m_cutreeStorageMode)
{
- char *tmpFile = strcatFilename(fileName, ".cutree");
- if (!tmpFile)
+ m_expectedBitsSum = 0;
+ char *p, *statsIn, *statsBuf;
+ /* read 1st pass stats */
+ statsIn = statsBuf = x265_slurp_file(fileName);
+ if (!statsBuf)
return false;
- m_cutreeStatFileIn = x265_fopen(tmpFile, "rb");
- X265_FREE(tmpFile);
- if (!m_cutreeStatFileIn)
+ if (m_param->rc.cuTree)
{
- x265_log_file(m_param, X265_LOG_ERROR, "can't open
stats file %s.cutree\n", fileName);
- return false;
+ char *tmpFile = strcatFilename(fileName, ".cutree");
+ if (!tmpFile)
+ return false;
+ m_cutreeStatFileIn = x265_fopen(tmpFile, "rb");
+ X265_FREE(tmpFile);
+ if (!m_cutreeStatFileIn)
+ {
+ x265_log_file(m_param, X265_LOG_ERROR, "can't open
stats file %s.cutree\n", fileName);
+ return false;
+ }
}
- }
- /* check whether 1st pass options were compatible with current
options */
- if (strncmp(statsBuf, "#options:", 9))
- {
- x265_log(m_param, X265_LOG_ERROR,"options list in stats
file not valid\n");
- return false;
- }
- {
- int i, j, m;
- uint32_t k , l;
- bool bErr = false;
- char *opts = statsBuf;
- statsIn = strchr(statsBuf, '\n');
- if (!statsIn)
+ /* check whether 1st pass options were compatible with
current options */
+ if (strncmp(statsBuf, "#options:", 9))
{
- x265_log(m_param, X265_LOG_ERROR, "Malformed stats
file\n");
+ x265_log(m_param, X265_LOG_ERROR, "options list in
stats file not valid\n");
return false;
}
- *statsIn = '\0';
- statsIn++;
- if ((p = strstr(opts, " input-res=")) == 0 || sscanf(p, "
input-res=%dx%d", &i, &j) != 2)
{
- x265_log(m_param, X265_LOG_ERROR, "Resolution
specified in stats file not valid\n");
+ int i, j, m;
+ uint32_t k, l;
+ bool bErr = false;
+ char *opts = statsBuf;
+ statsIn = strchr(statsBuf, '\n');
+ if (!statsIn)
+ {
+ x265_log(m_param, X265_LOG_ERROR, "Malformed stats
file\n");
+ return false;
+ }
+ *statsIn = '\0';
+ statsIn++;
+ if ((p = strstr(opts, " input-res=")) == 0 ||
sscanf(p, " input-res=%dx%d", &i, &j) != 2)
+ {
+ x265_log(m_param, X265_LOG_ERROR, "Resolution
specified in stats file not valid\n");
+ return false;
+ }
+ if ((p = strstr(opts, " fps=")) == 0 || sscanf(p, "
fps=%u/%u", &k, &l) != 2)
+ {
+ x265_log(m_param, X265_LOG_ERROR, "fps specified
in stats file not valid\n");
+ return false;
+ }
+ if (((p = strstr(opts, " vbv-maxrate=")) == 0 ||
sscanf(p, " vbv-maxrate=%d", &m) != 1) && m_param->rc.rateControlMode ==
X265_RC_CRF)
+ {
+ x265_log(m_param, X265_LOG_ERROR, "Constant
rate-factor is incompatible with 2pass without vbv-maxrate in the previous
pass\n");
+ return false;
+ }
+ if (k != m_param->fpsNum || l != m_param->fpsDenom)
+ {
+ x265_log(m_param, X265_LOG_ERROR, "fps mismatch
with 1st pass (%u/%u vs %u/%u)\n",
+ m_param->fpsNum, m_param->fpsDenom, k, l);
+ return false;
+ }
+ if (m_param->analysisMultiPassRefine)
+ {
+ p = strstr(opts, "ref=");
+ sscanf(p, "ref=%d", &i);
+ if (i > m_param->maxNumReferences)
+ {
+ x265_log(m_param, X265_LOG_ERROR,
"maxNumReferences cannot be less than 1st pass (%d vs %d)\n",
+ i, m_param->maxNumReferences);
+ return false;
+ }
+ }
+ if (m_param->analysisMultiPassRefine ||
m_param->analysisMultiPassDistortion)
+ {
+ p = strstr(opts, "ctu=");
+ sscanf(p, "ctu=%u", &k);
+ if (k != m_param->maxCUSize)
+ {
+ x265_log(m_param, X265_LOG_ERROR, "maxCUSize
mismatch with 1st pass (%u vs %u)\n",
+ k, m_param->maxCUSize);
+ return false;
+ }
+ }
+ CMP_OPT_FIRST_PASS("bitdepth",
m_param->internalBitDepth);
+ CMP_OPT_FIRST_PASS("weightp",
m_param->bEnableWeightedPred);
+ CMP_OPT_FIRST_PASS("bframes", m_param->bframes);
+ CMP_OPT_FIRST_PASS("b-pyramid", m_param->bBPyramid);
+ CMP_OPT_FIRST_PASS("open-gop", m_param->bOpenGOP);
+ CMP_OPT_FIRST_PASS(" keyint", m_param->keyframeMax);
+ CMP_OPT_FIRST_PASS("scenecut",
m_param->scenecutThreshold);
+ CMP_OPT_FIRST_PASS("intra-refresh",
m_param->bIntraRefresh);
+ CMP_OPT_FIRST_PASS("frame-dup",
m_param->bEnableFrameDuplication);
+ if (m_param->bMultiPassOptRPS)
+ {
+ CMP_OPT_FIRST_PASS("multi-pass-opt-rps",
m_param->bMultiPassOptRPS);
+ CMP_OPT_FIRST_PASS("repeat-headers",
m_param->bRepeatHeaders);
+ CMP_OPT_FIRST_PASS("min-keyint",
m_param->keyframeMin);
+ }
+
+ if ((p = strstr(opts, "b-adapt=")) != 0 && sscanf(p,
"b-adapt=%d", &i) && i >= X265_B_ADAPT_NONE && i <= X265_B_ADAPT_TRELLIS)
+ {
+ m_param->bFrameAdaptive = i;
+ }
+ else if (m_param->bframes)
+ {
+ x265_log(m_param, X265_LOG_ERROR, "b-adapt method
specified in stats file not valid\n");
+ return false;
+ }
+
+ if ((p = strstr(opts, "rc-lookahead=")) != 0 &&
sscanf(p, "rc-lookahead=%d", &i))
+ m_param->lookaheadDepth = i;
+ }
+ /* find number of pics */
+ p = statsIn;
+ int numEntries;
+ for (numEntries = -1; p; numEntries++)
+ p = strchr(p + 1, ';');
+ if (!numEntries)
+ {
+ x265_log(m_param, X265_LOG_ERROR, "empty stats
file\n");
return false;
}
- if ((p = strstr(opts, " fps=")) == 0 || sscanf(p, "
fps=%u/%u", &k, &l) != 2)
+ m_numEntries = numEntries;
+
+ if (m_param->totalFrames < m_numEntries &&
m_param->totalFrames > 0)
+ {
+ x265_log(m_param, X265_LOG_WARNING, "2nd pass has
fewer frames than 1st pass (%d vs %d)\n",
+ m_param->totalFrames, m_numEntries);
+ }
+ if (m_param->totalFrames > m_numEntries &&
!m_param->bEnableFrameDuplication)
{
- x265_log(m_param, X265_LOG_ERROR, "fps specified in
stats file not valid\n");
+ x265_log(m_param, X265_LOG_ERROR, "2nd pass has more
frames than 1st pass (%d vs %d)\n",
+ m_param->totalFrames, m_numEntries);
return false;
}
- if (((p = strstr(opts, " vbv-maxrate=")) == 0 || sscanf(p,
" vbv-maxrate=%d", &m) != 1) && m_param->rc.rateControlMode == X265_RC_CRF)
+
+ m_rce2Pass = X265_MALLOC(RateControlEntry, m_numEntries);
+ if (!m_rce2Pass)
{
- x265_log(m_param, X265_LOG_ERROR, "Constant
rate-factor is incompatible with 2pass without vbv-maxrate in the previous
pass\n");
+ x265_log(m_param, X265_LOG_ERROR, "Rce Entries for 2
pass cannot be allocated\n");
return false;
}
- if (k != m_param->fpsNum || l != m_param->fpsDenom)
+ m_encOrder = X265_MALLOC(int, m_numEntries);
+ if (!m_encOrder)
{
- x265_log(m_param, X265_LOG_ERROR, "fps mismatch with
1st pass (%u/%u vs %u/%u)\n",
- m_param->fpsNum, m_param->fpsDenom, k, l);
+ x265_log(m_param, X265_LOG_ERROR, "Encode order for 2
pass cannot be allocated\n");
return false;
}
- if (m_param->analysisMultiPassRefine)
+ /* init all to skipped p frames */
+ for (int i = 0; i < m_numEntries; i++)
{
- p = strstr(opts, "ref=");
- sscanf(p, "ref=%d", &i);
- if (i > m_param->maxNumReferences)
+ RateControlEntry *rce = &m_rce2Pass[i];
+ rce->sliceType = P_SLICE;
+ rce->qScale = rce->newQScale = x265_qp2qScale(20);
+ rce->miscBits = m_ncu + 10;
+ rce->newQp = 0;
+ }
+ /* read stats */
+ p = statsIn;
+ double totalQpAq = 0;
+ for (int i = 0; i < m_numEntries; i++)
+ {
+ RateControlEntry *rce, *rcePocOrder;
+ int frameNumber;
+ int encodeOrder;
+ char picType;
+ int e;
+ char *next;
+ double qpRc, qpAq, qNoVbv, qRceq;
+ next = strstr(p, ";");
+ if (next)
+ *next++ = 0;
+ e = sscanf(p, " in:%d out:%d", &frameNumber,
&encodeOrder);
+ if (frameNumber < 0 || frameNumber >= m_numEntries)
{
- x265_log(m_param, X265_LOG_ERROR,
"maxNumReferences cannot be less than 1st pass (%d vs %d)\n",
- i, m_param->maxNumReferences);
+ x265_log(m_param, X265_LOG_ERROR, "bad frame
number (%d) at stats line %d\n", frameNumber, i);
return false;
}
- }
- if (m_param->analysisMultiPassRefine ||
m_param->analysisMultiPassDistortion)
- {
- p = strstr(opts, "ctu=");
- sscanf(p, "ctu=%u", &k);
- if (k != m_param->maxCUSize)
+ rce = &m_rce2Pass[encodeOrder];
+ rcePocOrder = &m_rce2Pass[frameNumber];
+ m_encOrder[frameNumber] = encodeOrder;
+ if (!m_param->bMultiPassOptRPS)
{
- x265_log(m_param, X265_LOG_ERROR, "maxCUSize
mismatch with 1st pass (%u vs %u)\n",
- k, m_param->maxCUSize);
+ int scenecut = 0;
+ e += sscanf(p, " in:%*d out:%*d type:%c q:%lf
q-aq:%lf q-noVbv:%lf q-Rceq:%lf tex:%d mv:%d misc:%d icu:%lf pcu:%lf
scu:%lf sc:%d",
+ &picType, &qpRc, &qpAq, &qNoVbv, &qRceq,
&rce->coeffBits,
+ &rce->mvBits, &rce->miscBits, &rce->iCuCount,
&rce->pCuCount,
+ &rce->skipCuCount, &scenecut);
+ rcePocOrder->scenecut = scenecut != 0;
+ }
+ else
+ {
+ char deltaPOC[128];
+ char bUsed[40];
+ memset(deltaPOC, 0, sizeof(deltaPOC));
+ memset(bUsed, 0, sizeof(bUsed));
+ e += sscanf(p, " in:%*d out:%*d type:%c q:%lf
q-aq:%lf q-noVbv:%lf q-Rceq:%lf tex:%d mv:%d misc:%d icu:%lf pcu:%lf
scu:%lf nump:%d numnegp:%d numposp:%d deltapoc:%s bused:%s",
+ &picType, &qpRc, &qpAq, &qNoVbv, &qRceq,
&rce->coeffBits,
+ &rce->mvBits, &rce->miscBits, &rce->iCuCount,
&rce->pCuCount,
+ &rce->skipCuCount,
&rce->rpsData.numberOfPictures, &rce->rpsData.numberOfNegativePictures,
&rce->rpsData.numberOfPositivePictures, deltaPOC, bUsed);
+ splitdeltaPOC(deltaPOC, rce);
+ splitbUsed(bUsed, rce);
+ rce->rpsIdx = -1;
+ }
+ rce->keptAsRef = true;
+ rce->isIdr = false;
+ if (picType == 'b' || picType == 'p')
+ rce->keptAsRef = false;
+ if (picType == 'I')
+ rce->isIdr = true;
+ if (picType == 'I' || picType == 'i')
+ rce->sliceType = I_SLICE;
+ else if (picType == 'P' || picType == 'p')
+ rce->sliceType = P_SLICE;
+ else if (picType == 'B' || picType == 'b')
+ rce->sliceType = B_SLICE;
+ else
+ e = -1;
+ if (e < 10)
+ {
+ x265_log(m_param, X265_LOG_ERROR, "statistics are
damaged at line %d, parser out=%d\n", i, e);
return false;
}
+ rce->qScale = rce->newQScale = x265_qp2qScale(qpRc);
+ totalQpAq += qpAq;
+ rce->qpNoVbv = qNoVbv;
+ rce->qpaRc = qpRc;
+ rce->qpAq = qpAq;
+ rce->qRceq = qRceq;
+ p = next;
}
- CMP_OPT_FIRST_PASS("bitdepth", m_param->internalBitDepth);
- CMP_OPT_FIRST_PASS("weightp",
m_param->bEnableWeightedPred);
- CMP_OPT_FIRST_PASS("bframes", m_param->bframes);
- CMP_OPT_FIRST_PASS("b-pyramid", m_param->bBPyramid);
- CMP_OPT_FIRST_PASS("open-gop", m_param->bOpenGOP);
- CMP_OPT_FIRST_PASS(" keyint", m_param->keyframeMax);
- CMP_OPT_FIRST_PASS("scenecut", m_param->scenecutThreshold);
- CMP_OPT_FIRST_PASS("intra-refresh",
m_param->bIntraRefresh);
- CMP_OPT_FIRST_PASS("frame-dup",
m_param->bEnableFrameDuplication);
- if (m_param->bMultiPassOptRPS)
- {
- CMP_OPT_FIRST_PASS("multi-pass-opt-rps",
m_param->bMultiPassOptRPS);
- CMP_OPT_FIRST_PASS("repeat-headers",
m_param->bRepeatHeaders);
- CMP_OPT_FIRST_PASS("min-keyint", m_param->keyframeMin);
- }
-
- if ((p = strstr(opts, "b-adapt=")) != 0 && sscanf(p,
"b-adapt=%d", &i) && i >= X265_B_ADAPT_NONE && i <= X265_B_ADAPT_TRELLIS)
+ X265_FREE(statsBuf);
+ if (m_param->rc.rateControlMode != X265_RC_CQP)
{
- m_param->bFrameAdaptive = i;
- }
- else if (m_param->bframes)
- {
- x265_log(m_param, X265_LOG_ERROR, "b-adapt method
specified in stats file not valid\n");
- return false;
- }
-
- if ((p = strstr(opts, "rc-lookahead=")) != 0 && sscanf(p,
"rc-lookahead=%d", &i))
- m_param->lookaheadDepth = i;
- }
- /* find number of pics */
- p = statsIn;
- int numEntries;
- for (numEntries = -1; p; numEntries++)
- p = strchr(p + 1, ';');
- if (!numEntries)
- {
- x265_log(m_param, X265_LOG_ERROR, "empty stats file\n");
- return false;
- }
- m_numEntries = numEntries;
-
- if (m_param->totalFrames < m_numEntries &&
m_param->totalFrames > 0)
- {
- x265_log(m_param, X265_LOG_WARNING, "2nd pass has fewer
frames than 1st pass (%d vs %d)\n",
- m_param->totalFrames, m_numEntries);
- }
- if (m_param->totalFrames > m_numEntries &&
!m_param->bEnableFrameDuplication)
- {
- x265_log(m_param, X265_LOG_ERROR, "2nd pass has more
frames than 1st pass (%d vs %d)\n",
- m_param->totalFrames, m_numEntries);
- return false;
+ m_start = 0;
+ m_isQpModified = true;
+ if (!initPass2())
+ return false;
+ } /* else we're using constant quant, so no need to run
the bitrate allocation */
}
-
- m_rce2Pass = X265_MALLOC(RateControlEntry, m_numEntries);
- if (!m_rce2Pass)
+ else if (SHARED_MODE_MEM == m_cutreeStorageMode)
{
- x265_log(m_param, X265_LOG_ERROR, "Rce Entries for 2 pass
cannot be allocated\n");
- return false;
+ if (m_param->rc.cuTree)
+ {
+ if (!initCUTreeSharedMem())
+ {
+ return false;
+ }
+ }
}
- m_encOrder = X265_MALLOC(int, m_numEntries);
- if (!m_encOrder)
+ else
{
- x265_log(m_param, X265_LOG_ERROR, "Encode order for 2 pass
cannot be allocated\n");
return false;
}
- /* init all to skipped p frames */
- for (int i = 0; i < m_numEntries; i++)
- {
- RateControlEntry *rce = &m_rce2Pass[i];
- rce->sliceType = P_SLICE;
- rce->qScale = rce->newQScale = x265_qp2qScale(20);
- rce->miscBits = m_ncu + 10;
- rce->newQp = 0;
- }
- /* read stats */
- p = statsIn;
- double totalQpAq = 0;
- for (int i = 0; i < m_numEntries; i++)
- {
- RateControlEntry *rce, *rcePocOrder;
- int frameNumber;
- int encodeOrder;
- char picType;
- int e;
- char *next;
- double qpRc, qpAq, qNoVbv, qRceq;
- next = strstr(p, ";");
- if (next)
- *next++ = 0;
- e = sscanf(p, " in:%d out:%d", &frameNumber, &encodeOrder);
- if (frameNumber < 0 || frameNumber >= m_numEntries)
- {
- x265_log(m_param, X265_LOG_ERROR, "bad frame number
(%d) at stats line %d\n", frameNumber, i);
- return false;
- }
- rce = &m_rce2Pass[encodeOrder];
- rcePocOrder = &m_rce2Pass[frameNumber];
- m_encOrder[frameNumber] = encodeOrder;
- if (!m_param->bMultiPassOptRPS)
- {
- int scenecut = 0;
- e += sscanf(p, " in:%*d out:%*d type:%c q:%lf q-aq:%lf
q-noVbv:%lf q-Rceq:%lf tex:%d mv:%d misc:%d icu:%lf pcu:%lf scu:%lf sc:%d",
- &picType, &qpRc, &qpAq, &qNoVbv, &qRceq,
&rce->coeffBits,
- &rce->mvBits, &rce->miscBits, &rce->iCuCount,
&rce->pCuCount,
- &rce->skipCuCount, &scenecut);
- rcePocOrder->scenecut = scenecut != 0;
- }
- else
- {
- char deltaPOC[128];
- char bUsed[40];
- memset(deltaPOC, 0, sizeof(deltaPOC));
- memset(bUsed, 0, sizeof(bUsed));
- e += sscanf(p, " in:%*d out:%*d type:%c q:%lf q-aq:%lf
q-noVbv:%lf q-Rceq:%lf tex:%d mv:%d misc:%d icu:%lf pcu:%lf scu:%lf nump:%d
numnegp:%d numposp:%d deltapoc:%s bused:%s",
- &picType, &qpRc, &qpAq, &qNoVbv, &qRceq,
&rce->coeffBits,
- &rce->mvBits, &rce->miscBits, &rce->iCuCount,
&rce->pCuCount,
- &rce->skipCuCount, &rce->rpsData.numberOfPictures,
&rce->rpsData.numberOfNegativePictures,
&rce->rpsData.numberOfPositivePictures, deltaPOC, bUsed);
- splitdeltaPOC(deltaPOC, rce);
- splitbUsed(bUsed, rce);
- rce->rpsIdx = -1;
- }
- rce->keptAsRef = true;
- rce->isIdr = false;
- if (picType == 'b' || picType == 'p')
- rce->keptAsRef = false;
- if (picType == 'I')
- rce->isIdr = true;
- if (picType == 'I' || picType == 'i')
- rce->sliceType = I_SLICE;
- else if (picType == 'P' || picType == 'p')
- rce->sliceType = P_SLICE;
- else if (picType == 'B' || picType == 'b')
- rce->sliceType = B_SLICE;
- else
- e = -1;
- if (e < 10)
- {
- x265_log(m_param, X265_LOG_ERROR, "statistics are
damaged at line %d, parser out=%d\n", i, e);
- return false;
- }
- rce->qScale = rce->newQScale = x265_qp2qScale(qpRc);
- totalQpAq += qpAq;
- rce->qpNoVbv = qNoVbv;
- rce->qpaRc = qpRc;
- rce->qpAq = qpAq;
- rce->qRceq = qRceq;
- p = next;
- }
- X265_FREE(statsBuf);
- if (m_param->rc.rateControlMode != X265_RC_CQP)
- {
- m_start = 0;
- m_isQpModified = true;
- if (!initPass2())
- return false;
- } /* else we're using constant quant, so no need to run the
bitrate allocation */
}
/* Open output file */
/* If input and output files are the same, output to a temp file
@@ -682,19 +778,33 @@ bool RateControl::init(const SPS& sps)
X265_FREE(p);
if (m_param->rc.cuTree && !m_param->rc.bStatRead)
{
- statFileTmpname = strcatFilename(fileName, ".cutree.temp");
- if (!statFileTmpname)
- return false;
- m_cutreeStatFileOut = x265_fopen(statFileTmpname, "wb");
- X265_FREE(statFileTmpname);
- if (!m_cutreeStatFileOut)
+ if (SHARED_MODE_FILE == m_cutreeStorageMode)
+ {
+ statFileTmpname = strcatFilename(fileName,
".cutree.temp");
+ if (!statFileTmpname)
+ return false;
+ m_cutreeStatFileOut = x265_fopen(statFileTmpname,
"wb");
+ X265_FREE(statFileTmpname);
+ if (!m_cutreeStatFileOut)
+ {
+ x265_log_file(m_param, X265_LOG_ERROR, "can't open
mbtree stats file %s.cutree.temp\n", fileName);
+ return false;
+ }
+ }
+ else if (SHARED_MODE_MEM == m_cutreeStorageMode)
+ {
+ if (!initCUTreeSharedMem())
+ {
+ return false;
+ }
+ }
+ else
{
- x265_log_file(m_param, X265_LOG_ERROR, "can't open
mbtree stats file %s.cutree.temp\n", fileName);
return false;
}
}
}
- if (m_param->rc.cuTree)
+ if (m_param->rc.cuTree && !m_cuTreeStats.qpBuffer[0])
{
if (m_param->rc.qgSize == 8)
{
@@ -714,6 +824,10 @@ bool RateControl::init(const SPS& sps)
return true;
}
+void RateControl::skipCUTreeSharedMemRead(int32_t cnt)
+{
+ m_cutreeShrMem->skipRead(cnt);
+}
void RateControl::reconfigureRC()
{
if (m_isVbv)
@@ -1670,10 +1784,28 @@ bool RateControl::cuTreeReadFor2Pass(Frame* frame)
{
m_cuTreeStats.qpBufPos++;
- if (!fread(&type, 1, 1, m_cutreeStatFileIn))
- goto fail;
- if (fread(m_cuTreeStats.qpBuffer[m_cuTreeStats.qpBufPos],
sizeof(uint16_t), ncu, m_cutreeStatFileIn) != (size_t)ncu)
- goto fail;
+ if (SHARED_MODE_FILE == m_cutreeStorageMode)
+ {
+ if (!fread(&type, 1, 1, m_cutreeStatFileIn))
+ goto fail;
+ if
(fread(m_cuTreeStats.qpBuffer[m_cuTreeStats.qpBufPos], sizeof(uint16_t),
ncu, m_cutreeStatFileIn) != (size_t)ncu)
+ goto fail;
+ }
+ else
+ {
+ if (SHARED_MODE_MEM == m_cutreeStorageMode)
+ {
+ if (!m_cutreeShrMem)
+ {
+ goto fail;
+ }
+
+ CUTreeSharedDataItem shrItem;
+ shrItem.type = &type;
+ shrItem.stats =
m_cuTreeStats.qpBuffer[m_cuTreeStats.qpBufPos];
+ m_cutreeShrMem->readNext(&shrItem,
ReadSharedCUTreeData);
+ }
+ }
if (type != sliceTypeActual && m_cuTreeStats.qpBufPos == 1)
{
@@ -3064,10 +3196,34 @@ int RateControl::writeRateControlFrameStats(Frame*
curFrame, RateControlEntry* r
{
uint8_t sliceType = (uint8_t)rce->sliceType;
primitives.fix8Pack(m_cuTreeStats.qpBuffer[0],
curFrame->m_lowres.qpCuTreeOffset, ncu);
- if (fwrite(&sliceType, 1, 1, m_cutreeStatFileOut) < 1)
- goto writeFailure;
- if (fwrite(m_cuTreeStats.qpBuffer[0], sizeof(uint16_t), ncu,
m_cutreeStatFileOut) < (size_t)ncu)
+
+ if (SHARED_MODE_FILE == m_cutreeStorageMode)
+ {
+ if (fwrite(&sliceType, 1, 1, m_cutreeStatFileOut) < 1)
+ goto writeFailure;
+ if (fwrite(m_cuTreeStats.qpBuffer[0], sizeof(uint16_t), ncu,
m_cutreeStatFileOut) < (size_t)ncu)
+ goto writeFailure;
+ }
+ else if (SHARED_MODE_MEM == m_cutreeStorageMode)
+ {
+ if (SHARED_MODE_MEM == m_cutreeStorageMode)
+ {
+ if (!m_cutreeShrMem)
+ {
+ goto writeFailure;
+ }
+
+ CUTreeSharedDataItem shrItem;
+ shrItem.type = &sliceType;
+ shrItem.stats = m_cuTreeStats.qpBuffer[0];
+ m_cutreeShrMem->writeData(&shrItem, WriteSharedCUTreeData);
+ }
+ }
+ else
+ {
goto writeFailure;
+ }
+
}
return 0;
@@ -3143,6 +3299,13 @@ void RateControl::destroy()
if (m_cutreeStatFileIn)
fclose(m_cutreeStatFileIn);
+ if (m_cutreeShrMem)
+ {
+ m_cutreeShrMem->release();
+ delete m_cutreeShrMem;
+ m_cutreeShrMem = NULL;
+ }
+
X265_FREE(m_rce2Pass);
X265_FREE(m_encOrder);
for (int i = 0; i < 2; i++)
@@ -3151,6 +3314,8 @@ void RateControl::destroy()
if (m_relativeComplexity)
X265_FREE(m_relativeComplexity);
+ free(m_shrname);
+
}
void RateControl::splitdeltaPOC(char deltapoc[], RateControlEntry *rce)
diff --git a/source/encoder/ratecontrol.h b/source/encoder/ratecontrol.h
index 204bd71e1..19c8676b4 100644
--- a/source/encoder/ratecontrol.h
+++ b/source/encoder/ratecontrol.h
@@ -28,6 +28,7 @@
#include "common.h"
#include "sei.h"
+#include "ringmem.h"
namespace X265_NS {
// encoder namespace
@@ -126,6 +127,13 @@ struct RateControlEntry
bool isFadeEnd;
};
+enum DataSharedMode
+{
+ SHARED_MODE_FILE = 0,
+ SHARED_MODE_MEM,
+ SHARED_MODE_CNT
+};
+
class RateControl
{
public:
@@ -237,9 +245,16 @@ public:
int m_numEntries;
int m_start;
int m_reencode;
+ ///< store the cutree data in file or shared memory
+ ///< it is not necessary to store the cutree in shared memory.
+ ///< However, for further use, shared memeory is a better choice
+ DataSharedMode m_cutreeStorageMode;
+ char *m_shrname;
FILE* m_statFileOut;
FILE* m_cutreeStatFileOut;
FILE* m_cutreeStatFileIn;
+ ///< store the cutree data in memory instead of file
+ RingMem *m_cutreeShrMem;
double m_lastAccumPNorm;
double m_expectedBitsSum; /* sum of qscale2bits after rceq,
ratefactor, and overflow, only includes finished frames */
int64_t m_predictedBits;
@@ -254,7 +269,7 @@ public:
* This value is the current position (0 or
1). */
} m_cuTreeStats;
- RateControl(x265_param& p, Encoder *enc);
+ RateControl(x265_param& p, Encoder *enc, char *dataShr);
bool init(const SPS& sps);
void initHRD(SPS& sps);
void reconfigureRC();
@@ -274,6 +289,9 @@ public:
int writeRateControlFrameStats(Frame* curFrame, RateControlEntry* rce);
bool initPass2();
+ bool initCUTreeSharedMem();
+ void skipCUTreeSharedMemRead(int32_t cnt);
+
double forwardMasking(Frame* curFrame, double q);
double backwardMasking(Frame* curFrame, double q);
--
2.22.0.windows.1
*encoding only the focused frames in the crf 2 pass*
>From d4ac086b18c41e6d146cd70524c581138b1186ec Mon Sep 17 00:00:00 2001
From: lwWang <liwei at multicorewareinc.com>
Date: Wed, 15 Sep 2021 10:39:51 +0800
Subject: [PATCH 1/2] encoding only the focused frames in the crf 2 pass
---
source/common/frame.h | 1 +
source/common/framedata.cpp | 2 +-
source/common/param.cpp | 1 +
source/encoder/encoder.cpp | 1 +
source/encoder/ratecontrol.cpp | 240 ++++++++++++++++++---------------
source/encoder/ratecontrol.h | 3 +
source/x265.h | 3 +
7 files changed, 138 insertions(+), 113 deletions(-)
diff --git a/source/common/frame.h b/source/common/frame.h
index dc5bbacf7..ac1185e81 100644
--- a/source/common/frame.h
+++ b/source/common/frame.h
@@ -70,6 +70,7 @@ struct RcStats
double count[4];
double offset[4];
double bufferFillFinal;
+ int64_t currentSatd;
};
class Frame
diff --git a/source/common/framedata.cpp b/source/common/framedata.cpp
index c77b017a8..70af8a248 100644
--- a/source/common/framedata.cpp
+++ b/source/common/framedata.cpp
@@ -62,7 +62,7 @@ bool FrameData::create(const x265_param& param, const
SPS& sps, int csp)
}
else
return false;
- CHECKED_MALLOC_ZERO(m_cuStat, RCStatCU, sps.numCUsInFrame);
+ CHECKED_MALLOC_ZERO(m_cuStat, RCStatCU, sps.numCUsInFrame + 1);
CHECKED_MALLOC(m_rowStat, RCStatRow, sps.numCuInHeight);
reinit(sps);
diff --git a/source/common/param.cpp b/source/common/param.cpp
index 6751e0aa7..2c1583d93 100755
--- a/source/common/param.cpp
+++ b/source/common/param.cpp
@@ -282,6 +282,7 @@ void x265_param_default(x265_param* param)
param->rc.bStatRead = 0;
param->rc.bStatWrite = 0;
param->rc.statFileName = NULL;
+ param->rc.bEncFocusedFramesOnly = 0;
param->rc.complexityBlur = 20;
param->rc.qblur = 0.5;
param->rc.zoneCount = 0;
diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp
index dee8fd85d..cc014a740 100644
--- a/source/encoder/encoder.cpp
+++ b/source/encoder/encoder.cpp
@@ -2249,6 +2249,7 @@ int Encoder::encode(const x265_picture* pic_in,
x265_picture* pic_out)
outFrame->m_rcData->iCuCount =
outFrame->m_encData->m_frameStats.percent8x8Intra * m_rateControl->m_ncu;
outFrame->m_rcData->pCuCount =
outFrame->m_encData->m_frameStats.percent8x8Inter * m_rateControl->m_ncu;
outFrame->m_rcData->skipCuCount =
outFrame->m_encData->m_frameStats.percent8x8Skip * m_rateControl->m_ncu;
+ outFrame->m_rcData->currentSatd =
curEncoder->m_rce.coeffBits;
}
/* Allow this frame to be recycled if no frame encoders are
using it for reference */
diff --git a/source/encoder/ratecontrol.cpp b/source/encoder/ratecontrol.cpp
index a4756de39..2c211022d 100644
--- a/source/encoder/ratecontrol.cpp
+++ b/source/encoder/ratecontrol.cpp
@@ -1002,123 +1002,106 @@ bool RateControl::initPass2()
uint64_t allAvailableBits = uint64_t(m_param->rc.bitrate * 1000. *
m_numEntries * m_frameDuration);
int startIndex, framesCount, endIndex;
int fps = X265_MIN(m_param->keyframeMax, (int)(m_fps + 0.5));
+ int distance = fps << 1;
+ distance = distance > m_param->keyframeMax ? (m_param->keyframeMax <<
1) : m_param->keyframeMax;
startIndex = endIndex = framesCount = 0;
- int diffQp = 0;
double targetBits = 0;
double expectedBits = 0;
- for (startIndex = m_start, endIndex = m_start; endIndex <
m_numEntries; endIndex++)
+ double targetBits2 = 0;
+ double expectedBits2 = 0;
+ double cpxSum = 0;
+ double cpxSum2 = 0;
+
+ if (m_param->rc.rateControlMode == X265_RC_ABR)
{
- allConstBits += m_rce2Pass[endIndex].miscBits;
- allCodedBits += m_rce2Pass[endIndex].coeffBits +
m_rce2Pass[endIndex].mvBits;
- if (m_param->rc.rateControlMode == X265_RC_CRF)
+ for (startIndex = m_start, endIndex = m_start; endIndex <
m_numEntries; endIndex++)
{
- framesCount = endIndex - startIndex + 1;
- diffQp += int (m_rce2Pass[endIndex].qpaRc -
m_rce2Pass[endIndex].qpNoVbv);
- if (framesCount > fps)
- diffQp -= int (m_rce2Pass[endIndex - fps].qpaRc -
m_rce2Pass[endIndex - fps].qpNoVbv);
- if (framesCount >= fps)
- {
- if (diffQp >= 1)
- {
- if (!m_isQpModified && endIndex > fps)
- {
- double factor = 2;
- double step = 0;
- if (endIndex + fps >= m_numEntries)
- {
- m_start = endIndex - (endIndex % fps);
- return true;
- }
- for (int start = endIndex + 1; start <= endIndex +
fps && start < m_numEntries; start++)
- {
- RateControlEntry *rce = &m_rce2Pass[start];
- targetBits += qScale2bits(rce,
x265_qp2qScale(rce->qpNoVbv));
- expectedBits += qScale2bits(rce, rce->qScale);
- }
- if (expectedBits < 0.95 * targetBits)
- {
- m_isQpModified = true;
- m_isGopReEncoded = true;
- while (endIndex + fps < m_numEntries)
- {
- step = pow(2, factor / 6.0);
- expectedBits = 0;
- for (int start = endIndex + 1; start <=
endIndex + fps; start++)
- {
- RateControlEntry *rce =
&m_rce2Pass[start];
- rce->newQScale = rce->qScale / step;
- X265_CHECK(rce->newQScale >= 0, "new
Qscale is negative\n");
- expectedBits += qScale2bits(rce,
rce->newQScale);
- rce->newQp =
x265_qScale2qp(rce->newQScale);
- }
- if (expectedBits >= targetBits && step > 1)
- factor *= 0.90;
- else
- break;
- }
-
- if (m_isVbv && endIndex + fps < m_numEntries)
- if (!vbv2Pass((uint64_t)targetBits,
endIndex + fps, endIndex + 1))
- return false;
-
- targetBits = 0;
- expectedBits = 0;
-
- for (int start = endIndex - fps + 1; start <=
endIndex; start++)
- {
- RateControlEntry *rce = &m_rce2Pass[start];
- targetBits += qScale2bits(rce,
x265_qp2qScale(rce->qpNoVbv));
- }
- while (1)
- {
- step = pow(2, factor / 6.0);
- expectedBits = 0;
- for (int start = endIndex - fps + 1; start
<= endIndex; start++)
- {
- RateControlEntry *rce =
&m_rce2Pass[start];
- rce->newQScale = rce->qScale * step;
- X265_CHECK(rce->newQScale >= 0, "new
Qscale is negative\n");
- expectedBits += qScale2bits(rce,
rce->newQScale);
- rce->newQp =
x265_qScale2qp(rce->newQScale);
- }
- if (expectedBits > targetBits && step > 1)
- factor *= 1.1;
- else
- break;
- }
- if (m_isVbv)
- if (!vbv2Pass((uint64_t)targetBits,
endIndex, endIndex - fps + 1))
- return false;
- diffQp = 0;
- m_reencode = endIndex - fps + 1;
- endIndex = endIndex + fps;
- startIndex = endIndex + 1;
- m_start = startIndex;
- targetBits = expectedBits = 0;
- }
- else
- targetBits = expectedBits = 0;
- }
- }
- else
- m_isQpModified = false;
- }
+ allConstBits += m_rce2Pass[endIndex].miscBits;
+ allCodedBits += m_rce2Pass[endIndex].coeffBits +
m_rce2Pass[endIndex].mvBits;
}
- }
- if (m_param->rc.rateControlMode == X265_RC_ABR)
- {
if (allAvailableBits < allConstBits)
{
x265_log(m_param, X265_LOG_ERROR, "requested bitrate is too
low. estimated minimum is %d kbps\n",
- (int)(allConstBits * m_fps / framesCount * 1000.));
+ (int)(allConstBits * m_fps / framesCount * 1000.));
return false;
}
if (!analyseABR2Pass(allAvailableBits))
return false;
+
+ return true;
+ }
+
+ if (m_param->rc.rateControlMode != X265_RC_CRF)
+ {
+ return true;
}
- m_start = X265_MAX(m_start, endIndex - fps);
+ if (m_isQpModified)
+ {
+ return true;
+ }
+
+ if (m_start + (fps << 1) > m_numEntries)
+ {
+ return true;
+ }
+
+ for (startIndex = m_start, endIndex = m_numEntries - 1; startIndex <
endIndex; startIndex++, endIndex--)
+ {
+ cpxSum += m_rce2Pass[startIndex].qScale /
m_rce2Pass[startIndex].coeffBits;
+ cpxSum2 += m_rce2Pass[endIndex].qScale /
m_rce2Pass[endIndex].coeffBits;
+
+ RateControlEntry *rce = &m_rce2Pass[startIndex];
+ targetBits += qScale2bits(rce, x265_qp2qScale(rce->qpNoVbv));
+ expectedBits += qScale2bits(rce, rce->qScale);
+
+ rce = &m_rce2Pass[endIndex];
+ targetBits2 += qScale2bits(rce, x265_qp2qScale(rce->qpNoVbv));
+ expectedBits2 += qScale2bits(rce, rce->qScale);
+ }
+
+ if (expectedBits < 0.95 * targetBits || expectedBits2 < 0.95 *
targetBits2)
+ {
+ if (cpxSum / cpxSum2 < 0.95 || cpxSum2 / cpxSum < 0.95)
+ {
+ m_isQpModified = true;
+ m_isGopReEncoded = true;
+
+ m_shortTermCplxSum = 0;
+ m_shortTermCplxCount = 0;
+ m_framesDone = m_start;
+
+ for (startIndex = m_start; startIndex < m_numEntries;
startIndex++)
+ {
+ m_shortTermCplxSum *= 0.5;
+ m_shortTermCplxCount *= 0.5;
+ m_shortTermCplxSum += m_rce2Pass[startIndex].currentSatd /
(CLIP_DURATION(m_frameDuration) / BASE_FRAME_DURATION);
+ m_shortTermCplxCount++;
+ }
+
+ m_bufferFill = m_rce2Pass[m_start - 1].bufferFill;
+ m_bufferFillFinal = m_rce2Pass[m_start - 1].bufferFillFinal;
+ m_bufferFillActual = m_rce2Pass[m_start - 1].bufferFillActual;
+
+ m_reencode = m_start;
+ m_start = m_numEntries;
+ }
+ else
+ {
+
+ m_isQpModified = false;
+ m_isGopReEncoded = false;
+ }
+ }
+ else
+ {
+
+ m_isQpModified = false;
+ m_isGopReEncoded = false;
+ }
+
+ m_start = X265_MAX(m_start, m_numEntries - distance +
m_param->keyframeMax);
return true;
}
@@ -1391,15 +1374,47 @@ int RateControl::rateControlStart(Frame* curFrame,
RateControlEntry* rce, Encode
rce->frameSizeMaximum *= m_param->maxAUSizeFactor;
}
}
+
+ ///< regenerate the qp
if (!m_isAbr && m_2pass && m_param->rc.rateControlMode == X265_RC_CRF)
{
- rce->qpPrev = x265_qScale2qp(rce->qScale);
- rce->qScale = rce->newQScale;
- rce->qpaRc = curEncData.m_avgQpRc = curEncData.m_avgQpAq =
x265_qScale2qp(rce->newQScale);
- m_qp = int(rce->qpaRc + 0.5);
- rce->frameSizePlanned = qScale2bits(rce, rce->qScale);
- m_framesDone++;
- return m_qp;
+ if (!m_param->rc.bEncFocusedFramesOnly)
+ {
+ rce->qpPrev = x265_qScale2qp(rce->qScale);
+ rce->qScale = rce->newQScale;
+ rce->qpaRc = curEncData.m_avgQpRc = curEncData.m_avgQpAq =
x265_qScale2qp(rce->newQScale);
+ m_qp = int(rce->qpaRc + 0.5);
+ rce->frameSizePlanned = qScale2bits(rce, rce->qScale);
+ m_framesDone++;
+ return m_qp;
+ }
+ else
+ {
+ int index = m_encOrder[rce->poc];
+ index++;
+ double totalDuration = m_frameDuration;
+ for (int j = 0; totalDuration < 1.0 && index < m_numEntries;
j++)
+ {
+ switch (m_rce2Pass[index].sliceType)
+ {
+ case B_SLICE:
+ curFrame->m_lowres.plannedType[j] =
m_rce2Pass[index].keptAsRef ? X265_TYPE_BREF : X265_TYPE_B;
+ break;
+ case P_SLICE:
+ curFrame->m_lowres.plannedType[j] = X265_TYPE_P;
+ break;
+ case I_SLICE:
+ curFrame->m_lowres.plannedType[j] = m_param->bOpenGOP
? X265_TYPE_I : X265_TYPE_IDR;
+ break;
+ default:
+ break;
+ }
+
+ curFrame->m_lowres.plannedSatd[j] =
m_rce2Pass[index].currentSatd;
+ totalDuration += m_frameDuration;
+ index++;
+ }
+ }
}
if (m_isAbr || m_2pass) // ABR,CRF
@@ -1890,7 +1905,7 @@ double RateControl::rateEstimateQscale(Frame*
curFrame, RateControlEntry *rce)
qScale = x265_clip3(lqmin, lqmax, qScale);
}
- if (!m_2pass || m_param->bliveVBV2pass)
+ if (!m_2pass || m_param->bliveVBV2pass || (m_2pass &&
m_param->rc.rateControlMode == X265_RC_CRF &&
m_param->rc.bEncFocusedFramesOnly))
{
/* clip qp to permissible range after vbv-lookahead
estimation to avoid possible
* mispredictions by initial frame size predictors */
@@ -1927,7 +1942,7 @@ double RateControl::rateEstimateQscale(Frame*
curFrame, RateControlEntry *rce)
else
{
double abrBuffer = 2 * m_rateTolerance * m_bitrate;
- if (m_2pass)
+ if (m_2pass && (m_param->rc.rateControlMode != X265_RC_CRF ||
!m_param->rc.bEncFocusedFramesOnly))
{
double lmin = m_lmin[m_sliceType];
double lmax = m_lmax[m_sliceType];
@@ -2828,7 +2843,7 @@ int RateControl::rateControlEnd(Frame* curFrame,
int64_t bits, RateControlEntry*
if (m_param->rc.aqMode || m_isVbv || m_param->bAQMotion ||
bEnableDistOffset)
{
- if (m_isVbv && !(m_2pass && m_param->rc.rateControlMode ==
X265_RC_CRF))
+ if (m_isVbv && !(m_2pass && m_param->rc.rateControlMode ==
X265_RC_CRF && !m_param->rc.bEncFocusedFramesOnly))
{
double avgQpRc = 0;
/* determine avg QP decided by VBV rate control */
@@ -2862,8 +2877,9 @@ int RateControl::rateControlEnd(Frame* curFrame,
int64_t bits, RateControlEntry*
if (m_param->rc.rateControlMode == X265_RC_CRF)
{
double crfVal, qpRef = curEncData.m_avgQpRc;
+
bool is2passCrfChange = false;
- if (m_2pass)
+ if (m_2pass && !m_param->rc.bEncFocusedFramesOnly)
{
if (fabs(curEncData.m_avgQpRc - rce->qpPrev) > 0.1)
{
diff --git a/source/encoder/ratecontrol.h b/source/encoder/ratecontrol.h
index 996465eeb..204bd71e1 100644
--- a/source/encoder/ratecontrol.h
+++ b/source/encoder/ratecontrol.h
@@ -73,6 +73,7 @@ struct RateControlEntry
Predictor rowPreds[3][2];
Predictor* rowPred[2];
+ int64_t currentSatd;
int64_t lastSatd; /* Contains the picture cost of the previous
frame, required for resetAbr and VBV */
int64_t leadingNoBSatd;
int64_t rowTotalBits; /* update cplxrsum and totalbits at the end of
2 rows */
@@ -87,6 +88,8 @@ struct RateControlEntry
double rowCplxrSum;
double qpNoVbv;
double bufferFill;
+ double bufferFillFinal;
+ double bufferFillActual;
double targetFill;
bool vbvEndAdj;
double frameDuration;
diff --git a/source/x265.h b/source/x265.h
index 3f3133536..6bb893c98 100644
--- a/source/x265.h
+++ b/source/x265.h
@@ -1443,6 +1443,9 @@ typedef struct x265_param
* encoder will default to using x265_2pass.log */
const char* statFileName;
+ /* if only the focused frames would be re-encode or not */
+ int bEncFocusedFramesOnly;
+
/* temporally blur quants */
double qblur;
--
2.22.0.windows.1
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20210919/26f698ed/attachment-0001.html>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0001-encoding-only-the-focused-frames-in-the-crf-2-pass.patch
Type: application/octet-stream
Size: 17164 bytes
Desc: not available
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20210919/26f698ed/attachment-0002.obj>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0001-Enable-sharing-cutree-data-among-encoders.patch
Type: application/octet-stream
Size: 53479 bytes
Desc: not available
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20210919/26f698ed/attachment-0003.obj>
More information about the x265-devel
mailing list