<div dir="ltr"><div dir="ltr"><br></div><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Wed, Oct 6, 2021 at 12:10 PM Dakshinya T R S <<a href="mailto:dakshinya@multicorewareinc.com">dakshinya@multicorewareinc.com</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"><div dir="ltr">From 69d41f124d14347ec71cc61f3290d3eae7369631 Mon Sep 17 00:00:00 2001<br>From: lwWang <<a href="mailto:liwei@multicorewareinc.com" target="_blank">liwei@multicorewareinc.com</a>><br>Date: Wed, 8 Sep 2021 13:38:37 +0800<br>Subject: [PATCH] Enable sharing cutree data among encoders<br></div></blockquote><div>[AM] Commit message doesn't seem to reflect the purpose of the patch. Please update. </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"><div dir="ltr"><br>---<br> source/common/CMakeLists.txt   |   3 +-<br> source/common/ringmem.cpp      | 357 +++++++++++++++++++<br> source/common/ringmem.h        |  85 +++++<br> source/common/threading.h      | 137 ++++++++<br> source/encoder/encoder.cpp     |   4 +-<br> source/encoder/encoder.h       |   2 +-<br> source/encoder/ratecontrol.cpp | 603 +++++++++++++++++++++------------<br> source/encoder/ratecontrol.h   |  20 +-<br> 8 files changed, 987 insertions(+), 224 deletions(-)<br> create mode 100644 source/common/ringmem.cpp<br> create mode 100644 source/common/ringmem.h<br><br>diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt<br>index 12b643ad5..8752c6199 100644<br>--- a/source/common/CMakeLists.txt<br>+++ b/source/common/CMakeLists.txt<br>@@ -169,4 +169,5 @@ add_library(common OBJECT<br>     scalinglist.cpp scalinglist.h<br>     quant.cpp quant.h contexts.h<br>     deblock.cpp deblock.h<br>-    scaler.cpp scaler.h)<br>+    scaler.cpp scaler.h<br>+    ringmem.cpp ringmem.h)<br>diff --git a/source/common/ringmem.cpp b/source/common/ringmem.cpp<br>new file mode 100644<br>index 000000000..a4f191c90<br>--- /dev/null<br>+++ b/source/common/ringmem.cpp<br>@@ -0,0 +1,357 @@<br>+/*****************************************************************************<br>+ * Copyright (C) 2013-2017 MulticoreWare, Inc<br>+ *<br>+ * Authors: liwei <<a href="mailto:liwei@multicorewareinc.com" target="_blank">liwei@multicorewareinc.com</a>><br>+ *<br>+ * This program is free software; you can redistribute it and/or modify<br>+ * it under the terms of the GNU General Public License as published by<br>+ * the Free Software Foundation; either version 2 of the License, or<br>+ * (at your option) any later version.<br>+ *<br>+ * This program is distributed in the hope that it will be useful,<br>+ * but WITHOUT ANY WARRANTY; without even the implied warranty of<br>+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the<br>+ * GNU General Public License for more details.<br>+ *<br>+ * You should have received a copy of the GNU General Public License<br>+ * along with this program; if not, write to the Free Software<br>+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.<br>+ *<br>+ * This program is also available under a commercial proprietary license.<br>+ * For more information, contact us at license @ <a href="http://x265.com" target="_blank">x265.com</a><br>+ *****************************************************************************/<br>+<br>+#include "ringmem.h"<br>+<br>+#ifndef _WIN32<br>+#include <sys/mman.h><br>+#endif ////< _WIN32<br>+<br>+#ifdef _WIN32<br>+#define X265_SHARED_MEM_NAME                    "Local\\_x265_shr_mem_"<br>+#define X265_SEMAPHORE_RINGMEM_WRITER_NAME      "_x265_semW_"<br>+#define X265_SEMAPHORE_RINGMEM_READER_NAME          "_x265_semR_"<br>+#else /* POSIX / pthreads */<br>+#define X265_SHARED_MEM_NAME                    "/tmp/_x265_shr_mem_"<br>+#define X265_SEMAPHORE_RINGMEM_WRITER_NAME       "/tmp/_x265_semW_"<br>+#define X265_SEMAPHORE_RINGMEM_READER_NAME     "/tmp/_x265_semR_"<br>+#endif<br>+<br>+#define RINGMEM_ALLIGNMENT                       64<br>+<br>+namespace X265_NS {<br>+    RingMem::RingMem() <br>+        : m_initialized(false)<br>+        , m_protectRW(false)<br>+        , m_itemSize(0)<br>+        , m_itemCnt(0)<br>+        , m_dataPool(NULL)<br>+        , m_shrMem(NULL)<br>+#ifdef _WIN32<br>+        , m_handle(NULL)<br>+#else //_WIN32<br>+        , m_filepath(NULL)<br>+#endif //_WIN32<br>+        , m_writeSem(NULL)<br>+        , m_readSem(NULL)<br>+    {<br>+    }<br>+<br>+<br>+    RingMem::~RingMem()<br>+    {<br>+    }<br>+<br>+    bool RingMem::skipRead(int32_t cnt) {<br>+        if (!m_initialized)<br>+        {<br>+            return false;<br>+        }<br>+<br>+        if (m_protectRW)<br>+        {<br>+            for (int i = 0; i < cnt; i++)<br>+            {<br>+                m_readSem->take();<br>+            }<br>+        }<br>+        <br>+        ATOMIC_ADD(&m_shrMem->m_read, cnt);<br>+<br>+        if (m_protectRW)<br>+        {<br>+            m_writeSem->give(cnt);<br>+        }<br>+<br>+        return true;<br>+    }<br>+<br>+    bool RingMem::skipWrite(int32_t cnt) {<br>+        if (!m_initialized)<br>+        {<br>+            return false;<br>+        }<br>+<br>+        if (m_protectRW)<br>+        {<br>+            for (int i = 0; i < cnt; i++)<br>+            {<br>+                m_writeSem->take();<br>+            }<br>+        }<br>+<br>+        ATOMIC_ADD(&m_shrMem->m_write, cnt);<br>+<br>+        if (m_protectRW)<br>+        {<br>+            m_readSem->give(cnt);<br>+        }<br>+<br>+        return true;<br>+    }<br>+<br>+    ///< initialize<br>+    bool RingMem::init(int32_t itemSize, int32_t itemCnt, const char *name, bool protectRW)<br>+    {<br>+        ///< check parameters<br>+        if (itemSize <= 0 || itemCnt <= 0 || NULL == name)<br>+        {<br>+            ///< invalid parameters <br>+            return false;<br>+        }<br>+<br>+        if (!m_initialized)<br>+        {<br>+            ///< formating names<br>+            char nameBuf[MAX_SHR_NAME_LEN] = { 0 };<br>+<br>+            ///< shared memory name<br>+            snprintf(nameBuf, sizeof(nameBuf) - 1, "%s%s", X265_SHARED_MEM_NAME, name);<br>+<br>+            ///< create or open shared memory<br>+            bool newCreated = false;<br>+<br>+            ///< calculate the size of the shared memory<br>+            int32_t shrMemSize = (itemSize * itemCnt + sizeof(ShrMemCtrl) + RINGMEM_ALLIGNMENT - 1) & ~(RINGMEM_ALLIGNMENT - 1);<br>+<br>+#ifdef _WIN32<br>+            HANDLE h = OpenFileMappingA(FILE_MAP_WRITE | FILE_MAP_READ, FALSE, nameBuf);<br>+            if (!h)<br>+            {<br>+                h = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, shrMemSize, nameBuf);<br>+<br>+                if (!h)<br>+                {<br>+                    return false;<br>+                }<br>+<br>+                newCreated = true;<br>+            }<br>+<br>+            void *pool = MapViewOfFile(h, FILE_MAP_ALL_ACCESS, 0, 0, 0);<br>+<br>+            ///< should not close the handle here, otherwise the OpenFileMapping would fail<br>+            //CloseHandle(h);<br>+            m_handle = h;<br>+<br>+            if (!pool)<br>+            {<br>+                return false;<br>+            }<br>+<br>+#else /* POSIX / pthreads */<br>+            mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;<br>+            int flag = O_RDWR;<br>+            int shrfd = -1;<br>+            if ((shrfd = open(nameBuf, flag, mode)) < 0)<br>+            {<br>+                flag |= O_CREAT;<br>+                <br>+                shrfd = open(nameBuf, flag, mode);<br>+                if (shrfd < 0)<br>+                {<br>+                    return false;<br>+                }<br>+                newCreated = true;<br>+<br>+                lseek(shrfd, shrMemSize - 1, SEEK_SET);<br>+<br>+                if (-1 == write(shrfd, "\0", 1))<br>+                {<br>+                    close(shrfd);<br>+                    return false;<br>+                }<br>+<br>+                if (lseek(shrfd, 0, SEEK_END) < shrMemSize)<br>+                {<br>+                    close(shrfd);<br>+                    return false;<br>+                }<br>+            }<br>+<br>+            void *pool = mmap(0,<br>+                shrMemSize,<br>+                PROT_READ | PROT_WRITE,<br>+                MAP_SHARED,<br>+                shrfd,<br>+                0);<br>+<br>+            close(shrfd);<br>+            if (pool == MAP_FAILED)<br>+            {               <br>+                return false;<br>+            }<br>+<br>+            m_filepath = strdup(nameBuf);<br>+#endif ///< _WIN32<br>+<br>+            if (newCreated)<br>+            {<br>+                memset(pool, 0, shrMemSize);<br>+            }<br>+            <br>+            m_shrMem = reinterpret_cast<ShrMemCtrl *>(pool);<br>+            m_dataPool = reinterpret_cast<uint8_t *>(pool) + sizeof(ShrMemCtrl);<br>+            m_itemSize = itemSize;<br>+            m_itemCnt = itemCnt;<br>+            m_initialized = true;<br>+<br>+            if (protectRW)<br>+            {<br>+                m_protectRW = true;<br>+                m_writeSem = new NamedSemaphore();<br>+                if (!m_writeSem)<br>+                {<br>+                    release();<br>+                    return false;<br>+                }<br>+<br>+                ///< shared memory name<br>+                snprintf(nameBuf, sizeof(nameBuf - 1), "%s%s", X265_SEMAPHORE_RINGMEM_WRITER_NAME, name);<br>+                if (!m_writeSem->create(nameBuf, m_itemCnt, m_itemCnt))<br>+                {<br>+                    release();<br>+                    return false;<br>+                }<br>+<br>+                m_readSem = new NamedSemaphore();<br>+                if (!m_readSem)<br>+                {<br>+                    release();<br>+                    return false;<br>+                }<br>+<br>+                ///< shared memory name<br>+                snprintf(nameBuf, sizeof(nameBuf - 1), "%s%s", X265_SEMAPHORE_RINGMEM_READER_NAME, name);<br>+                if (!m_readSem->create(nameBuf, 0, m_itemCnt))<br>+                {<br>+                    release();<br>+                    return false;<br>+                }<br>+            }<br>+        }<br>+<br>+        return true;<br>+    }<br>+    ///< finalize<br>+    void RingMem::release()<br>+    {<br>+        if (m_initialized)<br>+        {<br>+            m_initialized = false;<br>+<br>+            if (m_shrMem)<br>+            {<br>+#ifdef _WIN32<br>+                UnmapViewOfFile(m_shrMem);<br>+                CloseHandle(m_handle);<br>+                m_handle = NULL;<br>+#else /* POSIX / pthreads */<br>+                int32_t shrMemSize = (m_itemSize * m_itemCnt + sizeof(ShrMemCtrl) + RINGMEM_ALLIGNMENT - 1) & (~RINGMEM_ALLIGNMENT - 1);<br>+                munmap(m_shrMem, shrMemSize);<br>+                unlink(m_filepath);<br>+                free(m_filepath);<br>+                m_filepath = NULL;<br>+#endif ///< _WIN32<br>+                m_shrMem = NULL;<br>+                m_dataPool = NULL;<br>+                m_itemSize = 0;<br>+                m_itemCnt = 0;<br>+            }<br>+            <br>+            if (m_protectRW)<br>+            {<br>+                m_protectRW = false;<br>+                if (m_writeSem)<br>+                {<br>+                    m_writeSem->release();<br>+<br>+                    delete m_writeSem;<br>+                    m_writeSem = NULL;<br>+                }<br>+<br>+                if (m_readSem)<br>+                {<br>+                    m_readSem->release();<br>+<br>+                    delete m_readSem;<br>+                    m_readSem = NULL;<br>+                }<br>+            }<br>+<br>+        }<br>+    }<br>+<br>+    ///< data read<br>+    bool RingMem::readNext(void* dst, fnRWSharedData callback)<br>+    {<br>+        if (!m_initialized || !callback || !dst)<br>+        {<br>+            return false;<br>+        }<br>+<br>+        if (m_protectRW)<br>+        {<br>+            if (!m_readSem->take())<br>+            {<br>+                return false;<br>+            }<br>+        }<br>+<br>+        int32_t index = ATOMIC_ADD(&m_shrMem->m_read, 1) % m_itemCnt;<br>+        (*callback)(dst, reinterpret_cast<uint8_t *>(m_dataPool) + index * m_itemSize, m_itemSize);<br>+<br>+        if (m_protectRW)<br>+        {<br>+            m_writeSem->give(1);<br>+        }<br>+<br>+        return true;<br>+    }<br>+    ///< data write<br>+    bool RingMem::writeData(void *data, fnRWSharedData callback)<br>+    {<br>+        if (!m_initialized || !data || !callback)<br>+        {<br>+            return false;<br>+        }<br>+<br>+        if (m_protectRW)<br>+        {<br>+            if (!m_writeSem->take())<br>+            {<br>+                return false;<br>+            }<br>+        }<br>+<br>+        int32_t index = ATOMIC_ADD(&m_shrMem->m_write, 1) % m_itemCnt;<br>+        (*callback)(reinterpret_cast<uint8_t *>(m_dataPool) + index * m_itemSize, data, m_itemSize);<br>+<br>+        if (m_protectRW)<br>+        {<br>+            m_readSem->give(1);<br>+        }<br>+<br>+        return true;<br>+    }<br>+}<br>diff --git a/source/common/ringmem.h b/source/common/ringmem.h<br>new file mode 100644<br>index 000000000..76f54bb19<br>--- /dev/null<br>+++ b/source/common/ringmem.h<br>@@ -0,0 +1,85 @@<br>+/*****************************************************************************<br>+ * Copyright (C) 2013-2017 MulticoreWare, Inc<br>+ *<br>+ * Authors: liwei <<a href="mailto:liwei@multicorewareinc.com" target="_blank">liwei@multicorewareinc.com</a>><br>+ *<br>+ * This program is free software; you can redistribute it and/or modify<br>+ * it under the terms of the GNU General Public License as published by<br>+ * the Free Software Foundation; either version 2 of the License, or<br>+ * (at your option) any later version.<br>+ *<br>+ * This program is distributed in the hope that it will be useful,<br>+ * but WITHOUT ANY WARRANTY; without even the implied warranty of<br>+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the<br>+ * GNU General Public License for more details.<br>+ *<br>+ * You should have received a copy of the GNU General Public License<br>+ * along with this program; if not, write to the Free Software<br>+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.<br>+ *<br>+ * This program is also available under a commercial proprietary license.<br>+ * For more information, contact us at license @ <a href="http://x265.com" target="_blank">x265.com</a><br>+ *****************************************************************************/<br>+<br>+#ifndef X265_RINGMEM_H<br>+#define X265_RINGMEM_H<br>+<br>+#include "common.h"<br>+#include "threading.h"<br>+<br>+namespace X265_NS {<br>+<br>+#define MAX_SHR_NAME_LEN                         256<br>+<br>+    class RingMem {<br>+    public:<br>+        RingMem();<br>+        ~RingMem();<br>+<br>+        bool skipRead(int32_t cnt);<br>+<br>+        bool skipWrite(int32_t cnt);<br>+<br>+        ///< initialize<br>+        ///< protectRW: if use the semaphore the protect the write and read operation.<br>+        bool init(int32_t itemSize, int32_t itemCnt, const char *name, bool protectRW = false);<br>+        ///< finalize<br>+        void release();<br>+<br>+        typedef void(*fnRWSharedData)(void *dst, void *src, int32_t size);<br>+<br>+        ///< data read<br>+        bool readNext(void* dst, fnRWSharedData callback);<br>+        ///< data write<br>+        bool writeData(void *data, fnRWSharedData callback);<br>+<br>+    private:        <br>+        bool    m_initialized;<br>+        bool    m_protectRW;<br>+<br>+        int32_t m_itemSize;<br>+        int32_t m_itemCnt;<br>+        ///< data pool<br>+        void   *m_dataPool;<br>+        typedef struct {<br>+            ///< index to write<br>+            int32_t m_write;<br>+            ///< index to read<br>+            int32_t m_read;<br>+            <br>+        }ShrMemCtrl;<br>+<br>+        ShrMemCtrl *m_shrMem;<br>+#ifdef _WIN32<br>+        void       *m_handle;<br>+#else // _WIN32<br>+        char       *m_filepath;<br>+#endif // _WIN32<br>+<br>+        ///< Semaphores<br>+        NamedSemaphore *m_writeSem;<br>+        NamedSemaphore *m_readSem;<br>+    };<br>+};<br>+<br>+#endif // ifndef X265_RINGMEM_H<br>diff --git a/source/common/threading.h b/source/common/threading.h<br>index 53a63beaf..dcf6081e3 100644<br>--- a/source/common/threading.h<br>+++ b/source/common/threading.h<br>@@ -3,6 +3,7 @@<br>  *<br>  * Authors: Steve Borho <<a href="mailto:steve@borho.org" target="_blank">steve@borho.org</a>><br>  *          Min Chen <<a href="mailto:chenm003@163.com" target="_blank">chenm003@163.com</a>><br>+            liwei <<a href="mailto:liwei@multicorewareinc.com" target="_blank">liwei@multicorewareinc.com</a>><br>  *<br>  * This program is free software; you can redistribute it and/or modify<br>  * it under the terms of the GNU General Public License as published by<br>@@ -253,6 +254,47 @@ protected:<br>     int                m_val;<br> };<br> <br>+class NamedSemaphore<br>+{<br>+public:<br>+    NamedSemaphore() : m_sem(NULL)<br>+    {<br>+    }<br>+<br>+    ~NamedSemaphore()<br>+    {<br>+    }<br>+<br>+    bool create(const char* name, const int initcnt, const int maxcnt)<br>+    {<br>+        if(!m_sem)<br>+        {<br>+            m_sem = CreateSemaphoreA(NULL, initcnt, maxcnt, name);<br>+        }<br>+        return m_sem != NULL;<br>+    }<br>+<br>+    bool give(const int32_t cnt)<br>+    {<br>+        return ReleaseSemaphore(m_sem, (LONG)cnt, NULL) != FALSE;<br>+    }<br>+<br>+    bool take(const uint32_t time_out = INFINITE)<br>+    {<br>+        int32_t rt = WaitForSingleObject(m_sem, time_out);<br>+        return rt != WAIT_TIMEOUT && rt != WAIT_FAILED;<br>+    }<br>+<br>+    void release()<br>+    {<br>+        CloseHandle(m_sem);<br>+        m_sem = NULL;<br>+    }<br>+<br>+private:<br>+    HANDLE m_sem;<br>+};<br>+<br> #else /* POSIX / pthreads */<br> <br> typedef pthread_t ThreadHandle;<br>@@ -459,6 +501,101 @@ protected:<br>     int             m_val;<br> };<br> <br>+#define TIMEOUT_INFINITE 0xFFFFFFFF<br>+<br>+class NamedSemaphore<br>+{<br>+public:<br>+    NamedSemaphore() <br>+        : m_sem(NULL)<br>+        , m_name(NULL)<br>+    {<br>+    }<br>+<br>+    ~NamedSemaphore()<br>+    {<br>+    }<br>+<br>+    bool create(const char* name, const int initcnt, const int maxcnt)<br>+    {<br>+        bool ret = false;<br>+<br>+        if (initcnt >= maxcnt)<br>+        {<br>+            return false;<br>+        }<br>+<br>+        m_sem = sem_open(name, O_CREAT | O_EXCL, 0666, initcnt);<br>+        if (m_sem != SEM_FAILED) <br>+        {<br>+            m_name = strdup(name);<br>+            ret = true;<br>+        }<br>+        else <br>+        {<br>+            if (EEXIST == errno) <br>+            {<br>+                m_sem = sem_open(name, 0);<br>+                if (m_sem != SEM_FAILED) <br>+                {<br>+                    m_name = strdup(name);<br>+                    ret = true;<br>+                }<br>+            }<br>+        }<br>+<br>+        return ret;<br>+    }<br>+<br>+    bool give(const int32_t cnt)<br>+    {<br>+        int ret = 0;<br>+        int32_t curCnt = cnt;<br>+        while (curCnt-- && !ret) {<br>+            ret = sem_post(m_sem);<br>+        }<br>+<br>+        return 0 == ret;<br>+    }<br>+<br>+    bool take(const uint32_t time_out = TIMEOUT_INFINITE)<br>+    {<br>+        if (TIMEOUT_INFINITE == time_out) {<br>+            return 0 == sem_wait(m_sem);<br>+        }<br>+        else <br>+        {<br>+            if (0 == time_out)<br>+            {<br>+                return 0 == sem_trywait(m_sem);<br>+            }<br>+            else<br>+            {<br>+                struct timespec ts;<br>+                ts.tv_sec = time_out / 1000L;<br>+                ts.tv_nsec = (time_out * 1000000L) - ts.tv_sec * 1000 * 1000 * 1000;<br>+                return 0 == sem_timedwait(m_sem, &ts);<br>+            }<br>+        }<br>+    }<br>+<br>+    void release()<br>+    {<br>+        if (m_sem)<br>+        {<br>+            sem_close(m_sem);<br>+            sem_unlink(m_name);<br>+            m_sem = NULL;<br>+            free(m_name);<br>+            m_name = NULL;<br>+        }<br>+    }<br>+<br>+private:<br>+    sem_t *m_sem;<br>+    char  *m_name;<br>+};<br>+<br> #endif // ifdef _WIN32<br> <br> class ScopedLock<br>diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp<br>index cc014a740..19551b1f2 100644<br>--- a/source/encoder/encoder.cpp<br>+++ b/source/encoder/encoder.cpp<br>@@ -187,7 +187,7 @@ inline char *strcatFilename(const char *input, const char *suffix)<br>     return output;<br> }<br> <br>-void Encoder::create()<br>+void Encoder::create(char *dataShr)<br> {<br>     if (!primitives.pu[0].sad)<br>     {<br>@@ -389,7 +389,7 @@ void Encoder::create()<br>             lookAheadThreadPool[i].start();<br>     m_lookahead->m_numPools = pools;<br>     m_dpb = new DPB(m_param);<br>-    m_rateControl = new RateControl(*m_param, this);<br>+    m_rateControl = new RateControl(*m_param, this, dataShr);<br>     if (!m_param->bResetZoneConfig)<br>     {<br>         zoneReadCount = new ThreadSafeInteger[m_param->rc.zonefileCount];<br>diff --git a/source/encoder/encoder.h b/source/encoder/encoder.h<br>index 2ee5bdaee..4b8478f16 100644<br>--- a/source/encoder/encoder.h<br>+++ b/source/encoder/encoder.h<br>@@ -305,7 +305,7 @@ public:<br> #endif<br>     };<br> <br>-    void create();<br>+    void create(char *dataShr = NULL);<br>     void stopJobs();<br>     void destroy();<br> <br>diff --git a/source/encoder/ratecontrol.cpp b/source/encoder/ratecontrol.cpp<br>index 71f08a73e..e2f514172 100644<br>--- a/source/encoder/ratecontrol.cpp<br>+++ b/source/encoder/ratecontrol.cpp<br>@@ -41,6 +41,10 @@<br> #define BR_SHIFT  6<br> #define CPB_SHIFT 4<br> <br>+#define SHARED_DATA_ALIGNMENT      4 ///< 4btye, 32bit<br>+#define CUTREE_SHARED_MEM_NAME     "cutree"<br>+#define GOP_CNT_CU_TREE            3<br>+<br></div></blockquote><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"><div dir="ltr"> using namespace X265_NS;<br> <br> /* Amortize the partial cost of I frames over the next N frames */<br>@@ -104,6 +108,37 @@ inline char *strcatFilename(const char *input, const char *suffix)<br>     return output;<br> }<br> <br>+typedef struct CUTreeSharedDataItem<br>+{<br>+    uint8_t  *type;<br>+    uint16_t *stats;<br>+}CUTreeSharedDataItem;<br>+<br>+void ReadSharedCUTreeData(void *dst, void *src, int32_t size)<br>+{<br>+    CUTreeSharedDataItem *statsDst = reinterpret_cast<CUTreeSharedDataItem *>(dst);<br>+    uint8_t *typeSrc = reinterpret_cast<uint8_t *>(src);<br>+    *statsDst->type = *typeSrc;<br>+<br>+    ///< for memory alignment, the type will take 32bit in the shared memory<br>+    int32_t offset = (sizeof(*statsDst->type) + SHARED_DATA_ALIGNMENT - 1) & ~(SHARED_DATA_ALIGNMENT - 1);<br>+    uint16_t *statsSrc = reinterpret_cast<uint16_t *>(typeSrc + offset);<br>+    memcpy(statsDst->stats, statsSrc, size - offset);<br>+}<br>+<br>+void WriteSharedCUTreeData(void *dst, void *src, int32_t size)<br>+{<br>+    CUTreeSharedDataItem *statsSrc = reinterpret_cast<CUTreeSharedDataItem *>(src);<br>+    uint8_t *typeDst = reinterpret_cast<uint8_t *>(dst);<br>+    *typeDst = *statsSrc->type;<br>+<br>+    ///< for memory alignment, the type will take 32bit in the shared memory<br>+    int32_t offset = (sizeof(*statsSrc->type) + SHARED_DATA_ALIGNMENT - 1) & ~(SHARED_DATA_ALIGNMENT - 1);<br>+    uint16_t *statsDst = reinterpret_cast<uint16_t *>(typeDst + offset);<br>+    memcpy(statsDst, statsSrc->stats, size - offset);<br>+}<br>+<br>+<br> inline double qScale2bits(RateControlEntry *rce, double qScale)<br> {<br>     if (qScale < 0.1)<br>@@ -146,7 +181,7 @@ x265_zone* RateControl::getZone()<br>     return NULL;<br> }<br> <br>-RateControl::RateControl(x265_param& p, Encoder *top)<br>+RateControl::RateControl(x265_param& p, Encoder *top, char * dataShr)<br> {<br>     m_param = &p;<br>     m_top = top;<br>@@ -209,6 +244,14 @@ RateControl::RateControl(x265_param& p, Encoder *top)<br>     m_lastAbrResetPoc = -1;<br>     m_statFileOut = NULL;<br>     m_cutreeStatFileOut = m_cutreeStatFileIn = NULL;<br>+    ///< store the cutree data in file by default<br>+    m_cutreeStorageMode = !dataShr ? SHARED_MODE_FILE : SHARED_MODE_MEM;<br></div></blockquote><div>[AM] Why not a param option (API) to toggle between file-based and shared-memory based cutree sharing?  The external application has to violate the API if it has to operate through this macro.</div><div><br></div><div>Is this shared memory only for cutree data sharing? If so, can file-based and shared-memory based RC data sharing coexist?  </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"><div dir="ltr">+    m_shrname = NULL;<br>+    if (dataShr)<br>+    {<br>+        m_shrname = strdup(dataShr);<br>+    }<br>+    m_cutreeShrMem = NULL;<br>     m_rce2Pass = NULL;<br>     m_encOrder = NULL;<br>     m_lastBsliceSatdCost = 0;<br>@@ -320,6 +363,42 @@ RateControl::RateControl(x265_param& p, Encoder *top)<br>         m_cuTreeStats.qpBuffer[i] = NULL;<br> }<br> <br>+bool RateControl::initCUTreeSharedMem()<br>+{<br>+    if (!m_cutreeShrMem) {<br>+        m_cutreeShrMem = new RingMem();<br>+        if (!m_cutreeShrMem)<br>+        {<br>+            return false;<br>+        }<br>+<br>+        ///< now cutree data form at most 3 gops would be stored in the shared memory at the same time<br>+        int32_t itemSize = (sizeof(uint8_t) + SHARED_DATA_ALIGNMENT - 1) & ~(SHARED_DATA_ALIGNMENT - 1);<br>+        if (m_param->rc.qgSize == 8)<br>+        {<br>+            itemSize += sizeof(uint16_t) * m_ncu * 4;<br>+        }<br>+        else<br>+        {<br>+            itemSize += sizeof(uint16_t) * m_ncu;<br>+        }<br>+<br>+        int32_t itemCnt = X265_MIN(m_param->keyframeMax, (int)(m_fps + 0.5));<br>+        itemCnt *= GOP_CNT_CU_TREE;<br>+<br>+        char shrname[MAX_SHR_NAME_LEN] = { 0 };<br>+        strcpy(shrname, m_shrname);<br>+        strcat(shrname, CUTREE_SHARED_MEM_NAME);<br>+<br>+        if (!m_cutreeShrMem->init(itemSize, itemCnt, shrname))<br>+        {<br>+            return false;<br>+        }<br>+    }<br>+<br>+    return true;<br>+}<br>+<br> bool RateControl::init(const SPS& sps)<br> {<br>     if (m_isVbv && !m_initVbv)<br>@@ -421,244 +500,261 @@ bool RateControl::init(const SPS& sps)<br>         /* Load stat file and init 2pass algo */<br>         if (m_param->rc.bStatRead)<br>         {<br>-            m_expectedBitsSum = 0;<br>-            char *p, *statsIn, *statsBuf;<br>-            /* read 1st pass stats */<br>-            statsIn = statsBuf = x265_slurp_file(fileName);<br>-            if (!statsBuf)<br>-                return false;<br>-            if (m_param->rc.cuTree)<br>+            if (SHARED_MODE_FILE == m_cutreeStorageMode)<br>             {<br>-                char *tmpFile = strcatFilename(fileName, ".cutree");<br>-                if (!tmpFile)<br>+                m_expectedBitsSum = 0;<br>+                char *p, *statsIn, *statsBuf;<br>+                /* read 1st pass stats */<br>+                statsIn = statsBuf = x265_slurp_file(fileName);<br>+                if (!statsBuf)<br>                     return false;<br>-                m_cutreeStatFileIn = x265_fopen(tmpFile, "rb");<br>-                X265_FREE(tmpFile);<br>-                if (!m_cutreeStatFileIn)<br>+                if (m_param->rc.cuTree)<br>                 {<br>-                    x265_log_file(m_param, X265_LOG_ERROR, "can't open stats file %s.cutree\n", fileName);<br>-                    return false;<br>+                    char *tmpFile = strcatFilename(fileName, ".cutree");<br>+                    if (!tmpFile)<br>+                        return false;<br>+                    m_cutreeStatFileIn = x265_fopen(tmpFile, "rb");<br>+                    X265_FREE(tmpFile);<br>+                    if (!m_cutreeStatFileIn)<br>+                    {<br>+                        x265_log_file(m_param, X265_LOG_ERROR, "can't open stats file %s.cutree\n", fileName);<br>+                        return false;<br>+                    }<br>                 }<br>-            }<br> <br>-            /* check whether 1st pass options were compatible with current options */<br>-            if (strncmp(statsBuf, "#options:", 9))<br>-            {<br>-                x265_log(m_param, X265_LOG_ERROR,"options list in stats file not valid\n");<br>-                return false;<br>-            }<br>-            {<br>-                int i, j, m;<br>-                uint32_t k , l;<br>-                bool bErr = false;<br>-                char *opts = statsBuf;<br>-                statsIn = strchr(statsBuf, '\n');<br>-                if (!statsIn)<br>+                /* check whether 1st pass options were compatible with current options */<br>+                if (strncmp(statsBuf, "#options:", 9))<br>                 {<br>-                    x265_log(m_param, X265_LOG_ERROR, "Malformed stats file\n");<br>+                    x265_log(m_param, X265_LOG_ERROR, "options list in stats file not valid\n");<br>                     return false;<br>                 }<br>-                *statsIn = '\0';<br>-                statsIn++;<br>-                if ((p = strstr(opts, " input-res=")) == 0 || sscanf(p, " input-res=%dx%d", &i, &j) != 2)<br>                 {<br>-                    x265_log(m_param, X265_LOG_ERROR, "Resolution specified in stats file not valid\n");<br>+                    int i, j, m;<br>+                    uint32_t k, l;<br>+                    bool bErr = false;<br>+                    char *opts = statsBuf;<br>+                    statsIn = strchr(statsBuf, '\n');<br>+                    if (!statsIn)<br>+                    {<br>+                        x265_log(m_param, X265_LOG_ERROR, "Malformed stats file\n");<br>+                        return false;<br>+                    }<br>+                    *statsIn = '\0';<br>+                    statsIn++;<br>+                    if ((p = strstr(opts, " input-res=")) == 0 || sscanf(p, " input-res=%dx%d", &i, &j) != 2)<br>+                    {<br>+                        x265_log(m_param, X265_LOG_ERROR, "Resolution specified in stats file not valid\n");<br>+                        return false;<br>+                    }<br>+                    if ((p = strstr(opts, " fps=")) == 0 || sscanf(p, " fps=%u/%u", &k, &l) != 2)<br>+                    {<br>+                        x265_log(m_param, X265_LOG_ERROR, "fps specified in stats file not valid\n");<br>+                        return false;<br>+                    }<br>+                    if (((p = strstr(opts, " vbv-maxrate=")) == 0 || sscanf(p, " vbv-maxrate=%d", &m) != 1) && m_param->rc.rateControlMode == X265_RC_CRF)<br>+                    {<br>+                        x265_log(m_param, X265_LOG_ERROR, "Constant rate-factor is incompatible with 2pass without vbv-maxrate in the previous pass\n");<br>+                        return false;<br>+                    }<br>+                    if (k != m_param->fpsNum || l != m_param->fpsDenom)<br>+                    {<br>+                        x265_log(m_param, X265_LOG_ERROR, "fps mismatch with 1st pass (%u/%u vs %u/%u)\n",<br>+                            m_param->fpsNum, m_param->fpsDenom, k, l);<br>+                        return false;<br>+                    }<br>+                    if (m_param->analysisMultiPassRefine)<br>+                    {<br>+                        p = strstr(opts, "ref=");<br>+                        sscanf(p, "ref=%d", &i);<br>+                        if (i > m_param->maxNumReferences)<br>+                        {<br>+                            x265_log(m_param, X265_LOG_ERROR, "maxNumReferences cannot be less than 1st pass (%d vs %d)\n",<br>+                                i, m_param->maxNumReferences);<br>+                            return false;<br>+                        }<br>+                    }<br>+                    if (m_param->analysisMultiPassRefine || m_param->analysisMultiPassDistortion)<br>+                    {<br>+                        p = strstr(opts, "ctu=");<br>+                        sscanf(p, "ctu=%u", &k);<br>+                        if (k != m_param->maxCUSize)<br>+                        {<br>+                            x265_log(m_param, X265_LOG_ERROR, "maxCUSize mismatch with 1st pass (%u vs %u)\n",<br>+                                k, m_param->maxCUSize);<br>+                            return false;<br>+                        }<br>+                    }<br>+                    CMP_OPT_FIRST_PASS("bitdepth", m_param->internalBitDepth);<br>+                    CMP_OPT_FIRST_PASS("weightp", m_param->bEnableWeightedPred);<br>+                    CMP_OPT_FIRST_PASS("bframes", m_param->bframes);<br>+                    CMP_OPT_FIRST_PASS("b-pyramid", m_param->bBPyramid);<br>+                    CMP_OPT_FIRST_PASS("open-gop", m_param->bOpenGOP);<br>+                    CMP_OPT_FIRST_PASS(" keyint", m_param->keyframeMax);<br>+                    CMP_OPT_FIRST_PASS("scenecut", m_param->scenecutThreshold);<br>+                    CMP_OPT_FIRST_PASS("intra-refresh", m_param->bIntraRefresh);<br>+                    CMP_OPT_FIRST_PASS("frame-dup", m_param->bEnableFrameDuplication);<br>+                    if (m_param->bMultiPassOptRPS)<br>+                    {<br>+                        CMP_OPT_FIRST_PASS("multi-pass-opt-rps", m_param->bMultiPassOptRPS);<br>+                        CMP_OPT_FIRST_PASS("repeat-headers", m_param->bRepeatHeaders);<br>+                        CMP_OPT_FIRST_PASS("min-keyint", m_param->keyframeMin);<br>+                    }<br>+<br>+                    if ((p = strstr(opts, "b-adapt=")) != 0 && sscanf(p, "b-adapt=%d", &i) && i >= X265_B_ADAPT_NONE && i <= X265_B_ADAPT_TRELLIS)<br>+                    {<br>+                        m_param->bFrameAdaptive = i;<br>+                    }<br>+                    else if (m_param->bframes)<br>+                    {<br>+                        x265_log(m_param, X265_LOG_ERROR, "b-adapt method specified in stats file not valid\n");<br>+                        return false;<br>+                    }<br>+<br>+                    if ((p = strstr(opts, "rc-lookahead=")) != 0 && sscanf(p, "rc-lookahead=%d", &i))<br>+                        m_param->lookaheadDepth = i;<br>+                }<br>+                /* find number of pics */<br>+                p = statsIn;<br>+                int numEntries;<br>+                for (numEntries = -1; p; numEntries++)<br>+                    p = strchr(p + 1, ';');<br>+                if (!numEntries)<br>+                {<br>+                    x265_log(m_param, X265_LOG_ERROR, "empty stats file\n");<br>                     return false;<br>                 }<br>-                if ((p = strstr(opts, " fps=")) == 0 || sscanf(p, " fps=%u/%u", &k, &l) != 2)<br>+                m_numEntries = numEntries;<br>+<br>+                if (m_param->totalFrames < m_numEntries && m_param->totalFrames > 0)<br>+                {<br>+                    x265_log(m_param, X265_LOG_WARNING, "2nd pass has fewer frames than 1st pass (%d vs %d)\n",<br>+                        m_param->totalFrames, m_numEntries);<br>+                }<br>+                if (m_param->totalFrames > m_numEntries && !m_param->bEnableFrameDuplication)<br>                 {<br>-                    x265_log(m_param, X265_LOG_ERROR, "fps specified in stats file not valid\n");<br>+                    x265_log(m_param, X265_LOG_ERROR, "2nd pass has more frames than 1st pass (%d vs %d)\n",<br>+                        m_param->totalFrames, m_numEntries);<br>                     return false;<br>                 }<br>-                if (((p = strstr(opts, " vbv-maxrate=")) == 0 || sscanf(p, " vbv-maxrate=%d", &m) != 1) && m_param->rc.rateControlMode == X265_RC_CRF)<br>+<br>+                m_rce2Pass = X265_MALLOC(RateControlEntry, m_numEntries);<br>+                if (!m_rce2Pass)<br>                 {<br>-                    x265_log(m_param, X265_LOG_ERROR, "Constant rate-factor is incompatible with 2pass without vbv-maxrate in the previous pass\n");<br>+                    x265_log(m_param, X265_LOG_ERROR, "Rce Entries for 2 pass cannot be allocated\n");<br>                     return false;<br>                 }<br>-                if (k != m_param->fpsNum || l != m_param->fpsDenom)<br>+                m_encOrder = X265_MALLOC(int, m_numEntries);<br>+                if (!m_encOrder)<br>                 {<br>-                    x265_log(m_param, X265_LOG_ERROR, "fps mismatch with 1st pass (%u/%u vs %u/%u)\n",<br>-                              m_param->fpsNum, m_param->fpsDenom, k, l);<br>+                    x265_log(m_param, X265_LOG_ERROR, "Encode order for 2 pass cannot be allocated\n");<br>                     return false;<br>                 }<br>-                if (m_param->analysisMultiPassRefine)<br>+                /* init all to skipped p frames */<br>+                for (int i = 0; i < m_numEntries; i++)<br>                 {<br>-                    p = strstr(opts, "ref=");<br>-                    sscanf(p, "ref=%d", &i);<br>-                    if (i > m_param->maxNumReferences)<br>+                    RateControlEntry *rce = &m_rce2Pass[i];<br>+                    rce->sliceType = P_SLICE;<br>+                    rce->qScale = rce->newQScale = x265_qp2qScale(20);<br>+                    rce->miscBits = m_ncu + 10;<br>+                    rce->newQp = 0;<br>+                }<br>+                /* read stats */<br>+                p = statsIn;<br>+                double totalQpAq = 0;<br>+                for (int i = 0; i < m_numEntries; i++)<br>+                {<br>+                    RateControlEntry *rce, *rcePocOrder;<br>+                    int frameNumber;<br>+                    int encodeOrder;<br>+                    char picType;<br>+                    int e;<br>+                    char *next;<br>+                    double qpRc, qpAq, qNoVbv, qRceq;<br>+                    next = strstr(p, ";");<br>+                    if (next)<br>+                        *next++ = 0;<br>+                    e = sscanf(p, " in:%d out:%d", &frameNumber, &encodeOrder);<br>+                    if (frameNumber < 0 || frameNumber >= m_numEntries)<br>                     {<br>-                        x265_log(m_param, X265_LOG_ERROR, "maxNumReferences cannot be less than 1st pass (%d vs %d)\n",<br>-                            i, m_param->maxNumReferences);<br>+                        x265_log(m_param, X265_LOG_ERROR, "bad frame number (%d) at stats line %d\n", frameNumber, i);<br>                         return false;<br>                     }<br>-                }<br>-                if (m_param->analysisMultiPassRefine || m_param->analysisMultiPassDistortion)<br>-                {<br>-                    p = strstr(opts, "ctu=");<br>-                    sscanf(p, "ctu=%u", &k);<br>-                    if (k != m_param->maxCUSize)<br>+                    rce = &m_rce2Pass[encodeOrder];<br>+                    rcePocOrder = &m_rce2Pass[frameNumber];<br>+                    m_encOrder[frameNumber] = encodeOrder;<br>+                    if (!m_param->bMultiPassOptRPS)<br>                     {<br>-                        x265_log(m_param, X265_LOG_ERROR, "maxCUSize mismatch with 1st pass (%u vs %u)\n",<br>-                            k, m_param->maxCUSize);<br>+                        int scenecut = 0;<br>+                        e += sscanf(p, " in:%*d out:%*d type:%c q:%lf q-aq:%lf q-noVbv:%lf q-Rceq:%lf tex:%d mv:%d misc:%d icu:%lf pcu:%lf scu:%lf sc:%d",<br>+                            &picType, &qpRc, &qpAq, &qNoVbv, &qRceq, &rce->coeffBits,<br>+                            &rce->mvBits, &rce->miscBits, &rce->iCuCount, &rce->pCuCount,<br>+                            &rce->skipCuCount, &scenecut);<br>+                        rcePocOrder->scenecut = scenecut != 0;<br>+                    }<br>+                    else<br>+                    {<br>+                        char deltaPOC[128];<br>+                        char bUsed[40];<br>+                        memset(deltaPOC, 0, sizeof(deltaPOC));<br>+                        memset(bUsed, 0, sizeof(bUsed));<br>+                        e += sscanf(p, " in:%*d out:%*d type:%c q:%lf q-aq:%lf q-noVbv:%lf q-Rceq:%lf tex:%d mv:%d misc:%d icu:%lf pcu:%lf scu:%lf nump:%d numnegp:%d numposp:%d deltapoc:%s bused:%s",<br>+                            &picType, &qpRc, &qpAq, &qNoVbv, &qRceq, &rce->coeffBits,<br>+                            &rce->mvBits, &rce->miscBits, &rce->iCuCount, &rce->pCuCount,<br>+                            &rce->skipCuCount, &rce->rpsData.numberOfPictures, &rce->rpsData.numberOfNegativePictures, &rce->rpsData.numberOfPositivePictures, deltaPOC, bUsed);<br>+                        splitdeltaPOC(deltaPOC, rce);<br>+                        splitbUsed(bUsed, rce);<br>+                        rce->rpsIdx = -1;<br>+                    }<br>+                    rce->keptAsRef = true;<br>+                    rce->isIdr = false;<br>+                    if (picType == 'b' || picType == 'p')<br>+                        rce->keptAsRef = false;<br>+                    if (picType == 'I')<br>+                        rce->isIdr = true;<br>+                    if (picType == 'I' || picType == 'i')<br>+                        rce->sliceType = I_SLICE;<br>+                    else if (picType == 'P' || picType == 'p')<br>+                        rce->sliceType = P_SLICE;<br>+                    else if (picType == 'B' || picType == 'b')<br>+                        rce->sliceType = B_SLICE;<br>+                    else<br>+                        e = -1;<br>+                    if (e < 10)<br>+                    {<br>+                        x265_log(m_param, X265_LOG_ERROR, "statistics are damaged at line %d, parser out=%d\n", i, e);<br>                         return false;<br>                     }<br>+                    rce->qScale = rce->newQScale = x265_qp2qScale(qpRc);<br>+                    totalQpAq += qpAq;<br>+                    rce->qpNoVbv = qNoVbv;<br>+                    rce->qpaRc = qpRc;<br>+                    rce->qpAq = qpAq;<br>+                    rce->qRceq = qRceq;<br>+                    p = next;<br>                 }<br>-                CMP_OPT_FIRST_PASS("bitdepth", m_param->internalBitDepth);<br>-                CMP_OPT_FIRST_PASS("weightp", m_param->bEnableWeightedPred);<br>-                CMP_OPT_FIRST_PASS("bframes", m_param->bframes);<br>-                CMP_OPT_FIRST_PASS("b-pyramid", m_param->bBPyramid);<br>-                CMP_OPT_FIRST_PASS("open-gop", m_param->bOpenGOP);<br>-                CMP_OPT_FIRST_PASS(" keyint", m_param->keyframeMax);<br>-                CMP_OPT_FIRST_PASS("scenecut", m_param->scenecutThreshold);<br>-                CMP_OPT_FIRST_PASS("intra-refresh", m_param->bIntraRefresh);<br>-                CMP_OPT_FIRST_PASS("frame-dup", m_param->bEnableFrameDuplication);<br>-                if (m_param->bMultiPassOptRPS)<br>-                {<br>-                    CMP_OPT_FIRST_PASS("multi-pass-opt-rps", m_param->bMultiPassOptRPS);<br>-                    CMP_OPT_FIRST_PASS("repeat-headers", m_param->bRepeatHeaders);<br>-                    CMP_OPT_FIRST_PASS("min-keyint", m_param->keyframeMin);<br>-                }<br>-<br>-                if ((p = strstr(opts, "b-adapt=")) != 0 && sscanf(p, "b-adapt=%d", &i) && i >= X265_B_ADAPT_NONE && i <= X265_B_ADAPT_TRELLIS)<br>+                X265_FREE(statsBuf);<br>+                if (m_param->rc.rateControlMode != X265_RC_CQP)<br>                 {<br>-                    m_param->bFrameAdaptive = i;<br>-                }<br>-                else if (m_param->bframes)<br>-                {<br>-                    x265_log(m_param, X265_LOG_ERROR, "b-adapt method specified in stats file not valid\n");<br>-                    return false;<br>-                }<br>-<br>-                if ((p = strstr(opts, "rc-lookahead=")) != 0 && sscanf(p, "rc-lookahead=%d", &i))<br>-                    m_param->lookaheadDepth = i;<br>-            }<br>-            /* find number of pics */<br>-            p = statsIn;<br>-            int numEntries;<br>-            for (numEntries = -1; p; numEntries++)<br>-                p = strchr(p + 1, ';');<br>-            if (!numEntries)<br>-            {<br>-                x265_log(m_param, X265_LOG_ERROR, "empty stats file\n");<br>-                return false;<br>-            }<br>-            m_numEntries = numEntries;<br>-<br>-            if (m_param->totalFrames < m_numEntries && m_param->totalFrames > 0)<br>-            {<br>-                x265_log(m_param, X265_LOG_WARNING, "2nd pass has fewer frames than 1st pass (%d vs %d)\n",<br>-                         m_param->totalFrames, m_numEntries);<br>-            }<br>-            if (m_param->totalFrames > m_numEntries && !m_param->bEnableFrameDuplication)<br>-            {<br>-                x265_log(m_param, X265_LOG_ERROR, "2nd pass has more frames than 1st pass (%d vs %d)\n",<br>-                         m_param->totalFrames, m_numEntries);<br>-                return false;<br>+                    m_start = 0;<br>+                    m_isQpModified = true;<br>+                    if (!initPass2())<br>+                        return false;<br>+                } /* else we're using constant quant, so no need to run the bitrate allocation */<br>             }<br>-<br>-            m_rce2Pass = X265_MALLOC(RateControlEntry, m_numEntries);<br>-            if (!m_rce2Pass)<br>+            else if (SHARED_MODE_MEM == m_cutreeStorageMode)<br>             {<br>-                 x265_log(m_param, X265_LOG_ERROR, "Rce Entries for 2 pass cannot be allocated\n");<br>-                 return false;<br>+                if (m_param->rc.cuTree)<br>+                {<br>+                    if (!initCUTreeSharedMem())<br>+                    {<br>+                        return false;<br>+                    }<br>+                }<br>             }<br>-            m_encOrder = X265_MALLOC(int, m_numEntries);<br>-            if (!m_encOrder)<br>+            else<br></div></blockquote><div>[AM] Why this else condition? m_cutreeStorageMode can either be SHARED_MODE_FILE or SHARED_MODE_MEM, right? </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"><div dir="ltr">             {<br>-                x265_log(m_param, X265_LOG_ERROR, "Encode order for 2 pass cannot be allocated\n");<br>                 return false;<br>             }<br>-            /* init all to skipped p frames */<br>-            for (int i = 0; i < m_numEntries; i++)<br>-            {<br>-                RateControlEntry *rce = &m_rce2Pass[i];<br>-                rce->sliceType = P_SLICE;<br>-                rce->qScale = rce->newQScale = x265_qp2qScale(20);<br>-                rce->miscBits = m_ncu + 10;<br>-                rce->newQp = 0;<br>-            }<br>-            /* read stats */<br>-            p = statsIn;<br>-            double totalQpAq = 0;<br>-            for (int i = 0; i < m_numEntries; i++)<br>-            {<br>-                RateControlEntry *rce, *rcePocOrder;<br>-                int frameNumber;<br>-                int encodeOrder;<br>-                char picType;<br>-                int e;<br>-                char *next;<br>-                double qpRc, qpAq, qNoVbv, qRceq;<br>-                next = strstr(p, ";");<br>-                if (next)<br>-                    *next++ = 0;<br>-                e = sscanf(p, " in:%d out:%d", &frameNumber, &encodeOrder);<br>-                if (frameNumber < 0 || frameNumber >= m_numEntries)<br>-                {<br>-                    x265_log(m_param, X265_LOG_ERROR, "bad frame number (%d) at stats line %d\n", frameNumber, i);<br>-                    return false;<br>-                }<br>-                rce = &m_rce2Pass[encodeOrder];<br>-                rcePocOrder = &m_rce2Pass[frameNumber];<br>-                m_encOrder[frameNumber] = encodeOrder;<br>-                if (!m_param->bMultiPassOptRPS)<br>-                {<br>-                    int scenecut = 0;<br>-                    e += sscanf(p, " in:%*d out:%*d type:%c q:%lf q-aq:%lf q-noVbv:%lf q-Rceq:%lf tex:%d mv:%d misc:%d icu:%lf pcu:%lf scu:%lf sc:%d",<br>-                        &picType, &qpRc, &qpAq, &qNoVbv, &qRceq, &rce->coeffBits,<br>-                        &rce->mvBits, &rce->miscBits, &rce->iCuCount, &rce->pCuCount,<br>-                        &rce->skipCuCount, &scenecut);<br>-                    rcePocOrder->scenecut = scenecut != 0;<br>-                }<br>-                else<br>-                {<br>-                    char deltaPOC[128];<br>-                    char bUsed[40];<br>-                    memset(deltaPOC, 0, sizeof(deltaPOC));<br>-                    memset(bUsed, 0, sizeof(bUsed));<br>-                    e += sscanf(p, " in:%*d out:%*d type:%c q:%lf q-aq:%lf q-noVbv:%lf q-Rceq:%lf tex:%d mv:%d misc:%d icu:%lf pcu:%lf scu:%lf nump:%d numnegp:%d numposp:%d deltapoc:%s bused:%s",<br>-                        &picType, &qpRc, &qpAq, &qNoVbv, &qRceq, &rce->coeffBits,<br>-                        &rce->mvBits, &rce->miscBits, &rce->iCuCount, &rce->pCuCount,<br>-                        &rce->skipCuCount, &rce->rpsData.numberOfPictures, &rce->rpsData.numberOfNegativePictures, &rce->rpsData.numberOfPositivePictures, deltaPOC, bUsed);<br>-                    splitdeltaPOC(deltaPOC, rce);<br>-                    splitbUsed(bUsed, rce);<br>-                    rce->rpsIdx = -1;<br>-                }<br>-                rce->keptAsRef = true;<br>-                rce->isIdr = false;<br>-                if (picType == 'b' || picType == 'p')<br>-                    rce->keptAsRef = false;<br>-                if (picType == 'I')<br>-                    rce->isIdr = true;<br>-                if (picType == 'I' || picType == 'i')<br>-                    rce->sliceType = I_SLICE;<br>-                else if (picType == 'P' || picType == 'p')<br>-                    rce->sliceType = P_SLICE;<br>-                else if (picType == 'B' || picType == 'b')<br>-                    rce->sliceType = B_SLICE;<br>-                else<br>-                    e = -1;<br>-                if (e < 10)<br>-                {<br>-                    x265_log(m_param, X265_LOG_ERROR, "statistics are damaged at line %d, parser out=%d\n", i, e);<br>-                    return false;<br>-                }<br>-                rce->qScale = rce->newQScale = x265_qp2qScale(qpRc);<br>-                totalQpAq += qpAq;<br>-                rce->qpNoVbv = qNoVbv;<br>-                rce->qpaRc = qpRc;<br>-                rce->qpAq = qpAq;<br>-                rce->qRceq = qRceq;<br>-                p = next;<br>-            }<br>-            X265_FREE(statsBuf);<br>-            if (m_param->rc.rateControlMode != X265_RC_CQP)<br>-            {<br>-                m_start = 0;<br>-                m_isQpModified = true;<br>-                if (!initPass2())<br>-                    return false;<br>-            } /* else we're using constant quant, so no need to run the bitrate allocation */<br>         }<br>         /* Open output file */<br>         /* If input and output files are the same, output to a temp file<br>@@ -682,19 +778,33 @@ bool RateControl::init(const SPS& sps)<br>             X265_FREE(p);<br>             if (m_param->rc.cuTree && !m_param->rc.bStatRead)<br>             {<br>-                statFileTmpname = strcatFilename(fileName, ".cutree.temp");<br>-                if (!statFileTmpname)<br>-                    return false;<br>-                m_cutreeStatFileOut = x265_fopen(statFileTmpname, "wb");<br>-                X265_FREE(statFileTmpname);<br>-                if (!m_cutreeStatFileOut)<br>+                if (SHARED_MODE_FILE == m_cutreeStorageMode)<br>+                {<br>+                    statFileTmpname = strcatFilename(fileName, ".cutree.temp");<br>+                    if (!statFileTmpname)<br>+                        return false;<br>+                    m_cutreeStatFileOut = x265_fopen(statFileTmpname, "wb");<br>+                    X265_FREE(statFileTmpname);<br>+                    if (!m_cutreeStatFileOut)<br>+                    {<br>+                        x265_log_file(m_param, X265_LOG_ERROR, "can't open mbtree stats file %s.cutree.temp\n", fileName);<br>+                        return false;<br>+                    }<br>+                }<br>+                else if (SHARED_MODE_MEM == m_cutreeStorageMode)<br>+                {<br>+                    if (!initCUTreeSharedMem())<br>+                    {<br>+                        return false;<br>+                    }<br>+                }<br>+                else<br>                 {<br>-                    x265_log_file(m_param, X265_LOG_ERROR, "can't open mbtree stats file %s.cutree.temp\n", fileName);<br>                     return false;<br>                 }<br>             }<br>         }<br>-        if (m_param->rc.cuTree)<br>+        if (m_param->rc.cuTree && !m_cuTreeStats.qpBuffer[0])<br>         {<br>             if (m_param->rc.qgSize == 8)<br>             {<br>@@ -714,6 +824,10 @@ bool RateControl::init(const SPS& sps)<br>     return true;<br> }<br> <br>+void RateControl::skipCUTreeSharedMemRead(int32_t cnt)<br>+{<br>+    m_cutreeShrMem->skipRead(cnt);<br>+}<br> void RateControl::reconfigureRC()<br> {<br>     if (m_isVbv)<br>@@ -1670,10 +1784,28 @@ bool RateControl::cuTreeReadFor2Pass(Frame* frame)<br>             {<br>                 m_cuTreeStats.qpBufPos++;<br> <br>-                if (!fread(&type, 1, 1, m_cutreeStatFileIn))<br>-                    goto fail;<br>-                if (fread(m_cuTreeStats.qpBuffer[m_cuTreeStats.qpBufPos], sizeof(uint16_t), ncu, m_cutreeStatFileIn) != (size_t)ncu)<br>-                    goto fail;<br>+                if (SHARED_MODE_FILE == m_cutreeStorageMode)<br>+                {<br>+                    if (!fread(&type, 1, 1, m_cutreeStatFileIn))<br>+                        goto fail;<br>+                    if (fread(m_cuTreeStats.qpBuffer[m_cuTreeStats.qpBufPos], sizeof(uint16_t), ncu, m_cutreeStatFileIn) != (size_t)ncu)<br>+                        goto fail;<br>+                }<br>+                else<br>+                {<br>+                    if (SHARED_MODE_MEM == m_cutreeStorageMode)<br>+                    {<br>+                        if (!m_cutreeShrMem)<br>+                        {<br>+                            goto fail;<br>+                        }<br>+<br>+                        CUTreeSharedDataItem shrItem;<br>+                        shrItem.type = &type;<br>+                        shrItem.stats = m_cuTreeStats.qpBuffer[m_cuTreeStats.qpBufPos];<br>+                        m_cutreeShrMem->readNext(&shrItem, ReadSharedCUTreeData);<br>+                    }<br>+                }<br> <br>                 if (type != sliceTypeActual && m_cuTreeStats.qpBufPos == 1)<br>                 {<br>@@ -3064,10 +3196,34 @@ int RateControl::writeRateControlFrameStats(Frame* curFrame, RateControlEntry* r<br>     {<br>         uint8_t sliceType = (uint8_t)rce->sliceType;<br>         primitives.fix8Pack(m_cuTreeStats.qpBuffer[0], curFrame->m_lowres.qpCuTreeOffset, ncu);<br>-        if (fwrite(&sliceType, 1, 1, m_cutreeStatFileOut) < 1)<br>-            goto writeFailure;<br>-        if (fwrite(m_cuTreeStats.qpBuffer[0], sizeof(uint16_t), ncu, m_cutreeStatFileOut) < (size_t)ncu)<br>+<br>+        if (SHARED_MODE_FILE == m_cutreeStorageMode)<br>+        {<br>+            if (fwrite(&sliceType, 1, 1, m_cutreeStatFileOut) < 1)<br>+                goto writeFailure;<br>+            if (fwrite(m_cuTreeStats.qpBuffer[0], sizeof(uint16_t), ncu, m_cutreeStatFileOut) < (size_t)ncu)<br>+                goto writeFailure;<br>+        }<br>+        else if (SHARED_MODE_MEM == m_cutreeStorageMode)<br>+        {<br>+            if (SHARED_MODE_MEM == m_cutreeStorageMode)<br></div></blockquote><div>[AM] duplicate nested check condition </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"><div dir="ltr">+            {<br>+                if (!m_cutreeShrMem)<br>+                {<br>+                    goto writeFailure;<br>+                }<br>+<br>+                CUTreeSharedDataItem shrItem;<br>+                shrItem.type = &sliceType;<br>+                shrItem.stats = m_cuTreeStats.qpBuffer[0];<br>+                m_cutreeShrMem->writeData(&shrItem, WriteSharedCUTreeData);<br>+            }<br>+        }<br>+        else<br></div></blockquote><div>[AM] Why this else? m_cutreeStorageMode can either be SHARED_MODE_FILE or SHARED_MODE_MEM, right?</div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"><div dir="ltr">+        {<br>             goto writeFailure;<br>+        }<br>+       <br>     }<br>     return 0;<br> <br>@@ -3143,6 +3299,13 @@ void RateControl::destroy()<br>     if (m_cutreeStatFileIn)<br>         fclose(m_cutreeStatFileIn);<br> <br>+    if (m_cutreeShrMem)<br>+    {<br>+        m_cutreeShrMem->release();<br>+        delete m_cutreeShrMem;<br>+        m_cutreeShrMem = NULL;<br>+    }<br>+<br>     X265_FREE(m_rce2Pass);<br>     X265_FREE(m_encOrder);<br>     for (int i = 0; i < 2; i++)<br>@@ -3151,6 +3314,8 @@ void RateControl::destroy()<br>     if (m_relativeComplexity)<br>         X265_FREE(m_relativeComplexity);<br> <br>+    free(m_shrname);<br>+<br> }<br> <br> void RateControl::splitdeltaPOC(char deltapoc[], RateControlEntry *rce)<br>diff --git a/source/encoder/ratecontrol.h b/source/encoder/ratecontrol.h<br>index 204bd71e1..19c8676b4 100644<br>--- a/source/encoder/ratecontrol.h<br>+++ b/source/encoder/ratecontrol.h<br>@@ -28,6 +28,7 @@<br> <br> #include "common.h"<br> #include "sei.h"<br>+#include "ringmem.h"<br> <br> namespace X265_NS {<br> // encoder namespace<br>@@ -126,6 +127,13 @@ struct RateControlEntry<br>     bool     isFadeEnd;<br> };<br> <br>+enum DataSharedMode<br>+{<br>+    SHARED_MODE_FILE = 0,<br>+    SHARED_MODE_MEM,<br>+    SHARED_MODE_CNT<br>+};<br>+<br> class RateControl<br> {<br> public:<br>@@ -237,9 +245,16 @@ public:<br>     int     m_numEntries;<br>     int     m_start;<br>     int     m_reencode;<br>+    ///< store the cutree data in file or shared memory<br>+    ///< it is not necessary to store the cutree in shared memory.<br>+    ///< However, for further use, shared memeory is a better choice<br>+    DataSharedMode m_cutreeStorageMode;<br>+    char   *m_shrname;<br>     FILE*   m_statFileOut;<br>     FILE*   m_cutreeStatFileOut;<br>     FILE*   m_cutreeStatFileIn;<br>+    ///< store the cutree data in memory instead of file<br>+    RingMem *m_cutreeShrMem;<br>     double  m_lastAccumPNorm;<br>     double  m_expectedBitsSum;   /* sum of qscale2bits after rceq, ratefactor, and overflow, only includes finished frames */<br>     int64_t m_predictedBits;<br>@@ -254,7 +269,7 @@ public:<br>                                 * This value is the current position (0 or 1). */<br>     } m_cuTreeStats;<br> <br>-    RateControl(x265_param& p, Encoder *enc);<br>+    RateControl(x265_param& p, Encoder *enc, char *dataShr);<br>     bool init(const SPS& sps);<br>     void initHRD(SPS& sps);<br>     void reconfigureRC();<br>@@ -274,6 +289,9 @@ public:<br>     int writeRateControlFrameStats(Frame* curFrame, RateControlEntry* rce);<br>     bool   initPass2();<br> <br>+    bool initCUTreeSharedMem();<br>+    void skipCUTreeSharedMemRead(int32_t cnt);<br>+<br>     double forwardMasking(Frame* curFrame, double q);<br>     double backwardMasking(Frame* curFrame, double q);<br> <br>-- <br>2.22.0.windows.1<br><br></div>
_______________________________________________<br>
x265-devel mailing list<br>
<a href="mailto:x265-devel@videolan.org" target="_blank">x265-devel@videolan.org</a><br>
<a href="https://mailman.videolan.org/listinfo/x265-devel" rel="noreferrer" target="_blank">https://mailman.videolan.org/listinfo/x265-devel</a><br>
</blockquote></div><br clear="all"><div><br></div>-- <br><div dir="ltr" class="gmail_signature"><div dir="ltr"><div><div dir="ltr"><div><div dir="ltr"><div><div dir="ltr"><font face="georgia, serif">Regards,</font><div><b><font face="georgia, serif">Aruna Matheswaran,</font></b></div><div><font face="georgia, serif">Video Codec Engineer,</font></div><div><font face="georgia, serif">Media & AI analytics BU,</font></div><div><span><span style="font-size:11pt;font-family:Arial;color:rgb(0,0,0);background-color:transparent;vertical-align:baseline;white-space:pre-wrap"><span style="border:none;display:inline-block;overflow:hidden;width:153px;height:58px"><img src="https://lh5.googleusercontent.com/gjX5cPNIZgwUrhfqkTwQUZWztIKmmo0qs3kbwvkS5H-bDVE2ftte9pMTVnFLSjOcjYWLtfc6_OGpxW4vraLg2r5QAIf1Q3MpldFDgWtzK_gXi8ptw5B3joIbsGL6mxj-JRdjHzT5" width="96" height="36" style="margin-left: 0px; margin-top: 0px;"></span></span></span><font face="georgia, serif"><br></font></div><div><span><span style="font-size:11pt;font-family:Arial;color:rgb(0,0,0);background-color:transparent;vertical-align:baseline;white-space:pre-wrap"><span style="border:none;display:inline-block;overflow:hidden;width:153px;height:58px"><img src="https://lh5.googleusercontent.com/gjX5cPNIZgwUrhfqkTwQUZWztIKmmo0qs3kbwvkS5H-bDVE2ftte9pMTVnFLSjOcjYWLtfc6_OGpxW4vraLg2r5QAIf1Q3MpldFDgWtzK_gXi8ptw5B3joIbsGL6mxj-JRdjHzT5" style="margin-left: 0px; margin-top: 0px;"></span></span></span><font face="georgia, serif"><br></font></div><div><font face="georgia, serif"><br></font></div></div></div></div></div></div></div></div></div></div>