[x265] [PATCH] Add csv feature into libx265
Pradeep Ramachandran
pradeep at multicorewareinc.com
Mon Jun 12 13:26:32 CEST 2017
On Mon, Jun 12, 2017 at 3:17 PM, Divya Manivannan <
divya at multicorewareinc.com> wrote:
> # HG changeset patch
> # User Divya Manivannan <divya at multicorewareinc.com>
> # Date 1497012715 -19800
> # Fri Jun 09 18:21:55 2017 +0530
> # Node ID 12097d65c3cc2d0624ca7a2a70cee2d53241c71d
> # Parent b19bc0c7b4eccfe336ecd381df25203ee6f57d05
> Add csv feature into libx265
>
This is a good feature to pull into the library. It should help folks
integrating other applications with libx265 to get stats about the encode.
Pushed into default branch.
> diff -r b19bc0c7b4ec -r 12097d65c3cc doc/reST/api.rst
> --- a/doc/reST/api.rst Tue Jun 06 11:13:19 2017 +0530
> +++ b/doc/reST/api.rst Fri Jun 09 18:21:55 2017 +0530
> @@ -347,6 +347,14 @@
> Cleanup
> =======
>
> +At the end of the encode, the application will want to trigger logging
> +of the final encode statistics, if :option:`--csv` had been specified::
> +
> + /* x265_encoder_log:
> + * write a line to the configured CSV file. If a CSV
> filename was not
> + * configured, or file open failed, this function will
> perform no write. */
> + void x265_encoder_log(x265_encoder *encoder, int argc, char
> **argv);
> +
> Finally, the encoder must be closed in order to free all of its
> resources. An encoder that has been flushed cannot be restarted and
> reused. Once **x265_encoder_close()** has been called, the encoder
> diff -r b19bc0c7b4ec -r 12097d65c3cc doc/reST/cli.rst
> --- a/doc/reST/cli.rst Tue Jun 06 11:13:19 2017 +0530
> +++ b/doc/reST/cli.rst Fri Jun 09 18:21:55 2017 +0530
> @@ -52,8 +52,7 @@
> 2. unable to open encoder
> 3. unable to generate stream headers
> 4. encoder abort
> - 5. unable to open csv file
> -
> +
> Logging/Statistic Options
> =========================
>
> @@ -182,8 +181,6 @@
> is more of a problem for P frames where some blocks are much more
> expensive than others.
>
> - **CLI ONLY**
> -
> .. option:: --csv-log-level <integer>
>
> Controls the level of detail (and size) of --csv log files
> @@ -192,8 +189,6 @@
> 1. frame level logging
> 2. frame level logging with performance statistics
>
> - **CLI ONLY**
> -
> .. option:: --ssim, --no-ssim
>
> Calculate and report Structural Similarity values. It is
> diff -r b19bc0c7b4ec -r 12097d65c3cc source/CMakeLists.txt
> --- a/source/CMakeLists.txt Tue Jun 06 11:13:19 2017 +0530
> +++ b/source/CMakeLists.txt Fri Jun 09 18:21:55 2017 +0530
> @@ -29,7 +29,7 @@
> option(STATIC_LINK_CRT "Statically link C runtime for release builds" OFF)
> mark_as_advanced(FPROFILE_USE FPROFILE_GENERATE NATIVE_BUILD)
> # X265_BUILD must be incremented each time the public API is changed
> -set(X265_BUILD 121)
> +set(X265_BUILD 122)
> configure_file("${PROJECT_SOURCE_DIR}/x265.def.in"
> "${PROJECT_BINARY_DIR}/x265.def")
> configure_file("${PROJECT_SOURCE_DIR}/x265_config.h.in"
> @@ -650,16 +650,16 @@
> # Xcode seems unable to link the CLI with libs, so link as one
> targget
> if(ENABLE_DYNAMIC_HDR10)
> add_executable(cli ../COPYING ${InputFiles} ${OutputFiles}
> ${GETOPT}
> - x265.cpp x265.h x265cli.h x265-extras.h
> x265-extras.cpp
> + x265.cpp x265.h x265cli.h
> $<TARGET_OBJECTS:encoder>
> $<TARGET_OBJECTS:common> $<TARGET_OBJECTS:dynamicHDR10> ${ASM_OBJS}
> ${ASM_SRCS})
> else()
> add_executable(cli ../COPYING ${InputFiles} ${OutputFiles}
> ${GETOPT}
> - x265.cpp x265.h x265cli.h x265-extras.h
> x265-extras.cpp
> + x265.cpp x265.h x265cli.h
> $<TARGET_OBJECTS:encoder>
> $<TARGET_OBJECTS:common> ${ASM_OBJS} ${ASM_SRCS})
> endif()
> else()
> add_executable(cli ../COPYING ${InputFiles} ${OutputFiles}
> ${GETOPT} ${X265_RC_FILE}
> - ${ExportDefs} x265.cpp x265.h x265cli.h
> x265-extras.h x265-extras.cpp)
> + ${ExportDefs} x265.cpp x265.h x265cli.h)
> if(WIN32 OR NOT ENABLE_SHARED OR INTEL_CXX)
> # The CLI cannot link to the shared library on Windows, it
> # requires internal APIs not exported from the DLL
> diff -r b19bc0c7b4ec -r 12097d65c3cc source/common/param.cpp
> --- a/source/common/param.cpp Tue Jun 06 11:13:19 2017 +0530
> +++ b/source/common/param.cpp Fri Jun 09 18:21:55 2017 +0530
> @@ -110,6 +110,7 @@
> param->frameNumThreads = 0;
>
> param->logLevel = X265_LOG_INFO;
> + param->csvLogLevel = 0;
> param->csvfn = NULL;
> param->rc.lambdaFileName = NULL;
> param->bLogCuStats = 0;
> @@ -927,6 +928,8 @@
> if (bExtraParams)
> {
> if (0) ;
> + OPT("csv") p->csvfn = strdup(value);
> + OPT("csv-log-level") p->csvLogLevel = atoi(value);
> OPT("qpmin") p->rc.qpMin = atoi(value);
> OPT("analyze-src-pics") p->bSourceReferenceEstimation =
> atobool(value);
> OPT("log2-max-poc-lsb") p->log2MaxPocLsb = atoi(value);
> @@ -1516,6 +1519,8 @@
> BOOL(p->bEnablePsnr, "psnr");
> BOOL(p->bEnableSsim, "ssim");
> s += sprintf(s, " log-level=%d", p->logLevel);
> + if (p->csvfn)
> + s += sprintf(s, " csvfn=%s csv-log-level=%d", p->csvfn,
> p->csvLogLevel);
> s += sprintf(s, " bitdepth=%d", p->internalBitDepth);
> s += sprintf(s, " input-csp=%d", p->internalCsp);
> s += sprintf(s, " fps=%u/%u", p->fpsNum, p->fpsDenom);
> diff -r b19bc0c7b4ec -r 12097d65c3cc source/encoder/CMakeLists.txt
> --- a/source/encoder/CMakeLists.txt Tue Jun 06 11:13:19 2017 +0530
> +++ b/source/encoder/CMakeLists.txt Fri Jun 09 18:21:55 2017 +0530
> @@ -43,4 +43,5 @@
> reference.cpp reference.h
> encoder.cpp encoder.h
> api.cpp
> - weightPrediction.cpp)
> + weightPrediction.cpp
> + ../x265-extras.cpp ../x265-extras.h)
> diff -r b19bc0c7b4ec -r 12097d65c3cc source/encoder/api.cpp
> --- a/source/encoder/api.cpp Tue Jun 06 11:13:19 2017 +0530
> +++ b/source/encoder/api.cpp Fri Jun 09 18:21:55 2017 +0530
> @@ -30,6 +30,7 @@
> #include "level.h"
> #include "nal.h"
> #include "bitcost.h"
> +#include "x265-extras.h"
>
> /* multilib namespace reflectors */
> #if LINKED_8BIT
> @@ -119,6 +120,17 @@
> }
>
> encoder->create();
> + /* Try to open CSV file handle */
> + if (encoder->m_param->csvfn)
> + {
> + encoder->m_csvfpt = x265_csvlog_open(*encoder->m_param,
> encoder->m_param->csvfn, encoder->m_param->csvLogLevel);
> + if (!encoder->m_csvfpt)
> + {
> + x265_log(encoder->m_param, X265_LOG_ERROR, "Unable to open
> CSV log file <%s>, aborting\n", encoder->m_param->csvfn);
> + encoder->m_aborted = true;
> + }
> + }
> +
> encoder->m_latestParam = latestParam;
> memcpy(latestParam, param, sizeof(x265_param));
> if (encoder->m_aborted)
> @@ -144,7 +156,10 @@
> if (encoder->m_param->rc.bStatRead && encoder->m_param->
> bMultiPassOptRPS)
> {
> if (!encoder->computeSPSRPSIndex())
> + {
> + encoder->m_aborted = true;
> return -1;
> + }
> }
> encoder->getStreamHeaders(encoder->m_nalList, sbacCoder, bs);
> *pp_nal = &encoder->m_nalList.m_nal[0];
> @@ -152,6 +167,11 @@
> return encoder->m_nalList.m_occupancy;
> }
>
> + if (enc)
> + {
> + Encoder *encoder = static_cast<Encoder*>(enc);
> + encoder->m_aborted = true;
> + }
> return -1;
> }
>
> @@ -251,6 +271,12 @@
> else if (pi_nal)
> *pi_nal = 0;
>
> + if (numEncoded && encoder->m_param->csvLogLevel)
> + x265_csvlog_frame(encoder->m_csvfpt, *encoder->m_param,
> *pic_out, encoder->m_param->csvLogLevel);
> +
> + if (numEncoded < 0)
> + encoder->m_aborted = true;
> +
> return numEncoded;
> }
>
> @@ -263,12 +289,16 @@
> }
> }
>
> -void x265_encoder_log(x265_encoder* enc, int, char **)
> +void x265_encoder_log(x265_encoder* enc, int argc, char **argv)
> {
> if (enc)
> {
> Encoder *encoder = static_cast<Encoder*>(enc);
> - x265_log(encoder->m_param, X265_LOG_WARNING, "x265_encoder_log is
> now deprecated\n");
> + x265_stats stats;
> + int padx = encoder->m_sps.conformanceWindow.rightOffset;
> + int pady = encoder->m_sps.conformanceWindow.bottomOffset;
> + encoder->fetchStats(&stats, sizeof(stats));
> + x265_csvlog_encode(encoder->m_csvfpt, x265_version_str,
> *encoder->m_param, padx, pady, stats, encoder->m_param->csvLogLevel, argc,
> argv);
> }
> }
>
> diff -r b19bc0c7b4ec -r 12097d65c3cc source/encoder/encoder.cpp
> --- a/source/encoder/encoder.cpp Tue Jun 06 11:13:19 2017 +0530
> +++ b/source/encoder/encoder.cpp Fri Jun 09 18:21:55 2017 +0530
> @@ -70,6 +70,7 @@
> m_exportedPic = NULL;
> m_numDelayedPic = 0;
> m_outputCount = 0;
> + m_csvfpt = NULL;
> m_param = NULL;
> m_latestParam = NULL;
> m_threadPool = NULL;
> @@ -468,6 +469,9 @@
> if (m_analysisFile)
> fclose(m_analysisFile);
>
> + if (m_csvfpt)
> + fclose(m_csvfpt);
> +
> if (m_latestParam != NULL && m_latestParam != m_param)
> {
> if (m_latestParam->scalingLists != m_param->scalingLists)
> @@ -504,6 +508,7 @@
> free((char*)m_param->rc.statFileName);
> free((char*)m_param->analysisFileName);
> free((char*)m_param->scalingLists);
> + free((char*)m_param->csvfn);
> free((char*)m_param->numaPools);
> free((char*)m_param->masteringDisplayColorVolume);
> free((char*)m_param->toneMapFile);
> @@ -2605,9 +2610,6 @@
> if (p->bLogCuStats)
> x265_log(p, X265_LOG_WARNING, "--cu-stats option is now
> deprecated\n");
>
> - if (p->csvfn)
> - x265_log(p, X265_LOG_WARNING, "libx265 no longer supports CSV
> file statistics\n");
> -
> if (p->log2MaxPocLsb < 4)
> {
> x265_log(p, X265_LOG_WARNING, "maximum of the picture order count
> can not be less than 4\n");
> diff -r b19bc0c7b4ec -r 12097d65c3cc source/encoder/encoder.h
> --- a/source/encoder/encoder.h Tue Jun 06 11:13:19 2017 +0530
> +++ b/source/encoder/encoder.h Fri Jun 09 18:21:55 2017 +0530
> @@ -137,6 +137,7 @@
> x265_param* m_latestParam; // Holds latest param during a
> reconfigure
> RateControl* m_rateControl;
> Lookahead* m_lookahead;
> + FILE* m_csvfpt;
>
> /* Collect statistics globally */
> EncStats m_analyzeAll;
> diff -r b19bc0c7b4ec -r 12097d65c3cc source/x265-extras.cpp
> --- a/source/x265-extras.cpp Tue Jun 06 11:13:19 2017 +0530
> +++ b/source/x265-extras.cpp Fri Jun 09 18:21:55 2017 +0530
> @@ -25,7 +25,7 @@
>
> #include "x265.h"
> #include "x265-extras.h"
> -
> +#include "param.h"
> #include "common.h"
>
> using namespace X265_NS;
> @@ -38,14 +38,8 @@
> "B count, B ave-QP, B kbps, B-PSNR Y, B-PSNR U, B-PSNR V, B-SSIM
> (dB), "
> "MaxCLL, MaxFALL, Version\n";
>
> -FILE* x265_csvlog_open(const x265_api& api, const x265_param& param,
> const char* fname, int level)
> +FILE* x265_csvlog_open(const x265_param& param, const char* fname, int
> level)
> {
> - if (sizeof(x265_stats) != api.sizeof_stats || sizeof(x265_picture) !=
> api.sizeof_picture)
> - {
> - fprintf(stderr, "extras [error]: structure size skew, unable to
> create CSV logfile\n");
> - return NULL;
> - }
> -
> FILE *csvfp = x265_fopen(fname, "r");
> if (csvfp)
> {
> @@ -255,7 +249,7 @@
> fflush(stderr);
> }
>
> -void x265_csvlog_encode(FILE* csvfp, const char* version, const
> x265_param& param, const x265_stats& stats, int level, int argc, char**
> argv)
> +void x265_csvlog_encode(FILE* csvfp, const char* version, const
> x265_param& param, int padx, int pady, const x265_stats& stats, int level,
> int argc, char** argv)
> {
> if (!csvfp)
> return;
> @@ -268,13 +262,27 @@
> }
>
> // CLI arguments or other
> - fputc('"', csvfp);
> - for (int i = 1; i < argc; i++)
> + if (argc)
> {
> - fputc(' ', csvfp);
> - fputs(argv[i], csvfp);
> + fputc('"', csvfp);
> + for (int i = 1; i < argc; i++)
> + {
> + fputc(' ', csvfp);
> + fputs(argv[i], csvfp);
> + }
> + fputc('"', csvfp);
> }
> - fputc('"', csvfp);
> + else
> + {
> + const x265_param* paramTemp = ¶m;
> + char *opts = x265_param2string((x265_param*)paramTemp, padx,
> pady);
> + if (opts)
> + {
> + fputc('"', csvfp);
> + fputs(opts, csvfp);
> + fputc('"', csvfp);
> + }
> + }
>
> // current date and time
> time_t now;
> diff -r b19bc0c7b4ec -r 12097d65c3cc source/x265-extras.h
> --- a/source/x265-extras.h Tue Jun 06 11:13:19 2017 +0530
> +++ b/source/x265-extras.h Fri Jun 09 18:21:55 2017 +0530
> @@ -44,7 +44,7 @@
> * closed by the caller using fclose(). If level is 0, then no frame
> logging
> * header is written to the file. This function will return NULL if it is
> unable
> * to open the file for write or if it detects a structure size skew */
> -LIBAPI FILE* x265_csvlog_open(const x265_api& api, const x265_param&
> param, const char* fname, int level);
> +LIBAPI FILE* x265_csvlog_open(const x265_param& param, const char* fname,
> int level);
>
> /* Log frame statistics to the CSV file handle. level should have been
> non-zero
> * in the call to x265_csvlog_open() if this function is called. */
> @@ -53,7 +53,7 @@
> /* Log final encode statistics to the CSV file handle. 'argc' and 'argv'
> are
> * intended to be command line arguments passed to the encoder. Encode
> * statistics should be queried from the encoder just prior to closing
> it. */
> -LIBAPI void x265_csvlog_encode(FILE* csvfp, const char* version, const
> x265_param& param, const x265_stats& stats, int level, int argc, char**
> argv);
> +LIBAPI void x265_csvlog_encode(FILE* csvfp, const char* version, const
> x265_param& param, int padx, int pady, const x265_stats& stats, int level,
> int argc, char** argv);
>
> /* In-place downshift from a bit-depth greater than 8 to a bit-depth of
> 8, using
> * the residual bits to dither each row. */
> diff -r b19bc0c7b4ec -r 12097d65c3cc source/x265.cpp
> --- a/source/x265.cpp Tue Jun 06 11:13:19 2017 +0530
> +++ b/source/x265.cpp Fri Jun 09 18:21:55 2017 +0530
> @@ -73,15 +73,12 @@
> ReconFile* recon;
> OutputFile* output;
> FILE* qpfile;
> - FILE* csvfpt;
> - const char* csvfn;
> const char* reconPlayCmd;
> const x265_api* api;
> x265_param* param;
> bool bProgress;
> bool bForceY4m;
> bool bDither;
> - int csvLogLevel;
> uint32_t seek; // number of frames to skip from the
> beginning
> uint32_t framesToBeEncoded; // number of frames to encode
> uint64_t totalbytes;
> @@ -97,8 +94,6 @@
> recon = NULL;
> output = NULL;
> qpfile = NULL;
> - csvfpt = NULL;
> - csvfn = NULL;
> reconPlayCmd = NULL;
> api = NULL;
> param = NULL;
> @@ -109,7 +104,6 @@
> startTime = x265_mdate();
> prevUpdateTime = 0;
> bDither = false;
> - csvLogLevel = 0;
> }
>
> void destroy();
> @@ -129,9 +123,6 @@
> if (qpfile)
> fclose(qpfile);
> qpfile = NULL;
> - if (csvfpt)
> - fclose(csvfpt);
> - csvfpt = NULL;
> if (output)
> output->release();
> output = NULL;
> @@ -292,8 +283,6 @@
> if (0) ;
> OPT2("frame-skip", "seek") this->seek =
> (uint32_t)x265_atoi(optarg, bError);
> OPT("frames") this->framesToBeEncoded =
> (uint32_t)x265_atoi(optarg, bError);
> - OPT("csv") this->csvfn = optarg;
> - OPT("csv-log-level") this->csvLogLevel = x265_atoi(optarg,
> bError);
> OPT("no-progress") this->bProgress = false;
> OPT("output") outputfn = optarg;
> OPT("input") inputfn = optarg;
> @@ -530,8 +519,7 @@
> * 1 - unable to parse command line
> * 2 - unable to open encoder
> * 3 - unable to generate stream headers
> - * 4 - encoder abort
> - * 5 - unable to open csv file */
> + * 4 - encoder abort */
>
> int main(int argc, char **argv)
> {
> @@ -586,20 +574,7 @@
> /* get the encoder parameters post-initialization */
> api->encoder_parameters(encoder, param);
>
> - if (cliopt.csvfn)
> - {
> - cliopt.csvfpt = x265_csvlog_open(*api, *param, cliopt.csvfn,
> cliopt.csvLogLevel);
> - if (!cliopt.csvfpt)
> - {
> - x265_log_file(param, X265_LOG_ERROR, "Unable to open CSV log
> file <%s>, aborting\n", cliopt.csvfn);
> - cliopt.destroy();
> - if (cliopt.api)
> - cliopt.api->param_free(cliopt.param);
> - exit(5);
> - }
> - }
> -
> - /* Control-C handler */
> + /* Control-C handler */
> if (signal(SIGINT, sigint_handler) == SIG_ERR)
> x265_log(param, X265_LOG_ERROR, "Unable to register CTRL+C
> handler: %s\n", strerror(errno));
>
> @@ -607,7 +582,7 @@
> x265_picture *pic_in = &pic_orig;
> /* Allocate recon picture if analysisMode is enabled */
> std::priority_queue<int64_t>* pts_queue = cliopt.output->needPTS() ?
> new std::priority_queue<int64_t>() : NULL;
> - x265_picture *pic_recon = (cliopt.recon || !!param->analysisMode ||
> pts_queue || reconPlay || cliopt.csvLogLevel) ? &pic_out : NULL;
> + x265_picture *pic_recon = (cliopt.recon || !!param->analysisMode ||
> pts_queue || reconPlay || param->csvLogLevel) ? &pic_out : NULL;
> uint32_t inFrameCount = 0;
> uint32_t outFrameCount = 0;
> x265_nal *p_nal;
> @@ -698,8 +673,6 @@
> }
>
> cliopt.printStatus(outFrameCount);
> - if (numEncoded && cliopt.csvLogLevel)
> - x265_csvlog_frame(cliopt.csvfpt, *param, *pic_recon,
> cliopt.csvLogLevel);
> }
>
> /* Flush the encoder */
> @@ -730,8 +703,6 @@
> }
>
> cliopt.printStatus(outFrameCount);
> - if (numEncoded && cliopt.csvLogLevel)
> - x265_csvlog_frame(cliopt.csvfpt, *param, *pic_recon,
> cliopt.csvLogLevel);
>
> if (!numEncoded)
> break;
> @@ -746,8 +717,8 @@
> delete reconPlay;
>
> api->encoder_get_stats(encoder, &stats, sizeof(stats));
> - if (cliopt.csvfpt && !b_ctrl_c)
> - x265_csvlog_encode(cliopt.csvfpt, api->version_str, *param,
> stats, cliopt.csvLogLevel, argc, argv);
> + if (param->csvfn && !b_ctrl_c)
> + api->encoder_log(encoder, argc, argv);
> api->encoder_close(encoder);
>
> int64_t second_largest_pts = 0;
> diff -r b19bc0c7b4ec -r 12097d65c3cc source/x265.h
> --- a/source/x265.h Tue Jun 06 11:13:19 2017 +0530
> +++ b/source/x265.h Fri Jun 09 18:21:55 2017 +0530
> @@ -654,7 +654,14 @@
> * X265_LOG_FULL, default is X265_LOG_INFO */
> int logLevel;
>
> - /* Filename of CSV log. Now deprecated */
> + /* Level of csv logging. 0 is summary, 1 is frame level logging,
> + * 2 is frame level logging with performance statistics */
> + int csvLogLevel;
> +
> + /* filename of CSV log. If csvLogLevel is non-zero, the encoder will
> emit
> + * per-slice statistics to this log file in encode order. Otherwise
> the
> + * encoder will emit per-stream statistics into the log file when
> + * x265_encoder_log is called (presumably at the end of the encode) */
> const char* csvfn;
>
> /*== Internal Picture Specification ==*/
> @@ -1620,7 +1627,8 @@
> void x265_encoder_get_stats(x265_encoder *encoder, x265_stats *,
> uint32_t statsSizeBytes);
>
> /* x265_encoder_log:
> - * This function is deprecated */
> + * write a line to the configured CSV file. If a CSV filename was
> not
> + * configured, or file open failed, this function will perform no
> write. */
> void x265_encoder_log(x265_encoder *encoder, int argc, char **argv);
>
> /* x265_encoder_close:
>
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20170612/a4b286f0/attachment-0001.html>
More information about the x265-devel
mailing list