From 5d051f38c95bb6f2f72588d55696a14939bc5fb5 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Thu, 30 Nov 2023 10:49:57 -0500 Subject: [PATCH 66/68] v4l2codecs: h264enc: Rewrite profile/level negotiation This aligns the SPS/PPS with caps and properly negotiate the level. Upstream-Status: Pending --- sys/v4l2codecs/gstv4l2codech264enc.c | 213 ++++++++++++++++----------- 1 file changed, 124 insertions(+), 89 deletions(-) diff --git a/sys/v4l2codecs/gstv4l2codech264enc.c b/sys/v4l2codecs/gstv4l2codech264enc.c index 7d52fba..911ed34 100644 --- a/sys/v4l2codecs/gstv4l2codech264enc.c +++ b/sys/v4l2codecs/gstv4l2codech264enc.c @@ -80,10 +80,11 @@ struct _GstV4l2CodecH264Enc gint width_in_macroblocks; gint height_in_macroblocks; gint qp_init, qp_max, qp_min; - guint64 targeted_bitrate; gboolean cabac; guint cabac_init_idc; + gchar *profile_name; + guint level_idc; GstV4l2CodecAllocator *sink_allocator; GstV4l2CodecAllocator *src_allocator; @@ -387,56 +388,10 @@ get_sps_aspect_ratio_idc (guint par_n, guint par_d) return 255; // Extended_SAR for custom ratios } -/* Begin of code taken from VA plugin */ -typedef struct -{ - const char *level; - int idc; - int max_macroblocks_per_second; - int max_frame_size_in_macroblocks; - int max_dpb_macroblocks; - int maximum_bitrate; - int max_cpb_size; - int minimum_compression_ratio; -} GstVaH264LevelLimits; - -static const GstVaH264LevelLimits _h264_level_limits[] = { - /* *INDENT-OFF* */ - /*level, idc, max_macroblocks_per_second, max_frame_size_in_macroblocks, - max_dpb_macroblocks, max_bitrate, max_cpb_size, minimum_compression_ratio */ - {"1", GST_H264_LEVEL_L1, 1485, 99, 396, 64, 175, 2}, - {"1b", GST_H264_LEVEL_L1B, 1485, 99, 396, 128, 350, 2}, - {"1.1", GST_H264_LEVEL_L1_1, 3000, 396, 900, 192, 500, 2}, - {"1.2", GST_H264_LEVEL_L1_2, 6000, 396, 2376, 384, 1000, 2}, - {"1.3", GST_H264_LEVEL_L1_3, 11880, 396, 2376, 768, 2000, 2}, - {"2", GST_H264_LEVEL_L2, 11880, 396, 2376, 2000, 2000, 2}, - {"2.1", GST_H264_LEVEL_L2_1, 19800, 792, 4752, 4000, 4000, 2}, - {"2.2", GST_H264_LEVEL_L2_2, 20250, 1620, 8100, 4000, 4000, 2}, - {"3", GST_H264_LEVEL_L3, 40500, 1620, 8100, 10000, 10000, 2}, - {"3.1", GST_H264_LEVEL_L3_1, 108000, 3600, 18000, 14000, 14000, 4}, - {"3.2", GST_H264_LEVEL_L3_2, 216000, 5120, 20480, 20000, 20000, 4}, - {"4", GST_H264_LEVEL_L4, 245760, 8192, 32768, 20000, 25000, 4}, - {"4.1", GST_H264_LEVEL_L4_1, 245760, 8192, 32768, 50000, 62500, 2}, - {"4.2", GST_H264_LEVEL_L4_2, 522240, 8704, 34816, 50000, 62500, 2}, - {"5", GST_H264_LEVEL_L5, 589824, 22080, 110400, 135000, 135000, 2}, - {"5.1", GST_H264_LEVEL_L5_1, 983040, 36864, 184320, 240000, 240000, 2}, - {"5.2", GST_H264_LEVEL_L5_2, 2073600, 36864, 184320, 240000, 240000, 2}, - {"6", GST_H264_LEVEL_L6, 4177920, 139264, 696320, 240000, 240000, 2}, - {"6.1", GST_H264_LEVEL_L6_1, 8355840, 139264, 696320, 480000, 480000, 2}, - {"6.2", GST_H264_LEVEL_L6_2, 16711680, 139264, 696320, 800000, 800000, 2}, - /* *INDENT-ON* */ -}; - -/* Enf of code taken from VA Plugin */ - static void gst_v4l2_codec_h264_enc_init_sps_pps (GstV4l2CodecH264Enc * self, GstVideoCodecState * state) { - gint maximum_bitrate, frame_size_in_macroblocks, - macroblocks_per_second, dpb_macroblocks, cpb_size, compression_ratio, - minimum_level_index; - memset (&self->sps, 0, sizeof (self->sps)); memset (&self->sps, 0, sizeof (self->pps)); @@ -474,41 +429,10 @@ gst_v4l2_codec_h264_enc_init_sps_pps (GstV4l2CodecH264Enc * self, self->sps.direct_8x8_inference_flag = 1; self->sps.frame_mbs_only_flag = 1; - // Calculate lowest acceptable level - gint n_levels = G_N_ELEMENTS (_h264_level_limits); - GValue bitrate = G_VALUE_INIT; - g_object_get_property (G_OBJECT (self), "bitrate", &bitrate); - self->targeted_bitrate = g_value_get_uint (&bitrate); - maximum_bitrate = self->targeted_bitrate; - frame_size_in_macroblocks = self->width * self->height / 256; - macroblocks_per_second = frame_size_in_macroblocks * state->info.fps_n / state->info.fps_d; // Assuming each macroblock is 16x16 - dpb_macroblocks = frame_size_in_macroblocks; - cpb_size = maximum_bitrate; - compression_ratio = state->info.fps_n / state->info.fps_d; - minimum_level_index = 0; - for (; minimum_level_index < n_levels; ++minimum_level_index) { - const GstVaH264LevelLimits *level = - &_h264_level_limits[minimum_level_index]; - if (macroblocks_per_second <= level->max_macroblocks_per_second - && frame_size_in_macroblocks <= level->max_frame_size_in_macroblocks - && dpb_macroblocks <= level->max_dpb_macroblocks - && maximum_bitrate <= level->maximum_bitrate - && cpb_size <= level->max_cpb_size - && compression_ratio >= level->minimum_compression_ratio) { - break; - } - } - self->sps.level_idc = _h264_level_limits[minimum_level_index].idc; - if (self->sps.level_idc == GST_H264_LEVEL_L1B) { + /* Add level specific constraint */ + self->sps.level_idc = self->level_idc; + if (self->sps.level_idc == GST_H264_LEVEL_L1B) self->sps.constraint_set3_flag = 1; - } - GST_DEBUG_OBJECT (self, "width:%d height:%d targeted_bitrate:%ld", - self->height, self->height, self->targeted_bitrate); - GST_DEBUG_OBJECT (self, - "macroblocks_per_second=%d,frame_size_in_macroblocks=%d,dpb_macroblocks=%d,maximum_bitrate=%d,cpb_size=%d,compression_ratio=%d", - macroblocks_per_second, frame_size_in_macroblocks, dpb_macroblocks, - maximum_bitrate, cpb_size, compression_ratio); - GST_DEBUG_OBJECT (self, "level_idc: %d", self->sps.level_idc); /* Crop unaligned videos */ if (self->width & 15 || self->height & 15) { @@ -582,13 +506,57 @@ gst_v4l2_codec_h264_enc_init_sps_pps (GstV4l2CodecH264Enc * self, self->pps.entropy_coding_mode_flag = self->cabac; } -static gchar * -gst_v4l2_codec_h264_enc_decide_profile (GstV4l2CodecH264Enc * self) +/* Begin of code taken from VA plugin */ +typedef struct +{ + const char *level; + int idc; + int max_mbs_per_second; + int max_frame_size_in_mbs; + int max_dpb_mbs; + int max_bitrate; + int max_cpb_size; + int minimum_compression_ratio; +} GstH264LevelLimits; + +static const GstH264LevelLimits _h264_level_limits[] = { + /* *INDENT-OFF* */ + // level, idc, mbs, frame, dpb, bitrate, cpb, cr + {"1", GST_H264_LEVEL_L1, 1485, 99, 396, 64, 175, 2}, + {"1b", GST_H264_LEVEL_L1B, 1485, 99, 396, 128, 350, 2}, + {"1.1", GST_H264_LEVEL_L1_1, 3000, 396, 900, 192, 500, 2}, + {"1.2", GST_H264_LEVEL_L1_2, 6000, 396, 2376, 384, 1000, 2}, + {"1.3", GST_H264_LEVEL_L1_3, 11880, 396, 2376, 768, 2000, 2}, + {"2", GST_H264_LEVEL_L2, 11880, 396, 2376, 2000, 2000, 2}, + {"2.1", GST_H264_LEVEL_L2_1, 19800, 792, 4752, 4000, 4000, 2}, + {"2.2", GST_H264_LEVEL_L2_2, 20250, 1620, 8100, 4000, 4000, 2}, + {"3", GST_H264_LEVEL_L3, 40500, 1620, 8100, 10000, 10000, 2}, + {"3.1", GST_H264_LEVEL_L3_1, 108000, 3600, 18000, 14000, 14000, 4}, + {"3.2", GST_H264_LEVEL_L3_2, 216000, 5120, 20480, 20000, 20000, 4}, + {"4", GST_H264_LEVEL_L4, 245760, 8192, 32768, 20000, 25000, 4}, + {"4.1", GST_H264_LEVEL_L4_1, 245760, 8192, 32768, 50000, 62500, 2}, + {"4.2", GST_H264_LEVEL_L4_2, 522240, 8704, 34816, 50000, 62500, 2}, + {"5", GST_H264_LEVEL_L5, 589824, 22080, 110400, 135000, 135000, 2}, + {"5.1", GST_H264_LEVEL_L5_1, 983040, 36864, 184320, 240000, 240000, 2}, + {"5.2", GST_H264_LEVEL_L5_2, 2073600, 36864, 184320, 240000, 240000, 2}, + {"6", GST_H264_LEVEL_L6, 4177920, 139264, 696320, 240000, 240000, 2}, + {"6.1", GST_H264_LEVEL_L6_1, 8355840, 139264, 696320, 480000, 480000, 2}, + {"6.2", GST_H264_LEVEL_L6_2, 16711680, 139264, 696320, 800000, 800000, 2}, + /* *INDENT-ON* */ +}; + +/* Enf of code taken from VA Plugin */ + +static gboolean +gst_v4l2_codec_h264_enc_decide_profile_and_level (GstV4l2CodecH264Enc * self, + GstVideoCodecState * state) { GstCaps *allowed_caps = NULL; const gchar *profile_name; + const gchar *level_name; GstStructure *structure; - gchar *ret; + gint minimum_level_index, caps_level_index; + guint bitrate, frame_size_in_mbs, mbs_per_second, dpb_mbs; g_object_get (self, "cabac", &self->cabac, "cabac-init-idc", &self->cabac_init_idc, NULL); @@ -613,10 +581,77 @@ gst_v4l2_codec_h264_enc_decide_profile (GstV4l2CodecH264Enc * self) } } - ret = g_strdup (profile_name); - gst_caps_unref (allowed_caps); + g_free (self->profile_name); + self->profile_name = g_strdup (profile_name); - return ret; + /* Calculate lowest acceptable level */ + g_object_get (self, "bitrate", &bitrate, NULL); + if (bitrate == G_MAXUINT) + bitrate = 0; + + /* table is in Kb/s, translate that number */ + bitrate /= 1000; + + frame_size_in_mbs = self->width_in_macroblocks * self->height_in_macroblocks; + mbs_per_second = frame_size_in_mbs * state->info.fps_n / state->info.fps_d; + + /* Only one reference is supported at the moment */ + dpb_mbs = frame_size_in_mbs; + + /* Ignoring CPB as we don't implement HRD */ + + for (minimum_level_index = 0; + minimum_level_index < G_N_ELEMENTS (_h264_level_limits); + minimum_level_index++) { + const GstH264LevelLimits *level = &_h264_level_limits[minimum_level_index]; + if (mbs_per_second <= level->max_mbs_per_second + && frame_size_in_mbs <= level->max_frame_size_in_mbs + && dpb_mbs <= level->max_dpb_mbs && bitrate <= level->max_bitrate) { + break; + } + } + + if (minimum_level_index >= G_N_ELEMENTS (_h264_level_limits)) { + GST_ELEMENT_ERROR (self, CORE, NEGOTIATION, ("Unsupported H.264 level."), + ("The minimum level required for this stream is not supported.")); + gst_caps_unref (allowed_caps); + return FALSE; + } + + level_name = gst_structure_get_string (structure, "level"); + caps_level_index = 0; + if (level_name) { + for (; caps_level_index < G_N_ELEMENTS (_h264_level_limits); + caps_level_index++) { + const GstH264LevelLimits *level = &_h264_level_limits[caps_level_index]; + if (g_str_equal (level->level, level_name)) + break; + } + + if (caps_level_index >= G_N_ELEMENTS (_h264_level_limits)) { + GST_ELEMENT_ERROR (self, CORE, NEGOTIATION, + ("Unsupported H.264 level '%s'.", level_name), + ("The H.264 level provided in caps is not supported.")); + gst_caps_unref (allowed_caps); + return FALSE; + } + + if (caps_level_index < minimum_level_index) { + GST_ELEMENT_ERROR (self, CORE, NEGOTIATION, + ("H.264 level '%s' too low. At least '%s' is needed.", level_name, + _h264_level_limits[minimum_level_index].level), + ("The H.264 level provided in caps is too low.")); + gst_caps_unref (allowed_caps); + return FALSE; + } + + self->level_idc = _h264_level_limits[caps_level_index].idc; + } else { + self->level_idc = _h264_level_limits[minimum_level_index].idc; + } + + gst_caps_unref (allowed_caps); + return TRUE; } static gboolean @@ -663,8 +698,8 @@ gst_v4l2_codec_h264_enc_set_format (GstVideoEncoder * encoder, if (self->output_state) gst_video_codec_state_unref (self->output_state); - g_free (self->profile_name); - self->profile_name = gst_v4l2_codec_h264_enc_decide_profile (self); + if (!gst_v4l2_codec_h264_enc_decide_profile_and_level (self, state)) + return FALSE; caps = gst_caps_new_simple ("video/x-h264", "stream-format", G_TYPE_STRING, "byte-stream", -- 2.25.1