diff --git a/src/video_core/command_classes/codecs/h264.cpp b/src/video_core/command_classes/codecs/h264.cpp
index 5fb6d45ee..51ee14c13 100644
--- a/src/video_core/command_classes/codecs/h264.cpp
+++ b/src/video_core/command_classes/codecs/h264.cpp
@@ -95,7 +95,8 @@ const std::vector<u8>& H264::ComposeFrameHeader(const NvdecCommon::NvdecRegister
     const s32 pic_height = context.h264_parameter_set.frame_height_in_map_units /
                            (context.h264_parameter_set.frame_mbs_only_flag ? 1 : 2);
 
-    writer.WriteUe(16);
+    // TODO (ameerj): Where do we get this number, it seems to be particular for each stream
+    writer.WriteUe(6); // Max number of reference frames
     writer.WriteBit(false);
     writer.WriteUe(context.h264_parameter_set.pic_width_in_mbs - 1);
     writer.WriteUe(pic_height - 1);